From 5b88f1711c2645b1e33c9a4e6ac4ced0002bc57e Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Fri, 11 Sep 2026 14:38:45 +0200 Subject: [PATCH 01/25] workflow: image simulations as an input mode of the real-data workflow Image-sim m-bias ran through the legacy bash runner and example/cfis_image_sims, a config chain frozen at an older pipeline state (no background vignets, no bkg-rms ngmix, no neighbour/mom-fail flags; its default.* symlinks dangle since e32c4fb9). An m measured that way does not calibrate the pipeline that makes the real catalogue. This makes the simulations an input of the same workflow: - input_type: data | image_sims selects $SP_CONFIG. config/cfis_image_sims is an overlay: real files only for the ingestion INIs whose input naming differs (tile Git/Uz/Fe, exposure Gie/Sp -- the sim get_images writes the real-data output names, so every later stage reads identical patterns), everything else symlinked into config/cfis. - psf_model: fake -- the simulations' true PSF. The exposure stage runs only SExtractor (background maps for the vignets); tile_vignets runs fake_interp_runner, which writes galaxy_psf from `psf_dict` under the name the shared configs read (${SP_PSF}_interp_runner). Star-bearing sims can run psfex/mccd unchanged. - unit_pre writes dashed tile numbers for sims (get_images substitutes them verbatim) and exports PSF_DICT when set. A data run's prologue is byte-identical (checked by diffing dry-run shell commands against the base), so no finished data unit is rerun by the params trigger. - bin/sp: SP_PROFILE selects profiles/; SP_RUN_CONFIG replaces (not layers on) workflow/config.yaml and is snapshotted with the code; the venv is optional when snakemake is on PATH. container.py follows both. - profiles/candide: SLURM profile for candide (node-local /tmp bound as /local/scratch for the tile store). Note: completeness.py gains the `fake` tables, and its content hash is a param on every rule, so this lands at a campaign boundary like any completeness edit. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01KNJYtm9z6VeoChxtDLYn4W --- profiles/candide/config.yaml | 34 ++++ src/shapepipe/modules/fake_interp_runner.py | 44 +++++ workflow/README.md | 28 +++ workflow/Snakefile | 62 ++++++- workflow/bin/sp | 40 ++++- workflow/config.yaml | 10 +- .../config/cfis_image_sims/config_MCCD.ini | 1 + .../config/cfis_image_sims/config_exp_Gie.ini | 104 +++++++++++ .../config/cfis_image_sims/config_exp_Sp.ini | 81 +++++++++ .../cfis_image_sims/config_exp_fake.ini | 129 ++++++++++++++ .../cfis_image_sims/config_exp_mccd.ini | 1 + .../cfis_image_sims/config_exp_psfex.ini | 1 + .../config/cfis_image_sims/config_tile_Fe.ini | 80 +++++++++ .../cfis_image_sims/config_tile_Git.ini | 98 ++++++++++ .../config/cfis_image_sims/config_tile_Mc.ini | 1 + .../cfis_image_sims/config_tile_Mh_exp.ini | 1 + .../config/cfis_image_sims/config_tile_Ms.ini | 1 + .../config_tile_Ng_template.ini | 1 + .../config_tile_PiViVi_fake.ini | 167 ++++++++++++++++++ .../config_tile_PiViVi_mccd.ini | 1 + .../config_tile_PiViVi_psfex.ini | 1 + .../config/cfis_image_sims/config_tile_Sx.ini | 1 + .../config/cfis_image_sims/config_tile_Uz.ini | 77 ++++++++ workflow/config/cfis_image_sims/default.conv | 1 + workflow/config/cfis_image_sims/default.param | 1 + workflow/config/cfis_image_sims/default.psfex | 1 + .../config/cfis_image_sims/default_exp.sex | 1 + .../cfis_image_sims/default_noimaflags.param | 1 + .../config/cfis_image_sims/default_tile.sex | 1 + .../config/cfis_image_sims/final_cat.param | 1 + .../cfis_image_sims/star_selection.setools | 1 + workflow/scripts/completeness.py | 16 +- workflow/scripts/container.py | 8 +- 33 files changed, 981 insertions(+), 15 deletions(-) create mode 100644 profiles/candide/config.yaml create mode 100644 src/shapepipe/modules/fake_interp_runner.py create mode 120000 workflow/config/cfis_image_sims/config_MCCD.ini create mode 100644 workflow/config/cfis_image_sims/config_exp_Gie.ini create mode 100644 workflow/config/cfis_image_sims/config_exp_Sp.ini create mode 100644 workflow/config/cfis_image_sims/config_exp_fake.ini create mode 120000 workflow/config/cfis_image_sims/config_exp_mccd.ini create mode 120000 workflow/config/cfis_image_sims/config_exp_psfex.ini create mode 100644 workflow/config/cfis_image_sims/config_tile_Fe.ini create mode 100644 workflow/config/cfis_image_sims/config_tile_Git.ini create mode 120000 workflow/config/cfis_image_sims/config_tile_Mc.ini create mode 120000 workflow/config/cfis_image_sims/config_tile_Mh_exp.ini create mode 120000 workflow/config/cfis_image_sims/config_tile_Ms.ini create mode 120000 workflow/config/cfis_image_sims/config_tile_Ng_template.ini create mode 100644 workflow/config/cfis_image_sims/config_tile_PiViVi_fake.ini create mode 120000 workflow/config/cfis_image_sims/config_tile_PiViVi_mccd.ini create mode 120000 workflow/config/cfis_image_sims/config_tile_PiViVi_psfex.ini create mode 120000 workflow/config/cfis_image_sims/config_tile_Sx.ini create mode 100644 workflow/config/cfis_image_sims/config_tile_Uz.ini create mode 120000 workflow/config/cfis_image_sims/default.conv create mode 120000 workflow/config/cfis_image_sims/default.param create mode 120000 workflow/config/cfis_image_sims/default.psfex create mode 120000 workflow/config/cfis_image_sims/default_exp.sex create mode 120000 workflow/config/cfis_image_sims/default_noimaflags.param create mode 120000 workflow/config/cfis_image_sims/default_tile.sex create mode 120000 workflow/config/cfis_image_sims/final_cat.param create mode 120000 workflow/config/cfis_image_sims/star_selection.setools diff --git a/profiles/candide/config.yaml b/profiles/candide/config.yaml new file mode 100644 index 000000000..67ea1bfe5 --- /dev/null +++ b/profiles/candide/config.yaml @@ -0,0 +1,34 @@ +# Snakemake profile for the ShapePipe workflow on candide (IAP). Selected by +# `SP_PROFILE=candide workflow/bin/sp run`. Mirrors profiles/nibi/config.yaml; +# only the cluster facts differ, and each is stated where it is set. +executor: slurm +# candide is a shared ~25-node cluster: cap concurrent submissions well below +# what nibi takes. +jobs: 150 +default-resources: + mem_mb: 2000 + runtime: 120 # minutes + slurm_account: cusers + # comp nodes have TmpDisk=0, so the tile_shape group (--tmp, see tile.smk's + # TILE_SLURM_EXTRA) lands on pscomp; everything else may use either. + slurm_partition: "comp,pscomp" + # n09/n17/n36: excluded in the sp_validation candide profile; n23: slurm jobs + # failed there (sp_validation 2b40c99). + slurm_extra: "'--exclude=n09,n17,n23,n36'" +software-deployment-method: [apptainer] +# /local/scratch is where tile.smk puts the node-local tile store (TILE_LOCAL, +# ~5.6 GB per real-data tile). candide has no /local, and its node-local /scratch +# is writable only per job (/scratch/$USER/$SLURM_JOB_ID, made by the prolog), +# which this line cannot name: snakemake passes it through literally, and +# --cleanenv drops SLURM_JOB_ID inside the container. So the node's /tmp (local +# disk, ~29 GB free, sticky) is bound there instead. That holds ~5 concurrent +# tile stores per node; if a campaign packs more tile_shape groups onto one node, +# this is the line to revisit. The PYTHONPATH pin is rewritten by bin/sp to the +# launch snapshot's src/; the value here only has to be a checkout's src/. +apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/n17data/mkilbing/astro/repositories/github/shapepipe/src --bind /home,/automnt,/n17data,/n23data1,/n09data --bind /tmp:/local/scratch" +latency-wait: 60 # NFS: wait for outputs to appear after a job +keep-going: true # a failed job poisons only its cone; siblings run on +rerun-incomplete: true # re-do jobs left incomplete by an unclean death +show-failed-logs: true +printshellcmds: true +rerun-triggers: [mtime, params, code, software-env] diff --git a/src/shapepipe/modules/fake_interp_runner.py b/src/shapepipe/modules/fake_interp_runner.py new file mode 100644 index 000000000..bc167f924 --- /dev/null +++ b/src/shapepipe/modules/fake_interp_runner.py @@ -0,0 +1,44 @@ +"""FAKE INTERP RUNNER. + +Module runner for ``fake_psf`` under the ``_interp_runner`` name. + +The Snakemake workflow's configs read the galaxy PSF from +``${SP_PSF}_interp_runner`` for every PSF model, so with ``psf_model: fake`` +(image simulations, true PSF) this runner stands where ``psfex_interp_runner`` +and ``mccd_interp_runner`` stand for the real data. It writes the same +``galaxy_psf`` SqliteDict, taken from the simulation's PSF dictionary instead +of a fitted model. ``fake_psf_runner`` is the same module under its original +name, used by the legacy bash job scripts. + +:Author: Martin Kilbinger + +""" + +from shapepipe.modules.fake_psf_package import fake_psf +from shapepipe.modules.module_decorator import module_runner + + +@module_runner( + version="1.0", + file_pattern=["sexcat"], + file_ext=".fits", + depends=["numpy", "astropy", "sqlitedict"], + numbering_scheme="-000-000", +) +def fake_interp_runner( + input_file_list, + run_dirs, + file_number_string, + config, + module_config_sec, + w_log, +): + """Define The Fake Interp Runner.""" + sexcat_path = input_file_list[0] + psf_dict_path = config.getexpanded(module_config_sec, "PSF_DICT_PATH") + output_path = f'{run_dirs["output"]}/galaxy_psf{file_number_string}.sqlite' + + inst = fake_psf.FakePsf(sexcat_path, psf_dict_path, output_path, w_log) + inst.process() + + return None, None diff --git a/workflow/README.md b/workflow/README.md index 55611cedc..97510c901 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -45,6 +45,34 @@ Anything other than `run`, `report`, `container`, `cancel` passes straight throu snakemake with the workflow's profile and state dir — the direct command path for `sp --unlock`, `sp --dag`, `sp exp_psf ...`. +## Image simulations + +The same workflow runs the SKiLLS image simulations used to measure the shear +multiplicative bias, so that m calibrates the pipeline that makes the real +catalogue rather than a frozen copy of it. A simulation run sets +`input_type: image_sims`, which points `$SP_CONFIG` at +`config/cfis_image_sims/`. That directory holds real files only for the stages +whose input naming differs (tile Git/Uz/Fe, exposure Gie/Sp) and for the true-PSF +model; everything else is a symlink into `config/cfis/`, so a change to the +real-data chain reaches the simulations with no second edit. Keep the diff of +each overlay file to its `cfis/` original confined to input naming. + +`psf_model: fake` is the simulations' true PSF: the exposure stage runs only +SExtractor (for the background maps the vignets read), and `tile_vignets` runs +`fake_interp_runner`, which writes the `galaxy_psf` product from `psf_dict`. +Simulations that contain stars can run `psfex` or `mccd` exactly as the data do. + +One campaign per shear branch, each with its own run config: + +```bash +SP_PROFILE=candide SP_RUN_CONFIG=/path/run_1p2z_grid_1.yaml workflow/bin/sp run +``` + +`SP_RUN_CONFIG` replaces `workflow/config.yaml` (it is not layered on it, so no +key falls back to the committed run's paths) and is snapshotted with the code; +`SP_PROFILE` picks `profiles//`. sp_validation's image-simulation workflow +drives these campaigns and measures m from their final catalogues. + ## The container image `sp container` owns which image the jobs run inside. Two layers, and the second diff --git a/workflow/Snakefile b/workflow/Snakefile index bbdfeefb6..94e52f7f8 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -44,7 +44,13 @@ from snakemake.exceptions import WorkflowError # Resolved relative to THIS file, not the working directory: snakemake runs with # --directory on /scratch (bin/sp) so .snakemake/ state never lands on /project # (group quota is a hard 27/27 TiB — a metadata write mid-run died on it live). -configfile: str(Path(workflow.snakefile).parent / "config.yaml") +# +# SP_RUN_CONFIG (set by bin/sp) REPLACES the committed config.yaml rather than +# layering on it: a run driven from outside -- one image-simulation shear branch, +# say -- must not inherit the committed run's paths for any key it omits. +_RUN_CONFIG = os.environ.get("SP_RUN_CONFIG") or str( + Path(workflow.snakefile).parent / "config.yaml") +configfile: _RUN_CONFIG # Every job's shell runs inside this container (apptainer software-deployment in # the profile); the user never types apptainer. WHICH image is the one resolution @@ -71,12 +77,40 @@ if _kind == "none": container: _image -PSF_MODELS = {"psfex", "mccd"} +# --- input type and PSF model ---------------------------------------------- +# input_type selects WHERE the pixels come from; everything downstream of the +# ingestion stages is the same chain. `image_sims` runs the SKiLLS image +# simulations through the real-data configs: the overlay config dir +# (config/cfis_image_sims) holds real files only for the ingestion INIs whose +# input naming differs, and symlinks the rest into config/cfis, so an m-bias +# measured on the sims calibrates the pipeline that makes the real catalogue. +INPUT_TYPES = {"data", "image_sims"} +INPUT_TYPE = config.get("input_type", "data") +if INPUT_TYPE not in INPUT_TYPES: + raise WorkflowError( + f"Invalid input_type={INPUT_TYPE!r}; expected one of " + f"{sorted(INPUT_TYPES)}.") + +# `fake` is the image-simulation true PSF: no exposure PSF fit, and +# fake_interp_runner writes the tile's galaxy_psf from the simulation's PSF +# dictionary (config `psf_dict`) -- named so that the configs' shared +# `${SP_PSF}_interp_runner` reads it exactly as it reads psfex_interp_runner. Simulations that contain stars can instead run +# psfex or mccd exactly as the real data does. +PSF_MODELS = {"psfex", "mccd", "fake"} PSF_MODEL = config.get("psf_model", "psfex") if PSF_MODEL not in PSF_MODELS: raise WorkflowError( f"Invalid psf_model={PSF_MODEL!r}; expected one of " f"{sorted(PSF_MODELS)}.") +if PSF_MODEL == "fake" and INPUT_TYPE != "image_sims": + raise WorkflowError( + "psf_model=fake is the image-simulation true PSF; it needs " + "input_type=image_sims.") +PSF_DICT = config.get("psf_dict") or "" +if PSF_MODEL == "fake" and not PSF_DICT: + raise WorkflowError("psf_model=fake needs `psf_dict:` (the simulation's " + "pickled PSF dictionary) in the run config.") + # --- paths ----------------------------------------------------------------- # RUN_DIR is the scratch root: bulk intermediates, sized so a batch finishes @@ -95,8 +129,9 @@ INDEX_DB = Path(OUTPUTS["index_db"]) SCRIPTS = Path(workflow.basedir) / "scripts" # The config chain is the repo's committed directory (D2). The configs and # rules that set their environment variables must be versioned together. There -# is no `config_src` knob. -CONFIG_DIR = Path(workflow.basedir) / "config" / "cfis" +# is no `config_src` knob; input_type picks between the two committed dirs. +CONFIG_DIR = Path(workflow.basedir) / "config" / { + "data": "cfis", "image_sims": "cfis_image_sims"}[INPUT_TYPE] sys.path.insert(0, str(SCRIPTS)) import build_index # noqa: E402 @@ -510,7 +545,19 @@ def unit_pre(stage, unit, *, exp_name=None, forest=None, env=None, It also exports the configured input roots as ``SP_INPUT_TILES`` and ``SP_INPUT_EXPOSURES`` and the PSF choice as ``SP_PSF`` for the committed - ini chain. + ini chain (plus ``PSF_DICT`` for the image-simulation true PSF, which the + configs read as ``${SP_PSF}_interp_runner`` = ``fake_interp_runner``). + + Every line here is part of each rule's ``params.pre`` and so of the + ``params`` rerun trigger: a line added for every rule reruns every finished + unit of a campaign on its next ``sp run``. The image-simulation lines are + therefore conditional, and a data run's prologue is byte-identical to what + it was before input_type existed. + + ``tile_numbers.txt`` carries the number in the form the INPUT files are + named with, because get_images substitutes it verbatim into + INPUT_FILE_PATTERN: dot format for the survey tiles (``CFIS.210.282.r``), + dash format for the simulations (``CFIS_simu_image-210-282``). Finally it ``rm -rf``s this stage's own fixed run dir — ShapePipe's FileHandler raises on an existing run dir, and it is how a rerun never sees @@ -535,13 +582,16 @@ def unit_pre(stage, unit, *, exp_name=None, forest=None, env=None, f"export {_THREAD_CAPS}", 'mkdir -p "$SP_RUN/output" "$SP_RUN/manifests" "$SP_RUN/logs"', ] + if PSF_DICT: + lines.append(f"export PSF_DICT='{PSF_DICT}'") if forest: lines.append(f"export SP_EXP='{forest}'") for k, v in (env or {}).items(): lines.append(f"export {k}='{v}'") if level == "tile": - lines.append(f"printf '%s\\n' '{unit}' > \"$SP_RUN/tile_numbers.txt\"") + number = unit.replace(".", "-") if INPUT_TYPE == "image_sims" else unit + lines.append(f"printf '%s\\n' '{number}' > \"$SP_RUN/tile_numbers.txt\"") else: fe = "$SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output" lines += [f'mkdir -p "{fe}"', diff --git a/workflow/bin/sp b/workflow/bin/sp index fdc3d4f59..37eefa1e6 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -24,17 +24,37 @@ # executor re-invokes it inside jobs, so it cannot live on a node-local path). # One entry point, so a fresh tmux or a restart after a crash always launches # with the right state. +# +# Three environment variables make it drivable off nibi and from outside: +# +# SP_PROFILE profiles// to launch with (default: nibi). +# SP_SNAKEMAKE_ENV venv to activate; skipped when it does not exist and a +# snakemake is already on PATH (e.g. candide). +# SP_RUN_CONFIG a run config that REPLACES workflow/config.yaml (it is not +# layered on it, so no key falls back to the committed run's +# paths). Snapshotted with the code by `sp run`. This is how +# sp_validation drives one campaign per image-simulation +# shear branch. set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # workflow/ REPO="$(dirname "$HERE")" VENV="${SP_SNAKEMAKE_ENV:-/project/def-mjhudson/cdaley/snakemake-env}" SCRIPTS="$HERE/scripts" -CONFIG="$HERE/config.yaml" +CONFIG="${SP_RUN_CONFIG:-$HERE/config.yaml}" +PROFILE="${SP_PROFILE:-nibi}" +[ -f "$CONFIG" ] || { echo "sp: run config $CONFIG does not exist" >&2; exit 2; } +[ -d "$REPO/profiles/$PROFILE" ] || { + echo "sp: no profile $REPO/profiles/$PROFILE (SP_PROFILE=$PROFILE)" >&2; exit 2; } module load apptainer/1.4.5 2>/dev/null || true -# shellcheck disable=SC1091 -source "$VENV/bin/activate" +if [ -f "$VENV/bin/activate" ]; then + # shellcheck disable=SC1091 + source "$VENV/bin/activate" +elif ! command -v snakemake >/dev/null 2>&1; then + echo "sp: no venv at $VENV and no snakemake on PATH (set SP_SNAKEMAKE_ENV)" >&2 + exit 2 +fi # Scalar reader for workflow/config.yaml. The venv is active by this point and # snakemake depends on PyYAML, so this parses the file rather than pattern-matching @@ -104,7 +124,7 @@ snapshot_code() { # rewritten HERE, in the snapshot's copy, and `sm` points --profile at that copy. # The checked-in profile keeps its literal checkout path and stays the source of # truth for the FLAGS; exactly one path is substituted, by one line of sed-work. - python - "$SNAPSHOT/profiles/nibi/config.yaml" "$SNAPSHOT/src" <<'PY' + python - "$SNAPSHOT/profiles/$PROFILE/config.yaml" "$SNAPSHOT/src" <<'PY' import pathlib, re, sys f, src = pathlib.Path(sys.argv[1]), sys.argv[2] text, n = re.subn(r'(--env PYTHONPATH=)\S+', lambda m: m.group(1) + src, @@ -137,6 +157,11 @@ out.write_text(json.dumps({ "dirty_files": status.splitlines() if status else [], }, indent=2) + "\n") PY + # An external run config is part of what is launched: snapshot it with the + # code, so a job re-parsing hours later reads the config the run started with. + if [ -n "${SP_RUN_CONFIG:-}" ]; then + cp "$SP_RUN_CONFIG" "$SNAPSHOT/run_config.yaml" + fi echo "sp: code snapshot refreshed at $SNAPSHOT" >&2 } @@ -164,9 +189,14 @@ code_root() { [ -f "$SNAPSHOT/workflow/Snakefile" ] && echo "$SNAPSHOT" || echo # index without building it and schedules no side effects. sm() { local root; root="$(code_root)" + # The run config the Snakefile reads: the snapshot's copy when this run has + # one, so every parse of the campaign -- head and jobs -- sees the same file. + if [ -n "${SP_RUN_CONFIG:-}" ] && [ -f "$root/run_config.yaml" ]; then + export SP_RUN_CONFIG="$root/run_config.yaml" + fi SP_PHASE="${SP_PHASE:-passthrough}" \ snakemake --snakefile "$root/workflow/Snakefile" \ - --profile "$root/profiles/nibi" \ + --profile "$root/profiles/$PROFILE" \ --directory "$STATE_DIR" "$@" } diff --git a/workflow/config.yaml b/workflow/config.yaml index 63b29413c..336cda5d6 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -28,9 +28,17 @@ inputs: # The container every job runs inside (apptainer software-deployment in the profile). container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif -# PSF model used by the exposure and tile interpolation stages. +# PSF model used by the exposure and tile interpolation stages: psfex, mccd, or +# fake (image simulations only: the true PSF from `psf_dict`, no exposure fit). psf_model: psfex +# Where the pixels come from: `data` (survey tiles/exposures, config/cfis) or +# `image_sims` (SKiLLS simulations, config/cfis_image_sims -- an overlay that +# differs from config/cfis only in the ingestion stages). An image_sims run +# normally sets `psf_model: fake` and `psf_dict:`, and is driven with its own run +# config through SP_RUN_CONFIG (see README, "Image simulations"). +input_type: data + # THE TWO ROOTS (D5). # # run_dir is the SCRATCH root and the $SP_RUN every config interpolates: bulk diff --git a/workflow/config/cfis_image_sims/config_MCCD.ini b/workflow/config/cfis_image_sims/config_MCCD.ini new file mode 120000 index 000000000..3ba64d26e --- /dev/null +++ b/workflow/config/cfis_image_sims/config_MCCD.ini @@ -0,0 +1 @@ +../cfis/config_MCCD.ini \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/config_exp_Gie.ini b/workflow/config/cfis_image_sims/config_exp_Gie.ini new file mode 100644 index 000000000..d2c782672 --- /dev/null +++ b/workflow/config/cfis_image_sims/config_exp_Gie.ini @@ -0,0 +1,104 @@ +# IMAGE SIMULATIONS overlay of ../cfis/config_exp_Gie.ini: identical except the lines +# marked below. Keep the diff to ../cfis/config_exp_Gie.ini confined to input naming. +# ShapePipe configuration file for: get images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Gie + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = get_images_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get exposures +[GET_IMAGES_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output + +FILE_PATTERN = exp_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + + +# Paths + +# Output path (optional, default is [FILE]:OUTPUT_DIR +# OUTPUT_PATH = input_images + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = $SP_INPUT_EXPOSURES, $SP_INPUT_EXPOSURES, $SP_INPUT_EXPOSURES + +# Input file pattern including tile number as dummy template +# sims +INPUT_FILE_PATTERN = simu_image-0000000, simu_weight-0000000, simu_flag-0000000 + +# Input file extensions +# sims +INPUT_FILE_EXT = .fits, .fits, .fits + +# Input numbering scheme, python regexp +# sims: 7-digit exposure ids +INPUT_NUMBERING = \d{7} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = image-, weight-, flag- + +# Method to retrieve images, one in 'vos', 'symlink' +RETRIEVE = symlink + +# If RETRIEVE=vos, number of attempts to download +# Optional, default=3 +N_TRY = 3 + +# Retrieve command options, optional +RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem + +#CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_Gie_prev diff --git a/workflow/config/cfis_image_sims/config_exp_Sp.ini b/workflow/config/cfis_image_sims/config_exp_Sp.ini new file mode 100644 index 000000000..9783189f3 --- /dev/null +++ b/workflow/config/cfis_image_sims/config_exp_Sp.ini @@ -0,0 +1,81 @@ +# IMAGE SIMULATIONS overlay of ../cfis/config_exp_Sp.ini: identical except the lines +# marked below. Keep the diff to ../cfis/config_exp_Sp.ini confined to input naming. +# ShapePipe configuration file for single-exposures, +# split images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Sp + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = split_exp_runner + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed exposure ID (exp split is a "tile-scheme" stage per sp_rule.py). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SPLIT_EXP_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_exp_Gie/get_images_runner/output + +FILE_PATTERN = image, weight, flag + +# Matches compressed single-exposure files +# sims: uncompressed exposures +FILE_EXT = .fits, .fits, .fits + +NUMBERING_SCHEME = -0000000 + +# OUTPUT_SUFFIX, actually file name prefixes. +# Expected keyword "flag" will lead to a behavior where the data are saved as int. +# The code also expects the image data to use the "image" suffix +# (default value in the pipeline). +OUTPUT_SUFFIX = image, weight, flag + +# Number of HDUs/CCDs of mosaic +N_HDU = 40 diff --git a/workflow/config/cfis_image_sims/config_exp_fake.ini b/workflow/config/cfis_image_sims/config_exp_fake.ini new file mode 100644 index 000000000..f09a4054b --- /dev/null +++ b/workflow/config/cfis_image_sims/config_exp_fake.ini @@ -0,0 +1,129 @@ +# ShapePipe configuration file for single-exposures. IMAGE SIMULATIONS, true PSF +# (psf_model: fake). No PSF fit: the simulations' PSF is known, and the tile's +# fake_interp_runner reads it from the PSF dictionary. What remains is the +# SExtractor pass, whose background and background_rms checkimages the tile +# vignets read. Derived from ../cfis/config_exp_psfex.ini: the [SEXTRACTOR_RUNNER] +# section is kept identical to it, and RUN_NAME stays the exp_psf stage's run dir +# (workflow/scripts/completeness.py STAGE_DIR). + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_SxSePsf + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = sextractor_runner + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = $SP_RUN/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SEXTRACTOR_RUNNER] + +# The split CCDs, and nothing else: ShapePipe generates no masks +INPUT_DIR = $SP_RUN/output/run_sp_exp_Sp/split_exp_runner/output + +# Read the instrument flag image split_exp wrote per CCD +FILE_PATTERN = image, weight, flag + +# Explicit extensions: a 3-entry FILE_PATTERN override must not fall back on +# the decorator's 4-entry FILE_EXT default (length check fails at startup) +FILE_EXT = .fits, .fits, .fits + +NUMBERING_SCHEME = -0000000-0 + +# SExtractor executable path +EXEC_PATH = source-extractor + +# SExtractor configuration files +DOT_SEX_FILE = $SP_CONFIG/default_exp.sex +DOT_PARAM_FILE = $SP_CONFIG//default.param +DOT_CONV_FILE = $SP_CONFIG/default.conv + +# Use input weight image if True +WEIGHT_IMAGE = True + +# Use input flag image if True +FLAG_IMAGE = True + +# Use input PSF file if True +PSF_FILE = False + +# Use distinct image for detection (SExtractor in +# dual-image mode) if True. +DETECTION_IMAGE = False + +# Distinct weight image for detection (SExtractor +# in dual-image mode) +DETECTION_WEIGHT = False + +# True if photometry zero-point is to be read from exposure image header +ZP_FROM_HEADER = True + +# If ZP_FROM_HEADER is True, zero-point key name +ZP_KEY = PHOTZP + +# Background information from image header. +# If BKG_FROM_HEADER is True, background value will be read from header. +# In that case, the value of BACK_TYPE will be set atomatically to MANUAL. +# This is used e.g. for the LSB images. +BKG_FROM_HEADER = False +# LSB images: +# BKG_FROM_HEADER = True + +# If BKG_FROM_HEADER is True, background value key name +# LSB images: +#BKG_KEY = IMMODE + +# Type of image check (optional), default not used, can be a list of +# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, MINIBACK_RMS, -BACKGROUND, +# FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, APERTURES +CHECKIMAGE = BACKGROUND, BACKGROUND_RMS + +# File name suffix for the output sextractor files (optional) SUFFIX = tile +SUFFIX = sexcat + +## Post-processing + +# Not required for single exposures +MAKE_POST_PROCESS = FALSE diff --git a/workflow/config/cfis_image_sims/config_exp_mccd.ini b/workflow/config/cfis_image_sims/config_exp_mccd.ini new file mode 120000 index 000000000..58166d49c --- /dev/null +++ b/workflow/config/cfis_image_sims/config_exp_mccd.ini @@ -0,0 +1 @@ +../cfis/config_exp_mccd.ini \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/config_exp_psfex.ini b/workflow/config/cfis_image_sims/config_exp_psfex.ini new file mode 120000 index 000000000..ea86cb50f --- /dev/null +++ b/workflow/config/cfis_image_sims/config_exp_psfex.ini @@ -0,0 +1 @@ +../cfis/config_exp_psfex.ini \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/config_tile_Fe.ini b/workflow/config/cfis_image_sims/config_tile_Fe.ini new file mode 100644 index 000000000..2a003ea9c --- /dev/null +++ b/workflow/config/cfis_image_sims/config_tile_Fe.ini @@ -0,0 +1,80 @@ +# IMAGE SIMULATIONS overlay of ../cfis/config_tile_Fe.ini: identical except the lines +# marked below. Keep the diff to ../cfis/config_tile_Fe.ini confined to input naming. +# ShapePipe configuration file for: find exposures + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Fe + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = find_exposures_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get tiles +[FIND_EXPOSURES_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Git/get_images_runner/output + +FILE_PATTERN = CFIS_image + +FILE_EXT = .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Column number of exposure name in FITS header +# sims: HISTORY layout of the simulated tiles +COLNUM = 2 + +# Prefix to remove from exposure name +# sims +EXP_PREFIX = simu_image- + diff --git a/workflow/config/cfis_image_sims/config_tile_Git.ini b/workflow/config/cfis_image_sims/config_tile_Git.ini new file mode 100644 index 000000000..69fcff540 --- /dev/null +++ b/workflow/config/cfis_image_sims/config_tile_Git.ini @@ -0,0 +1,98 @@ +# IMAGE SIMULATIONS overlay of ../cfis/config_tile_Git.ini: identical except the lines +# marked below. Keep the diff to ../cfis/config_tile_Git.ini confined to input naming. +# ShapePipe configuration file for: get tile images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Git + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = get_images_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get tiles +[GET_IMAGES_RUNNER] + +FILE_PATTERN = tile_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = + +# Paths + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = $SP_INPUT_TILES, $SP_INPUT_TILES + +# Input file pattern including tile number as dummy template +# sims +INPUT_FILE_PATTERN = CFIS_simu_image-000-000, CFIS_simu_weight-000-000 + +# Input file extensions +# sims: weight uncompressed +INPUT_FILE_EXT = .fits, .fits + +# Input numbering scheme, python regexp +# sims: dash-numbered tiles +INPUT_NUMBERING = \d{3}-\d{3} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight- + +# Copy/download method, one in 'vos', 'symlink' +RETRIEVE = symlink + +# If RETRIEVE=vos, number of attempts to download +# Optional, default=3 +N_TRY = 3 + +# Copy command options, optional +RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem + +#CHECK_EXISTING_DIR = $SP_RUN/data_tiles diff --git a/workflow/config/cfis_image_sims/config_tile_Mc.ini b/workflow/config/cfis_image_sims/config_tile_Mc.ini new file mode 120000 index 000000000..eeba9f53e --- /dev/null +++ b/workflow/config/cfis_image_sims/config_tile_Mc.ini @@ -0,0 +1 @@ +../cfis/config_tile_Mc.ini \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/config_tile_Mh_exp.ini b/workflow/config/cfis_image_sims/config_tile_Mh_exp.ini new file mode 120000 index 000000000..5e8e91716 --- /dev/null +++ b/workflow/config/cfis_image_sims/config_tile_Mh_exp.ini @@ -0,0 +1 @@ +../cfis/config_tile_Mh_exp.ini \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/config_tile_Ms.ini b/workflow/config/cfis_image_sims/config_tile_Ms.ini new file mode 120000 index 000000000..f301e8673 --- /dev/null +++ b/workflow/config/cfis_image_sims/config_tile_Ms.ini @@ -0,0 +1 @@ +../cfis/config_tile_Ms.ini \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/config_tile_Ng_template.ini b/workflow/config/cfis_image_sims/config_tile_Ng_template.ini new file mode 120000 index 000000000..c00711e7b --- /dev/null +++ b/workflow/config/cfis_image_sims/config_tile_Ng_template.ini @@ -0,0 +1 @@ +../cfis/config_tile_Ng_template.ini \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/config_tile_PiViVi_fake.ini b/workflow/config/cfis_image_sims/config_tile_PiViVi_fake.ini new file mode 100644 index 000000000..4651c4e7c --- /dev/null +++ b/workflow/config/cfis_image_sims/config_tile_PiViVi_fake.ini @@ -0,0 +1,167 @@ +# ShapePipe configuration file for tile, from detection up to shape measurement. +# IMAGE SIMULATIONS, true PSF (psf_model: fake). Derived from +# ../cfis/config_tile_PiViVi_psfex.ini: only the PSF section differs. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_PiViVi + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +#MODULE = psfex_interp_runner, + +MODULE = ${SP_PSF}_interp_runner, vignetmaker_runner, vignetmaker_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory. +# +# NODE-LOCAL, not $SP_RUN/output. This run produces the tile's ~5.6 GB vignette +# store, which is read only by ngmix and make_cat -- both of which run in the +# same fused group job on the same node (workflow/rules/tile.smk, TILE_GROUP). +# Writing it to the node's NVMe instead of NFS scratch is the whole point: it +# removes the store from the per-tile scratch high-water AND removes ~163 GB of +# small random NFS reads per tile. +# +# $SP_VIGNET_OUT is exported by every member of that group (TILE_LOCAL) as +# $SLURM_TMPDIR/sp-tile/output. Set it to $SP_RUN/output to keep the store on +# shared storage. It is NOT optional: ShapePipe's config expansion is strict +# (pipeline/config.py::_expandvars_strict), so an unset value is a loud error. +# +# The INPUT_DIRs below stay on $SP_RUN -- they are Sx / Mh_exp / Fe products on +# scratch, and every path here is absolute, so nothing resolves through a run +# log that this split would break. +OUTPUT_DIR = $SP_VIGNET_OUT + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[FAKE_INTERP_RUNNER] + +# The tile's multi-epoch SExtractor catalogue: its EPOCH HDUs carry, per object, +# the exposure and CCD each epoch fell on, which key the PSF dictionary. +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output + +FILE_PATTERN = sexcat + +FILE_EXT = .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# The simulation's pickled PSF dictionary ("-" -> PSF), exported by +# the workflow from the run config's `psf_dict:` +PSF_DICT_PATH = $PSF_DICT + + +# Create vignets for tiles weights +[VIGNETMAKER_RUNNER_RUN_1] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Uz/uncompress_fits_runner/output + +FILE_PATTERN = sexcat, CFIS_weight + +FILE_EXT = .fits, .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +MASKING = False +MASK_VALUE = 0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = CLASSIC + +# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) +COORD = PIX +POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE + +# Vignet size in pixels +STAMP_SIZE = 51 + +# Output file name prefix, file name is _vignet.fits +PREFIX = weight + + +[VIGNETMAKER_RUNNER_RUN_2] + +# Create multi-epoch vignets for tiles corresponding to +# positions on single-exposures + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output, $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output + +FILE_PATTERN = sexcat, log_exp_headers, exp_numbers + +FILE_EXT = .fits, .sqlite, .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +MASKING = False +MASK_VALUE = 0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = MULTI-EPOCH + +# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) +COORD = SPHE +POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD + +# Vignet size in pixels +STAMP_SIZE = 51 + +# Output file name prefix, file name is vignet.fits +PREFIX = + +# Additional parameters for path and file pattern corresponding to single-exposure +# run outputs. ME_IMAGE_EXP_DIR/ME_IMAGE_EXP_RUNNERS replace ME_IMAGE_DIR for +# the v2.0 per-exposure pipeline; output dirs are discovered by scanning $SP_EXP. +ME_IMAGE_EXP_DIR = $SP_EXP +ME_IMAGE_EXP_RUNNERS = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner, sextractor_runner +ME_IMAGE_PATTERN = flag, image, weight, background, background_rms diff --git a/workflow/config/cfis_image_sims/config_tile_PiViVi_mccd.ini b/workflow/config/cfis_image_sims/config_tile_PiViVi_mccd.ini new file mode 120000 index 000000000..6271fb312 --- /dev/null +++ b/workflow/config/cfis_image_sims/config_tile_PiViVi_mccd.ini @@ -0,0 +1 @@ +../cfis/config_tile_PiViVi_mccd.ini \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/config_tile_PiViVi_psfex.ini b/workflow/config/cfis_image_sims/config_tile_PiViVi_psfex.ini new file mode 120000 index 000000000..5f120bca9 --- /dev/null +++ b/workflow/config/cfis_image_sims/config_tile_PiViVi_psfex.ini @@ -0,0 +1 @@ +../cfis/config_tile_PiViVi_psfex.ini \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/config_tile_Sx.ini b/workflow/config/cfis_image_sims/config_tile_Sx.ini new file mode 120000 index 000000000..81637de66 --- /dev/null +++ b/workflow/config/cfis_image_sims/config_tile_Sx.ini @@ -0,0 +1 @@ +../cfis/config_tile_Sx.ini \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/config_tile_Uz.ini b/workflow/config/cfis_image_sims/config_tile_Uz.ini new file mode 100644 index 000000000..18a13e6eb --- /dev/null +++ b/workflow/config/cfis_image_sims/config_tile_Uz.ini @@ -0,0 +1,77 @@ +# IMAGE SIMULATIONS overlay of ../cfis/config_tile_Uz.ini: identical except the lines +# marked below. Keep the diff to ../cfis/config_tile_Uz.ini confined to input naming. +# ShapePipe configuration file for: uncompress FITS image + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Uz + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = uncompress_fits_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options +[UNCOMPRESS_FITS_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Git/get_images_runner/output + +FILE_PATTERN = CFIS_weight + +# sims: the weight is not compressed +FILE_EXT = .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Input HDU of image data, optional, default=0 +# sims: plain FITS, data in the primary HDU +HDU_DATA = 0 + +# Output file pattern +OUTPUT_PATTERN = CFIS_weight diff --git a/workflow/config/cfis_image_sims/default.conv b/workflow/config/cfis_image_sims/default.conv new file mode 120000 index 000000000..bd71df850 --- /dev/null +++ b/workflow/config/cfis_image_sims/default.conv @@ -0,0 +1 @@ +../cfis/default.conv \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/default.param b/workflow/config/cfis_image_sims/default.param new file mode 120000 index 000000000..49e000314 --- /dev/null +++ b/workflow/config/cfis_image_sims/default.param @@ -0,0 +1 @@ +../cfis/default.param \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/default.psfex b/workflow/config/cfis_image_sims/default.psfex new file mode 120000 index 000000000..65dd716d1 --- /dev/null +++ b/workflow/config/cfis_image_sims/default.psfex @@ -0,0 +1 @@ +../cfis/default.psfex \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/default_exp.sex b/workflow/config/cfis_image_sims/default_exp.sex new file mode 120000 index 000000000..4c83dea8e --- /dev/null +++ b/workflow/config/cfis_image_sims/default_exp.sex @@ -0,0 +1 @@ +../cfis/default_exp.sex \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/default_noimaflags.param b/workflow/config/cfis_image_sims/default_noimaflags.param new file mode 120000 index 000000000..75451801e --- /dev/null +++ b/workflow/config/cfis_image_sims/default_noimaflags.param @@ -0,0 +1 @@ +../cfis/default_noimaflags.param \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/default_tile.sex b/workflow/config/cfis_image_sims/default_tile.sex new file mode 120000 index 000000000..8770da87e --- /dev/null +++ b/workflow/config/cfis_image_sims/default_tile.sex @@ -0,0 +1 @@ +../cfis/default_tile.sex \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/final_cat.param b/workflow/config/cfis_image_sims/final_cat.param new file mode 120000 index 000000000..c286947d0 --- /dev/null +++ b/workflow/config/cfis_image_sims/final_cat.param @@ -0,0 +1 @@ +../cfis/final_cat.param \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/star_selection.setools b/workflow/config/cfis_image_sims/star_selection.setools new file mode 120000 index 000000000..11a570187 --- /dev/null +++ b/workflow/config/cfis_image_sims/star_selection.setools @@ -0,0 +1 @@ +../cfis/star_selection.setools \ No newline at end of file diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index ff776e6e7..0a22179f5 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -118,6 +118,13 @@ # config_exp_mccd enables the ten meanshape and six histogram plots. "mccd_plots_runner": dict(expect=16, warn=True), }, + # Image simulations with the true PSF (psf_model: fake): no PSF fit on + # the exposures, only the SExtractor pass whose background/background_rms + # checkimages the tile vignets read (config_exp_fake.ini in + # config/cfis_image_sims). Same 40 CCDs x (sexcat, background, rms). + "fake": { + "sextractor_runner": dict(expect=120), + }, }, # --- tile post --- @@ -138,6 +145,13 @@ "vignetmaker_runner_run_1": dict(expect=1, warn=True), "vignetmaker_runner_run_2": dict(expect=5, warn=True), }, + # Image simulations: fake_interp_runner writes the same galaxy_psf + # sqlite psfex_interp_runner writes, from the simulation's PSF dictionary. + "fake": { + "fake_interp_runner": dict(expect=1), + "vignetmaker_runner_run_1": dict(expect=1), + "vignetmaker_runner_run_2": dict(expect=5), + }, }, # One check runs inside run_sp_tile_ngmix_Ng${SP_NGMIX_CHUNK}u per chunk, # so expect=1 is the correct per-chunk count. @@ -188,7 +202,7 @@ def check_counts(stage, run_dir): table = table[psf_model] except KeyError as exc: raise ValueError( - f"Invalid SP_PSF={psf_model!r}; expected one of psfex, mccd." + f"Invalid SP_PSF={psf_model!r}; expected one of {sorted(table)}." ) from exc details, ok = [], True for runner, spec in table.items(): diff --git a/workflow/scripts/container.py b/workflow/scripts/container.py index afc7019ed..c35f91ad8 100644 --- a/workflow/scripts/container.py +++ b/workflow/scripts/container.py @@ -59,14 +59,18 @@ class ContainerError(Exception): # The single source of truth for the fallback image: the workflow's own # `container:` key, which is also what the Snakefile reads. Written down once, # here, so the CLI and the workflow cannot disagree about the default. -CONFIG_FILE = Path(__file__).resolve().parents[1] / "config.yaml" +# SP_RUN_CONFIG replaces it for a run driven with its own config (bin/sp). +CONFIG_FILE = Path(os.environ.get("SP_RUN_CONFIG") + or Path(__file__).resolve().parents[1] / "config.yaml") CONFIG_KEY = "container" # The profile whose `apptainer-args:` every workflow job runs under. `exec` reads # it at runtime rather than restating it, so a one-off `sp container exec` and a # job see the same environment (the PYTHONPATH pin above all: a divergence there # means the one-off imports a different src/ than the workflow does). -PROFILE_FILE = Path(__file__).resolve().parents[2] / "profiles" / "nibi" / "config.yaml" +# SP_PROFILE names it (profiles//, default nibi), as it does for bin/sp. +PROFILE_FILE = (Path(__file__).resolve().parents[2] / "profiles" + / os.environ.get("SP_PROFILE", "nibi") / "config.yaml") # ~/.cache/shapepipe by default; SP_CACHE_DIR moves the whole cache (e.g. onto # a filesystem with room), XDG_CACHE_HOME moves it with everything else. From 4e17feee3e1246a4fdcdd8ebdb9a085ddefaf3ad Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Fri, 11 Sep 2026 21:03:07 +0200 Subject: [PATCH 02/25] workflow: image-sims merge column list, and the merge reads the workflow layout create_final_cat.py -I globbed run_sp_tile_Mc_*, the legacy bash runner's dated run dir; the workflow writes an undated run_sp_tile_Mc. Accept both. The overlay's final_cat.param was a symlink to the real-data list, which names columns make_cat does not write (IMAFLAGS_ISO: the tile SExtractor has no flag image; NGMIX_MOM_FAIL: written as NGMIX_MCAL_TYPES_FAIL) and omits NUMBER and NGMIX_MCAL_TYPES_FAIL, which sp_validation's image-sim extract reads. It is now a real file: the legacy example/cfis_image_sims list plus NGMIX_NEIGHBOUR_FLAG, checked column by column against a smoke-run final_cat. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01RSjZrhgsGCLAJCLb5Zjmc3 --- scripts/python/create_final_cat.py | 4 +- workflow/Snakefile | 3 +- .../config/cfis_image_sims/final_cat.param | 114 +++++++++++++++++- 3 files changed, 118 insertions(+), 3 deletions(-) mode change 120000 => 100644 workflow/config/cfis_image_sims/final_cat.param diff --git a/scripts/python/create_final_cat.py b/scripts/python/create_final_cat.py index 2b583b857..d98232818 100755 --- a/scripts/python/create_final_cat.py +++ b/scripts/python/create_final_cat.py @@ -391,7 +391,9 @@ def process(params): if params["image_sims"]: patch_name = params["patch"] - run_prefix = "run_sp_tile_Mc_*" + # The Snakemake workflow writes an undated run_sp_tile_Mc; the legacy + # bash runner wrote run_sp_tile_Mc_. + run_prefix = "run_sp_tile_Mc*" else: patch_name = rf"P{params['patch']}" run_prefix = "run_sp_tile_Mc_*" diff --git a/workflow/Snakefile b/workflow/Snakefile index 94e52f7f8..3162276b7 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -82,7 +82,8 @@ container: _image # ingestion stages is the same chain. `image_sims` runs the SKiLLS image # simulations through the real-data configs: the overlay config dir # (config/cfis_image_sims) holds real files only for the ingestion INIs whose -# input naming differs, and symlinks the rest into config/cfis, so an m-bias +# input naming differs, the fake-PSF INIs, and the merge column list +# (final_cat.param), and symlinks the rest into config/cfis, so an m-bias # measured on the sims calibrates the pipeline that makes the real catalogue. INPUT_TYPES = {"data", "image_sims"} INPUT_TYPE = config.get("input_type", "data") diff --git a/workflow/config/cfis_image_sims/final_cat.param b/workflow/config/cfis_image_sims/final_cat.param deleted file mode 120000 index c286947d0..000000000 --- a/workflow/config/cfis_image_sims/final_cat.param +++ /dev/null @@ -1 +0,0 @@ -../cfis/final_cat.param \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/final_cat.param b/workflow/config/cfis_image_sims/final_cat.param new file mode 100644 index 000000000..5bb0bdd17 --- /dev/null +++ b/workflow/config/cfis_image_sims/final_cat.param @@ -0,0 +1,113 @@ +# Final-catalogue column selection for the image simulations. +# +# create_final_cat.py -I selects exactly these columns from each simulated +# tile's make_cat output (tiles///output/run_sp_tile_Mc/...) into +# final_cat_{sim}.hdf5, and sp_validation's extract step reads the same list; +# every entry must exist in that catalogue. +# +# A real file in this overlay dir, not a symlink into ../cfis/, because the +# real-data list names columns make_cat does not write for the simulations: +# - IMAFLAGS_ISO : the tile SExtractor runs without a flag image +# (config_tile_Sx.ini: FLAG_IMAGE = False) +# - NGMIX_MOM_FAIL : make_cat writes it as NGMIX_MCAL_TYPES_FAIL +# and it omits two that sp_validation's image-sim extract step reads: NUMBER and +# NGMIX_MCAL_TYPES_FAIL (the metacal moments-failure cut). + +# coordinates +XWIN_WORLD +YWIN_WORLD + +# tile ID, for plot of tile-dependent additive bias. +TILE_ID + +# SExtractor number +NUMBER + +# flags +FLAGS +NGMIX_MCAL_FLAGS +NGMIX_MCAL_TYPES_FAIL +NGMIX_NEIGHBOUR_FLAG + +# PSF ellipticity (original image PSF) +NGMIX_G1_PSF_ORIG_NOSHEAR +NGMIX_G2_PSF_ORIG_NOSHEAR + +# Number of epochs (exposures) +N_EPOCH +NGMIX_N_EPOCH + +## Shape measurement outputs +## Ngmix: model fitting + +# galaxy ellipticity +NGMIX_G1_1M +NGMIX_G2_1M +NGMIX_G1_1P +NGMIX_G2_1P +NGMIX_G1_2M +NGMIX_G2_2M +NGMIX_G1_2P +NGMIX_G2_2P +NGMIX_G1_NOSHEAR +NGMIX_G2_NOSHEAR +NGMIX_G1_ERR_NOSHEAR +NGMIX_G2_ERR_NOSHEAR + +# flags +NGMIX_FLAGS_1M +NGMIX_FLAGS_1P +NGMIX_FLAGS_2M +NGMIX_FLAGS_2P +NGMIX_FLAGS_NOSHEAR + +# size and error +NGMIX_T_1M +NGMIX_T_1P +NGMIX_T_2M +NGMIX_T_2P +NGMIX_T_NOSHEAR +NGMIX_T_ERR_1M +NGMIX_T_ERR_1P +NGMIX_T_ERR_2M +NGMIX_T_ERR_2P +NGMIX_T_ERR_NOSHEAR + +# reconvolved PSF size (used by the metacal size cut / Tpsf) +NGMIX_T_PSF_RECONV_1M +NGMIX_T_PSF_RECONV_1P +NGMIX_T_PSF_RECONV_2M +NGMIX_T_PSF_RECONV_2P +NGMIX_T_PSF_RECONV_NOSHEAR + +# flux and error +NGMIX_FLUX_1M +NGMIX_FLUX_1P +NGMIX_FLUX_2M +NGMIX_FLUX_2P +NGMIX_FLUX_NOSHEAR +NGMIX_FLUX_ERR_1M +NGMIX_FLUX_ERR_1P +NGMIX_FLUX_ERR_2M +NGMIX_FLUX_ERR_2P +NGMIX_FLUX_ERR_NOSHEAR + +# magnitudes +MAG_AUTO +MAGERR_AUTO +MAG_WIN +MAGERR_WIN +FLUX_AUTO +FLUXERR_AUTO +FLUX_APER +FLUXERR_APER +FLUX_RADIUS + +# SNR from SExtractor +SNR_WIN + +FWHM_IMAGE +FWHM_WORLD + +# PSF size measured on original image +NGMIX_T_PSF_ORIG_NOSHEAR From 716b160798362b75ee447709646a9fe4ae823d38 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Fri, 11 Sep 2026 21:22:37 +0200 Subject: [PATCH 03/25] workflow: campaign-unique node-local tile store for image sims; candide excludes The node-local tile store was /local/scratch/sp-, unique within a campaign but not across concurrent ones. The image-simulation shear branches are concurrent campaigns over the same tile IDs, and on candide /local/scratch is the node's shared /tmp: two branches' tile_shape jobs on n09 shared one store, one tile_vignets wiped it under the other's ngmix, and both failed. The failure can also be silent -- ngmix reading the other branch's vignets. image_sims now prefixes the store name with a hash of the run dir; data campaigns keep the bare name, so their shell commands (a rerun trigger) are unchanged. The tile_shape group's own slurm_extra (the --tmp floor) replaced the profile default and dropped candide's node excludes; its jobs ran on n09 and n17. profiles/candide restates both for the four members via set-resources (slurm_extra only; the rules keep their threads and attempt-scaled mem_mb, checked in a dry run). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01RSjZrhgsGCLAJCLb5Zjmc3 --- profiles/candide/config.yaml | 14 ++++++++++++++ workflow/rules/tile.smk | 17 +++++++++++++++-- 2 files changed, 29 insertions(+), 2 deletions(-) diff --git a/profiles/candide/config.yaml b/profiles/candide/config.yaml index 67ea1bfe5..b4b1d63c2 100644 --- a/profiles/candide/config.yaml +++ b/profiles/candide/config.yaml @@ -32,3 +32,17 @@ rerun-incomplete: true # re-do jobs left incomplete by an unclean deat show-failed-logs: true printshellcmds: true rerun-triggers: [mtime, params, code, software-env] +# The tile_shape group (tile.smk) sets its own slurm_extra (TILE_SLURM_EXTRA, +# the --tmp floor), which REPLACES the default above and so drops the node +# excludes: its jobs landed on n09 and n17. Restate both, identically on all +# four members (a group's string resources must agree). Only slurm_extra is +# overridden; the rules keep their own threads and attempt-scaled mem_mb. +set-resources: + tile_vignets: + slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n36'" + tile_ngmix: + slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n36'" + tile_merge_cats: + slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n36'" + tile_make_cat: + slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n36'" diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 68d6da9ba..8e3cf5171 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -130,6 +130,11 @@ is what downstream selections cut on. # `--rerun-triggers mtime code software-env`, accepting that clean_exposure's # consumer-set staleness detection (which rides on params) is off for that # invocation. +# Per-campaign prefix of the node-local store name; see tile_local(). +LOCAL_TAG = (hashlib.sha1(str(RUN_DIR).encode()).hexdigest()[:8] + "-" + if INPUT_TYPE == "image_sims" else "") + + def tile_local(tile): """The node-local prologue, as bash, for one tile. @@ -144,7 +149,15 @@ def tile_local(tile): the container (probe job 20798618) -- and the tile id is a wildcard snakemake substitutes at DAG time, so the shell string carries a concrete path with no `$` left for anything to escape. One group job per tile means - the name cannot collide; the sticky bit means nobody else can remove it. + the name cannot collide within a campaign; the sticky bit means nobody else + can remove it. Across campaigns it can: the image-simulation shear branches + are concurrent campaigns over the SAME tile IDs, and on candide + `/local/scratch` is the node's shared `/tmp`. Two branches' fused jobs on one + node then share one store -- a second `tile_vignets` wipes and rewrites it + under the first branch's ngmix, and the first `tile_make_cat`'s EXIT trap + deletes it under the second (seen on candide n09). So image_sims prefixes + the name with LOCAL_TAG, a hash of the run dir. Data campaigns keep the + bare tile name, so their shell commands -- a rerun trigger -- are unchanged. What we give up is Slurm's own cleanup of `$SLURM_TMPDIR`. TILE_VIGNET_FRESH reclaims a stale directory on the next attempt for the same tile, the trap @@ -201,7 +214,7 @@ if [ ! -d /local/scratch ]; then echo " profiles/nibi/config.yaml apptainer-args must carry --bind /local" >&2 exit 1 fi -export SP_LOCAL="/local/scratch/sp-{tile}" +export SP_LOCAL="/local/scratch/sp-{LOCAL_TAG}{tile}" export SP_VIGNET_OUT="$SP_LOCAL/output" export NGMIX_VIGNET_DIR="$SP_LOCAL/output/run_sp_tile_PiViVi" export SP_WCS_DIR="$SP_LOCAL/wcs" From fdc9db408b10e6444648cf435482e4af76826149 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Sat, 12 Sep 2026 00:32:05 +0200 Subject: [PATCH 04/25] workflow: port the MCCD exposure chain to the workflow grammar config_exp_mccd.ini was never ported: it read split_exp and mask_runner outputs through INPUT_MODULE (mask_runner is gone since #847, and with it the pipeline_flag files), had no mask_query, chained setools and MCCD through last:, and left RUN_DATETIME at its default, so the run dir was dated and nothing downstream could find it. psf_model: mccd could not run. It is now the PSFEx chain up to the star selection -- same split-CCD inputs, flag image, mask_query and setools, so both models fit the same stars -- followed by mccd_preprocessing (all CCDs of the exposure into one training and one test catalogue) and mccd_fit_val (fitted_model-.npy, which the tiles' mccd_interp reads, plus validation_psf-.fits). merge_starcat and mccd_plots are dropped: they are campaign-level diagnostics, not per-exposure products. The completeness table's MCCD branch had preprocessing at 80 (per CCD; it is 2 per exposure), listed merge_starcat and the plots, and was all warning-only. It now carries the counts this chain produces, mandatory as for PSFEx: an exposure without a model has no PSF on any tile it overlaps. Checked on SKiLLS star sim 1z2z (exposure 2086792): 120/40/80/2 through preprocessing. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VSU9pPZjpBjzWorAL88aKQ --- workflow/config/cfis/config_exp_mccd.ini | 155 ++++++++++------------- workflow/scripts/completeness.py | 26 ++-- 2 files changed, 80 insertions(+), 101 deletions(-) diff --git a/workflow/config/cfis/config_exp_mccd.ini b/workflow/config/cfis/config_exp_mccd.ini index 635952ca7..9a6a6f7bf 100644 --- a/workflow/config/cfis/config_exp_mccd.ini +++ b/workflow/config/cfis/config_exp_mccd.ini @@ -1,5 +1,8 @@ -# ShapePipe configuration file for single-exposures, MCCD PSF model. -# Process exposures after masking, from star detection to PSF model. +# ShapePipe configuration file for single-exposures. MCCD PSF model. +# Process exposures after splitting, from star detection to PSF model. +# ShapePipe generates no masks: SExtractor reads the instrument flag image +# delivered with the exposure, and mask_query flags detections against the +# external healsparse maps (see [MASK_QUERY_RUNNER] below). ## Default ShapePipe options @@ -12,16 +15,15 @@ VERBOSE = True RUN_NAME = run_sp_exp_SxSePsf # Add date and time to RUN_NAME, optional, default: True -; RUN_DATETIME = False +RUN_DATETIME = False ## ShapePipe execution options [EXECUTION] # Module name, single string or comma-separated list of valid module runner names -MODULE = sextractor_runner, setools_runner, - mccd_preprocessing_runner, mccd_fit_val_runner, - merge_starcat_runner, mccd_plots_runner +MODULE = sextractor_runner, mask_query_runner, setools_runner, + mccd_preprocessing_runner, mccd_fit_val_runner # Run mode, SMP or MPI MODE = SMP @@ -47,7 +49,7 @@ OUTPUT_DIR = $SP_RUN/output [JOB] # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 +SMP_BATCH_SIZE = 8 # Timeout value (optional), default is None, i.e. no timeout limit applied TIMEOUT = 96:00:00 @@ -57,16 +59,15 @@ TIMEOUT = 96:00:00 [SEXTRACTOR_RUNNER] -# Somehow this works but not -# - omitting -# - $SP_RUN/output -#INPUT_DIR = . +# The split CCDs, and nothing else: ShapePipe generates no masks +INPUT_DIR = $SP_RUN/output/run_sp_exp_Sp/split_exp_runner/output -# Input from two modules -INPUT_MODULE = split_exp_runner, mask_runner +# Read the instrument flag image split_exp wrote per CCD +FILE_PATTERN = image, weight, flag -# Read pipeline flag files created by mask module -FILE_PATTERN = image, weight, pipeline_flag +# Explicit extensions: a 3-entry FILE_PATTERN override must not fall back on +# the decorator's 4-entry FILE_EXT default (length check fails at startup) +FILE_EXT = .fits, .fits, .fits NUMBERING_SCHEME = -0000000-0 @@ -127,52 +128,72 @@ SUFFIX = sexcat MAKE_POST_PROCESS = FALSE -[SETOOLS_RUNNER] +[MASK_QUERY_RUNNER] -INPUT_MODULE = last:sextractor_runner +INPUT_DIR = $SP_RUN/output/run_sp_exp_SxSePsf/sextractor_runner/output -# Note: Make sure this doe not match the SExtractor background images +# Note: Make sure this does not match the SExtractor background images # (sexcat_background*) -FILE_PATTERN = sexcat_sexcat +FILE_PATTERN = sexcat NUMBERING_SCHEME = -0000000-0 -# SETools config file -SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools - - -[MCCD_PREPROCESSING_RUNNER] +# The PSF-star diet, and it is deliberately NARROW: instrument flags (read by +# SExtractor as IMAFLAGS_ISO) plus the healsparse star-body map (UNIONS bit 2), +# and nothing else. Halo bits 0 and 1 are excluded on purpose — halos flag +# objects for the final catalogue, they do not reject PSF stars (mask-force +# telecon, 2026-07-21) — and MaxiMask is not in the diet either. Widen it by +# adding paths here; every map that is True (boolean) or nonzero (integer) at a +# detection sets MASK_EXT. Comma-separated. +# +# UNSET BY DEFAULT, and the module is a strict no-op without it: the catalogue +# is passed through with no MASK_EXT column, the same gating make_cat gives +# MASK_EXT_PATHS. mask_query stays in the MODULE chain either way, so turning +# the query on is uncommenting one line and never editing the chain. +# +# Even with it set, NOTHING CUTS ON IT: the star selection ships permissive +# (instrument flags only) and the column is carried for transparency and +# measurement — see star_selection.setools' header for the one-line change +# that would impose it. +; MASK_PATHS = $SP_CONFIG/mask_ugriz_nside131072_n4.hsp + +# Optional: restrict integer maps to these bits (value & MASK_BITS). Absent, +# any nonzero value flags. Boolean maps — the UNIONS per-bit products, one map +# per bit — ignore it, which is why the diet above is a path list and not a +# bit mask. +; MASK_BITS = 4 -# Path to MCCD config file -CONFIG_PATH = $SP_CONFIG/config_MCCD.ini -MODE = FIT_VALIDATION +[SETOOLS_RUNNER] -VERBOSE = False +INPUT_DIR = $SP_RUN/output/run_sp_exp_SxSePsf/mask_query_runner/output -INPUT_DIR = last:setools_runner +FILE_PATTERN = sexcat_ext -# Input are individual CCDs, thus single-exposure single-HDU images NUMBERING_SCHEME = -0000000-0 -FILE_PATTERN = star_split_ratio_80, star_split_ratio_20 - -FILE_EXT = .fits, .fits - - -[MCCD_FIT_VAL_RUNNER] +# SETools config file +SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools -# Path to MCCD config file -CONFIG_PATH = $SP_CONFIG/config_MCCD.ini -MODE = FIT_VALIDATION +# The MCCD chain differs from the PSFEx one only from here on; everything up +# to the star selection is the same, so the two PSF models are fitted on the +# same stars. Every module below chains from the one before it in this run +# (the decorators' input_module), so none needs an INPUT_DIR. +# +# Not in this chain: merge_starcat_runner and mccd_plots_runner. Both are +# campaign-level diagnostics over all exposures' validation stars, not +# per-exposure products; the tiles need only the fitted model. -VERBOSE = False +[MCCD_PREPROCESSING_RUNNER] -NUMBERING_SCHEME = -0000000 +# Serial runner: it receives every CCD's star catalogues of this exposure and +# writes one focal-plane training (80%) and one test (20%) catalogue for it. +FILE_PATTERN = star_split_ratio_80, star_split_ratio_20 +FILE_EXT = .fits, .fits -[MCCD_MERGE_STARCAT_RUNNER] +NUMBERING_SCHEME = -0000000-0 # Path to MCCD config file CONFIG_PATH = $SP_CONFIG/config_MCCD.ini @@ -181,10 +202,13 @@ MODE = FIT_VALIDATION VERBOSE = False -NUMBERING_SCHEME = -0000000 +[MCCD_FIT_VAL_RUNNER] -[MCCD_PLOTS_RUNNER] +# One focal-plane model per exposure: fitted_model-.npy, which the tiles' +# mccd_interp_runner reads (config_tile_PiViVi_mccd.ini, ME_DOT_PSF_PATTERN), +# and validation_psf-.fits, the model at the 20% test stars. +NUMBERING_SCHEME = -0000000 # Path to MCCD config file CONFIG_PATH = $SP_CONFIG/config_MCCD.ini @@ -192,46 +216,3 @@ CONFIG_PATH = $SP_CONFIG/config_MCCD.ini MODE = FIT_VALIDATION VERBOSE = False - -# Now MCCD has created a focal-plane PSF model, including all CCDS per images, -# thus single-exposure files -NUMBERING_SCHEME = -0000000 - -PLOT_MEANSHAPES = True - -# X_GRID, Y_GRID: correspond to the number of bins in each direction of each -# CCD from the focal plane. Ex: each CCD will be binned in 5x10 regular grids. -X_GRID = 5 -Y_GRID = 10 - -PLOT_HISTOGRAMS = True - -# REMOVE_OUTLIERS: Remove validated stars that are outliers in terms of shape -# before drawing the plots. -REMOVE_OUTLIERS = False - - - -[MCCD_INTERP_RUNNER] - -# MODE: Define the way the MCCD interpolation will run. -# CLASSIC for classical run. -# MULTI-EPOCH for multi epoch. -MODE = CLASSIC - -# Position parameter names -# for multi-epoch XWIN_WORLD,YWIN_WORLD -# For classical XWIN_IMAGE,YWIN_IMAGE: -POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE - -# Get PSF shapes calculated and saved on the output dict -GET_SHAPES = True - -# Directory with PSF models -PSF_MODEL_DIR = $SP_RUN/output - -# PSF model patterns -PSF_MODEL_PATTERN = fitted_model - -# PSF model separator -PSF_MODEL_SEPARATOR = - diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index 0a22179f5..00a9d09b0 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -102,21 +102,19 @@ "psfex_runner": dict(expect=80), "psfex_interp_runner": dict(expect=40, warn=True), }, - # MCCD counts are derived from config_exp_mccd.ini and its per-CCD - # runners, but this chain has not been exercised through this workflow. - # Keep the expected counts visible while making the unverified branch - # warning-only until a real campaign validates its counts. + # MCCD shares the chain up to setools with PSFEx, then fits one + # focal-plane model per exposure. Preprocessing is a serial runner that + # merges the 40 CCDs' split catalogues into one training and one test + # catalogue; fit_val writes the model (fitted_model-.npy, what the + # tiles interpolate) and its validation catalogue. Both are + # exposure-wide and all-or-nothing: an exposure without a model has no + # PSF on any tile it overlaps. "mccd": { - "sextractor_runner": dict(expect=120, warn=True), - "setools_runner": dict(expect=80, warn=True, - subpath="rand_split"), - "mccd_preprocessing_runner": dict(expect=80, warn=True), - # Fit/validation is exposure-wide: one model and one validation - # catalogue, unlike the per-CCD preprocessing outputs. - "mccd_fit_val_runner": dict(expect=2, warn=True), - "merge_starcat_runner": dict(expect=1, warn=True), - # config_exp_mccd enables the ten meanshape and six histogram plots. - "mccd_plots_runner": dict(expect=16, warn=True), + "sextractor_runner": dict(expect=120), + "mask_query_runner": dict(expect=40), + "setools_runner": dict(expect=80, subpath="rand_split"), + "mccd_preprocessing_runner": dict(expect=2), + "mccd_fit_val_runner": dict(expect=2), }, # Image simulations with the true PSF (psf_model: fake): no PSF fit on # the exposures, only the SExtractor pass whose background/background_rms From 836f91dcad7eaf7c14235f85321e68356b35d529 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Sat, 12 Sep 2026 02:19:29 +0200 Subject: [PATCH 05/25] workflow: exp_psf reserves 2 cores, not 8, for the MCCD chain MCCD fits one focal-plane model per exposure after the per-CCD stages: 85 min single-threaded for ~2500 stars and 8.3 GB (SKiLLS star sim 1z2z_1, exposure 2086792), and 48 min on 8 BLAS threads -- 1.75x for 8x the cores, which the thread caps forbid anyway. With threads: 8 the fit leaves 7 reserved cores idle for its whole length. The mccd chain now takes 2: the per-CCD stages run 2 wide (minutes), the fit is unchanged, and an exposure reserves about a quarter of the core-hours. PSFEx and fake keep 8. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VSU9pPZjpBjzWorAL88aKQ --- workflow/rules/exposure.smk | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 4e1ef45a4..bc35e70b9 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -81,6 +81,13 @@ rule exp_split: # -> psfex_interp, per CCD. # setools may reject a sparse CCD (~0.2% attrition) — tolerated by the floor's # :warn on psfex_interp_runner. +# +# MCCD instead fits ONE focal-plane model per exposure after the per-CCD +# stages: ~85 min single-threaded for ~2500 stars (SKiLLS star sim, 8.3 GB), +# and only 1.75x faster on 8 BLAS threads (which the thread caps forbid anyway). +# With 8 cores reserved the fit would idle 7 of them for its whole length, so +# the mccd chain takes 2: the per-CCD stages run 2 wide (minutes), the fit is +# unchanged, and the exposure reserves a quarter of the core-hours. rule exp_psf: input: rules.exp_split.output.manifest @@ -91,7 +98,7 @@ rule exp_psf: params: pre = lambda wc: unit_pre("exp_psf", wc.exp), script_hash = SCRIPT_HASH - threads: 8 + threads: 2 if PSF_MODEL == "mccd" else 8 retries: 2 benchmark: # BESIDE manifests/, not inside it: clean_exposure deletes manifests/ From ad80f23fc0ac474c24cb48503663f48ed11c7ec8 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Sat, 12 Sep 2026 02:58:03 +0200 Subject: [PATCH 06/25] mccd_interp: test for N_EPOCH among the column names Multi-epoch MCCD interpolation checked `"N_EPOCH" not in cat.get_data()`, a membership test against the FITS record array, which current numpy rejects ("Cannot compare structured or void to non-void arrays"): every tile failed in tile_vignets with psf_model: mccd. psfex_interp already tests `.dtype.names`; so does this now. With it, tile 233.293 of SKiLLS star sim 1z2z_1 interpolates the seven exposures' MCCD models (a few sparse CCDs lose an epoch, as intended). Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VSU9pPZjpBjzWorAL88aKQ --- src/shapepipe/modules/mccd_package/mccd_interpolation_script.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shapepipe/modules/mccd_package/mccd_interpolation_script.py b/src/shapepipe/modules/mccd_package/mccd_interpolation_script.py index b6183f4e1..0942d93ac 100644 --- a/src/shapepipe/modules/mccd_package/mccd_interpolation_script.py +++ b/src/shapepipe/modules/mccd_package/mccd_interpolation_script.py @@ -442,7 +442,7 @@ def _interpolate_me(self): all_id = np.copy(cat.get_data()["NUMBER"]) key_ne = "N_EPOCH" - if key_ne not in cat.get_data(): + if key_ne not in cat.get_data().dtype.names: raise KeyError( f"Key {key_ne} not found in input galaxy catalogue, needed for" + " PSF interpolation to multi-epoch data; run previous module" From 2082dcf8d23fe3fb6123eeeec59e7d5e106a5668 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Sat, 12 Sep 2026 03:57:46 +0200 Subject: [PATCH 07/25] workflow: MCCD validated through the full chain; drop the warn-only flags With the ported exposure chain, the mccd fixes and the mccd_interp fix, psf_model: mccd ran SKiLLS star sim 1z2z_1 tile 233.293 end to end: seven exposures' focal-plane models (120/40/80/2/2 each), mccd_interp's galaxy_psf store, vignets, ngmix and a final_cat of 21857 objects. tile_vignets' mccd counts are now mandatory as for psfex, and the README no longer calls mccd "wired but unvalidated". Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01VSU9pPZjpBjzWorAL88aKQ --- workflow/README.md | 3 ++- workflow/scripts/completeness.py | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/workflow/README.md b/workflow/README.md index 97510c901..d404d1b16 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -25,7 +25,8 @@ uv pip install 'snakemake>=9,<10' 'snakemake-executor-plugin-slurm>=2.7,<3' # Edit workflow/config.yaml: tile_list, inputs.tiles/exposures, outputs.run_dir, # outputs.products_dir/index_db, and container. -# `psf_model` is `psfex` or `mccd`; mccd is wired but unvalidated here, while psfex is exercised by smk-g4 through smk-g6. +# `psf_model` is `psfex` or `mccd`. psfex is exercised by smk-g4 through smk-g6; mccd has run the full chain on +# an image-sim star tile (one focal-plane model per exposure, ~1.5 CPU-hours each). # The committed launcher loads apptainer/1.4.5 + the /project venv, so a # fresh shell always has the right state. diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index 00a9d09b0..e400e14c6 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -136,12 +136,12 @@ # v2.0's 4 was the canfar flavor. every vignette feeds ngmix, so the expected count is all-or-nothing. "vignetmaker_runner_run_2": dict(expect=5), }, - # MCCD is wired but unvalidated here; retain the expected runner names - # and counts as warnings until a workflow campaign exercises them. + # As psfex: mccd_interp writes the tile's galaxy_psf store from the + # exposures' focal-plane models (SKiLLS star sim 1z2z_1, 233.293). "mccd": { - "mccd_interp_runner": dict(expect=1, warn=True), - "vignetmaker_runner_run_1": dict(expect=1, warn=True), - "vignetmaker_runner_run_2": dict(expect=5, warn=True), + "mccd_interp_runner": dict(expect=1), + "vignetmaker_runner_run_1": dict(expect=1), + "vignetmaker_runner_run_2": dict(expect=5), }, # Image simulations: fake_interp_runner writes the same galaxy_psf # sqlite psfex_interp_runner writes, from the simulation's PSF dictionary. From 9085cd0b877abf9c4c8a61f87fb702aff30b37bf Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Sat, 12 Sep 2026 10:21:03 +0200 Subject: [PATCH 08/25] workflow: tile_store_root run-config key moves the tile store's bind, per campaign bin/sp rewrites the source of the `/local/scratch` bind in the launch snapshot's profile (appending one when the profile has none) and creates the directory. tile_local() is untouched: its output is a params rerun trigger, a bind is not, so the key can be set on a resume. candide's 31 GB node /tmp cannot hold dense image-sim tile stores; image-sim campaigns there point this at NFS. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01Qv5MwRiVKa8wnH665dWBia --- profiles/candide/config.yaml | 5 +++-- workflow/README.md | 6 ++++++ workflow/bin/sp | 36 ++++++++++++++++++++++++++++++++++++ workflow/config.yaml | 9 +++++++++ workflow/rules/tile.smk | 4 +++- 5 files changed, 57 insertions(+), 3 deletions(-) diff --git a/profiles/candide/config.yaml b/profiles/candide/config.yaml index b4b1d63c2..bacf48071 100644 --- a/profiles/candide/config.yaml +++ b/profiles/candide/config.yaml @@ -22,8 +22,9 @@ software-deployment-method: [apptainer] # which this line cannot name: snakemake passes it through literally, and # --cleanenv drops SLURM_JOB_ID inside the container. So the node's /tmp (local # disk, ~29 GB free, sticky) is bound there instead. That holds ~5 concurrent -# tile stores per node; if a campaign packs more tile_shape groups onto one node, -# this is the line to revisit. The PYTHONPATH pin is rewritten by bin/sp to the +# tile stores per node, too few for dense image-sim tiles: such a campaign sets +# `tile_store_root:` in its run config, and bin/sp replaces this bind's source in +# the launch snapshot (see workflow/config.yaml). The PYTHONPATH pin is rewritten by bin/sp to the # launch snapshot's src/; the value here only has to be a checkout's src/. apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/n17data/mkilbing/astro/repositories/github/shapepipe/src --bind /home,/automnt,/n17data,/n23data1,/n09data --bind /tmp:/local/scratch" latency-wait: 60 # NFS: wait for outputs to appear after a job diff --git a/workflow/README.md b/workflow/README.md index d404d1b16..43a7d018d 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -74,6 +74,12 @@ key falls back to the committed run's paths) and is snapshotted with the code; `SP_PROFILE` picks `profiles//`. sp_validation's image-simulation workflow drives these campaigns and measures m from their final catalogues. +On candide, the node-local tile store (bound from the node's 31 GB `/tmp`) does not +hold several dense image-sim tiles at once. Set `tile_store_root:` in the run +config to a shared directory; `sp run` binds it to `/local/scratch` for that +campaign. The store names carry a per-campaign hash, so all branches can share one +root. + ## The container image `sp container` owns which image the jobs run inside. Two layers, and the second diff --git a/workflow/bin/sp b/workflow/bin/sp index 37eefa1e6..b64905181 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -135,6 +135,42 @@ if not n: f.write_text(text) PY + # `tile_store_root:` in the run config moves the node-local tile store + # (tile.smk's tile_local(): the vignette + WCS stores of each tile_shape group) + # off the profile's `/local/scratch` bind, for this campaign only. It is the + # same substitution as the pin above -- the BIND SOURCE of `/local/scratch` in + # the snapshot's profile -- and deliberately not a path in tile_local(): that + # function's output is a params rerun trigger, so a path there would reschedule + # every finished tile of a resumed campaign (its warning block says how badly), + # while a bind is not a trigger at all (a dry run after changing it schedules + # only the missing tiles). Why a campaign wants it: candide binds the node's + # 31 GB /tmp, which a few dense image-sim tile stores or failed groups' orphans + # fill, after which every later group on that node fails in seconds. A shared + # directory (NFS) trades ngmix read latency for room; the campaign-unique store + # names (LOCAL_TAG) keep concurrent campaigns apart in one root. Profiles with + # no `X:/local/scratch` bind (nibi's `--bind /local`) get one appended. + python - "$SNAPSHOT/profiles/$PROFILE/config.yaml" "$CONFIG" <<'PY' +import os, pathlib, re, sys, yaml +f, run_config = pathlib.Path(sys.argv[1]), sys.argv[2] +root = (yaml.safe_load(open(run_config)) or {}).get("tile_store_root") +if root: + root = str(root) + if not os.path.isabs(root): + sys.exit(f"sp: tile_store_root must be an absolute path, got {root!r}") + os.makedirs(root, exist_ok=True) + bind = f"--bind {root}:/local/scratch" + text, n = re.subn(r'--bind \S+:/local/scratch(?=[\s"])', lambda m: bind, + f.read_text(), count=1) + if not n: + text, n = re.subn(r'(apptainer-args:\s*"[^"]*)"', + lambda m: m.group(1) + " " + bind + '"', text, count=1) + if not n: + sys.exit("sp: tile_store_root is set but the profile has no quoted " + "apptainer-args to bind it with") + f.write_text(text) + print(f"sp: tile store root {root} (bound to /local/scratch)", file=sys.stderr) +PY + python - "$SNAPSHOT/snapshot.json" "$REPO" <<'PY' import json, pathlib, subprocess, sys, time out, repo = pathlib.Path(sys.argv[1]), sys.argv[2] diff --git a/workflow/config.yaml b/workflow/config.yaml index 336cda5d6..58dde3d5b 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -144,6 +144,15 @@ clean_ignore_tiles: [] # per-tile, in-job, from the tile's own sexcat). ngmix_chunks: 8 +# Where the node-local tile store lives (tile.smk's tile_local(): each tile_shape +# group's vignette and WCS stores). Unset means the profile's `/local/scratch` +# bind: nibi's /local NVMe, candide's 31 GB node /tmp. Set it to an absolute +# directory to bind that instead, for this campaign only; bin/sp rewrites the +# snapshot's profile and creates the directory. Not a rerun trigger, so it may be +# set on a resume. Image-sim campaigns on candide want a shared directory (dense +# tiles and failed groups' orphans fill /tmp); the cost is ngmix read latency. +# tile_store_root: /n23data1//shapepipe/tilestore + # The container's installed shapepipe is overridden by the prod worktree via # --env PYTHONPATH in profiles/nibi (settled call 3); no config knob here. # build_index.py's missing-tile fraction threshold is passed by workflow/bin/sp diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 8e3cf5171..ccbc83fbf 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -91,7 +91,9 @@ is what downstream selections cut on. # # HOW THE PATH GETS IN HERE. Not through the environment: profiles/nibi passes # --bind /local and tile_local() below DERIVES the path from the tile wildcard. -# Why nothing can be communicated instead is on that profile line. +# Why nothing can be communicated instead is on that profile line. A campaign +# that needs the store elsewhere (candide's 31 GB /tmp) moves the BIND, via the +# run config's `tile_store_root` (bin/sp), never the path here -- see below. # # THE COST WE ACCEPT: a failure anywhere in the tile re-runs the WHOLE tile, # not one chunk, because the store dies with the job. At ~1 h per fused tile From 7637b390527a5763366f1078b387630e8079500f Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Tue, 15 Sep 2026 10:36:48 +0200 Subject: [PATCH 09/25] completeness.py: atomic write for stage log/manifest JSON write_if_changed() used Path.write_text(), which truncates before writing. run_report.py reads every logs/*.json right after compute finishes (the onsuccess hook) and can catch one mid-write as an empty file -- observed during the grid_3 image-sims acceptance test (exp_get_images.json for one exposure came back "Expecting value: line 1 column 1"). Write to a same-directory temp file and os.replace() it into place instead, which is atomic and closes the race. --- workflow/scripts/completeness.py | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index e400e14c6..bc8975064 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -354,10 +354,23 @@ def build_manifest(stage, run_dir, unit, stage_subdir=None): def write_if_changed(path: Path, text: str) -> None: - """Write only when the bytes differ — see the module docstring on mtime.""" + """Write only when the bytes differ — see the module docstring on mtime. + + Via a same-directory temp file + ``os.replace``, not ``path.write_text``: + the latter truncates before writing, so a reader (``run_report.py``, run + automatically at compute's end) can catch the file empty mid-write. Same + directory keeps the replace on one filesystem, which is what makes it + atomic; a losing writer's temp file is unlinked rather than left behind. + """ path.parent.mkdir(parents=True, exist_ok=True) if not path.exists() or path.read_text() != text: - path.write_text(text) + tmp = path.with_name(f".{path.name}.tmp{os.getpid()}") + try: + tmp.write_text(text) + os.replace(tmp, path) + except BaseException: + tmp.unlink(missing_ok=True) + raise def _unit_from_run_dir(run_dir): From 7aababf93ad70942fe5ba9f5c6392c8bf9764767 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Tue, 15 Sep 2026 16:25:41 +0200 Subject: [PATCH 10/25] workflow/config.yaml: universal template, no cluster-specific paths The committed default carried one person's real nibi paths (tile_list, inputs, container, outputs) baked in. That means `sp run` was not actually the same command on every machine: anyone who forgot to set SP_RUN_CONFIG on a different cluster would either hit a confusing filesystem error somewhere downstream, or silently point at paths that happen to also resolve on their machine but belong to someone else's campaign. config.yaml now carries a REPLACE_ME sentinel for every machine- or campaign-specific path, with a top-of-file explanation that a real run always supplies its own via SP_RUN_CONFIG. The Snakefile checks the keys that have no safe fallback (tile_list, inputs.tiles/exposures, outputs.run_dir/index_db) right after the config file is read and raises one clear, identical error on any cluster if one is still the placeholder, rather than failing confusingly later or not at all. `container:` is left out of that check: a local sandbox or cached SIF is resolved before it, so a placeholder there is often harmless. Also cleans up several stale comments left over from earlier campaign states (smk-g4/g5 narrative, `clean`/`clean_tiles` comments describing "OFF" for a value that had already been changed to `true`) and two "knob" -> "setting" wording fixes. --- workflow/Snakefile | 25 ++++++++++++++ workflow/config.yaml | 78 +++++++++++++++++++++++++------------------- 2 files changed, 70 insertions(+), 33 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 3162276b7..a6a51b92e 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -52,6 +52,31 @@ _RUN_CONFIG = os.environ.get("SP_RUN_CONFIG") or str( Path(workflow.snakefile).parent / "config.yaml") configfile: _RUN_CONFIG +# The committed config.yaml is a universal template (no machine- or +# campaign-specific path in it), so the same `sp run` invocation works on +# every cluster: whoever runs it always supplies their own paths through +# SP_RUN_CONFIG (see config.yaml's own top comment and the README). Left +# unset, the template's REPLACE_ME sentinel would otherwise read as a real, +# oddly-named path -- silently missing tiles, or a job dying hours in on a +# node that can't see "/REPLACE_ME" -- rather than the one-line, same-message- +# everywhere error this check gives instead. `container:` is checked +# separately (container.py resolves a local sandbox or cached SIF first, so a +# placeholder there is often harmless) and so is left out here. +_PLACEHOLDER = "REPLACE_ME" +for _key, _value in { + "tile_list": config.get("tile_list"), + "inputs.tiles": (config.get("inputs") or {}).get("tiles"), + "inputs.exposures": (config.get("inputs") or {}).get("exposures"), + "outputs.run_dir": (config.get("outputs") or {}).get("run_dir"), + "outputs.index_db": (config.get("outputs") or {}).get("index_db"), +}.items(): + if _value == _PLACEHOLDER: + raise WorkflowError( + f"{_key!r} in {_RUN_CONFIG} is still the template placeholder " + f"{_PLACEHOLDER!r}. workflow/config.yaml ships as a universal " + f"template with no real paths in it; point SP_RUN_CONFIG at your " + f"own run config instead (see the README, 'Run configuration').") + # Every job's shell runs inside this container (apptainer software-deployment in # the profile); the user never types apptainer. WHICH image is the one resolution # order `sp container` exposes — this user's writable sandbox if they built one, diff --git a/workflow/config.yaml b/workflow/config.yaml index 58dde3d5b..31c45c038 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -1,32 +1,41 @@ # Run configuration for the ShapePipe Snakemake workflow. # +# THIS FILE IS A UNIVERSAL TEMPLATE, NOT A LIVE CAMPAIGN'S CONFIG. It carries +# no machine- or campaign-specific path, on purpose: `sp run` must be the same +# command on nibi, candide, or anywhere else, and a committed default that +# happened to bake in one person's real paths would silently run a stranger's +# campaign against the wrong cluster's filesystem the moment they forgot to +# override it. Every path below is REPLACE_ME; the Snakefile refuses to parse +# past one of the ones it cannot otherwise fail clearly on later (tile_list, +# inputs.tiles, inputs.exposures, outputs.run_dir, outputs.index_db -- see the +# check near the top of Snakefile). `container:` is exempt from that check: a +# local sandbox or cached SIF (`sp container status`) is resolved first and +# makes this key irrelevant for anyone who has one. +# +# A real run always supplies its own paths via SP_RUN_CONFIG, which REPLACES +# this file wholesale rather than layering over it (workflow/bin/sp; see the +# README's "Run configuration" section): +# +# SP_PROFILE= SP_RUN_CONFIG=/path/to/your_run.yaml \ +# workflow/bin/sp run +# # A "run" is declared by a tile list plus the paths below. Everything here is # read at parse time; none of it is a rule input, so editing it (e.g. appending # tiles) never invalidates completed work — it only changes which jobs exist. -# The tile list that scopes this run (one "IDra.IDdec" per line). -# The campaign grows by appending to this file — parse-time config, so -# completed work is never invalidated. Current contents: smk-g5, ONE tile -# (186.307), the paired control for the two fixes landed after smk-g4 — the -# node-local WCS sqlite and tile_ngmix's mem_mb 14000 -> 5000. That tile ran -# ALONE as smk-g4 job 20799387 (elapsed 8063 s, chunk median 6813 s, chunk mean -# load 59%, MaxRSS 26.6 GiB), so the baseline carries no concurrency confound -# and the contrast is same tile, same solitude, two commits apart. -# The 34-tile smk-g4 set is preserved at smk-g4/tiles34.txt. -# (The 210/211 quad used earlier is unusable: its tile images are symlinks into -# anaennis' moved processed_tiles tree — ~8.5k of the 10.3k staged tiles are -# broken.) -tile_list: /project/def-mjhudson/cdaley/sp-products/smk-g6/tiles.txt +# The tile list that scopes this run (one "IDra.IDdec" per line). The +# campaign grows by appending to this file — parse-time config, so completed +# work is never invalidated. +tile_list: REPLACE_ME # Inputs are the only site-specific data paths; all committed configs consume these # through SP_INPUT_TILES and SP_INPUT_EXPOSURES. inputs: - # Pre-staged P3 data (get_images RETRIEVE=symlink). - tiles: /project/def-mjhudson/unions-wl/tiles - exposures: /project/def-mjhudson/unions-wl/exposures + tiles: REPLACE_ME + exposures: REPLACE_ME # The container every job runs inside (apptainer software-deployment in the profile). -container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif +container: REPLACE_ME # PSF model used by the exposure and tile interpolation stages: psfex, mccd, or # fake (image simulations only: the true PSF from `psf_dict`, no exposure fit). @@ -46,7 +55,7 @@ input_type: data # sharded per-unit stores live under it: # /tiles/<2-char prefix>// and /exp/// outputs: - run_dir: /scratch/cdaley/shapepipe-output/smk-g6 + run_dir: REPLACE_ME # products_dir is the PERSISTENT root: the durable, low-volume products — the # final catalogues (/tiles///final_cat-.fits, @@ -59,31 +68,34 @@ outputs: # tile re-declare inputs against exposure stores reclamation already deleted. # # Unset means "one root": products land under run_dir, exactly the pre-D5 -# layout, which is what a fixture or smoke test wants. +# layout, which is what a fixture or smoke test wants. Delete this whole key +# (not REPLACE_ME) if that's what you want -- it's the one path in this file +# that's genuinely optional rather than templated. # # Snakemake's own state is the one durable-looking thing that stays on scratch # (-state; bin/sp explains why). - products_dir: /project/def-mjhudson/cdaley/sp-products/smk-g6 + products_dir: REPLACE_ME -# There is no config_src knob: the config chain is workflow/config/cfis, resolved -# relative to the Snakefile. The configs interpolate $SP_RUN / $SP_UNIT_NUM / -# $SP_CONFIG / $SP_EXP / $NGMIX_* and the rules export them -- configs and rules -# are one artefact and must version together, so the dir is fixed by construction. +# There is no separate config_src setting: the config chain is +# workflow/config/cfis, resolved relative to the Snakefile. The configs +# interpolate $SP_RUN / $SP_UNIT_NUM / $SP_CONFIG / $SP_EXP / $NGMIX_* and the +# rules export them -- configs and rules are one artefact and must version +# together, so the dir is fixed by construction. # The run index, and — sharing its directory — missing.json and run_report.json. # On the persistent root with the catalogues (D5): the index is the record of # which tile reads which exposure, so it is what a post-purge reconstruction # would otherwise have to rebuild from tile headers. - index_db: /project/def-mjhudson/cdaley/sp-products/smk-g6/index/run_index.sqlite + index_db: REPLACE_ME # Rolling exposure-store reclamation (D5). When true, the COMPUTE DAG grows one # `clean_exposure` job per exposure. It fires once every campaign tile that reads # that exposure has its vignets, deletes the exposure's store AND its manifests, # and leaves `cleaned.json`, which absorbs the manifests — `sp report` reads them -# back out of the tombstone and reports the exposure as `cleaned`. OFF for smk-g5: -# this is a one-tile control, its ~8 exposure stores cost ~54 GiB against 979 GiB -# free, reclamation is already measured exactly by smk-g4, and keeping the stores -# means a re-measurement re-runs only the shape chain instead of the whole tile. +# back out of the tombstone and reports the exposure as `cleaned`. Set it false +# for a small control run whose exposure stores you want to keep around (e.g. to +# re-measure only the shape chain rather than rebuild the whole tile) -- true is +# the right default for anything that isn't a deliberately-kept fixture. clean: true # Rolling TILE-store reclamation (D5). When true, the COMPUTE DAG grows one @@ -122,8 +134,8 @@ clean: true # against a 1M quota. Reclaimed, a tile costs 10 inodes and 12.9 KB — 231k inodes # and ~300 MB for the whole of DR6. # -# OFF here for the same reason `clean:` is: smk-g5 is a one-tile control, and its -# store is what a re-measurement would read. +# Set it false for the same kind of small control run `clean:` above would be +# false for, when the per-tile store is what a re-measurement needs to read. clean_tiles: true # Tiles that may NOT pin an exposure store (default: empty). @@ -153,7 +165,7 @@ ngmix_chunks: 8 # tiles and failed groups' orphans fill /tmp); the cost is ngmix read latency. # tile_store_root: /n23data1//shapepipe/tilestore -# The container's installed shapepipe is overridden by the prod worktree via -# --env PYTHONPATH in profiles/nibi (settled call 3); no config knob here. +# The container's installed shapepipe is overridden by the launching worktree +# via each profile's own --env PYTHONPATH; there is no setting for it here. # build_index.py's missing-tile fraction threshold is passed by workflow/bin/sp # (SP_MISSING_THRESHOLD, default 0.0 = any missing tile is fatal). From d64f88ed40afb1a56d509c33a58d3185f51bbe2f Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Tue, 15 Sep 2026 17:02:38 +0200 Subject: [PATCH 11/25] workflow: retrieve mode (symlink|vos) is a run-config key, not fixed in the ini RETRIEVE=symlink was hard-coded in config_tile_Git.ini and config_exp_Gie.ini (both config/cfis and config/cfis_image_sims), which assumes a pre-staged local mirror. That's nibi's layout; candide's real-data ingestion fetches tiles/exposures from VOSpace (RETRIEVE=vos) and has no local mirror. Add `retrieve: symlink|vos` to the run config, validated at parse time, exported as SP_RETRIEVE by unit_pre() and interpolated into the four ini files. input_type=image_sims always forces symlink regardless of the run config's value, since simulation output is generated locally and never lives in VOSpace -- so a `data` run config's `retrieve: vos` reused as a sims campaign's base can't leak into the sims ingestion stages. Verified with a dry run against both input_type=data (retrieve: vos) and the existing image_sims acceptance-test config (still forces symlink) that SP_RETRIEVE renders correctly in each case. --- workflow/Snakefile | 28 +++++++++++++++++-- workflow/config.yaml | 9 ++++++ workflow/config/cfis/config_exp_Gie.ini | 7 +++-- workflow/config/cfis/config_tile_Git.ini | 6 ++-- .../config/cfis_image_sims/config_exp_Gie.ini | 7 +++-- .../cfis_image_sims/config_tile_Git.ini | 7 +++-- 6 files changed, 53 insertions(+), 11 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index a6a51b92e..78672861d 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -137,6 +137,25 @@ if PSF_MODEL == "fake" and not PSF_DICT: raise WorkflowError("psf_model=fake needs `psf_dict:` (the simulation's " "pickled PSF dictionary) in the run config.") +# How tile/exposure images reach the run: `symlink` (a pre-staged local +# mirror -- nibi's layout) or `vos` (fetched from VOSpace per unit -- candide's +# real-data layout, no local mirror needed). A per-cluster choice, so it is a +# run-config key (`retrieve:`) rather than fixed in the committed ini files; +# `unit_pre` exports it as SP_RETRIEVE for config_tile_Git.ini / +# config_exp_Gie.ini. `image_sims` always overrides it to `symlink` below, +# regardless of what the run config says: simulation output is generated +# locally and never lives in VOSpace, so a `data` run config's `retrieve: vos` +# reused as a sims campaign's base must not leak into the sims ingestion +# stages. +RETRIEVE_MODES = {"symlink", "vos"} +RETRIEVE_MODE = config.get("retrieve", "symlink") +if RETRIEVE_MODE not in RETRIEVE_MODES: + raise WorkflowError( + f"Invalid retrieve={RETRIEVE_MODE!r}; expected one of " + f"{sorted(RETRIEVE_MODES)}.") +if INPUT_TYPE == "image_sims": + RETRIEVE_MODE = "symlink" + # --- paths ----------------------------------------------------------------- # RUN_DIR is the scratch root: bulk intermediates, sized so a batch finishes @@ -570,9 +589,11 @@ def unit_pre(stage, unit, *, exp_name=None, forest=None, env=None, the committed config dir. It also exports the configured input roots as ``SP_INPUT_TILES`` and - ``SP_INPUT_EXPOSURES`` and the PSF choice as ``SP_PSF`` for the committed - ini chain (plus ``PSF_DICT`` for the image-simulation true PSF, which the - configs read as ``${SP_PSF}_interp_runner`` = ``fake_interp_runner``). + ``SP_INPUT_EXPOSURES``, how to fetch them as ``SP_RETRIEVE`` (``symlink`` + or ``vos``; forced to ``symlink`` for ``image_sims``, see ``RETRIEVE_MODE`` + above), and the PSF choice as ``SP_PSF`` for the committed ini chain (plus + ``PSF_DICT`` for the image-simulation true PSF, which the configs read as + ``${SP_PSF}_interp_runner`` = ``fake_interp_runner``). Every line here is part of each rule's ``params.pre`` and so of the ``params`` rerun trigger: a line added for every rule reruns every finished @@ -602,6 +623,7 @@ def unit_pre(stage, unit, *, exp_name=None, forest=None, env=None, f"export SP_CONFIG='{CONFIG_DIR}'", f"export SP_INPUT_TILES='{INPUTS['tiles']}'", f"export SP_INPUT_EXPOSURES='{INPUTS['exposures']}'", + f"export SP_RETRIEVE='{RETRIEVE_MODE}'", f"export SP_PSF='{PSF_MODEL}'", # Also set via apptainer-args in the profile; kept here so a hand-run of # this same line outside snakemake behaves identically. diff --git a/workflow/config.yaml b/workflow/config.yaml index 31c45c038..0ca3b239f 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -34,6 +34,15 @@ inputs: tiles: REPLACE_ME exposures: REPLACE_ME +# How the tile/exposure images in `inputs:` above reach the run: `symlink` +# (a pre-staged local mirror) or `vos` (fetched from VOSpace per unit, no +# local mirror). This is cluster infrastructure, not a campaign choice -- +# nibi mirrors locally, candide fetches from VOSpace -- so pick the one that +# matches wherever `inputs.tiles`/`inputs.exposures` actually point. Ignored +# for `input_type: image_sims`, which is always `symlink` (simulation output +# is generated locally and never lives in VOSpace). +retrieve: symlink + # The container every job runs inside (apptainer software-deployment in the profile). container: REPLACE_ME diff --git a/workflow/config/cfis/config_exp_Gie.ini b/workflow/config/cfis/config_exp_Gie.ini index b0cce78e8..bfd65b84e 100644 --- a/workflow/config/cfis/config_exp_Gie.ini +++ b/workflow/config/cfis/config_exp_Gie.ini @@ -86,8 +86,11 @@ INPUT_NUMBERING = \d{6} # Output file pattern without number OUTPUT_FILE_PATTERN = image-, weight-, flag- -# Method to retrieve images, one in 'vos', 'symlink' -RETRIEVE = symlink +# Method to retrieve images, one in 'vos', 'symlink' -- set by the run +# config's `retrieve:` key (unit_pre exports SP_RETRIEVE), not fixed here: +# nibi mirrors the exposures locally (symlink), candide fetches them from +# VOSpace (vos). +RETRIEVE = $SP_RETRIEVE # If RETRIEVE=vos, number of attempts to download # Optional, default=3 diff --git a/workflow/config/cfis/config_tile_Git.ini b/workflow/config/cfis/config_tile_Git.ini index cccf0a8b9..87a189f07 100644 --- a/workflow/config/cfis/config_tile_Git.ini +++ b/workflow/config/cfis/config_tile_Git.ini @@ -80,8 +80,10 @@ INPUT_NUMBERING = \d{3}\.\d{3} # Output file pattern without number OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight- -# Copy/download method, one in 'vos', 'symlink' -RETRIEVE = symlink +# Copy/download method, one in 'vos', 'symlink' -- set by the run config's +# `retrieve:` key (unit_pre exports SP_RETRIEVE), not fixed here: nibi mirrors +# the tiles locally (symlink), candide fetches them from VOSpace (vos). +RETRIEVE = $SP_RETRIEVE # If RETRIEVE=vos, number of attempts to download # Optional, default=3 diff --git a/workflow/config/cfis_image_sims/config_exp_Gie.ini b/workflow/config/cfis_image_sims/config_exp_Gie.ini index d2c782672..c5eee0f4b 100644 --- a/workflow/config/cfis_image_sims/config_exp_Gie.ini +++ b/workflow/config/cfis_image_sims/config_exp_Gie.ini @@ -91,8 +91,11 @@ INPUT_NUMBERING = \d{7} # Output file pattern without number OUTPUT_FILE_PATTERN = image-, weight-, flag- -# Method to retrieve images, one in 'vos', 'symlink' -RETRIEVE = symlink +# Method to retrieve images, one in 'vos', 'symlink'. Not a sims/data +# difference (unlike the fields marked above): the Snakefile always exports +# SP_RETRIEVE=symlink for image_sims regardless of the run config's +# `retrieve:` key, since simulation output is always local, never VOSpace. +RETRIEVE = $SP_RETRIEVE # If RETRIEVE=vos, number of attempts to download # Optional, default=3 diff --git a/workflow/config/cfis_image_sims/config_tile_Git.ini b/workflow/config/cfis_image_sims/config_tile_Git.ini index 69fcff540..4489788cb 100644 --- a/workflow/config/cfis_image_sims/config_tile_Git.ini +++ b/workflow/config/cfis_image_sims/config_tile_Git.ini @@ -85,8 +85,11 @@ INPUT_NUMBERING = \d{3}-\d{3} # Output file pattern without number OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight- -# Copy/download method, one in 'vos', 'symlink' -RETRIEVE = symlink +# Copy/download method, one in 'vos', 'symlink'. Not a sims/data difference +# (unlike the fields marked above): the Snakefile always exports +# SP_RETRIEVE=symlink for image_sims regardless of the run config's +# `retrieve:` key, since simulation output is always local, never VOSpace. +RETRIEVE = $SP_RETRIEVE # If RETRIEVE=vos, number of attempts to download # Optional, default=3 From c7bcfddd3d9d7d273f662c73a83af7d1cbb00e74 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Tue, 15 Sep 2026 17:29:51 +0200 Subject: [PATCH 12/25] config.yaml: restore real nibi defaults, keep the placeholder check as a guard REPLACE_ME sentinels made the committed config unrunnable as-is on any cluster, including nibi -- its own actual mainline user. Restore the real, working nibi values (tile_list, inputs, container, outputs) so `sp run` with no other setup does something real and correct there, the same way it always used to. The REPLACE_ME check in the Snakefile stays: it now guards against a future edit that strips these defaults without supplying real ones, rather than gatekeeping every run. Running the committed default from candide now fails honestly on nibi's real, unreachable /project path (FileNotFoundError) rather than on the placeholder check -- correct, since candide cannot see nibi's filesystem and must supply its own paths via SP_RUN_CONFIG regardless. --- workflow/config.yaml | 58 +++++++++++++++++++++++--------------------- 1 file changed, 31 insertions(+), 27 deletions(-) diff --git a/workflow/config.yaml b/workflow/config.yaml index 0ca3b239f..2db875bab 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -1,38 +1,43 @@ # Run configuration for the ShapePipe Snakemake workflow. # -# THIS FILE IS A UNIVERSAL TEMPLATE, NOT A LIVE CAMPAIGN'S CONFIG. It carries -# no machine- or campaign-specific path, on purpose: `sp run` must be the same -# command on nibi, candide, or anywhere else, and a committed default that -# happened to bake in one person's real paths would silently run a stranger's -# campaign against the wrong cluster's filesystem the moment they forgot to -# override it. Every path below is REPLACE_ME; the Snakefile refuses to parse -# past one of the ones it cannot otherwise fail clearly on later (tile_list, -# inputs.tiles, inputs.exposures, outputs.run_dir, outputs.index_db -- see the -# check near the top of Snakefile). `container:` is exempt from that check: a -# local sandbox or cached SIF (`sp container status`) is resolved first and -# makes this key irrelevant for anyone who has one. -# -# A real run always supplies its own paths via SP_RUN_CONFIG, which REPLACES -# this file wholesale rather than layering over it (workflow/bin/sp; see the -# README's "Run configuration" section): +# This file is nibi's own real, working default (Cail's real-data campaign, +# the checkout's current mainline user) -- `sp run` with no other setup does +# something real and correct on nibi. It is NOT the right config for another +# cluster or a different campaign: `SP_RUN_CONFIG` points `sp run` at a run +# config that REPLACES this file wholesale rather than layering over it +# (workflow/bin/sp; see the README's "Run configuration" section), which is +# how a candide run, an image-sims branch, or a second nibi campaign supplies +# its own paths without touching this one: # # SP_PROFILE= SP_RUN_CONFIG=/path/to/your_run.yaml \ # workflow/bin/sp run # +# Every path a run genuinely cannot do without (tile_list, inputs.tiles, +# inputs.exposures, outputs.run_dir, outputs.index_db) is checked at parse +# time against the sentinel value REPLACE_ME (see the check near the top of +# Snakefile) -- so a config that strips these defaults without supplying real +# ones fails with one clear message instead of a confusing one downstream, or +# silently running against nibi's paths from a machine that can't see them. +# `container:` is exempt from that check: a local sandbox or cached SIF +# (`sp container status`) is resolved first and makes this key irrelevant for +# anyone who has one. +# # A "run" is declared by a tile list plus the paths below. Everything here is # read at parse time; none of it is a rule input, so editing it (e.g. appending # tiles) never invalidates completed work — it only changes which jobs exist. # The tile list that scopes this run (one "IDra.IDdec" per line). The # campaign grows by appending to this file — parse-time config, so completed -# work is never invalidated. -tile_list: REPLACE_ME +# work is never invalidated. This is the real, currently-growing list for +# nibi's smk-gN campaign; a different run always supplies its own. +tile_list: /project/def-mjhudson/cdaley/sp-products/smk-g6/tiles.txt # Inputs are the only site-specific data paths; all committed configs consume these -# through SP_INPUT_TILES and SP_INPUT_EXPOSURES. +# through SP_INPUT_TILES and SP_INPUT_EXPOSURES. Pre-staged P3 data on nibi +# (get_images RETRIEVE=symlink, below). inputs: - tiles: REPLACE_ME - exposures: REPLACE_ME + tiles: /project/def-mjhudson/unions-wl/tiles + exposures: /project/def-mjhudson/unions-wl/exposures # How the tile/exposure images in `inputs:` above reach the run: `symlink` # (a pre-staged local mirror) or `vos` (fetched from VOSpace per unit, no @@ -44,7 +49,7 @@ inputs: retrieve: symlink # The container every job runs inside (apptainer software-deployment in the profile). -container: REPLACE_ME +container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif # PSF model used by the exposure and tile interpolation stages: psfex, mccd, or # fake (image simulations only: the true PSF from `psf_dict`, no exposure fit). @@ -64,7 +69,7 @@ input_type: data # sharded per-unit stores live under it: # /tiles/<2-char prefix>// and /exp/// outputs: - run_dir: REPLACE_ME + run_dir: /scratch/cdaley/shapepipe-output/smk-g6 # products_dir is the PERSISTENT root: the durable, low-volume products — the # final catalogues (/tiles///final_cat-.fits, @@ -77,13 +82,12 @@ outputs: # tile re-declare inputs against exposure stores reclamation already deleted. # # Unset means "one root": products land under run_dir, exactly the pre-D5 -# layout, which is what a fixture or smoke test wants. Delete this whole key -# (not REPLACE_ME) if that's what you want -- it's the one path in this file -# that's genuinely optional rather than templated. +# layout, which is what a fixture or smoke test wants -- delete this whole +# key, rather than pointing it at run_dir, if that's what you want. # # Snakemake's own state is the one durable-looking thing that stays on scratch # (-state; bin/sp explains why). - products_dir: REPLACE_ME + products_dir: /project/def-mjhudson/cdaley/sp-products/smk-g6 # There is no separate config_src setting: the config chain is # workflow/config/cfis, resolved relative to the Snakefile. The configs @@ -95,7 +99,7 @@ outputs: # On the persistent root with the catalogues (D5): the index is the record of # which tile reads which exposure, so it is what a post-purge reconstruction # would otherwise have to rebuild from tile headers. - index_db: REPLACE_ME + index_db: /project/def-mjhudson/cdaley/sp-products/smk-g6/index/run_index.sqlite # Rolling exposure-store reclamation (D5). When true, the COMPUTE DAG grows one # `clean_exposure` job per exposure. It fires once every campaign tile that reads From 5be4c431a0578d1e7147a85b48c4b0bc8bec2952 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Tue, 15 Sep 2026 18:19:58 +0200 Subject: [PATCH 13/25] Snakefile: check machine: against SP_PROFILE at parse time machine: (the run config) and SP_PROFILE (the env var bin/sp reads to pick profiles//, default nibi) are two independent switches for the same fact -- which cluster this is -- set in two different places at two different times, with nothing stopping them from disagreeing: export SP_PROFILE=candide but leave a copied run config's `machine: nibi` unedited, and jobs run under candide's SLURM settings against nibi's data paths. Raise a clear WorkflowError at parse time if they don't match, skipped only when `machine:` is genuinely absent (a config predating this key). No code default for `machine:` -- guessing one on its behalf would defeat the point of the check. Also fixes a comment above the REPLACE_ME check left stale by the previous commit: it still called config.yaml "a universal template with no real paths in it" after nibi's real defaults were restored. --- workflow/Snakefile | 45 +++++++++++++++++++++++++++++++------------- workflow/config.yaml | 8 ++++++++ 2 files changed, 40 insertions(+), 13 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 78672861d..9a0951bf2 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -52,16 +52,16 @@ _RUN_CONFIG = os.environ.get("SP_RUN_CONFIG") or str( Path(workflow.snakefile).parent / "config.yaml") configfile: _RUN_CONFIG -# The committed config.yaml is a universal template (no machine- or -# campaign-specific path in it), so the same `sp run` invocation works on -# every cluster: whoever runs it always supplies their own paths through -# SP_RUN_CONFIG (see config.yaml's own top comment and the README). Left -# unset, the template's REPLACE_ME sentinel would otherwise read as a real, -# oddly-named path -- silently missing tiles, or a job dying hours in on a -# node that can't see "/REPLACE_ME" -- rather than the one-line, same-message- -# everywhere error this check gives instead. `container:` is checked -# separately (container.py resolves a local sandbox or cached SIF first, so a -# placeholder there is often harmless) and so is left out here. +# config.yaml ships nibi's own real, working paths (the checkout's current +# mainline campaign) so `sp run` does something correct there with no other +# setup; a different cluster or campaign supplies its own via SP_RUN_CONFIG +# (see config.yaml's own top comment and the README) rather than editing this +# file. This check guards against a FUTURE edit that strips these defaults +# without supplying real ones -- one clear, identical-everywhere error instead +# of a confusing failure downstream, or (worse) a job dying hours in on a node +# that can't see "/REPLACE_ME". `container:` is checked separately +# (container.py resolves a local sandbox or cached SIF first, so a placeholder +# there is often harmless) and so is left out here. _PLACEHOLDER = "REPLACE_ME" for _key, _value in { "tile_list": config.get("tile_list"), @@ -73,9 +73,28 @@ for _key, _value in { if _value == _PLACEHOLDER: raise WorkflowError( f"{_key!r} in {_RUN_CONFIG} is still the template placeholder " - f"{_PLACEHOLDER!r}. workflow/config.yaml ships as a universal " - f"template with no real paths in it; point SP_RUN_CONFIG at your " - f"own run config instead (see the README, 'Run configuration').") + f"{_PLACEHOLDER!r}. Point SP_RUN_CONFIG at your own run config " + f"instead of editing the committed one (see the README, " + f"'Run configuration').") + +# `machine:` (the run config) and SP_PROFILE (the environment variable +# bin/sp reads to pick profiles//, default "nibi") are two independent +# switches for the same fact -- which cluster this is -- set in two different +# places at two different times, and nothing stops them disagreeing: export +# SP_PROFILE=candide but leave a copied run config's `machine: nibi` unedited, +# and jobs run under candide's SLURM settings against nibi's data paths. +# `machine:` has no code default (a wrong SLURM profile is exactly the kind +# of mistake this check exists to catch, so guessing one on its behalf would +# defeat the point); the check is skipped only if `machine:` truly is unset +# (a config predating this key), never silently. +_SP_PROFILE = os.environ.get("SP_PROFILE", "nibi") +_MACHINE = config.get("machine") +if _MACHINE is not None and _MACHINE != _SP_PROFILE: + raise WorkflowError( + f"machine={_MACHINE!r} in {_RUN_CONFIG} does not match " + f"SP_PROFILE={_SP_PROFILE!r} (env, defaults to 'nibi' when unset). " + f"Set SP_PROFILE={_MACHINE} to launch on the machine this config " + f"declares, or fix `machine:` if the config is the stale one.") # Every job's shell runs inside this container (apptainer software-deployment in # the profile); the user never types apptainer. WHICH image is the one resolution diff --git a/workflow/config.yaml b/workflow/config.yaml index 2db875bab..a9f90332a 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -26,6 +26,14 @@ # read at parse time; none of it is a rule input, so editing it (e.g. appending # tiles) never invalidates completed work — it only changes which jobs exist. +# Which cluster this run config is for. Must match SP_PROFILE (the +# environment variable that picks profiles//, default "nibi") -- the +# Snakefile raises a clear error at parse time if they disagree, since that +# combination means a job would run under one cluster's SLURM settings +# against another cluster's data paths. +machine: nibi +#machine: candide + # The tile list that scopes this run (one "IDra.IDdec" per line). The # campaign grows by appending to this file — parse-time config, so completed # work is never invalidated. This is the real, currently-growing list for From 29442d3b631d2e656f7990db4ce19e685bc936c6 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Tue, 15 Sep 2026 18:45:13 +0200 Subject: [PATCH 14/25] config.yaml: machines: table drives per-machine (and per-input_type) defaults `machine: nibi` selects config.yaml's own real defaults; the same key now also indexes a `machines:` table carrying tile_list, retrieve, inputs, outputs and container for both nibi and candide, so switching `machine:` (kept consistent with SP_PROFILE by the existing check) is enough to point a bare `sp run` at the right cluster's paths. On candide, where the real survey (VOSpace, RETRIEVE=vos) and the SKiLLS simulations (local disk, RETRIEVE=symlink, a different container) need different values, a machine entry nests them under `data:`/ `image_sims:` instead of stating them flat. `$base_dir` in a string is substituted with that machine's own `base_dir:`, so nibi's block states its one common root once instead of four times. A value may be the sentinel `TBD` -- known to be needed, not yet known what it is (candide's real-data ingestion: no local mirror exists there, the real vos: paths and output roots aren't confirmed yet) -- checked at parse time with the same one-clear-error mechanism REPLACE_ME used to provide. Container resolution needed a specific bridge: container.py reads `container:` by regex off the run config FILE, independent of the Snakefile's `config` dict, so a machine-derived container (which only exists after this Python-side merge) would otherwise be invisible to it. Reused SP_CONTAINER (already the documented way to point resolve_image() at a specific image) instead of teaching container.py about `machines:` too -- set only when neither an explicit top-level `container:` nor a user-set SP_CONTAINER already exists, preserving the same override precedence as every other machine default. Also fixes SP_RUN_CONFIG to LAYER on top of config.yaml rather than replacing it wholesale (workflow.configfile(), called conditionally -- it's the plain method the `configfile:` directive itself compiles to, so it works inside an `if`; merges recursively per snakemake.utils.update_config). Necessary for `machines:` to reach the common case at all: every image-sims run and any one-off cluster override already goes through SP_RUN_CONFIG, and under the old wholesale-replace behaviour none of them would ever have seen the table. Safe against the original replace-wholesale concern (an external config silently inheriting an unrelated real campaign's paths) because `machines:` entries are well-scoped defaults gated on `machine:`/`input_type:` and independently checked against SP_PROFILE, not one campaign's actual values reused as another's fallback. Verified: the committed nibi default still fails honestly (on nibi's real, candide-unreachable path) rather than on a placeholder; a machine:/SP_PROFILE mismatch still raises before that; a candide+data run config (all TBD) raises the new clear placeholder error; and the grid_3 image-sims acceptance-test config, with only `machine: candide` added and its explicit retrieve:/container: removed, dry-runs identically to before except for the retrieve: addition already committed separately -- confirming the machines: table now supplies what that config used to have to state itself, without disturbing the already-completed run's resume state. --- workflow/Snakefile | 176 ++++++++++++++++++++++++++++++------------- workflow/config.yaml | 161 ++++++++++++++++++++++----------------- 2 files changed, 214 insertions(+), 123 deletions(-) diff --git a/workflow/Snakefile b/workflow/Snakefile index 9a0951bf2..0a81548c9 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -45,37 +45,42 @@ from snakemake.exceptions import WorkflowError # --directory on /scratch (bin/sp) so .snakemake/ state never lands on /project # (group quota is a hard 27/27 TiB — a metadata write mid-run died on it live). # -# SP_RUN_CONFIG (set by bin/sp) REPLACES the committed config.yaml rather than -# layering on it: a run driven from outside -- one image-simulation shear branch, -# say -- must not inherit the committed run's paths for any key it omits. -_RUN_CONFIG = os.environ.get("SP_RUN_CONFIG") or str( - Path(workflow.snakefile).parent / "config.yaml") -configfile: _RUN_CONFIG - -# config.yaml ships nibi's own real, working paths (the checkout's current -# mainline campaign) so `sp run` does something correct there with no other -# setup; a different cluster or campaign supplies its own via SP_RUN_CONFIG -# (see config.yaml's own top comment and the README) rather than editing this -# file. This check guards against a FUTURE edit that strips these defaults -# without supplying real ones -- one clear, identical-everywhere error instead -# of a confusing failure downstream, or (worse) a job dying hours in on a node -# that can't see "/REPLACE_ME". `container:` is checked separately -# (container.py resolves a local sandbox or cached SIF first, so a placeholder -# there is often harmless) and so is left out here. -_PLACEHOLDER = "REPLACE_ME" -for _key, _value in { - "tile_list": config.get("tile_list"), - "inputs.tiles": (config.get("inputs") or {}).get("tiles"), - "inputs.exposures": (config.get("inputs") or {}).get("exposures"), - "outputs.run_dir": (config.get("outputs") or {}).get("run_dir"), - "outputs.index_db": (config.get("outputs") or {}).get("index_db"), -}.items(): - if _value == _PLACEHOLDER: - raise WorkflowError( - f"{_key!r} in {_RUN_CONFIG} is still the template placeholder " - f"{_PLACEHOLDER!r}. Point SP_RUN_CONFIG at your own run config " - f"instead of editing the committed one (see the README, " - f"'Run configuration').") +# config.yaml's defaults -- including the per-machine `machines:` table -- +# are ALWAYS loaded first; SP_RUN_CONFIG (set by bin/sp), when given, is +# layered on top via a second, conditional load (workflow.configfile() is a +# plain method -- what the `configfile:` directive itself compiles to -- +# so it can be called from inside an `if`; it merges recursively via +# snakemake.utils.update_config, not by replacement). This matters +# specifically for `machines:`: an external run config (every image-sims +# run, any one-off nibi/candide override) must still see the per-machine +# defaults it exists to provide, or they would only ever apply to the one +# case nobody actually runs -- the bare committed config with no override at +# all. A key given in SP_RUN_CONFIG still wins over the same key from +# config.yaml (later configfile: wins in the merge), so an override remains +# exactly that -- this is the "layer, don't inherit blind" version of what +# used to be a wholesale replace: `machines:` supplies well-scoped defaults +# (gated on `machine:`/`input_type:`, checked against SP_PROFILE below), not +# one campaign's actual paths reused as another's fallback. +_CONFIG_YAML = str(Path(workflow.snakefile).parent / "config.yaml") +_RUN_CONFIG = os.environ.get("SP_RUN_CONFIG") or _CONFIG_YAML +configfile: _CONFIG_YAML +if os.environ.get("SP_RUN_CONFIG"): + workflow.configfile(os.environ["SP_RUN_CONFIG"]) + +# --- input type (needed below, before PSF model) ---------------------------- +# input_type selects WHERE the pixels come from; everything downstream of the +# ingestion stages is the same chain. `image_sims` runs the SKiLLS image +# simulations through the real-data configs: the overlay config dir +# (config/cfis_image_sims) holds real files only for the ingestion INIs whose +# input naming differs, the fake-PSF INIs, and the merge column list +# (final_cat.param), and symlinks the rest into config/cfis, so an m-bias +# measured on the sims calibrates the pipeline that makes the real catalogue. +INPUT_TYPES = {"data", "image_sims"} +INPUT_TYPE = config.get("input_type", "data") +if INPUT_TYPE not in INPUT_TYPES: + raise WorkflowError( + f"Invalid input_type={INPUT_TYPE!r}; expected one of " + f"{sorted(INPUT_TYPES)}.") # `machine:` (the run config) and SP_PROFILE (the environment variable # bin/sp reads to pick profiles//, default "nibi") are two independent @@ -87,15 +92,78 @@ for _key, _value in { # of mistake this check exists to catch, so guessing one on its behalf would # defeat the point); the check is skipped only if `machine:` truly is unset # (a config predating this key), never silently. -_SP_PROFILE = os.environ.get("SP_PROFILE", "nibi") -_MACHINE = config.get("machine") -if _MACHINE is not None and _MACHINE != _SP_PROFILE: +SP_PROFILE = os.environ.get("SP_PROFILE", "nibi") +MACHINE = config.get("machine") +if MACHINE is not None and MACHINE != SP_PROFILE: raise WorkflowError( - f"machine={_MACHINE!r} in {_RUN_CONFIG} does not match " - f"SP_PROFILE={_SP_PROFILE!r} (env, defaults to 'nibi' when unset). " - f"Set SP_PROFILE={_MACHINE} to launch on the machine this config " + f"machine={MACHINE!r} in {_RUN_CONFIG} does not match " + f"SP_PROFILE={SP_PROFILE!r} (env, defaults to 'nibi' when unset). " + f"Set SP_PROFILE={MACHINE} to launch on the machine this config " f"declares, or fix `machine:` if the config is the stale one.") +# --- per-machine defaults (config.yaml's `machines:` table) ----------------- +# tile_list / retrieve / inputs / outputs / container differ by machine, and +# on candide also by input_type (the real survey and the SKiLLS simulations +# live in entirely different places, fetched differently) -- `machines:` in +# the run config carries one entry per machine, itself carrying either those +# keys directly (uniform across input_type, e.g. nibi) or nested one level +# under `data:`/`image_sims:` (e.g. candide). Any of these keys given at the +# TOP LEVEL of the run config (outside `machines:`) overrides its machine +# default -- same precedence SP_RUN_CONFIG already has over config.yaml, one +# level down. `$base_dir` inside a string is replaced with that machine +# entry's own `base_dir:`, so nibi's block states the one path that actually +# is common to its tiles/exposures/products/container once, instead of four +# times. A machine default may itself be the sentinel `TBD` (see the check +# below) -- known to be needed, not yet known what it is. +def _expand_base_dir(value, base_dir): + if isinstance(value, str): + return value.replace("$base_dir", base_dir) if base_dir else value + if isinstance(value, dict): + return {k: _expand_base_dir(v, base_dir) for k, v in value.items()} + return value + +_mconf = (config.get("machines") or {}).get(MACHINE) or {} +_sub = _mconf.get(INPUT_TYPE) or {} +_base_dir = _mconf.get("base_dir", "") +_MACHINE_CONTAINER = None +for _key in ("tile_list", "retrieve"): + if _key not in config: + _val = _sub.get(_key, _mconf.get(_key)) + if _val is not None: + config[_key] = _expand_base_dir(_val, _base_dir) +_MACHINE_CONTAINER = _sub.get("container", _mconf.get("container")) +if _MACHINE_CONTAINER is not None: + _MACHINE_CONTAINER = _expand_base_dir(_MACHINE_CONTAINER, _base_dir) +for _section in ("inputs", "outputs"): + _val = _sub.get(_section, _mconf.get(_section)) + if _val: + _merged = _expand_base_dir(dict(_val), _base_dir) + _merged.update(config.get(_section) or {}) + config[_section] = _merged + +# Every path a run genuinely cannot do without (tile_list, inputs.tiles, +# inputs.exposures, outputs.run_dir, outputs.index_db) is checked, after the +# machine defaults above are folded in, against the sentinel value `TBD` -- +# known to be needed, not yet known what it is (typically: this machine +# entry's real value hasn't been confirmed yet, see config.yaml). One clear, +# identical-everywhere error instead of a confusing failure downstream, or +# (worse) a job dying hours in on a node that can't see a path literally +# named "TBD". `container:` is checked separately, below its own resolution. +_PLACEHOLDER = "TBD" +for _key, _value in { + "tile_list": config.get("tile_list"), + "inputs.tiles": (config.get("inputs") or {}).get("tiles"), + "inputs.exposures": (config.get("inputs") or {}).get("exposures"), + "outputs.run_dir": (config.get("outputs") or {}).get("run_dir"), + "outputs.index_db": (config.get("outputs") or {}).get("index_db"), +}.items(): + if _value == _PLACEHOLDER: + raise WorkflowError( + f"{_key!r} resolves to the placeholder {_PLACEHOLDER!r} for " + f"machine={MACHINE!r}, input_type={INPUT_TYPE!r} -- this value " + f"is not known yet (see config.yaml's `machines:` table). Supply " + f"it explicitly in your run config until it is.") + # Every job's shell runs inside this container (apptainer software-deployment in # the profile); the user never types apptainer. WHICH image is the one resolution # order `sp container` exposes — this user's writable sandbox if they built one, @@ -105,6 +173,21 @@ if _MACHINE is not None and _MACHINE != _SP_PROFILE: sys.path.insert(0, str(Path(workflow.snakefile).parent / "scripts")) import container as _container # noqa: E402 +# container.py's `configured_default()` reads `container:` by REGEX off the +# run config FILE, independent of the `config` dict above (it is deliberately +# stdlib-only, no yaml import) -- so a machine-derived container, which only +# exists after the Python-side merge above, is invisible to it. SP_CONTAINER +# is already the documented way to point resolve_image() at a specific image +# instead of the cache (`local_sif()` reads it), so reuse that rather than +# teach container.py about `machines:` too. Set only when nothing more +# specific already exists: an explicit top-level `container:` in the run +# config (which STILL reaches container.py, by file, the normal way) or an +# SP_CONTAINER the user set themselves both take precedence, matching every +# other machine default above. +if _MACHINE_CONTAINER and not os.environ.get("SP_CONTAINER") \ + and not config.get("container"): + os.environ["SP_CONTAINER"] = _MACHINE_CONTAINER + # Loud at PARSE time, both ways: a broken SP_CONTAINER override, or nothing to # resolve at all (empty cache AND no `container:` key). The empty string the # resolver returns for "none" is a valid-looking container directive that passes @@ -121,21 +204,7 @@ if _kind == "none": container: _image -# --- input type and PSF model ---------------------------------------------- -# input_type selects WHERE the pixels come from; everything downstream of the -# ingestion stages is the same chain. `image_sims` runs the SKiLLS image -# simulations through the real-data configs: the overlay config dir -# (config/cfis_image_sims) holds real files only for the ingestion INIs whose -# input naming differs, the fake-PSF INIs, and the merge column list -# (final_cat.param), and symlinks the rest into config/cfis, so an m-bias -# measured on the sims calibrates the pipeline that makes the real catalogue. -INPUT_TYPES = {"data", "image_sims"} -INPUT_TYPE = config.get("input_type", "data") -if INPUT_TYPE not in INPUT_TYPES: - raise WorkflowError( - f"Invalid input_type={INPUT_TYPE!r}; expected one of " - f"{sorted(INPUT_TYPES)}.") - +# --- PSF model --------------------------------------------------------------- # `fake` is the image-simulation true PSF: no exposure PSF fit, and # fake_interp_runner writes the tile's galaxy_psf from the simulation's PSF # dictionary (config `psf_dict`) -- named so that the configs' shared @@ -193,7 +262,8 @@ INDEX_DB = Path(OUTPUTS["index_db"]) SCRIPTS = Path(workflow.basedir) / "scripts" # The config chain is the repo's committed directory (D2). The configs and # rules that set their environment variables must be versioned together. There -# is no `config_src` knob; input_type picks between the two committed dirs. +# is no separate `config_src` setting; input_type picks between the two +# committed dirs. CONFIG_DIR = Path(workflow.basedir) / "config" / { "data": "cfis", "image_sims": "cfis_image_sims"}[INPUT_TYPE] diff --git a/workflow/config.yaml b/workflow/config.yaml index a9f90332a..a3dc1d1d7 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -12,16 +12,6 @@ # SP_PROFILE= SP_RUN_CONFIG=/path/to/your_run.yaml \ # workflow/bin/sp run # -# Every path a run genuinely cannot do without (tile_list, inputs.tiles, -# inputs.exposures, outputs.run_dir, outputs.index_db) is checked at parse -# time against the sentinel value REPLACE_ME (see the check near the top of -# Snakefile) -- so a config that strips these defaults without supplying real -# ones fails with one clear message instead of a confusing one downstream, or -# silently running against nibi's paths from a machine that can't see them. -# `container:` is exempt from that check: a local sandbox or cached SIF -# (`sp container status`) is resolved first and makes this key irrelevant for -# anyone who has one. -# # A "run" is declared by a tile list plus the paths below. Everything here is # read at parse time; none of it is a rule input, so editing it (e.g. appending # tiles) never invalidates completed work — it only changes which jobs exist. @@ -34,80 +24,111 @@ machine: nibi #machine: candide -# The tile list that scopes this run (one "IDra.IDdec" per line). The -# campaign grows by appending to this file — parse-time config, so completed -# work is never invalidated. This is the real, currently-growing list for -# nibi's smk-gN campaign; a different run always supplies its own. -tile_list: /project/def-mjhudson/cdaley/sp-products/smk-g6/tiles.txt - -# Inputs are the only site-specific data paths; all committed configs consume these -# through SP_INPUT_TILES and SP_INPUT_EXPOSURES. Pre-staged P3 data on nibi -# (get_images RETRIEVE=symlink, below). -inputs: - tiles: /project/def-mjhudson/unions-wl/tiles - exposures: /project/def-mjhudson/unions-wl/exposures - -# How the tile/exposure images in `inputs:` above reach the run: `symlink` -# (a pre-staged local mirror) or `vos` (fetched from VOSpace per unit, no -# local mirror). This is cluster infrastructure, not a campaign choice -- -# nibi mirrors locally, candide fetches from VOSpace -- so pick the one that -# matches wherever `inputs.tiles`/`inputs.exposures` actually point. Ignored -# for `input_type: image_sims`, which is always `symlink` (simulation output -# is generated locally and never lives in VOSpace). -retrieve: symlink - -# The container every job runs inside (apptainer software-deployment in the profile). -container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif - -# PSF model used by the exposure and tile interpolation stages: psfex, mccd, or -# fake (image simulations only: the true PSF from `psf_dict`, no exposure fit). -psf_model: psfex - # Where the pixels come from: `data` (survey tiles/exposures, config/cfis) or # `image_sims` (SKiLLS simulations, config/cfis_image_sims -- an overlay that # differs from config/cfis only in the ingestion stages). An image_sims run -# normally sets `psf_model: fake` and `psf_dict:`, and is driven with its own run -# config through SP_RUN_CONFIG (see README, "Image simulations"). +# normally sets `psf_model: fake` and `psf_dict:`, and is driven with its own +# run config through SP_RUN_CONFIG. There is no separate `config_src` +# setting: input_type alone picks between the two committed ini directories, +# which must version together with the rules that set their environment +# variables. input_type: data -# THE TWO ROOTS (D5). +# PSF model used by the exposure and tile interpolation stages: psfex, mccd, or +# fake (image simulations only: the true PSF from `psf_dict`, no exposure fit). +psf_model: psfex + +# Per-(machine, and on candide also input_type) settings: tile_list, retrieve, +# inputs, outputs, container -- see the Snakefile for the exact merge and the +# `$base_dir` substitution rule. Any of these keys given at the TOP LEVEL of +# this file (outside `machines:`) overrides its machine default; `#retrieve:` +# and `#container:` below are exactly that override point, commented out. A +# value of `TBD` means: known to be needed, not yet known what it is -- +# parsing refuses to proceed past one. +# +# THE TWO OUTPUT ROOTS EVERY MACHINE ENTRY DECLARES (D5). # # run_dir is the SCRATCH root and the $SP_RUN every config interpolates: bulk # intermediates, sized so a batch finishes inside the 60-day purge window. The # sharded per-unit stores live under it: # /tiles/<2-char prefix>// and /exp/// -outputs: - run_dir: /scratch/cdaley/shapepipe-output/smk-g6 - +# # products_dir is the PERSISTENT root: the durable, low-volume products — the # final catalogues (/tiles///final_cat-.fits, # mirroring the scratch tree shard for shard), the index, the report. ~32-46 MB -# per tile, so a full DR6 campaign is a few hundred GB. +# per tile, so a full DR6 campaign is a few hundred GB. The final catalogue is +# also the tile-finished MARKER that cuts a finished tile's exposure edges +# (D5), which is the second reason it cannot sit on scratch: a purge would not +# merely lose a product, it would make every finished tile re-declare inputs +# against exposure stores reclamation already deleted. Unset means "one +# root": products land under run_dir, exactly the pre-D5 layout, which is +# what a fixture or smoke test wants -- omit the key, rather than pointing it +# at run_dir, if that's what you want. # -# The final catalogue is also the tile-finished MARKER that cuts a finished -# tile's exposure edges (D5), which is the second reason it cannot sit on -# scratch: a purge would not merely lose a product, it would make every finished -# tile re-declare inputs against exposure stores reclamation already deleted. -# -# Unset means "one root": products land under run_dir, exactly the pre-D5 -# layout, which is what a fixture or smoke test wants -- delete this whole -# key, rather than pointing it at run_dir, if that's what you want. -# -# Snakemake's own state is the one durable-looking thing that stays on scratch -# (-state; bin/sp explains why). - products_dir: /project/def-mjhudson/cdaley/sp-products/smk-g6 - -# There is no separate config_src setting: the config chain is -# workflow/config/cfis, resolved relative to the Snakefile. The configs -# interpolate $SP_RUN / $SP_UNIT_NUM / $SP_CONFIG / $SP_EXP / $NGMIX_* and the -# rules export them -- configs and rules are one artefact and must version -# together, so the dir is fixed by construction. - -# The run index, and — sharing its directory — missing.json and run_report.json. -# On the persistent root with the catalogues (D5): the index is the record of -# which tile reads which exposure, so it is what a post-purge reconstruction -# would otherwise have to rebuild from tile headers. - index_db: /project/def-mjhudson/cdaley/sp-products/smk-g6/index/run_index.sqlite +# index_db is the run index, and — sharing its directory — missing.json and +# run_report.json. On the persistent root with the catalogues (D5): the index +# is the record of which tile reads which exposure, so it is what a +# post-purge reconstruction would otherwise have to rebuild from tile +# headers. Snakemake's own state is the one durable-looking thing that stays +# on scratch instead (-state; bin/sp explains why). +machines: + nibi: + base_dir: /project/def-mjhudson + + # The real, currently-growing tile list for nibi's smk-gN campaign; a + # different run always supplies its own. + tile_list: $base_dir/cdaley/sp-products/smk-g6/tiles.txt + + # Pre-staged P3 data (get_images RETRIEVE=symlink). + retrieve: symlink + inputs: + tiles: $base_dir/unions-wl/tiles + exposures: $base_dir/unions-wl/exposures + + outputs: + run_dir: /scratch/cdaley/shapepipe-output/smk-g6 + products_dir: $base_dir/cdaley/sp-products/smk-g6 + index_db: $base_dir/cdaley/sp-products/smk-g6/index/run_index.sqlite + + container: $base_dir/cdaley/containers/shapepipe-develop-runtime.sif + + candide: + base_dir: /n17data/UNIONS/WL + + data: + # candide has no pre-staged local mirror of the survey (unlike nibi): + # images are fetched from VOSpace per unit instead. + retrieve: vos + tile_list: TBD + inputs: + tiles: TBD # a vos: URL, once known + exposures: TBD + outputs: + run_dir: TBD + container: /n17data/cdaley/containers/shapepipe_develop-runtime-20260718.sif + + image_sims: + # SKiLLS simulation output already lives on local disk. + retrieve: symlink + container: /n17data/cdaley/containers/shapepipe_im_sims-runtime.sif + # tile_list, inputs and outputs are intentionally absent here: each + # SKiLLS grid/shear-branch campaign supplies its own via SP_RUN_CONFIG + # (e.g. inputs.tiles: /n09data/hervas/skills_out//images/SP_tiles) + # -- this entry only saves such a run from restating retrieve/container + # too. + +# How the tile/exposure images in `inputs:` above reach the run: `symlink` +# (a pre-staged local mirror) or `vos` (fetched from VOSpace per unit, no +# local mirror). Set per machine (and, on candide, per input_type) above; +# ignored either way for `input_type: image_sims`, which is always `symlink` +# (simulation output is generated locally and never lives in VOSpace). Give +# it here directly only to override the machine default for one run. +#retrieve: symlink + +# The container every job runs inside (apptainer software-deployment in the +# profile). Set per machine (and, on candide, per input_type) above; give it +# here directly only to override that default for one run. +#container: ... # Rolling exposure-store reclamation (D5). When true, the COMPUTE DAG grows one # `clean_exposure` job per exposure. It fires once every campaign tile that reads From 7293e3b2ef513dfd56b4c25b9b3fbc6c8aca4f3f Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Tue, 15 Sep 2026 19:00:43 +0200 Subject: [PATCH 15/25] workflow: one run-config resolver for Snakefile, bin/sp and container.py The machines: table was only understood by the Snakefile. bin/sp read outputs.run_dir/index_db and tile_store_root straight from the config file, so the committed config (outputs now under machines.nibi) broke `sp run` on nibi with a KeyError, and container.py only saw a top-level container: line. scripts/run_config.py now does the whole resolution (config.yaml, SP_RUN_CONFIG merged on top, then machines[machine] [input_type] for anything unset) and all three use it. - container.resolve_image() takes the resolved container explicitly; drops the SP_CONTAINER workaround, which had let a machine default outrank a user's cached SIF, against the documented order. - nibi's machine entry is nested under data: like candide's, so a nibi image_sims run cannot inherit real-data paths. - Unset required keys are reported like TBD, in one error. - README gains a "Run configuration" section; config.yaml, Snakefile and the ini comments are cut down. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013NzWDtdQbsK1VhvYbDnmTo --- workflow/README.md | 31 ++- workflow/Snakefile | 202 ++++------------- workflow/bin/sp | 30 ++- workflow/config.yaml | 213 ++++-------------- workflow/config/cfis/config_exp_Gie.ini | 5 +- workflow/config/cfis/config_tile_Git.ini | 4 +- .../config/cfis_image_sims/config_exp_Gie.ini | 5 +- .../cfis_image_sims/config_tile_Git.ini | 5 +- workflow/scripts/completeness.py | 9 +- workflow/scripts/container.py | 38 ++-- workflow/scripts/run_config.py | 83 +++++++ 11 files changed, 234 insertions(+), 391 deletions(-) create mode 100644 workflow/scripts/run_config.py diff --git a/workflow/README.md b/workflow/README.md index 43a7d018d..c0652c1eb 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -69,10 +69,33 @@ One campaign per shear branch, each with its own run config: SP_PROFILE=candide SP_RUN_CONFIG=/path/run_1p2z_grid_1.yaml workflow/bin/sp run ``` -`SP_RUN_CONFIG` replaces `workflow/config.yaml` (it is not layered on it, so no -key falls back to the committed run's paths) and is snapshotted with the code; -`SP_PROFILE` picks `profiles//`. sp_validation's image-simulation workflow -drives these campaigns and measures m from their final catalogues. +## Run configuration + +`SP_RUN_CONFIG` is merged on top of `workflow/config.yaml` and snapshotted with +the code. `machine:` (which must match `SP_PROFILE`, default `nibi`) and +`input_type:` then select an entry of the `machines:` table, which supplies +`tile_list`, `retrieve` (`symlink` or `vos`), `inputs`, `outputs` and +`container` for any of these the run config leaves unset (`$base_dir` expands +to that machine's `base_dir`). A value of `TBD` stops the run at parse time +until it is set. A run config therefore only needs what differs, e.g. for one +SKiLLS shear branch on candide: + +```yaml +machine: candide +input_type: image_sims +psf_model: fake +psf_dict: /home/hervas/fhervas/workdir_skills/input/psf_files/Full_psf_dict.pickle +tile_list: /path/to/tiles.txt +inputs: + tiles: /n09data/hervas/skills_out/1z2z_grid_3/images/SP_tiles + exposures: /n09data/hervas/skills_out/1z2z_grid_3/images/SP_exp +outputs: + run_dir: /path/to/run + index_db: /path/to/run/index.sqlite +``` + +sp_validation's image-simulation workflow drives these campaigns and measures m +from their final catalogues. On candide, the node-local tile store (bound from the node's 31 GB `/tmp`) does not hold several dense image-sim tiles at once. Set `tile_store_root:` in the run diff --git a/workflow/Snakefile b/workflow/Snakefile index 0a81548c9..b3612ed24 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -42,39 +42,22 @@ from pathlib import Path from snakemake.exceptions import WorkflowError # Resolved relative to THIS file, not the working directory: snakemake runs with -# --directory on /scratch (bin/sp) so .snakemake/ state never lands on /project -# (group quota is a hard 27/27 TiB — a metadata write mid-run died on it live). +# --directory on /scratch (bin/sp) so .snakemake/ state never lands on /project. # -# config.yaml's defaults -- including the per-machine `machines:` table -- -# are ALWAYS loaded first; SP_RUN_CONFIG (set by bin/sp), when given, is -# layered on top via a second, conditional load (workflow.configfile() is a -# plain method -- what the `configfile:` directive itself compiles to -- -# so it can be called from inside an `if`; it merges recursively via -# snakemake.utils.update_config, not by replacement). This matters -# specifically for `machines:`: an external run config (every image-sims -# run, any one-off nibi/candide override) must still see the per-machine -# defaults it exists to provide, or they would only ever apply to the one -# case nobody actually runs -- the bare committed config with no override at -# all. A key given in SP_RUN_CONFIG still wins over the same key from -# config.yaml (later configfile: wins in the merge), so an override remains -# exactly that -- this is the "layer, don't inherit blind" version of what -# used to be a wholesale replace: `machines:` supplies well-scoped defaults -# (gated on `machine:`/`input_type:`, checked against SP_PROFILE below), not -# one campaign's actual paths reused as another's fallback. -_CONFIG_YAML = str(Path(workflow.snakefile).parent / "config.yaml") -_RUN_CONFIG = os.environ.get("SP_RUN_CONFIG") or _CONFIG_YAML -configfile: _CONFIG_YAML +# config.yaml is always loaded; SP_RUN_CONFIG (bin/sp) is merged on top, then +# the `machines:` entry for (machine, input_type) fills whatever is still unset +# (scripts/run_config.py, shared with bin/sp and container.py). +sys.path.insert(0, str(Path(workflow.snakefile).parent / "scripts")) +import run_config # noqa: E402 + +_RUN_CONFIG = os.environ.get("SP_RUN_CONFIG") or str( + Path(workflow.snakefile).parent / "config.yaml") +configfile: str(Path(workflow.snakefile).parent / "config.yaml") if os.environ.get("SP_RUN_CONFIG"): workflow.configfile(os.environ["SP_RUN_CONFIG"]) -# --- input type (needed below, before PSF model) ---------------------------- -# input_type selects WHERE the pixels come from; everything downstream of the -# ingestion stages is the same chain. `image_sims` runs the SKiLLS image -# simulations through the real-data configs: the overlay config dir -# (config/cfis_image_sims) holds real files only for the ingestion INIs whose -# input naming differs, the fake-PSF INIs, and the merge column list -# (final_cat.param), and symlinks the rest into config/cfis, so an m-bias -# measured on the sims calibrates the pipeline that makes the real catalogue. +# input_type picks where the pixels come from (and config/cfis vs +# config/cfis_image_sims); everything after ingestion is the same chain. INPUT_TYPES = {"data", "image_sims"} INPUT_TYPE = config.get("input_type", "data") if INPUT_TYPE not in INPUT_TYPES: @@ -82,134 +65,44 @@ if INPUT_TYPE not in INPUT_TYPES: f"Invalid input_type={INPUT_TYPE!r}; expected one of " f"{sorted(INPUT_TYPES)}.") -# `machine:` (the run config) and SP_PROFILE (the environment variable -# bin/sp reads to pick profiles//, default "nibi") are two independent -# switches for the same fact -- which cluster this is -- set in two different -# places at two different times, and nothing stops them disagreeing: export -# SP_PROFILE=candide but leave a copied run config's `machine: nibi` unedited, -# and jobs run under candide's SLURM settings against nibi's data paths. -# `machine:` has no code default (a wrong SLURM profile is exactly the kind -# of mistake this check exists to catch, so guessing one on its behalf would -# defeat the point); the check is skipped only if `machine:` truly is unset -# (a config predating this key), never silently. +# `machine:` must agree with SP_PROFILE (bin/sp's profile choice, default +# nibi): otherwise jobs run under one cluster's SLURM profile against another +# cluster's paths. Skipped only if `machine:` is unset. SP_PROFILE = os.environ.get("SP_PROFILE", "nibi") MACHINE = config.get("machine") if MACHINE is not None and MACHINE != SP_PROFILE: raise WorkflowError( - f"machine={MACHINE!r} in {_RUN_CONFIG} does not match " - f"SP_PROFILE={SP_PROFILE!r} (env, defaults to 'nibi' when unset). " - f"Set SP_PROFILE={MACHINE} to launch on the machine this config " - f"declares, or fix `machine:` if the config is the stale one.") - -# --- per-machine defaults (config.yaml's `machines:` table) ----------------- -# tile_list / retrieve / inputs / outputs / container differ by machine, and -# on candide also by input_type (the real survey and the SKiLLS simulations -# live in entirely different places, fetched differently) -- `machines:` in -# the run config carries one entry per machine, itself carrying either those -# keys directly (uniform across input_type, e.g. nibi) or nested one level -# under `data:`/`image_sims:` (e.g. candide). Any of these keys given at the -# TOP LEVEL of the run config (outside `machines:`) overrides its machine -# default -- same precedence SP_RUN_CONFIG already has over config.yaml, one -# level down. `$base_dir` inside a string is replaced with that machine -# entry's own `base_dir:`, so nibi's block states the one path that actually -# is common to its tiles/exposures/products/container once, instead of four -# times. A machine default may itself be the sentinel `TBD` (see the check -# below) -- known to be needed, not yet known what it is. -def _expand_base_dir(value, base_dir): - if isinstance(value, str): - return value.replace("$base_dir", base_dir) if base_dir else value - if isinstance(value, dict): - return {k: _expand_base_dir(v, base_dir) for k, v in value.items()} - return value - -_mconf = (config.get("machines") or {}).get(MACHINE) or {} -_sub = _mconf.get(INPUT_TYPE) or {} -_base_dir = _mconf.get("base_dir", "") -_MACHINE_CONTAINER = None -for _key in ("tile_list", "retrieve"): - if _key not in config: - _val = _sub.get(_key, _mconf.get(_key)) - if _val is not None: - config[_key] = _expand_base_dir(_val, _base_dir) -_MACHINE_CONTAINER = _sub.get("container", _mconf.get("container")) -if _MACHINE_CONTAINER is not None: - _MACHINE_CONTAINER = _expand_base_dir(_MACHINE_CONTAINER, _base_dir) -for _section in ("inputs", "outputs"): - _val = _sub.get(_section, _mconf.get(_section)) - if _val: - _merged = _expand_base_dir(dict(_val), _base_dir) - _merged.update(config.get(_section) or {}) - config[_section] = _merged - -# Every path a run genuinely cannot do without (tile_list, inputs.tiles, -# inputs.exposures, outputs.run_dir, outputs.index_db) is checked, after the -# machine defaults above are folded in, against the sentinel value `TBD` -- -# known to be needed, not yet known what it is (typically: this machine -# entry's real value hasn't been confirmed yet, see config.yaml). One clear, -# identical-everywhere error instead of a confusing failure downstream, or -# (worse) a job dying hours in on a node that can't see a path literally -# named "TBD". `container:` is checked separately, below its own resolution. -_PLACEHOLDER = "TBD" -for _key, _value in { - "tile_list": config.get("tile_list"), - "inputs.tiles": (config.get("inputs") or {}).get("tiles"), - "inputs.exposures": (config.get("inputs") or {}).get("exposures"), - "outputs.run_dir": (config.get("outputs") or {}).get("run_dir"), - "outputs.index_db": (config.get("outputs") or {}).get("index_db"), -}.items(): - if _value == _PLACEHOLDER: - raise WorkflowError( - f"{_key!r} resolves to the placeholder {_PLACEHOLDER!r} for " - f"machine={MACHINE!r}, input_type={INPUT_TYPE!r} -- this value " - f"is not known yet (see config.yaml's `machines:` table). Supply " - f"it explicitly in your run config until it is.") - -# Every job's shell runs inside this container (apptainer software-deployment in -# the profile); the user never types apptainer. WHICH image is the one resolution -# order `sp container` exposes — this user's writable sandbox if they built one, -# else their cached SIF if they pulled one, else `container:` above. An empty -# cache therefore lands on exactly the shared /project image the workflow has -# always run, and a package installed into a sandbox reaches the jobs too. -sys.path.insert(0, str(Path(workflow.snakefile).parent / "scripts")) + f"machine={MACHINE!r} ({_RUN_CONFIG}) does not match " + f"SP_PROFILE={SP_PROFILE!r}. Set SP_PROFILE={MACHINE}, or fix " + f"`machine:`.") + +run_config.apply_machine_defaults(config) +_unresolved = run_config.unresolved(config) +if _unresolved: + raise WorkflowError( + f"Unset or {run_config.PLACEHOLDER!r} for machine={MACHINE!r}, " + f"input_type={INPUT_TYPE!r}: {', '.join(_unresolved)}. Set them in " + f"your run config (SP_RUN_CONFIG).") + +# The image every job runs in: this user's sandbox, else their cached SIF, +# else the run config's `container:`. Checked at parse time, since an empty +# container directive passes a dry run and only fails on a node. import container as _container # noqa: E402 -# container.py's `configured_default()` reads `container:` by REGEX off the -# run config FILE, independent of the `config` dict above (it is deliberately -# stdlib-only, no yaml import) -- so a machine-derived container, which only -# exists after the Python-side merge above, is invisible to it. SP_CONTAINER -# is already the documented way to point resolve_image() at a specific image -# instead of the cache (`local_sif()` reads it), so reuse that rather than -# teach container.py about `machines:` too. Set only when nothing more -# specific already exists: an explicit top-level `container:` in the run -# config (which STILL reaches container.py, by file, the normal way) or an -# SP_CONTAINER the user set themselves both take precedence, matching every -# other machine default above. -if _MACHINE_CONTAINER and not os.environ.get("SP_CONTAINER") \ - and not config.get("container"): - os.environ["SP_CONTAINER"] = _MACHINE_CONTAINER - -# Loud at PARSE time, both ways: a broken SP_CONTAINER override, or nothing to -# resolve at all (empty cache AND no `container:` key). The empty string the -# resolver returns for "none" is a valid-looking container directive that passes -# dry-run and only fails once jobs reach a node. try: - _image, _kind = _container.resolve_image() + _image, _kind = _container.resolve_image(config.get("container")) except _container.ContainerError as _exc: raise WorkflowError(str(_exc)) if _kind == "none": raise WorkflowError( - f"No container image resolved: no sandbox, no cached SIF, and no " - f"'{_container.CONFIG_KEY}:' key in {_container.CONFIG_FILE}. " - f"Run `sp container pull`, or restore the key.") + "No container image resolved: no sandbox, no cached SIF, and no " + "`container:` in the run config. Run `sp container pull`, or set it.") container: _image -# --- PSF model --------------------------------------------------------------- -# `fake` is the image-simulation true PSF: no exposure PSF fit, and -# fake_interp_runner writes the tile's galaxy_psf from the simulation's PSF -# dictionary (config `psf_dict`) -- named so that the configs' shared -# `${SP_PSF}_interp_runner` reads it exactly as it reads psfex_interp_runner. Simulations that contain stars can instead run -# psfex or mccd exactly as the real data does. +# `fake` is the image-simulation true PSF: no exposure PSF fit; the tile's +# galaxy_psf comes from `psf_dict` via fake_interp_runner (read by the +# configs as ${SP_PSF}_interp_runner). Sims with stars may use psfex/mccd. PSF_MODELS = {"psfex", "mccd", "fake"} PSF_MODEL = config.get("psf_model", "psfex") if PSF_MODEL not in PSF_MODELS: @@ -225,16 +118,8 @@ if PSF_MODEL == "fake" and not PSF_DICT: raise WorkflowError("psf_model=fake needs `psf_dict:` (the simulation's " "pickled PSF dictionary) in the run config.") -# How tile/exposure images reach the run: `symlink` (a pre-staged local -# mirror -- nibi's layout) or `vos` (fetched from VOSpace per unit -- candide's -# real-data layout, no local mirror needed). A per-cluster choice, so it is a -# run-config key (`retrieve:`) rather than fixed in the committed ini files; -# `unit_pre` exports it as SP_RETRIEVE for config_tile_Git.ini / -# config_exp_Gie.ini. `image_sims` always overrides it to `symlink` below, -# regardless of what the run config says: simulation output is generated -# locally and never lives in VOSpace, so a `data` run config's `retrieve: vos` -# reused as a sims campaign's base must not leak into the sims ingestion -# stages. +# How images reach the run (exported as SP_RETRIEVE for the Git/Gie configs): +# `symlink` to a local mirror, or `vos` download. Sims are always local. RETRIEVE_MODES = {"symlink", "vos"} RETRIEVE_MODE = config.get("retrieve", "symlink") if RETRIEVE_MODE not in RETRIEVE_MODES: @@ -262,8 +147,7 @@ INDEX_DB = Path(OUTPUTS["index_db"]) SCRIPTS = Path(workflow.basedir) / "scripts" # The config chain is the repo's committed directory (D2). The configs and # rules that set their environment variables must be versioned together. There -# is no separate `config_src` setting; input_type picks between the two -# committed dirs. +# is no separate `config_src` setting; input_type picks between the two dirs. CONFIG_DIR = Path(workflow.basedir) / "config" / { "data": "cfis", "image_sims": "cfis_image_sims"}[INPUT_TYPE] @@ -678,11 +562,9 @@ def unit_pre(stage, unit, *, exp_name=None, forest=None, env=None, the committed config dir. It also exports the configured input roots as ``SP_INPUT_TILES`` and - ``SP_INPUT_EXPOSURES``, how to fetch them as ``SP_RETRIEVE`` (``symlink`` - or ``vos``; forced to ``symlink`` for ``image_sims``, see ``RETRIEVE_MODE`` - above), and the PSF choice as ``SP_PSF`` for the committed ini chain (plus - ``PSF_DICT`` for the image-simulation true PSF, which the configs read as - ``${SP_PSF}_interp_runner`` = ``fake_interp_runner``). + ``SP_INPUT_EXPOSURES``, the retrieve mode as ``SP_RETRIEVE`` and the PSF + choice as ``SP_PSF`` for the committed ini chain (plus ``PSF_DICT`` for the + image-simulation true PSF). Every line here is part of each rule's ``params.pre`` and so of the ``params`` rerun trigger: a line added for every rule reruns every finished diff --git a/workflow/bin/sp b/workflow/bin/sp index b64905181..11ce102da 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -30,11 +30,10 @@ # SP_PROFILE profiles// to launch with (default: nibi). # SP_SNAKEMAKE_ENV venv to activate; skipped when it does not exist and a # snakemake is already on PATH (e.g. candide). -# SP_RUN_CONFIG a run config that REPLACES workflow/config.yaml (it is not -# layered on it, so no key falls back to the committed run's -# paths). Snapshotted with the code by `sp run`. This is how -# sp_validation drives one campaign per image-simulation -# shear branch. +# SP_RUN_CONFIG a run config merged on top of workflow/config.yaml +# (scripts/run_config.py). Snapshotted with the code by +# `sp run`. This is how sp_validation drives one campaign +# per image-simulation shear branch. set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # workflow/ @@ -56,15 +55,13 @@ elif ! command -v snakemake >/dev/null 2>&1; then exit 2 fi -# Scalar reader for workflow/config.yaml. The venv is active by this point and -# snakemake depends on PyYAML, so this parses the file rather than pattern-matching -# it -- quotes, inline comments and nesting are the parser's problem, not ours. -cfg() { python -c 'import sys, yaml -value = yaml.safe_load(open(sys.argv[1])) -for key in sys.argv[2].split("."): - value = value[key] -print(value or "")' "$CONFIG" "$1"; } +# Resolved run-config value (config.yaml + SP_RUN_CONFIG + machine defaults), +# the same resolution the Snakefile uses. +cfg() { python "$SCRIPTS/run_config.py" "$HERE/config.yaml" "${SP_RUN_CONFIG:-}" "$1"; } RUN_DIR="$(cfg outputs.run_dir)"; INDEX_DB="$(cfg outputs.index_db)" +if [ "${1:-}" != container ] && { [ -z "$RUN_DIR" ] || [ "$RUN_DIR" = TBD ]; }; then + echo "sp: outputs.run_dir is unset or TBD for this machine/input_type" >&2; exit 2 +fi # Snakemake state (.snakemake: metadata, locks, incomplete markers) lives NEXT TO # THE RUN on /scratch, never on the persistent root — the one exception to D5's @@ -149,10 +146,9 @@ PY # directory (NFS) trades ngmix read latency for room; the campaign-unique store # names (LOCAL_TAG) keep concurrent campaigns apart in one root. Profiles with # no `X:/local/scratch` bind (nibi's `--bind /local`) get one appended. - python - "$SNAPSHOT/profiles/$PROFILE/config.yaml" "$CONFIG" <<'PY' -import os, pathlib, re, sys, yaml -f, run_config = pathlib.Path(sys.argv[1]), sys.argv[2] -root = (yaml.safe_load(open(run_config)) or {}).get("tile_store_root") + python - "$SNAPSHOT/profiles/$PROFILE/config.yaml" "$(cfg tile_store_root)" <<'PY' +import os, pathlib, re, sys +f, root = pathlib.Path(sys.argv[1]), sys.argv[2] if root: root = str(root) if not os.path.isabs(root): diff --git a/workflow/config.yaml b/workflow/config.yaml index a3dc1d1d7..ab60bd20c 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -1,213 +1,80 @@ # Run configuration for the ShapePipe Snakemake workflow. # -# This file is nibi's own real, working default (Cail's real-data campaign, -# the checkout's current mainline user) -- `sp run` with no other setup does -# something real and correct on nibi. It is NOT the right config for another -# cluster or a different campaign: `SP_RUN_CONFIG` points `sp run` at a run -# config that REPLACES this file wholesale rather than layering over it -# (workflow/bin/sp; see the README's "Run configuration" section), which is -# how a candide run, an image-sims branch, or a second nibi campaign supplies -# its own paths without touching this one: +# A run config given via SP_RUN_CONFIG is merged on top of this file, then +# the `machines:` entry for (machine, input_type) fills anything still unset: # -# SP_PROFILE= SP_RUN_CONFIG=/path/to/your_run.yaml \ -# workflow/bin/sp run +# SP_PROFILE= SP_RUN_CONFIG=/path/to/run.yaml workflow/bin/sp run # -# A "run" is declared by a tile list plus the paths below. Everything here is -# read at parse time; none of it is a rule input, so editing it (e.g. appending -# tiles) never invalidates completed work — it only changes which jobs exist. +# Everything is read at parse time; editing it (e.g. appending tiles) never +# invalidates completed work. -# Which cluster this run config is for. Must match SP_PROFILE (the -# environment variable that picks profiles//, default "nibi") -- the -# Snakefile raises a clear error at parse time if they disagree, since that -# combination means a job would run under one cluster's SLURM settings -# against another cluster's data paths. +# Cluster; must match SP_PROFILE (default nibi). machine: nibi #machine: candide -# Where the pixels come from: `data` (survey tiles/exposures, config/cfis) or -# `image_sims` (SKiLLS simulations, config/cfis_image_sims -- an overlay that -# differs from config/cfis only in the ingestion stages). An image_sims run -# normally sets `psf_model: fake` and `psf_dict:`, and is driven with its own -# run config through SP_RUN_CONFIG. There is no separate `config_src` -# setting: input_type alone picks between the two committed ini directories, -# which must version together with the rules that set their environment -# variables. +# data (survey, config/cfis) or image_sims (SKiLLS, config/cfis_image_sims). input_type: data -# PSF model used by the exposure and tile interpolation stages: psfex, mccd, or -# fake (image simulations only: the true PSF from `psf_dict`, no exposure fit). +# psfex, mccd, or fake (image_sims only: true PSF from `psf_dict`). psf_model: psfex -# Per-(machine, and on candide also input_type) settings: tile_list, retrieve, -# inputs, outputs, container -- see the Snakefile for the exact merge and the -# `$base_dir` substitution rule. Any of these keys given at the TOP LEVEL of -# this file (outside `machines:`) overrides its machine default; `#retrieve:` -# and `#container:` below are exactly that override point, commented out. A -# value of `TBD` means: known to be needed, not yet known what it is -- -# parsing refuses to proceed past one. +# Per machine and input_type: tile_list, retrieve (symlink|vos), inputs, +# outputs, container. A key set at top level (or in SP_RUN_CONFIG) overrides +# it. `$base_dir` expands to the machine's base_dir. TBD = not known yet; the +# run refuses to start until it is set. # -# THE TWO OUTPUT ROOTS EVERY MACHINE ENTRY DECLARES (D5). -# -# run_dir is the SCRATCH root and the $SP_RUN every config interpolates: bulk -# intermediates, sized so a batch finishes inside the 60-day purge window. The -# sharded per-unit stores live under it: -# /tiles/<2-char prefix>// and /exp/// -# -# products_dir is the PERSISTENT root: the durable, low-volume products — the -# final catalogues (/tiles///final_cat-.fits, -# mirroring the scratch tree shard for shard), the index, the report. ~32-46 MB -# per tile, so a full DR6 campaign is a few hundred GB. The final catalogue is -# also the tile-finished MARKER that cuts a finished tile's exposure edges -# (D5), which is the second reason it cannot sit on scratch: a purge would not -# merely lose a product, it would make every finished tile re-declare inputs -# against exposure stores reclamation already deleted. Unset means "one -# root": products land under run_dir, exactly the pre-D5 layout, which is -# what a fixture or smoke test wants -- omit the key, rather than pointing it -# at run_dir, if that's what you want. -# -# index_db is the run index, and — sharing its directory — missing.json and -# run_report.json. On the persistent root with the catalogues (D5): the index -# is the record of which tile reads which exposure, so it is what a -# post-purge reconstruction would otherwise have to rebuild from tile -# headers. Snakemake's own state is the one durable-looking thing that stays -# on scratch instead (-state; bin/sp explains why). +# outputs: run_dir = scratch root for intermediates; products_dir = persistent +# root for final catalogues, index and report (omit to use run_dir); +# index_db = tile/exposure index. machines: nibi: base_dir: /project/def-mjhudson - - # The real, currently-growing tile list for nibi's smk-gN campaign; a - # different run always supplies its own. - tile_list: $base_dir/cdaley/sp-products/smk-g6/tiles.txt - - # Pre-staged P3 data (get_images RETRIEVE=symlink). - retrieve: symlink - inputs: - tiles: $base_dir/unions-wl/tiles - exposures: $base_dir/unions-wl/exposures - - outputs: - run_dir: /scratch/cdaley/shapepipe-output/smk-g6 - products_dir: $base_dir/cdaley/sp-products/smk-g6 - index_db: $base_dir/cdaley/sp-products/smk-g6/index/run_index.sqlite - - container: $base_dir/cdaley/containers/shapepipe-develop-runtime.sif + data: + tile_list: $base_dir/cdaley/sp-products/smk-g6/tiles.txt + retrieve: symlink + inputs: + tiles: $base_dir/unions-wl/tiles + exposures: $base_dir/unions-wl/exposures + outputs: + run_dir: /scratch/cdaley/shapepipe-output/smk-g6 + products_dir: $base_dir/cdaley/sp-products/smk-g6 + index_db: $base_dir/cdaley/sp-products/smk-g6/index/run_index.sqlite + container: $base_dir/cdaley/containers/shapepipe-develop-runtime.sif candide: base_dir: /n17data/UNIONS/WL - data: - # candide has no pre-staged local mirror of the survey (unlike nibi): - # images are fetched from VOSpace per unit instead. retrieve: vos tile_list: TBD inputs: - tiles: TBD # a vos: URL, once known + tiles: TBD # vos: URL exposures: TBD outputs: run_dir: TBD + index_db: TBD container: /n17data/cdaley/containers/shapepipe_develop-runtime-20260718.sif - image_sims: - # SKiLLS simulation output already lives on local disk. + # tile_list, inputs, outputs: per SKiLLS campaign, in its own run config, + # e.g. inputs.tiles: /n09data/hervas/skills_out//images/SP_tiles retrieve: symlink container: /n17data/cdaley/containers/shapepipe_im_sims-runtime.sif - # tile_list, inputs and outputs are intentionally absent here: each - # SKiLLS grid/shear-branch campaign supplies its own via SP_RUN_CONFIG - # (e.g. inputs.tiles: /n09data/hervas/skills_out//images/SP_tiles) - # -- this entry only saves such a run from restating retrieve/container - # too. - -# How the tile/exposure images in `inputs:` above reach the run: `symlink` -# (a pre-staged local mirror) or `vos` (fetched from VOSpace per unit, no -# local mirror). Set per machine (and, on candide, per input_type) above; -# ignored either way for `input_type: image_sims`, which is always `symlink` -# (simulation output is generated locally and never lives in VOSpace). Give -# it here directly only to override the machine default for one run. -#retrieve: symlink -# The container every job runs inside (apptainer software-deployment in the -# profile). Set per machine (and, on candide, per input_type) above; give it -# here directly only to override that default for one run. -#container: ... - -# Rolling exposure-store reclamation (D5). When true, the COMPUTE DAG grows one -# `clean_exposure` job per exposure. It fires once every campaign tile that reads -# that exposure has its vignets, deletes the exposure's store AND its manifests, -# and leaves `cleaned.json`, which absorbs the manifests — `sp report` reads them -# back out of the tombstone and reports the exposure as `cleaned`. Set it false -# for a small control run whose exposure stores you want to keep around (e.g. to -# re-measure only the shape chain rather than rebuild the whole tile) -- true is -# the right default for anything that isn't a deliberately-kept fixture. +# Per-exposure store reclamation once every tile reading it has its vignets. clean: true -# Rolling TILE-store reclamation (D5). When true, the COMPUTE DAG grows one -# `clean_tile` job per in-scope tile, ordered after that tile's own final_cat, so -# a tile's scratch store goes as soon as its chain lands rather than at the end -# of the batch. It deletes the tile's whole /tiles/// tree -# except four things another mechanism owns — `cleaned.json`, -# `manifests/tile_vignets.json` (clean_exposure's eligibility currency), -# `manifests/tile_find_exposures.json` (prepare_all_tiles' target) and the Fe -# exposure list under output/ (build_index re-reads it at every compute parse). -# clean_tile.py argues each one. -# -# A SEPARATE SWITCH FROM `clean:` ABOVE, AND NOT AN OVERSIGHT. Exposure -# reclamation is reversible in the only sense that matters: a tile appended later -# rebuilds the exposure chain from VOS, expensively but completely. TILE -# reclamation DESTROYS THE PER-TILE AUDIT TRAIL and nothing rebuilds it. -# TWO tools read it: sp-products/tools/sp_tilecost.py and sp_costmodel.py. Both -# attribute the fused tile_shape group job's cost per tile from that tile's -# scratch store — the SExtractor catalogue (NAXIS2 of the sexcat is the object -# count, the cost model's independent variable; sp_costmodel also reads its -# EPOCH_k extensions for the geometric epoch count) and the eight tile_ngmix -# benchmark TSVs — and that model is how every mem_mb and runtime number in -# tile.smk was derived. The sexcat is ~380 MB per tile and cannot be kept; the -# benchmark ROWS are absorbed into the tombstone, so the measurements survive -# but not at the paths the tools read. -# -# (The third artifact they read, run_sp_tile_ngmix_Ngu/, is NOT lost to this -# flag: tile_ngmix declares its chunk dir temp(), so snakemake reclaims it as -# soon as tile_merge_cats runs. It is already absent from every finished tile, -# clean_tiles or not.) -# -# The honest one-line summary: REQUIRED FOR ANY BATCH OVER ~850 TILES, AND IT -# TURNS OFF PER-TILE COST ATTRIBUTION. The arithmetic: a finished tile leaves -# 1.19 GiB across 137 inodes (measured on 186.307), so a 1 TiB scratch quota is -# full at 859 tiles, and DR6's 23,114 tiles would want 26.9 TiB and 3.17M inodes -# against a 1M quota. Reclaimed, a tile costs 10 inodes and 12.9 KB — 231k inodes -# and ~300 MB for the whole of DR6. -# -# Set it false for the same kind of small control run `clean:` above would be -# false for, when the per-tile store is what a re-measurement needs to read. +# Per-tile store reclamation after the tile's final_cat. Needed above ~850 +# tiles (scratch quota); deletes the per-tile data sp_tilecost/sp_costmodel +# read. Set both false for a small control run you want to re-measure. clean_tiles: true -# Tiles that may NOT pin an exposure store (default: empty). -# -# An exposure is eligible for cleaning only once EVERY consuming tile has its -# vignets. One permanently-failed tile therefore holds all its exposures -# (~8 measured on P3: 7.9 exposures/tile) for the life of the campaign. A tile listed here is dropped from the -# consumer sets, and its exposures become eligible. -# -# READ THIS BEFORE ADDING A TILE. Ignoring a tile is a decision to give up its -# exposures' stores. If you later retry that tile, those exposure chains are -# gone and will be REBUILT from scratch — get_images, split, psf, per -# exposure. That is correct, and expensive. Ignore a tile when you have decided -# it is dead, not while you are still debugging it. +# Failed tiles that should not keep their exposures from being cleaned. +# Their exposures are rebuilt from scratch if the tile is retried. clean_ignore_tiles: [] -# ngmix within-tile chunking: static N chunks (closed ID ranges computed -# per-tile, in-job, from the tile's own sexcat). +# ngmix chunks per tile. ngmix_chunks: 8 -# Where the node-local tile store lives (tile.smk's tile_local(): each tile_shape -# group's vignette and WCS stores). Unset means the profile's `/local/scratch` -# bind: nibi's /local NVMe, candide's 31 GB node /tmp. Set it to an absolute -# directory to bind that instead, for this campaign only; bin/sp rewrites the -# snapshot's profile and creates the directory. Not a rerun trigger, so it may be -# set on a resume. Image-sim campaigns on candide want a shared directory (dense -# tiles and failed groups' orphans fill /tmp); the cost is ngmix read latency. +# Node-local tile store (vignets, WCS). Unset: the profile's /local/scratch +# bind (nibi NVMe, candide node /tmp). Dense image-sim campaigns on candide +# want a shared directory instead. # tile_store_root: /n23data1//shapepipe/tilestore - -# The container's installed shapepipe is overridden by the launching worktree -# via each profile's own --env PYTHONPATH; there is no setting for it here. -# build_index.py's missing-tile fraction threshold is passed by workflow/bin/sp -# (SP_MISSING_THRESHOLD, default 0.0 = any missing tile is fatal). diff --git a/workflow/config/cfis/config_exp_Gie.ini b/workflow/config/cfis/config_exp_Gie.ini index bfd65b84e..9f69fbaee 100644 --- a/workflow/config/cfis/config_exp_Gie.ini +++ b/workflow/config/cfis/config_exp_Gie.ini @@ -86,10 +86,7 @@ INPUT_NUMBERING = \d{6} # Output file pattern without number OUTPUT_FILE_PATTERN = image-, weight-, flag- -# Method to retrieve images, one in 'vos', 'symlink' -- set by the run -# config's `retrieve:` key (unit_pre exports SP_RETRIEVE), not fixed here: -# nibi mirrors the exposures locally (symlink), candide fetches them from -# VOSpace (vos). +# 'vos' or 'symlink', from the run config's `retrieve:` RETRIEVE = $SP_RETRIEVE # If RETRIEVE=vos, number of attempts to download diff --git a/workflow/config/cfis/config_tile_Git.ini b/workflow/config/cfis/config_tile_Git.ini index 87a189f07..8aad4193b 100644 --- a/workflow/config/cfis/config_tile_Git.ini +++ b/workflow/config/cfis/config_tile_Git.ini @@ -80,9 +80,7 @@ INPUT_NUMBERING = \d{3}\.\d{3} # Output file pattern without number OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight- -# Copy/download method, one in 'vos', 'symlink' -- set by the run config's -# `retrieve:` key (unit_pre exports SP_RETRIEVE), not fixed here: nibi mirrors -# the tiles locally (symlink), candide fetches them from VOSpace (vos). +# 'vos' or 'symlink', from the run config's `retrieve:` RETRIEVE = $SP_RETRIEVE # If RETRIEVE=vos, number of attempts to download diff --git a/workflow/config/cfis_image_sims/config_exp_Gie.ini b/workflow/config/cfis_image_sims/config_exp_Gie.ini index c5eee0f4b..bbf1b98af 100644 --- a/workflow/config/cfis_image_sims/config_exp_Gie.ini +++ b/workflow/config/cfis_image_sims/config_exp_Gie.ini @@ -91,10 +91,7 @@ INPUT_NUMBERING = \d{7} # Output file pattern without number OUTPUT_FILE_PATTERN = image-, weight-, flag- -# Method to retrieve images, one in 'vos', 'symlink'. Not a sims/data -# difference (unlike the fields marked above): the Snakefile always exports -# SP_RETRIEVE=symlink for image_sims regardless of the run config's -# `retrieve:` key, since simulation output is always local, never VOSpace. +# 'vos' or 'symlink', from the run config's `retrieve:` RETRIEVE = $SP_RETRIEVE # If RETRIEVE=vos, number of attempts to download diff --git a/workflow/config/cfis_image_sims/config_tile_Git.ini b/workflow/config/cfis_image_sims/config_tile_Git.ini index 4489788cb..d638ff559 100644 --- a/workflow/config/cfis_image_sims/config_tile_Git.ini +++ b/workflow/config/cfis_image_sims/config_tile_Git.ini @@ -85,10 +85,7 @@ INPUT_NUMBERING = \d{3}-\d{3} # Output file pattern without number OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight- -# Copy/download method, one in 'vos', 'symlink'. Not a sims/data difference -# (unlike the fields marked above): the Snakefile always exports -# SP_RETRIEVE=symlink for image_sims regardless of the run config's -# `retrieve:` key, since simulation output is always local, never VOSpace. +# 'vos' or 'symlink', from the run config's `retrieve:` RETRIEVE = $SP_RETRIEVE # If RETRIEVE=vos, number of attempts to download diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index bc8975064..14a68b7ae 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -354,13 +354,10 @@ def build_manifest(stage, run_dir, unit, stage_subdir=None): def write_if_changed(path: Path, text: str) -> None: - """Write only when the bytes differ — see the module docstring on mtime. + """Write only when the bytes differ (see the module docstring on mtime). - Via a same-directory temp file + ``os.replace``, not ``path.write_text``: - the latter truncates before writing, so a reader (``run_report.py``, run - automatically at compute's end) can catch the file empty mid-write. Same - directory keeps the replace on one filesystem, which is what makes it - atomic; a losing writer's temp file is unlinked rather than left behind. + Atomic (temp file + os.replace): a reader such as run_report.py must never + see the file truncated mid-write. """ path.parent.mkdir(parents=True, exist_ok=True) if not path.exists() or path.read_text() != text: diff --git a/workflow/scripts/container.py b/workflow/scripts/container.py index c35f91ad8..defafb495 100644 --- a/workflow/scripts/container.py +++ b/workflow/scripts/container.py @@ -56,12 +56,10 @@ class ContainerError(Exception): # the slim variant the workflow runs. CONTAINER_URI = "docker://ghcr.io/cosmostat/shapepipe:develop-runtime" -# The single source of truth for the fallback image: the workflow's own -# `container:` key, which is also what the Snakefile reads. Written down once, -# here, so the CLI and the workflow cannot disagree about the default. -# SP_RUN_CONFIG replaces it for a run driven with its own config (bin/sp). -CONFIG_FILE = Path(os.environ.get("SP_RUN_CONFIG") - or Path(__file__).resolve().parents[1] / "config.yaml") +# The fallback image is the run config's resolved `container:` (config.yaml, +# SP_RUN_CONFIG on top, then the machine default; see run_config.py). +BASE_CONFIG = Path(__file__).resolve().parents[1] / "config.yaml" +CONFIG_FILE = Path(os.environ.get("SP_RUN_CONFIG") or BASE_CONFIG) CONFIG_KEY = "container" # The profile whose `apptainer-args:` every workflow job runs under. `exec` reads @@ -92,17 +90,22 @@ class ContainerError(Exception): def configured_default(): - """Return the ``container:`` path from workflow/config.yaml, or ``None``. + """Return the resolved ``container:`` of the run config, or ``None``. - A deliberately minimal scalar read (the same one bin/sp does in sed): this - module is stdlib-only, so there is no yaml to import. + Via run_config.py when PyYAML is available; on a bare host without it, + a top-level ``container:`` line in the run config file is still read. """ try: - text = CONFIG_FILE.read_text() - except OSError: - return None - match = re.search(rf"^{CONFIG_KEY}:[ \t]*(\S+)", text, re.MULTILINE) - return match.group(1) if match else None + import run_config + except ImportError: + try: + text = CONFIG_FILE.read_text() + except OSError: + return None + match = re.search(rf"^{CONFIG_KEY}:[ \t]*(\S+)", text, re.MULTILINE) + return match.group(1) if match else None + run = os.environ.get("SP_RUN_CONFIG") + return run_config.load(BASE_CONFIG, run).get(CONFIG_KEY) or None def profile_apptainer_args(): @@ -131,9 +134,12 @@ def local_sandbox(): return (Path(override) if override else DEFAULT_SANDBOX).expanduser() -def resolve_image(): +def resolve_image(configured=None): """Return ``(path, kind)`` for the image everything should run. + ``configured`` is the run config's resolved ``container:`` when the + caller already has it (the Snakefile); otherwise it is looked up. + ``kind`` is ``"sandbox"``, ``"sif"``, ``"configured"`` (the shared /project image named in config.yaml -- the default when the cache is empty) or ``"none"``. @@ -152,7 +158,7 @@ def resolve_image(): f"SP_CONTAINER={os.environ['SP_CONTAINER']} does not exist " f"(resolved to {sif}). Unset it, or point it at an image that does." ) - default = configured_default() + default = configured or configured_default() if default: return default, "configured" return "", "none" diff --git a/workflow/scripts/run_config.py b/workflow/scripts/run_config.py new file mode 100644 index 000000000..5daa69382 --- /dev/null +++ b/workflow/scripts/run_config.py @@ -0,0 +1,83 @@ +#!/usr/bin/env python3 +"""Resolve a run config: config.yaml, then SP_RUN_CONFIG merged on top, then +the `machines:` entry for (machine, input_type) filling whatever is still +unset. One definition shared by the Snakefile, bin/sp and container.py. + +CLI (used by bin/sp): run_config.py CONFIG_YAML RUN_CONFIG KEY[.SUBKEY] +prints the resolved value, or an empty line if unset. RUN_CONFIG may be "". +""" + +import sys + +import yaml + +PLACEHOLDER = "TBD" +MACHINE_KEYS = ("tile_list", "retrieve", "container", "inputs", "outputs") +REQUIRED = ("tile_list", "inputs.tiles", "inputs.exposures", + "outputs.run_dir", "outputs.index_db") + + +def merge(base, over): + """Recursive dict merge; `over` wins (as snakemake's update_config).""" + out = dict(base) + for key, value in over.items(): + if isinstance(value, dict) and isinstance(out.get(key), dict): + out[key] = merge(out[key], value) + else: + out[key] = value + return out + + +def _expand(value, base_dir): + if isinstance(value, str): + return value.replace("$base_dir", base_dir) + if isinstance(value, dict): + return {k: _expand(v, base_dir) for k, v in value.items()} + return value + + +def apply_machine_defaults(config): + """Fill unset MACHINE_KEYS from machines[machine][input_type], in place. + + A key already in `config` wins; for `inputs`/`outputs` the merge is per + sub-key. `$base_dir` expands to machines[machine].base_dir. + """ + entry = (config.get("machines") or {}).get(config.get("machine")) or {} + defaults = entry.get(config.get("input_type", "data")) or {} + base_dir = str(entry.get("base_dir", "")) + for key in MACHINE_KEYS: + if key not in defaults: + continue + value = _expand(defaults[key], base_dir) + if isinstance(value, dict): + config[key] = merge(value, config.get(key) or {}) + else: + config.setdefault(key, value) + return config + + +def get(config, dotted): + value = config + for part in dotted.split("."): + value = value.get(part) if isinstance(value, dict) else None + return value + + +def unresolved(config): + """REQUIRED keys that are unset or still the placeholder.""" + return [k for k in REQUIRED if get(config, k) in (None, "", PLACEHOLDER)] + + +def load(config_yaml, run_config=None): + with open(config_yaml) as f: + config = yaml.safe_load(f) or {} + if run_config: + with open(run_config) as f: + config = merge(config, yaml.safe_load(f) or {}) + return apply_machine_defaults(config) + + +if __name__ == "__main__": + config_yaml, run_config, key = sys.argv[1:4] + value = get(load(config_yaml, run_config or None), key) + print("" if value is None else value) From 084e305a6090f9ca65e2054eba1879601e471e99 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Tue, 15 Sep 2026 19:17:33 +0200 Subject: [PATCH 16/25] run_config: `run:` name, expanded as $run in run-config paths config.yaml gains `run: smk-g6`, and nibi's paths use $run instead of the literal campaign name. $base_dir and $run now expand in values set in the run config itself too, not only in machine defaults, so a sims run config can name its output directories by `run:`. A required path left holding an unexpanded $variable is reported like TBD. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013NzWDtdQbsK1VhvYbDnmTo --- workflow/README.md | 2 +- workflow/config.yaml | 15 +++++++++------ workflow/scripts/run_config.py | 35 ++++++++++++++++++++-------------- 3 files changed, 31 insertions(+), 21 deletions(-) diff --git a/workflow/README.md b/workflow/README.md index c0652c1eb..0ab08c0d4 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -76,7 +76,7 @@ the code. `machine:` (which must match `SP_PROFILE`, default `nibi`) and `input_type:` then select an entry of the `machines:` table, which supplies `tile_list`, `retrieve` (`symlink` or `vos`), `inputs`, `outputs` and `container` for any of these the run config leaves unset (`$base_dir` expands -to that machine's `base_dir`). A value of `TBD` stops the run at parse time +to that machine's `base_dir`, `$run` to the run config's `run:`). A value of `TBD` stops the run at parse time until it is set. A run config therefore only needs what differs, e.g. for one SKiLLS shear branch on candide: diff --git a/workflow/config.yaml b/workflow/config.yaml index ab60bd20c..59307fb2b 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -18,10 +18,13 @@ input_type: data # psfex, mccd, or fake (image_sims only: true PSF from `psf_dict`). psf_model: psfex +# Run name, available as `$run` in the paths below. +run: smk-g6 + # Per machine and input_type: tile_list, retrieve (symlink|vos), inputs, # outputs, container. A key set at top level (or in SP_RUN_CONFIG) overrides -# it. `$base_dir` expands to the machine's base_dir. TBD = not known yet; the -# run refuses to start until it is set. +# it. `$base_dir` expands to the machine's base_dir, `$run` to `run:`. +# TBD = not known yet; the run refuses to start until it is set. # # outputs: run_dir = scratch root for intermediates; products_dir = persistent # root for final catalogues, index and report (omit to use run_dir); @@ -30,15 +33,15 @@ machines: nibi: base_dir: /project/def-mjhudson data: - tile_list: $base_dir/cdaley/sp-products/smk-g6/tiles.txt + tile_list: $base_dir/cdaley/sp-products/$run/tiles.txt retrieve: symlink inputs: tiles: $base_dir/unions-wl/tiles exposures: $base_dir/unions-wl/exposures outputs: - run_dir: /scratch/cdaley/shapepipe-output/smk-g6 - products_dir: $base_dir/cdaley/sp-products/smk-g6 - index_db: $base_dir/cdaley/sp-products/smk-g6/index/run_index.sqlite + run_dir: /scratch/cdaley/shapepipe-output/$run + products_dir: $base_dir/cdaley/sp-products/$run + index_db: $base_dir/cdaley/sp-products/$run/index/run_index.sqlite container: $base_dir/cdaley/containers/shapepipe-develop-runtime.sif candide: diff --git a/workflow/scripts/run_config.py b/workflow/scripts/run_config.py index 5daa69382..481219f6e 100644 --- a/workflow/scripts/run_config.py +++ b/workflow/scripts/run_config.py @@ -7,6 +7,7 @@ prints the resolved value, or an empty line if unset. RUN_CONFIG may be "". """ +import re import sys import yaml @@ -28,11 +29,14 @@ def merge(base, over): return out -def _expand(value, base_dir): +def _expand(value, variables): + """Replace $name for each set name in `variables` (base_dir, run).""" if isinstance(value, str): - return value.replace("$base_dir", base_dir) + return re.sub(r"\$(\w+)", + lambda m: str(variables.get(m.group(1)) or m.group(0)), + value) if isinstance(value, dict): - return {k: _expand(v, base_dir) for k, v in value.items()} + return {k: _expand(v, variables) for k, v in value.items()} return value @@ -40,19 +44,20 @@ def apply_machine_defaults(config): """Fill unset MACHINE_KEYS from machines[machine][input_type], in place. A key already in `config` wins; for `inputs`/`outputs` the merge is per - sub-key. `$base_dir` expands to machines[machine].base_dir. + sub-key. In all of these, `$base_dir` expands to machines[machine].base_dir + and `$run` to the top-level `run:`. """ entry = (config.get("machines") or {}).get(config.get("machine")) or {} defaults = entry.get(config.get("input_type", "data")) or {} - base_dir = str(entry.get("base_dir", "")) + variables = {"base_dir": entry.get("base_dir"), "run": config.get("run")} for key in MACHINE_KEYS: - if key not in defaults: - continue - value = _expand(defaults[key], base_dir) - if isinstance(value, dict): - config[key] = merge(value, config.get(key) or {}) - else: - config.setdefault(key, value) + default = defaults.get(key) + if isinstance(default, dict): + config[key] = merge(default, config.get(key) or {}) + elif default is not None: + config.setdefault(key, default) + if key in config: + config[key] = _expand(config[key], variables) return config @@ -64,8 +69,10 @@ def get(config, dotted): def unresolved(config): - """REQUIRED keys that are unset or still the placeholder.""" - return [k for k in REQUIRED if get(config, k) in (None, "", PLACEHOLDER)] + """REQUIRED keys that are unset, the placeholder, or hold an unexpanded + $variable (e.g. `$run` with no `run:` set).""" + return [k for k in REQUIRED + if get(config, k) in (None, "", PLACEHOLDER) or "$" in str(get(config, k))] def load(config_yaml, run_config=None): From 78cc0ca2dd23ebd3e480c907dfa1f8bd829705c4 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Thu, 17 Sep 2026 07:18:07 +0200 Subject: [PATCH 17/25] added user run config template --- scripts/python/create_final_cat.py | 2 -- workflow/README.md | 4 +-- workflow/Snakefile | 10 +++---- workflow/config.yaml | 45 +++++++++++++++++++----------- workflow/run_template.yaml | 18 ++++++++++++ workflow/scripts/run_config.py | 7 ++++- 6 files changed, 60 insertions(+), 26 deletions(-) create mode 100644 workflow/run_template.yaml diff --git a/scripts/python/create_final_cat.py b/scripts/python/create_final_cat.py index d98232818..dc14a3324 100755 --- a/scripts/python/create_final_cat.py +++ b/scripts/python/create_final_cat.py @@ -391,8 +391,6 @@ def process(params): if params["image_sims"]: patch_name = params["patch"] - # The Snakemake workflow writes an undated run_sp_tile_Mc; the legacy - # bash runner wrote run_sp_tile_Mc_. run_prefix = "run_sp_tile_Mc*" else: patch_name = rf"P{params['patch']}" diff --git a/workflow/README.md b/workflow/README.md index 0ab08c0d4..c23bb2fab 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -72,8 +72,8 @@ SP_PROFILE=candide SP_RUN_CONFIG=/path/run_1p2z_grid_1.yaml workflow/bin/sp run ## Run configuration `SP_RUN_CONFIG` is merged on top of `workflow/config.yaml` and snapshotted with -the code. `machine:` (which must match `SP_PROFILE`, default `nibi`) and -`input_type:` then select an entry of the `machines:` table, which supplies +the code. `SP_PROFILE` (default `nibi`, or `machine:` in the run config, which must +agree with it) and `input_type:` then select an entry of the `machines:` table, which supplies `tile_list`, `retrieve` (`symlink` or `vos`), `inputs`, `outputs` and `container` for any of these the run config leaves unset (`$base_dir` expands to that machine's `base_dir`, `$run` to the run config's `run:`). A value of `TBD` stops the run at parse time diff --git a/workflow/Snakefile b/workflow/Snakefile index b3612ed24..9288053ec 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -65,12 +65,12 @@ if INPUT_TYPE not in INPUT_TYPES: f"Invalid input_type={INPUT_TYPE!r}; expected one of " f"{sorted(INPUT_TYPES)}.") -# `machine:` must agree with SP_PROFILE (bin/sp's profile choice, default -# nibi): otherwise jobs run under one cluster's SLURM profile against another -# cluster's paths. Skipped only if `machine:` is unset. +# The machine is SP_PROFILE (bin/sp's profile choice, default nibi). A run +# config may also state `machine:`, and must then agree with it: otherwise +# jobs run under one cluster's SLURM profile against another's paths. SP_PROFILE = os.environ.get("SP_PROFILE", "nibi") -MACHINE = config.get("machine") -if MACHINE is not None and MACHINE != SP_PROFILE: +MACHINE = config.get("machine") or SP_PROFILE +if config.get("machine") is not None and config["machine"] != SP_PROFILE: raise WorkflowError( f"machine={MACHINE!r} ({_RUN_CONFIG}) does not match " f"SP_PROFILE={SP_PROFILE!r}. Set SP_PROFILE={MACHINE}, or fix " diff --git a/workflow/config.yaml b/workflow/config.yaml index 59307fb2b..d5f31f039 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -1,5 +1,14 @@ -# Run configuration for the ShapePipe Snakemake workflow. -# +# Main configuration for the ShapePipe Snakemake workflow. + +# To run, first set a profile (machine/architecture): + +# SP_PROFILE=nibi|candide + +# Optional, set the path to your personal config file, to override default +# settings or add variables + +# SP_RUN_CONFIG=/path/to/my_run.yaml + # A run config given via SP_RUN_CONFIG is merged on top of this file, then # the `machines:` entry for (machine, input_type) fills anything still unset: # @@ -8,10 +17,6 @@ # Everything is read at parse time; editing it (e.g. appending tiles) never # invalidates completed work. -# Cluster; must match SP_PROFILE (default nibi). -machine: nibi -#machine: candide - # data (survey, config/cfis) or image_sims (SKiLLS, config/cfis_image_sims). input_type: data @@ -21,14 +26,17 @@ psf_model: psfex # Run name, available as `$run` in the paths below. run: smk-g6 -# Per machine and input_type: tile_list, retrieve (symlink|vos), inputs, -# outputs, container. A key set at top level (or in SP_RUN_CONFIG) overrides -# it. `$base_dir` expands to the machine's base_dir, `$run` to `run:`. -# TBD = not known yet; the run refuses to start until it is set. -# -# outputs: run_dir = scratch root for intermediates; products_dir = persistent -# root for final catalogues, index and report (omit to use run_dir); -# index_db = tile/exposure index. +# Entries per machine (SP_PROFILE, default nibi; implemented: nibi, candide) +# and input_type. A run config may instead state `machine:`, which must +# then agree with SP_PROFILE. +# - base_dir +# - tile_list: text file with tile IDs +# - retrieve: method to get input data, allowed are symlink, vos +# - inputs.tiles,, .exposures: path for input tile and exposure +# - outputs.run_dir: (scratch) run directory, where tmp files will be stored +# - outputs.products_dir: path to final products +# - outputs.index_db: path to bookkeeping index sqlite file + machines: nibi: base_dir: /project/def-mjhudson @@ -57,9 +65,14 @@ machines: index_db: TBD container: /n17data/cdaley/containers/shapepipe_develop-runtime-20260718.sif image_sims: - # tile_list, inputs, outputs: per SKiLLS campaign, in its own run config, - # e.g. inputs.tiles: /n09data/hervas/skills_out//images/SP_tiles retrieve: symlink + tile_list: TBD + inputs: + # Need to be pointed to appropriate grid + tiles: /n09data/hervas/skills_out/1z2z_grid_1/images/SP_tiles + exposures: /n09data/hervas/skills_out/1z2z_grid_1/images/SP_exp + # Dictionary of PSF model stamps + psf_dict: /home/hervas/fhervas/workdir_skills/input/psf_files/Full_psf_dict.pickle container: /n17data/cdaley/containers/shapepipe_im_sims-runtime.sif # Per-exposure store reclamation once every tile reading it has its vignets. diff --git a/workflow/run_template.yaml b/workflow/run_template.yaml new file mode 100644 index 000000000..c75bdb736 --- /dev/null +++ b/workflow/run_template.yaml @@ -0,0 +1,18 @@ +# Run config template file, can be specified along with main config.yaml for +# ShapePipe run. + +my_base_dir: + +tile_list: $my_base_dir/tiles.txt + +sim: 1z2z_grid_1 + +inputs: + tiles: /n09data/hervas/skills_out/$sim/images/SP_tiles + exposures: /n09data/hervas/skills_out/$sim/images/SP_exp + +outputs: + run_dir: $my_base_dir/$sim/run + products_dir: $my_base_dir/$sim/product + +shapepipe_repo: diff --git a/workflow/scripts/run_config.py b/workflow/scripts/run_config.py index 481219f6e..c6ed4f3d9 100644 --- a/workflow/scripts/run_config.py +++ b/workflow/scripts/run_config.py @@ -7,6 +7,7 @@ prints the resolved value, or an empty line if unset. RUN_CONFIG may be "". """ +import os import re import sys @@ -43,11 +44,15 @@ def _expand(value, variables): def apply_machine_defaults(config): """Fill unset MACHINE_KEYS from machines[machine][input_type], in place. + The machine is `machine:` when the run config states one, else SP_PROFILE + (default nibi) -- the same value bin/sp picks the SLURM profile with. + A key already in `config` wins; for `inputs`/`outputs` the merge is per sub-key. In all of these, `$base_dir` expands to machines[machine].base_dir and `$run` to the top-level `run:`. """ - entry = (config.get("machines") or {}).get(config.get("machine")) or {} + machine = config.get("machine") or os.environ.get("SP_PROFILE", "nibi") + entry = (config.get("machines") or {}).get(machine) or {} defaults = entry.get(config.get("input_type", "data")) or {} variables = {"base_dir": entry.get("base_dir"), "run": config.get("run")} for key in MACHINE_KEYS: From d3b088a8d02e920dd8226562af38afc91242f332 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Thu, 17 Sep 2026 07:46:58 +0200 Subject: [PATCH 18/25] bin/sp: -c/--config-file for the run config, instead of SP_RUN_CONFIG The run config was only settable through an exported variable: invisible in the command, absent from shell history and job scripts, and silently inherited by later commands. `sp -c my_run.yaml` now takes it on any verb; sp consumes the flag, makes the path absolute and exports SP_RUN_CONFIG itself, which is what the Snakefile, container.py and the snapshot read. An already-set SP_RUN_CONFIG is still honoured, and -c wins. -c is sp's own flag, not snakemake's --cores; cores pass through as --cores/-j. Closes the `sp run --config-file` half of #891 step 1. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013NzWDtdQbsK1VhvYbDnmTo --- workflow/README.md | 8 +++++--- workflow/bin/sp | 48 +++++++++++++++++++++++++++++++++++--------- workflow/config.yaml | 13 +++++------- 3 files changed, 49 insertions(+), 20 deletions(-) diff --git a/workflow/README.md b/workflow/README.md index c23bb2fab..16795aa32 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -66,13 +66,15 @@ Simulations that contain stars can run `psfex` or `mccd` exactly as the data do. One campaign per shear branch, each with its own run config: ```bash -SP_PROFILE=candide SP_RUN_CONFIG=/path/run_1p2z_grid_1.yaml workflow/bin/sp run +SP_PROFILE=candide workflow/bin/sp run -c /path/run_1p2z_grid_1.yaml ``` ## Run configuration -`SP_RUN_CONFIG` is merged on top of `workflow/config.yaml` and snapshotted with -the code. `SP_PROFILE` (default `nibi`, or `machine:` in the run config, which must +A run config passed with `-c/--config-file` is merged on top of +`workflow/config.yaml` and snapshotted with the code. (`-c` is `sp`'s own flag; +pass snakemake's cores as `--cores`/`-j`. `SP_RUN_CONFIG` still works and is what +the jobs read.) `SP_PROFILE` (default `nibi`, or `machine:` in the run config, which must agree with it) and `input_type:` then select an entry of the `machines:` table, which supplies `tile_list`, `retrieve` (`symlink` or `vos`), `inputs`, `outputs` and `container` for any of these the run config leaves unset (`$base_dir` expands diff --git a/workflow/bin/sp b/workflow/bin/sp index 11ce102da..bee176e26 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -3,7 +3,8 @@ # # Three verbs, nothing else: # -# sp run [ARGS...] bring the products on disk up to date with the tile list. +# sp run [-c FILE] [ARGS...] +# bring the products on disk up to date with the tile list. # Snapshots the code into the state dir first (see "the # launch code snapshot" below) and runs out of the copy, so # editing the checkout mid-campaign cannot reach the jobs. @@ -25,24 +26,53 @@ # One entry point, so a fresh tmux or a restart after a crash always launches # with the right state. # -# Three environment variables make it drivable off nibi and from outside: +# THE RUN CONFIG: -c/--config-file , on any verb. It is merged on top of +# workflow/config.yaml (scripts/run_config.py), so it states only what differs, +# and `sp run` snapshots it with the code. This is how one campaign per +# image-simulation shear branch is driven. NOTE: -c here is sp's own flag, not +# snakemake's --cores; pass cores as --cores/-j, which sp forwards. +# +# Environment: # # SP_PROFILE profiles// to launch with (default: nibi). # SP_SNAKEMAKE_ENV venv to activate; skipped when it does not exist and a -# snakemake is already on PATH (e.g. candide). -# SP_RUN_CONFIG a run config merged on top of workflow/config.yaml -# (scripts/run_config.py). Snapshotted with the code by -# `sp run`. This is how sp_validation drives one campaign -# per image-simulation shear branch. +# snakemake is already on PATH (e.g. candide). A site +# detail, not something a run needs to set. +# SP_RUN_CONFIG what -c sets, and what the jobs read. Honoured when +# already set, so older callers keep working; -c wins. set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # workflow/ REPO="$(dirname "$HERE")" VENV="${SP_SNAKEMAKE_ENV:-/project/def-mjhudson/cdaley/snakemake-env}" SCRIPTS="$HERE/scripts" -CONFIG="${SP_RUN_CONFIG:-$HERE/config.yaml}" PROFILE="${SP_PROFILE:-nibi}" -[ -f "$CONFIG" ] || { echo "sp: run config $CONFIG does not exist" >&2; exit 2; } + +# -c/--config-file is consumed HERE and never forwarded: snakemake has its own +# --configfile with different merge semantics, and this one also has to reach +# the jobs (as SP_RUN_CONFIG) and the snapshot. Everything else passes through. +RUN_CONFIG="${SP_RUN_CONFIG:-}" +_sp_args=() +while [ $# -gt 0 ]; do + case "$1" in + -c|--config|--config-file) + [ $# -ge 2 ] || { echo "sp: $1 needs a file" >&2; exit 2; } + RUN_CONFIG="$2"; shift 2 ;; + -c=*|--config=*|--config-file=*) + RUN_CONFIG="${1#*=}"; shift ;; + *) _sp_args+=("$1"); shift ;; + esac +done +set -- ${_sp_args[@]+"${_sp_args[@]}"} + +if [ -n "$RUN_CONFIG" ]; then + [ -f "$RUN_CONFIG" ] || { + echo "sp: run config $RUN_CONFIG does not exist" >&2; exit 2; } + # Absolute: the jobs and the snapshot resolve it from other directories. + RUN_CONFIG="$(cd "$(dirname "$RUN_CONFIG")" && pwd)/$(basename "$RUN_CONFIG")" + export SP_RUN_CONFIG="$RUN_CONFIG" +fi + [ -d "$REPO/profiles/$PROFILE" ] || { echo "sp: no profile $REPO/profiles/$PROFILE (SP_PROFILE=$PROFILE)" >&2; exit 2; } diff --git a/workflow/config.yaml b/workflow/config.yaml index d5f31f039..2ac109136 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -4,15 +4,12 @@ # SP_PROFILE=nibi|candide -# Optional, set the path to your personal config file, to override default -# settings or add variables - -# SP_RUN_CONFIG=/path/to/my_run.yaml - -# A run config given via SP_RUN_CONFIG is merged on top of this file, then -# the `machines:` entry for (machine, input_type) fills anything still unset: +# Your own run config, passed with -c, is merged on top of this file; then the +# `machines:` entry for (machine, input_type) fills anything still unset: +# +# SP_PROFILE= workflow/bin/sp run -c /path/to/my_run.yaml # -# SP_PROFILE= SP_RUN_CONFIG=/path/to/run.yaml workflow/bin/sp run +# It therefore states only what differs from the defaults below. # # Everything is read at parse time; editing it (e.g. appending tiles) never # invalidates completed work. From 746ca9c3c4da62d8ddc5f81b37829e9fa9352ca9 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Thu, 17 Sep 2026 07:53:21 +0200 Subject: [PATCH 19/25] bin/sp: document -c, the two-file merge and the environment in the header Typical invocations, the order the run config and config.yaml are read in (and that the machines: table fills the rest), the run_config.py one-liner for inspecting a resolved value, why -c is not snakemake's --cores, and the remaining SP_* variables including SP_PHASE, which sp sets itself. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_013NzWDtdQbsK1VhvYbDnmTo --- workflow/bin/sp | 40 ++++++++++++++++++++++++++++++++++------ 1 file changed, 34 insertions(+), 6 deletions(-) diff --git a/workflow/bin/sp b/workflow/bin/sp index bee176e26..dc3049ee3 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -26,20 +26,48 @@ # One entry point, so a fresh tmux or a restart after a crash always launches # with the right state. # -# THE RUN CONFIG: -c/--config-file , on any verb. It is merged on top of -# workflow/config.yaml (scripts/run_config.py), so it states only what differs, -# and `sp run` snapshots it with the code. This is how one campaign per -# image-simulation shear branch is driven. NOTE: -c here is sp's own flag, not -# snakemake's --cores; pass cores as --cores/-j, which sp forwards. +# Typical use, from the repo root: +# +# SP_PROFILE=candide workflow/bin/sp run -c ~/my_run.yaml # a campaign +# SP_PROFILE=candide workflow/bin/sp run -c ~/my_run.yaml -n # dry run +# SP_PROFILE=candide workflow/bin/sp report -c ~/my_run.yaml # status now +# workflow/bin/sp run # config.yaml only +# +# THE RUN CONFIG: -c/--config-file , accepted on any verb (also +# --config, and the =FILE forms). Two files are always read, in this order: +# +# 1. workflow/config.yaml committed defaults, including the `machines:` +# table (per machine and input_type) +# 2. the -c file merged on top, so it states ONLY what differs +# +# then anything still unset is filled from the `machines:` entry for this +# machine (SP_PROFILE, or `machine:`) and `input_type`. scripts/run_config.py +# does that resolution and is shared with container.py; to see what a pair of +# files resolves to without launching: +# +# python scripts/run_config.py workflow/config.yaml ~/my_run.yaml outputs.run_dir +# +# sp consumes -c, makes it absolute, exports it as SP_RUN_CONFIG (what the +# jobs and the snapshot read) and never forwards it: snakemake has its own +# --configfile with different merge semantics. +# +# NOTE: -c is sp's own flag, NOT snakemake's --cores. Cores pass through as +# --cores/-j, like every other snakemake argument. # # Environment: # -# SP_PROFILE profiles// to launch with (default: nibi). +# SP_PROFILE profiles// to launch with (default: nibi). Also +# selects the `machines:` entry, and must agree with a +# `machine:` key if the run config states one. # SP_SNAKEMAKE_ENV venv to activate; skipped when it does not exist and a # snakemake is already on PATH (e.g. candide). A site # detail, not something a run needs to set. # SP_RUN_CONFIG what -c sets, and what the jobs read. Honoured when # already set, so older callers keep working; -c wins. +# SP_STATE_DIR snakemake state + code snapshot (default -state). +# SP_CONTAINER image to run in, ahead of the run config's `container:`. +# SP_PHASE set by sp itself (prepare/compute/passthrough); the +# Snakefile refuses to parse without it. Never set it. set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # workflow/ From 4e0bedcb6106c525a8c4c24901f8029ffdda86f7 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Thu, 17 Sep 2026 07:58:31 +0200 Subject: [PATCH 20/25] improved commeents --- workflow/bin/sp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/workflow/bin/sp b/workflow/bin/sp index dc3049ee3..189ac89ff 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -1,7 +1,7 @@ #!/usr/bin/env bash -# sp — the committed launcher for the ShapePipe Snakemake workflow (PRD #848 D1). +# Main ShapePipe launch script for Snakemake workflow # -# Three verbs, nothing else: +# Usage: # # sp run [-c FILE] [ARGS...] # bring the products on disk up to date with the tile list. @@ -12,6 +12,9 @@ # 1. PREPARE snakemake prepare_all_tiles # 2. COMPUTE snakemake all <- its PARSE builds the index # ARGS (--jobs, -n, --forcerun, ...) pass through to BOTH. +# FILE is a user-specified config file that can be used +# to add and/or overwrite entries in the default config +# file workflow/config.yaml (read automatically) # sp report [ARGS...] emit run_report.json now (mid-run is fine). # sp container VERB manage the image every job runs inside: pull, status, # sandbox, exec, resolve. `sp container --help` documents From fbfdf3ba0e0f43c02143cb03f480af0d3e8e1c97 Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Wed, 23 Sep 2026 14:03:48 +0200 Subject: [PATCH 21/25] workflow smoothed; running until hdf5 file --- profiles/candide/config.yaml | 9 ++ scripts/python/create_final_cat.py | 110 ++++++++++++++++++--- src/shapepipe/modules/get_images_runner.py | 7 +- workflow/Snakefile | 50 ++++++++++ workflow/bin/sp | 72 +++++++------- workflow/config.yaml | 46 ++++----- workflow/scripts/run_config.py | 10 +- 7 files changed, 224 insertions(+), 80 deletions(-) diff --git a/profiles/candide/config.yaml b/profiles/candide/config.yaml index bacf48071..2932e64eb 100644 --- a/profiles/candide/config.yaml +++ b/profiles/candide/config.yaml @@ -29,6 +29,15 @@ software-deployment-method: [apptainer] apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/n17data/mkilbing/astro/repositories/github/shapepipe/src --bind /home,/automnt,/n17data,/n23data1,/n09data --bind /tmp:/local/scratch" latency-wait: 60 # NFS: wait for outputs to appear after a job keep-going: true # a failed job poisons only its cone; siblings run on +# Transient node-level failures are the common case on a shared cluster: +# a fused tile_shape group died at joblib's fork_exec with BlockingIOError +# [Errno 11] because the node was out of process slots, killing one tile of +# 39 outright. Without this, attempt is always 1, so every +# `mem_mb = lambda wc, attempt: N * attempt` in tile.smk is dead code and a +# one-off OOM or EAGAIN is fatal. Safe for the fused group: a retry +# resubmits the same member set, tile_vignets included (tile.smk's +# "one sharp edge" note). tile_ngmix keeps its own rule-level retries: 2. +retries: 2 rerun-incomplete: true # re-do jobs left incomplete by an unclean death show-failed-logs: true printshellcmds: true diff --git a/scripts/python/create_final_cat.py b/scripts/python/create_final_cat.py index dc14a3324..7571cb052 100755 --- a/scripts/python/create_final_cat.py +++ b/scripts/python/create_final_cat.py @@ -27,6 +27,56 @@ from cs_util import logging +def params_from_run_config(params, defaults): + """Fill unset paths from a workflow run config. + + The workflow already knows where a campaign writes, so a manual merge + should not have to restate it. Resolution goes through the workflow's own + resolver (workflow/scripts/run_config.py), layering the run config on + workflow/config.yaml and then the machines: table, so what lands here is + what the rules would have used. + + Only values still at their default are filled -- an explicit flag always + wins. Nothing is derived for the data path: its patch naming differs and + is not this function's business. + """ + repo = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + sys.path.insert(0, os.path.join(repo, "workflow", "scripts")) + import run_config as _rc + + cfg = _rc.load(os.path.join(repo, "workflow", "config.yaml"), + params["run_config"]) + if cfg.get("input_type") != "image_sims": + raise ValueError( + f"run config {params['run_config']} has input_type=" + f"{cfg.get('input_type', 'data')!r}; -c derives paths for " + "image_sims only") + + outputs = cfg.get("outputs") or {} + products = outputs.get("products_dir") or outputs.get("run_dir") + if not products: + raise ValueError(f"run config {params['run_config']} sets neither " + "outputs.products_dir nor outputs.run_dir") + + # The same derivation as the workflow's merge_final_cats rule: the patch + # dir is the branch dir holding product/tiles, and -i is its parent. + patch_dir = os.path.dirname(os.path.normpath(products)) + patch = os.path.basename(patch_dir) + derived = { + "image_sims": True, + "input_root_dir": os.path.dirname(patch_dir), + "patch": patch, + "merged_cat_path": os.path.join(products, f"final_cat_{patch}.hdf5"), + "output_summary": os.path.join(products, "n_tiles_final.txt"), + "param_path": os.path.join(repo, "workflow", "config", + "cfis_image_sims", "final_cat.param"), + } + for key, value in derived.items(): + if params.get(key) == defaults.get(key): + params[key] = value + return params + + def set_params_from_command_line(args): """Set Params From Command Line. @@ -48,6 +98,12 @@ def set_params_from_command_line(args): _params[key] = options[key] del options + + # A run config fills in whatever is still at its default + # (see the docstring); explicit flags always win. + if _params.get("run_config"): + _defaults, _, _, _ = params_default() + _params = params_from_run_config(_params, _defaults) # Save calling command logging.log_command(args) @@ -72,6 +128,7 @@ def params_default(): "ID": None, "single_op": None, "image_sims": False, + "run_config": None, } _short_options = { "input_root_dir": "-i", @@ -82,6 +139,7 @@ def params_default(): "output_summary": "-o", "single_op": "-s", "image_sims": "-I", + "run_config": "-c", } _types = { "hdu_num": "int", @@ -98,6 +156,9 @@ def params_default(): "ID": "ID for single-ID operation, default={}", "single_op": "single ID operation, allowed are 'check', 'add', 'remove'; default={}", "image_sims": "image simulations mode (different dir layout and run prefix), default={}", + "run_config": "workflow run config (e.g. sp_1p2z_grid_1.yaml); fills" + " in the paths below that were not given explicitly," + " default={}", } return _params, _short_options, _types, _help_strings @@ -373,9 +434,17 @@ def collect_tile_ids_image_sims(patch_path): list of (tile_id, tile_path) tuples """ id_pattern = re.compile(r"^\d+\.\d+$") - tiles_root = os.path.join(patch_path, "tiles") + # Two layouts. The Gen-2 / native runs keep tiles under the patch dir + # itself; the unified workflow (shapepipe #891) publishes to a separate + # products root, and clean_tile deletes the run-dir copies -- so on a + # reclaimed campaign /product/tiles holds the ONLY catalogues. result = [] - if not os.path.isdir(tiles_root): + for sub in ("tiles", os.path.join("product", "tiles"), + os.path.join("products", "tiles")): + tiles_root = os.path.join(patch_path, sub) + if os.path.isdir(tiles_root): + break + else: return result for prefix in os.listdir(tiles_root): prefix_path = os.path.join(tiles_root, prefix) @@ -387,6 +456,28 @@ def collect_tile_ids_image_sims(patch_path): return result +def find_final_cat(id, id_path, run_prefix): + """Path of this tile's final catalogue, or None. + + Flat layout first: the unified workflow copies one catalogue per tile to + /tiles///final_cat-.fits, in DOT form and with + no run sub-tree. Then the legacy layout, where the catalogue sits under the + tile's own shapepipe run dir in DASH form, newest run wins. + """ + flat = os.path.join(id_path, f"final_cat-{id}.fits") + if os.path.exists(flat): + return flat + + base_pattern = os.path.join(id_path, "output", run_prefix) + all_matches = [d for d in glob.glob(base_pattern) if os.path.isdir(d)] + if not all_matches: + return None + newest_dir = max(all_matches, key=os.path.getmtime) + id_dash = re.sub(r"\.", "-", id) + legacy = f"{newest_dir}/make_cat_runner/output/final_cat-{id_dash}.fits" + return legacy if os.path.exists(legacy) else None + + def process(params): if params["image_sims"]: @@ -446,22 +537,11 @@ def process(params): print(f"Skipping {id} (already processed)") continue - base_pattern = os.path.join(id_path, "output", run_prefix) - all_matches = [d for d in glob.glob(base_pattern) if os.path.isdir(d)] - if not all_matches: + fits_file = find_final_cat(id, id_path, run_prefix) + if fits_file is None: if params["verbose"]: print(f"Final cat for {id} not found, continuing") continue - newest_dir = max(all_matches, key=os.path.getmtime) - - id_dash = re.sub(r"\.", "-", id) - fits_file = f"{newest_dir}/make_cat_runner/output/final_cat-{id_dash}.fits" - - # Exclude unsuccessful run without output FITS file - if not os.path.exists(fits_file): - if params["verbose"]: - print(f"Run without output file found for {id}, skipping") - continue extracted_data, dtype = read_data(fits_file, params) diff --git a/src/shapepipe/modules/get_images_runner.py b/src/shapepipe/modules/get_images_runner.py index b1a03b15a..28e81b7d5 100644 --- a/src/shapepipe/modules/get_images_runner.py +++ b/src/shapepipe/modules/get_images_runner.py @@ -27,8 +27,11 @@ def get_images_runner( """Define The Get Images Runner.""" # Read config file section - # Copy/download method - retrieve_method = config.get(module_config_sec, "RETRIEVE") + # Copy/download method. getexpanded, not get: the workflow's ini sets + # RETRIEVE = $SP_RETRIEVE (a run-config key since d64f88ed), and plain get + # returns the literal "$SP_RETRIEVE", which then fails the check below. + # RETRIEVE_OPTIONS a few lines down already reads the same way. + retrieve_method = config.getexpanded(module_config_sec, "RETRIEVE") retrieve_ok = ["vos", "symlink"] if retrieve_method not in retrieve_ok: raise ValueError( diff --git a/workflow/Snakefile b/workflow/Snakefile index 9288053ec..2befdb5ec 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -672,9 +672,59 @@ include: "rules/tile.smk" # mask generation.) localrules: all, prepare_all_tiles, clean_exposure, clean_tile +# --- merged catalogue (image sims only) ------------------------------------ +# The sp_validation side starts from ONE hdf5 per branch, not 39 FITS tiles, so +# the campaign is not finished until they are merged. Image sims only: the data +# path's patches are merged separately, on a different naming convention. +# +# Paths are derived from products_dir rather than from `run:`, so they hold +# whatever the run config calls things. create_final_cat.py takes a root to +# scan (-i) and a patch name (-P) that is BOTH the directory to match under it +# and the hdf5 group, so the patch directory is the branch dir and the root is +# its parent -- exactly the `-i .. -P ` the old sp_validation rule ran +# from inside the branch. +MERGE_PATCH_DIR = PRODUCTS_DIR.parent # the branch dir, holding product/tiles +MERGE_ROOT = MERGE_PATCH_DIR.parent # -i +MERGE_PATCH = MERGE_PATCH_DIR.name # -P, and the hdf5 group +MERGED_CAT = PRODUCTS_DIR / f"final_cat_{MERGE_PATCH}.hdf5" +CREATE_FINAL_CAT = Path(workflow.basedir).parent / "scripts" / "python" / "create_final_cat.py" + + +def merge_targets(): + """The merged catalogue, once every ready tile has published.""" + return [str(MERGED_CAT)] if INPUT_TYPE == "image_sims" and TILES_READY else [] + + +rule merge_final_cats: + """Merge the per-tile final catalogues into one hdf5 (image sims). + + Reads the PRODUCTS root, not the run dir: clean_tile deletes the run-dir + copies, so on a reclaimed campaign the published ones are all that is left. + Ordering needs no edge to clean_tile for the same reason. + """ + input: + [final_cat(t) for t in TILES_READY], + output: + cat = str(MERGED_CAT), + n_tiles = str(PRODUCTS_DIR / "n_tiles_final.txt"), + params: + root = str(MERGE_ROOT), + patch = MERGE_PATCH, + param_file = str(CONFIG_DIR / "final_cat.param"), + threads: 1 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 120 + shell: + f"python {CREATE_FINAL_CAT} -I" + " -m {output.cat} -i {params.root} -p {params.param_file}" + " -P {params.patch} -o {output.n_tiles} -v" + + rule all: input: [final_cat(t) for t in TILES_READY], + merge_targets(), clean_targets(), clean_tile_targets(), diff --git a/workflow/bin/sp b/workflow/bin/sp index 189ac89ff..c4dc50995 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -36,41 +36,24 @@ # SP_PROFILE=candide workflow/bin/sp report -c ~/my_run.yaml # status now # workflow/bin/sp run # config.yaml only # -# THE RUN CONFIG: -c/--config-file , accepted on any verb (also -# --config, and the =FILE forms). Two files are always read, in this order: -# -# 1. workflow/config.yaml committed defaults, including the `machines:` -# table (per machine and input_type) -# 2. the -c file merged on top, so it states ONLY what differs -# -# then anything still unset is filled from the `machines:` entry for this -# machine (SP_PROFILE, or `machine:`) and `input_type`. scripts/run_config.py -# does that resolution and is shared with container.py; to see what a pair of -# files resolves to without launching: +# Run config: -c FILE, also --config-file or --config. Read after +# workflow/config.yaml and merged on top; anything still unset comes from the +# machines: entry for SP_PROFILE and input_type (scripts/run_config.py). +# To check a resolved value without running: # # python scripts/run_config.py workflow/config.yaml ~/my_run.yaml outputs.run_dir # -# sp consumes -c, makes it absolute, exports it as SP_RUN_CONFIG (what the -# jobs and the snapshot read) and never forwards it: snakemake has its own -# --configfile with different merge semantics. -# -# NOTE: -c is sp's own flag, NOT snakemake's --cores. Cores pass through as -# --cores/-j, like every other snakemake argument. +# -c is sp's flag, not snakemake's --cores; pass cores as --cores or -j. # # Environment: # -# SP_PROFILE profiles// to launch with (default: nibi). Also -# selects the `machines:` entry, and must agree with a -# `machine:` key if the run config states one. -# SP_SNAKEMAKE_ENV venv to activate; skipped when it does not exist and a -# snakemake is already on PATH (e.g. candide). A site -# detail, not something a run needs to set. -# SP_RUN_CONFIG what -c sets, and what the jobs read. Honoured when -# already set, so older callers keep working; -c wins. -# SP_STATE_DIR snakemake state + code snapshot (default -state). -# SP_CONTAINER image to run in, ahead of the run config's `container:`. -# SP_PHASE set by sp itself (prepare/compute/passthrough); the -# Snakefile refuses to parse without it. Never set it. +# SP_PROFILE profiles/ to use (default nibi); also selects the +# machines: entry, and must match `machine:` if set +# SP_SNAKEMAKE_ENV venv to activate; skipped if snakemake is on PATH +# SP_RUN_CONFIG same as -c, and what the jobs read; -c wins +# SP_STATE_DIR snakemake state and code snapshot (default -state) +# SP_CONTAINER image to use, before the config's container: +# SP_PHASE set by sp (prepare/compute/passthrough); do not set set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # workflow/ @@ -79,9 +62,8 @@ VENV="${SP_SNAKEMAKE_ENV:-/project/def-mjhudson/cdaley/snakemake-env}" SCRIPTS="$HERE/scripts" PROFILE="${SP_PROFILE:-nibi}" -# -c/--config-file is consumed HERE and never forwarded: snakemake has its own -# --configfile with different merge semantics, and this one also has to reach -# the jobs (as SP_RUN_CONFIG) and the snapshot. Everything else passes through. +# -c is consumed here, not forwarded: snakemake's own --configfile merges +# differently, and the jobs and the snapshot need it as SP_RUN_CONFIG. RUN_CONFIG="${SP_RUN_CONFIG:-}" _sp_args=() while [ $# -gt 0 ]; do @@ -99,7 +81,7 @@ set -- ${_sp_args[@]+"${_sp_args[@]}"} if [ -n "$RUN_CONFIG" ]; then [ -f "$RUN_CONFIG" ] || { echo "sp: run config $RUN_CONFIG does not exist" >&2; exit 2; } - # Absolute: the jobs and the snapshot resolve it from other directories. + # Absolute: jobs and the snapshot run from other directories. RUN_CONFIG="$(cd "$(dirname "$RUN_CONFIG")" && pwd)/$(basename "$RUN_CONFIG")" export SP_RUN_CONFIG="$RUN_CONFIG" fi @@ -107,7 +89,18 @@ fi [ -d "$REPO/profiles/$PROFILE" ] || { echo "sp: no profile $REPO/profiles/$PROFILE (SP_PROFILE=$PROFILE)" >&2; exit 2; } -module load apptainer/1.4.5 2>/dev/null || true +# snakemake resolves `apptainer` via PATH at job runtime, so all this has to +# do is put it there. candide already has it in /usr/bin and ships no +# modulefile for it, so probing for one only produces a scary ERROR line; +# skip the module entirely when apptainer is already usable. Both streams are +# dropped because the message arrives on stderr under environment-modules and +# can arrive on stdout under Lmod. +if ! command -v apptainer >/dev/null 2>&1; then + module load apptainer/1.4.5 >/dev/null 2>&1 || true + command -v apptainer >/dev/null 2>&1 || { + echo "sp: apptainer is not on PATH and 'module load apptainer/1.4.5' failed" >&2 + exit 2; } +fi if [ -f "$VENV/bin/activate" ]; then # shellcheck disable=SC1091 source "$VENV/bin/activate" @@ -116,8 +109,8 @@ elif ! command -v snakemake >/dev/null 2>&1; then exit 2 fi -# Resolved run-config value (config.yaml + SP_RUN_CONFIG + machine defaults), -# the same resolution the Snakefile uses. +# Resolved config value (config.yaml + run config + machine defaults), as the +# Snakefile resolves it. cfg() { python "$SCRIPTS/run_config.py" "$HERE/config.yaml" "${SP_RUN_CONFIG:-}" "$1"; } RUN_DIR="$(cfg outputs.run_dir)"; INDEX_DB="$(cfg outputs.index_db)" if [ "${1:-}" != container ] && { [ -z "$RUN_DIR" ] || [ "$RUN_DIR" = TBD ]; }; then @@ -150,7 +143,8 @@ STATE_DIR="${SP_STATE_DIR:-${RUN_DIR}-state}"; mkdir -p "$STATE_DIR" # WHAT. `sp run` copies the code it is about to launch into $STATE_DIR/code and # runs the campaign entirely out of that copy: the Snakefile, the rules, the # scripts, the ini chain (symlinks DEREFERENCED -- workflow/config/cfis points -# into example/, and the copy must be self-contained), src/, and the profile. +# into example/, and the copy must be self-contained), src/, the profile, and +# the repo's top-level scripts/ (576 KB), which merge_final_cats runs out of. # Every workflow-internal path hangs off `workflow.basedir`, which IS the # snapshot, so they all follow it for free; the profile's PYTHONPATH pin is the # one that cannot (YAML splices nothing) and is rewritten below. @@ -169,10 +163,10 @@ snapshot_code() { mkdir -p "$SNAPSHOT" if command -v rsync >/dev/null 2>&1; then rsync -a --delete --copy-links --exclude '__pycache__' --exclude '*.egg-info' \ - "$HERE" "$REPO/src" "$REPO/profiles" "$SNAPSHOT/" + "$HERE" "$REPO/src" "$REPO/profiles" "$REPO/scripts" "$SNAPSHOT/" else rm -rf "$SNAPSHOT"; mkdir -p "$SNAPSHOT" - cp -rL "$HERE" "$REPO/src" "$REPO/profiles" "$SNAPSHOT/" + cp -rL "$HERE" "$REPO/src" "$REPO/profiles" "$REPO/scripts" "$SNAPSHOT/" find "$SNAPSHOT" -name __pycache__ -type d -prune -exec rm -rf {} + fi diff --git a/workflow/config.yaml b/workflow/config.yaml index 2ac109136..76a7ee3d9 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -1,24 +1,25 @@ # Main configuration for the ShapePipe Snakemake workflow. -# To run, first set a profile (machine/architecture): +# To run, first set the profile env variable to the desired machine/architecture: # SP_PROFILE=nibi|candide -# Your own run config, passed with -c, is merged on top of this file; then the -# `machines:` entry for (machine, input_type) fills anything still unset: -# -# SP_PROFILE= workflow/bin/sp run -c /path/to/my_run.yaml -# -# It therefore states only what differs from the defaults below. +# Then, run in /path/to/shapepipe: + +# ./workflow/bin/sp run -c /path/to/my_run.yaml # -# Everything is read at parse time; editing it (e.g. appending tiles) never -# invalidates completed work. +# The config file my_run.yaml can contain user- or run-specific entries +# that overwrite the main config file workflow/config.yaml . -# data (survey, config/cfis) or image_sims (SKiLLS, config/cfis_image_sims). +# Main keys, global for all machines + +# input_tyle: allowed are data or image_sims input_type: data -# psfex, mccd, or fake (image_sims only: true PSF from `psf_dict`). -psf_model: psfex +# psf_model (psfex, mccd, or fake) and psf_dict are set per (machine, +# input_type) in the machines: table below, NOT here: a top-level key +# shadows that table. The Snakefile falls back to psfex if no entry +# supplies one. # Run name, available as `$run` in the paths below. run: smk-g6 @@ -33,11 +34,13 @@ run: smk-g6 # - outputs.run_dir: (scratch) run directory, where tmp files will be stored # - outputs.products_dir: path to final products # - outputs.index_db: path to bookkeeping index sqlite file +# - container: TBD machines: nibi: base_dir: /project/def-mjhudson data: + psf_model: psfex tile_list: $base_dir/cdaley/sp-products/$run/tiles.txt retrieve: symlink inputs: @@ -52,22 +55,19 @@ machines: candide: base_dir: /n17data/UNIONS/WL data: + psf_model: psfex retrieve: vos - tile_list: TBD - inputs: - tiles: TBD # vos: URL - exposures: TBD - outputs: - run_dir: TBD - index_db: TBD container: /n17data/cdaley/containers/shapepipe_develop-runtime-20260718.sif image_sims: retrieve: symlink - tile_list: TBD + # True simulation PSF: no exposure PSF fit; fake_interp_runner + # reads psf_dict below. + psf_model: fake inputs: - # Need to be pointed to appropriate grid - tiles: /n09data/hervas/skills_out/1z2z_grid_1/images/SP_tiles - exposures: /n09data/hervas/skills_out/1z2z_grid_1/images/SP_exp + # $run is the branch dir (e.g. 1p2z_grid_1), so a run config + # picks the grid by setting `run:` alone. + tiles: /n09data/hervas/skills_out/$run/images/SP_tiles + exposures: /n09data/hervas/skills_out/$run/images/SP_exp # Dictionary of PSF model stamps psf_dict: /home/hervas/fhervas/workdir_skills/input/psf_files/Full_psf_dict.pickle container: /n17data/cdaley/containers/shapepipe_im_sims-runtime.sif diff --git a/workflow/scripts/run_config.py b/workflow/scripts/run_config.py index c6ed4f3d9..2925bfca5 100644 --- a/workflow/scripts/run_config.py +++ b/workflow/scripts/run_config.py @@ -14,7 +14,15 @@ import yaml PLACEHOLDER = "TBD" -MACHINE_KEYS = ("tile_list", "retrieve", "container", "inputs", "outputs") +# Keys the machines: table may default, per (machine, input_type). +# psf_model/psf_dict belong here because they are per-input_type facts, +# not per-run ones: psf_model=fake is only legal with +# input_type=image_sims, and psf_dict is the sim PSF it reads. A key +# also present at the TOP level of config.yaml shadows the table (the +# setdefault below only fires when the key is absent), so a key listed +# here must not carry a top-level default as well. +MACHINE_KEYS = ("tile_list", "retrieve", "container", "inputs", "outputs", + "psf_model", "psf_dict") REQUIRED = ("tile_list", "inputs.tiles", "inputs.exposures", "outputs.run_dir", "outputs.index_db") From 0b5fae7cdcf1ef32d48a9b24ad80a1840a722797 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 01:06:52 +0200 Subject: [PATCH 22/25] bin/sp: don't swallow -c/--config meant for the delegated command The argv scan for sp's own -c/--config-file took every -c/--config/ --config-file anywhere in argv as the run config, including ones that belong to the command sp delegates to. This broke the README's own `sp container exec python -c ...` (container.py already reads SP_RUN_CONFIG, so its own -c is never sp's run config) and `sp run --config clean=false` (the override flag() in the Snakefile is written for). Stop the scan at `--` and at the command after `container exec`, and drop the `--config` alias so snakemake's own `--config k=v` passes through untouched. Co-Authored-By: Claude Opus 5.5 --- workflow/bin/sp | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/workflow/bin/sp b/workflow/bin/sp index c4dc50995..7d10b988d 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -36,9 +36,9 @@ # SP_PROFILE=candide workflow/bin/sp report -c ~/my_run.yaml # status now # workflow/bin/sp run # config.yaml only # -# Run config: -c FILE, also --config-file or --config. Read after -# workflow/config.yaml and merged on top; anything still unset comes from the -# machines: entry for SP_PROFILE and input_type (scripts/run_config.py). +# Run config: -c FILE, also --config-file. Read after workflow/config.yaml and +# merged on top; anything still unset comes from the machines: entry for +# SP_PROFILE and input_type (scripts/run_config.py). # To check a resolved value without running: # # python scripts/run_config.py workflow/config.yaml ~/my_run.yaml outputs.run_dir @@ -63,16 +63,30 @@ SCRIPTS="$HERE/scripts" PROFILE="${SP_PROFILE:-nibi}" # -c is consumed here, not forwarded: snakemake's own --configfile merges -# differently, and the jobs and the snapshot need it as SP_RUN_CONFIG. +# differently, and the jobs and the snapshot need it as SP_RUN_CONFIG. The scan +# stops at `--` and at the command after `container exec`, so a -c meant for +# the delegated command (`sp container exec python -c ...`) or for snakemake's +# own escape hatch (`sp -- ...`) is left alone. `--config` is NOT sp's: keeping +# it free lets snakemake's own `--config k=v` (what the Snakefile's flag() +# reads) pass through untouched. RUN_CONFIG="${SP_RUN_CONFIG:-}" _sp_args=() while [ $# -gt 0 ]; do case "$1" in - -c|--config|--config-file) + --) + _sp_args+=("$@"); break ;; + -c|--config-file) [ $# -ge 2 ] || { echo "sp: $1 needs a file" >&2; exit 2; } RUN_CONFIG="$2"; shift 2 ;; - -c=*|--config=*|--config-file=*) + -c=*|--config-file=*) RUN_CONFIG="${1#*=}"; shift ;; + container) + # Everything from `exec`'s own command on is the delegated command's, + # not sp's: `container.py` reads SP_RUN_CONFIG, so its own -c is never + # sp's run config. + _sp_args+=("$1"); shift + [ "${1:-}" = exec ] && { _sp_args+=("$@"); break; } + ;; *) _sp_args+=("$1"); shift ;; esac done From d22c4a0b8e295eb2b03145adde9e3cb524325e9a Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 01:06:57 +0200 Subject: [PATCH 23/25] workflow/README: add products_dir to the image-sims run-config example Without products_dir, merge_final_cats falls back to run_dir and create_final_cat.py finds no matching tile directory under it, so the rule writes an empty hdf5 and exits 0 -- a failure that only surfaces later, in sp_validation, as "No data found". Giving the example the template's /run + /product layout avoids that trap; hardening the rule itself is left to #879, which replaces merge_final_cats for sims. Co-Authored-By: Claude Opus 5.5 --- workflow/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/workflow/README.md b/workflow/README.md index 16795aa32..e3081ccca 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -93,6 +93,7 @@ inputs: exposures: /n09data/hervas/skills_out/1z2z_grid_3/images/SP_exp outputs: run_dir: /path/to/run + products_dir: /path/to/product index_db: /path/to/run/index.sqlite ``` From 5ded2ca88071e3370b6548ecd99138f22cfa3135 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 26 Sep 2026 01:11:23 +0200 Subject: [PATCH 24/25] run_template.yaml: make it resolve Three problems kept this from resolving: run_config.py only expands $base_dir and $run, so the template's own $my_base_dir and $sim stayed literal and the Snakefile rejected the paths; with no input_type, the run took the data branch (cfis configs, psfex, VOS retrieval) instead of image_sims; and outputs.index_db, a required key, was missing. shapepipe_repo: is also dropped -- nothing reads it. Puts the sim name in run: and uses $run throughout, which run_config.py already expands; the remaining placeholders are for hand-editing, same as before. Co-Authored-By: Claude Opus 5.5 --- workflow/run_template.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/workflow/run_template.yaml b/workflow/run_template.yaml index c75bdb736..5516aa58f 100644 --- a/workflow/run_template.yaml +++ b/workflow/run_template.yaml @@ -1,18 +1,18 @@ # Run config template file, can be specified along with main config.yaml for # ShapePipe run. -my_base_dir: +input_type: image_sims -tile_list: $my_base_dir/tiles.txt +# Run name: the SKiLLS grid to use. Available as $run in the paths below. +run: 1z2z_grid_1 -sim: 1z2z_grid_1 +tile_list: /tiles.txt inputs: - tiles: /n09data/hervas/skills_out/$sim/images/SP_tiles - exposures: /n09data/hervas/skills_out/$sim/images/SP_exp + tiles: /n09data/hervas/skills_out/$run/images/SP_tiles + exposures: /n09data/hervas/skills_out/$run/images/SP_exp outputs: - run_dir: $my_base_dir/$sim/run - products_dir: $my_base_dir/$sim/product - -shapepipe_repo: + run_dir: /$run/run + products_dir: /$run/product + index_db: /$run/run/index.sqlite From dc9e27de9e801bc67b169c24eb0d349a5d699cdc Mon Sep 17 00:00:00 2001 From: martinkilbinger Date: Mon, 28 Sep 2026 15:19:28 +0200 Subject: [PATCH 25/25] more variable expansion in config file --- docs/source/pipeline_canfar.md | 2 +- workflow/scripts/run_config.py | 45 ++++++++++++++++++++++++++++++---- 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/docs/source/pipeline_canfar.md b/docs/source/pipeline_canfar.md index 41e2c1301..21416d375 100644 --- a/docs/source/pipeline_canfar.md +++ b/docs/source/pipeline_canfar.md @@ -404,7 +404,7 @@ this requies a job with a large memory, e.g. with `mem=380000` ```bash -[squeue] python ~/astro/repositories/github/sp_validation/notebooks/extract_info.py +[squeue] python ~/astro/repositories/github/sp_validation/scripts/calibrate/extract_info.py ``` This creates a patch-wise comprehensive catalogue. diff --git a/workflow/scripts/run_config.py b/workflow/scripts/run_config.py index 2925bfca5..50cb9a417 100644 --- a/workflow/scripts/run_config.py +++ b/workflow/scripts/run_config.py @@ -38,12 +38,27 @@ def merge(base, over): return out +# $name or ${name}. The brace form is not decoration: `_` is a word character, +# so `$shear_$grid` parses its first name as `shear_` and silently fails to +# match a `shear` key. Anything adjacent to more word characters needs braces. +_VAR_RE = re.compile(r"\$\{(\w+)\}|\$(\w+)") + + def _expand(value, variables): - """Replace $name for each set name in `variables` (base_dir, run).""" + """Replace $name / ${name} for each set name in `variables`. + + Names come from every top-level scalar in the config plus `base_dir` from + the machines: entry, so a run config can name its own shorthands -- a long + output root written once and reused, rather than repeated per path. An + unknown name is left as-is, which is what makes the "unresolved" check + below able to see it. + """ if isinstance(value, str): - return re.sub(r"\$(\w+)", - lambda m: str(variables.get(m.group(1)) or m.group(0)), - value) + return _VAR_RE.sub( + lambda m: str(variables.get(m.group(1) or m.group(2)) + or m.group(0)), + value, + ) if isinstance(value, dict): return {k: _expand(v, variables) for k, v in value.items()} return value @@ -62,7 +77,27 @@ def apply_machine_defaults(config): machine = config.get("machine") or os.environ.get("SP_PROFILE", "nibi") entry = (config.get("machines") or {}).get(machine) or {} defaults = entry.get(config.get("input_type", "data")) or {} - variables = {"base_dir": entry.get("base_dir"), "run": config.get("run")} + # Every top-level scalar is a variable, so a run config can define its own + # shorthands. They are resolved AGAINST EACH OTHER first, to a fixpoint, so + # one shorthand may be written in terms of another + # (grid: grid_2 / base: .../${grid} / run: ${shear}_${grid}). One pass is + # not enough: re.sub does not rescan what it substitutes, so a nested name + # would survive into the paths and only surface at the unresolved check. + # The loop is capped, so a self-reference (a: $b, b: $a) stops rather than + # spinning -- the leftover $ is then caught by unresolved() by design. + variables = {k: v for k, v in config.items() + if isinstance(v, (str, int, float))} + variables["base_dir"] = entry.get("base_dir") + for _ in range(10): + resolved = {k: (_expand(v, variables) if isinstance(v, str) else v) + for k, v in variables.items()} + if resolved == variables: + break + variables = resolved + # `run` is consumed downstream (paths, the hdf5 group name), so the + # resolved value has to go back into the config, not just the table. + if isinstance(config.get("run"), str): + config["run"] = variables.get("run", config["run"]) for key in MACHINE_KEYS: default = defaults.get(key) if isinstance(default, dict):