diff --git a/docs/source/pipeline_canfar.md b/docs/source/pipeline_canfar.md index 41e2c1301..21416d375 100644 --- a/docs/source/pipeline_canfar.md +++ b/docs/source/pipeline_canfar.md @@ -404,7 +404,7 @@ this requies a job with a large memory, e.g. with `mem=380000` ```bash -[squeue] python ~/astro/repositories/github/sp_validation/notebooks/extract_info.py +[squeue] python ~/astro/repositories/github/sp_validation/scripts/calibrate/extract_info.py ``` This creates a patch-wise comprehensive catalogue. diff --git a/profiles/candide/config.yaml b/profiles/candide/config.yaml new file mode 100644 index 000000000..2932e64eb --- /dev/null +++ b/profiles/candide/config.yaml @@ -0,0 +1,58 @@ +# Snakemake profile for the ShapePipe workflow on candide (IAP). Selected by +# `SP_PROFILE=candide workflow/bin/sp run`. Mirrors profiles/nibi/config.yaml; +# only the cluster facts differ, and each is stated where it is set. +executor: slurm +# candide is a shared ~25-node cluster: cap concurrent submissions well below +# what nibi takes. +jobs: 150 +default-resources: + mem_mb: 2000 + runtime: 120 # minutes + slurm_account: cusers + # comp nodes have TmpDisk=0, so the tile_shape group (--tmp, see tile.smk's + # TILE_SLURM_EXTRA) lands on pscomp; everything else may use either. + slurm_partition: "comp,pscomp" + # n09/n17/n36: excluded in the sp_validation candide profile; n23: slurm jobs + # failed there (sp_validation 2b40c99). + slurm_extra: "'--exclude=n09,n17,n23,n36'" +software-deployment-method: [apptainer] +# /local/scratch is where tile.smk puts the node-local tile store (TILE_LOCAL, +# ~5.6 GB per real-data tile). candide has no /local, and its node-local /scratch +# is writable only per job (/scratch/$USER/$SLURM_JOB_ID, made by the prolog), +# which this line cannot name: snakemake passes it through literally, and +# --cleanenv drops SLURM_JOB_ID inside the container. So the node's /tmp (local +# disk, ~29 GB free, sticky) is bound there instead. That holds ~5 concurrent +# tile stores per node, too few for dense image-sim tiles: such a campaign sets +# `tile_store_root:` in its run config, and bin/sp replaces this bind's source in +# the launch snapshot (see workflow/config.yaml). The PYTHONPATH pin is rewritten by bin/sp to the +# launch snapshot's src/; the value here only has to be a checkout's src/. +apptainer-args: "--cleanenv --env OMP_NUM_THREADS=1 --env MALLOC_ARENA_MAX=2 --env PYTHONPATH=/n17data/mkilbing/astro/repositories/github/shapepipe/src --bind /home,/automnt,/n17data,/n23data1,/n09data --bind /tmp:/local/scratch" +latency-wait: 60 # NFS: wait for outputs to appear after a job +keep-going: true # a failed job poisons only its cone; siblings run on +# Transient node-level failures are the common case on a shared cluster: +# a fused tile_shape group died at joblib's fork_exec with BlockingIOError +# [Errno 11] because the node was out of process slots, killing one tile of +# 39 outright. Without this, attempt is always 1, so every +# `mem_mb = lambda wc, attempt: N * attempt` in tile.smk is dead code and a +# one-off OOM or EAGAIN is fatal. Safe for the fused group: a retry +# resubmits the same member set, tile_vignets included (tile.smk's +# "one sharp edge" note). tile_ngmix keeps its own rule-level retries: 2. +retries: 2 +rerun-incomplete: true # re-do jobs left incomplete by an unclean death +show-failed-logs: true +printshellcmds: true +rerun-triggers: [mtime, params, code, software-env] +# The tile_shape group (tile.smk) sets its own slurm_extra (TILE_SLURM_EXTRA, +# the --tmp floor), which REPLACES the default above and so drops the node +# excludes: its jobs landed on n09 and n17. Restate both, identically on all +# four members (a group's string resources must agree). Only slurm_extra is +# overridden; the rules keep their own threads and attempt-scaled mem_mb. +set-resources: + tile_vignets: + slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n36'" + tile_ngmix: + slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n36'" + tile_merge_cats: + slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n36'" + tile_make_cat: + slurm_extra: "'--tmp=16000 --exclude=n09,n17,n23,n36'" diff --git a/scripts/python/create_final_cat.py b/scripts/python/create_final_cat.py index 2b583b857..7571cb052 100755 --- a/scripts/python/create_final_cat.py +++ b/scripts/python/create_final_cat.py @@ -27,6 +27,56 @@ from cs_util import logging +def params_from_run_config(params, defaults): + """Fill unset paths from a workflow run config. + + The workflow already knows where a campaign writes, so a manual merge + should not have to restate it. Resolution goes through the workflow's own + resolver (workflow/scripts/run_config.py), layering the run config on + workflow/config.yaml and then the machines: table, so what lands here is + what the rules would have used. + + Only values still at their default are filled -- an explicit flag always + wins. Nothing is derived for the data path: its patch naming differs and + is not this function's business. + """ + repo = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + sys.path.insert(0, os.path.join(repo, "workflow", "scripts")) + import run_config as _rc + + cfg = _rc.load(os.path.join(repo, "workflow", "config.yaml"), + params["run_config"]) + if cfg.get("input_type") != "image_sims": + raise ValueError( + f"run config {params['run_config']} has input_type=" + f"{cfg.get('input_type', 'data')!r}; -c derives paths for " + "image_sims only") + + outputs = cfg.get("outputs") or {} + products = outputs.get("products_dir") or outputs.get("run_dir") + if not products: + raise ValueError(f"run config {params['run_config']} sets neither " + "outputs.products_dir nor outputs.run_dir") + + # The same derivation as the workflow's merge_final_cats rule: the patch + # dir is the branch dir holding product/tiles, and -i is its parent. + patch_dir = os.path.dirname(os.path.normpath(products)) + patch = os.path.basename(patch_dir) + derived = { + "image_sims": True, + "input_root_dir": os.path.dirname(patch_dir), + "patch": patch, + "merged_cat_path": os.path.join(products, f"final_cat_{patch}.hdf5"), + "output_summary": os.path.join(products, "n_tiles_final.txt"), + "param_path": os.path.join(repo, "workflow", "config", + "cfis_image_sims", "final_cat.param"), + } + for key, value in derived.items(): + if params.get(key) == defaults.get(key): + params[key] = value + return params + + def set_params_from_command_line(args): """Set Params From Command Line. @@ -48,6 +98,12 @@ def set_params_from_command_line(args): _params[key] = options[key] del options + + # A run config fills in whatever is still at its default + # (see the docstring); explicit flags always win. + if _params.get("run_config"): + _defaults, _, _, _ = params_default() + _params = params_from_run_config(_params, _defaults) # Save calling command logging.log_command(args) @@ -72,6 +128,7 @@ def params_default(): "ID": None, "single_op": None, "image_sims": False, + "run_config": None, } _short_options = { "input_root_dir": "-i", @@ -82,6 +139,7 @@ def params_default(): "output_summary": "-o", "single_op": "-s", "image_sims": "-I", + "run_config": "-c", } _types = { "hdu_num": "int", @@ -98,6 +156,9 @@ def params_default(): "ID": "ID for single-ID operation, default={}", "single_op": "single ID operation, allowed are 'check', 'add', 'remove'; default={}", "image_sims": "image simulations mode (different dir layout and run prefix), default={}", + "run_config": "workflow run config (e.g. sp_1p2z_grid_1.yaml); fills" + " in the paths below that were not given explicitly," + " default={}", } return _params, _short_options, _types, _help_strings @@ -373,9 +434,17 @@ def collect_tile_ids_image_sims(patch_path): list of (tile_id, tile_path) tuples """ id_pattern = re.compile(r"^\d+\.\d+$") - tiles_root = os.path.join(patch_path, "tiles") + # Two layouts. The Gen-2 / native runs keep tiles under the patch dir + # itself; the unified workflow (shapepipe #891) publishes to a separate + # products root, and clean_tile deletes the run-dir copies -- so on a + # reclaimed campaign /product/tiles holds the ONLY catalogues. result = [] - if not os.path.isdir(tiles_root): + for sub in ("tiles", os.path.join("product", "tiles"), + os.path.join("products", "tiles")): + tiles_root = os.path.join(patch_path, sub) + if os.path.isdir(tiles_root): + break + else: return result for prefix in os.listdir(tiles_root): prefix_path = os.path.join(tiles_root, prefix) @@ -387,11 +456,33 @@ def collect_tile_ids_image_sims(patch_path): return result +def find_final_cat(id, id_path, run_prefix): + """Path of this tile's final catalogue, or None. + + Flat layout first: the unified workflow copies one catalogue per tile to + /tiles///final_cat-.fits, in DOT form and with + no run sub-tree. Then the legacy layout, where the catalogue sits under the + tile's own shapepipe run dir in DASH form, newest run wins. + """ + flat = os.path.join(id_path, f"final_cat-{id}.fits") + if os.path.exists(flat): + return flat + + base_pattern = os.path.join(id_path, "output", run_prefix) + all_matches = [d for d in glob.glob(base_pattern) if os.path.isdir(d)] + if not all_matches: + return None + newest_dir = max(all_matches, key=os.path.getmtime) + id_dash = re.sub(r"\.", "-", id) + legacy = f"{newest_dir}/make_cat_runner/output/final_cat-{id_dash}.fits" + return legacy if os.path.exists(legacy) else None + + def process(params): if params["image_sims"]: patch_name = params["patch"] - run_prefix = "run_sp_tile_Mc_*" + run_prefix = "run_sp_tile_Mc*" else: patch_name = rf"P{params['patch']}" run_prefix = "run_sp_tile_Mc_*" @@ -446,22 +537,11 @@ def process(params): print(f"Skipping {id} (already processed)") continue - base_pattern = os.path.join(id_path, "output", run_prefix) - all_matches = [d for d in glob.glob(base_pattern) if os.path.isdir(d)] - if not all_matches: + fits_file = find_final_cat(id, id_path, run_prefix) + if fits_file is None: if params["verbose"]: print(f"Final cat for {id} not found, continuing") continue - newest_dir = max(all_matches, key=os.path.getmtime) - - id_dash = re.sub(r"\.", "-", id) - fits_file = f"{newest_dir}/make_cat_runner/output/final_cat-{id_dash}.fits" - - # Exclude unsuccessful run without output FITS file - if not os.path.exists(fits_file): - if params["verbose"]: - print(f"Run without output file found for {id}, skipping") - continue extracted_data, dtype = read_data(fits_file, params) diff --git a/src/shapepipe/modules/fake_interp_runner.py b/src/shapepipe/modules/fake_interp_runner.py new file mode 100644 index 000000000..bc167f924 --- /dev/null +++ b/src/shapepipe/modules/fake_interp_runner.py @@ -0,0 +1,44 @@ +"""FAKE INTERP RUNNER. + +Module runner for ``fake_psf`` under the ``_interp_runner`` name. + +The Snakemake workflow's configs read the galaxy PSF from +``${SP_PSF}_interp_runner`` for every PSF model, so with ``psf_model: fake`` +(image simulations, true PSF) this runner stands where ``psfex_interp_runner`` +and ``mccd_interp_runner`` stand for the real data. It writes the same +``galaxy_psf`` SqliteDict, taken from the simulation's PSF dictionary instead +of a fitted model. ``fake_psf_runner`` is the same module under its original +name, used by the legacy bash job scripts. + +:Author: Martin Kilbinger + +""" + +from shapepipe.modules.fake_psf_package import fake_psf +from shapepipe.modules.module_decorator import module_runner + + +@module_runner( + version="1.0", + file_pattern=["sexcat"], + file_ext=".fits", + depends=["numpy", "astropy", "sqlitedict"], + numbering_scheme="-000-000", +) +def fake_interp_runner( + input_file_list, + run_dirs, + file_number_string, + config, + module_config_sec, + w_log, +): + """Define The Fake Interp Runner.""" + sexcat_path = input_file_list[0] + psf_dict_path = config.getexpanded(module_config_sec, "PSF_DICT_PATH") + output_path = f'{run_dirs["output"]}/galaxy_psf{file_number_string}.sqlite' + + inst = fake_psf.FakePsf(sexcat_path, psf_dict_path, output_path, w_log) + inst.process() + + return None, None diff --git a/src/shapepipe/modules/get_images_runner.py b/src/shapepipe/modules/get_images_runner.py index b1a03b15a..28e81b7d5 100644 --- a/src/shapepipe/modules/get_images_runner.py +++ b/src/shapepipe/modules/get_images_runner.py @@ -27,8 +27,11 @@ def get_images_runner( """Define The Get Images Runner.""" # Read config file section - # Copy/download method - retrieve_method = config.get(module_config_sec, "RETRIEVE") + # Copy/download method. getexpanded, not get: the workflow's ini sets + # RETRIEVE = $SP_RETRIEVE (a run-config key since d64f88ed), and plain get + # returns the literal "$SP_RETRIEVE", which then fails the check below. + # RETRIEVE_OPTIONS a few lines down already reads the same way. + retrieve_method = config.getexpanded(module_config_sec, "RETRIEVE") retrieve_ok = ["vos", "symlink"] if retrieve_method not in retrieve_ok: raise ValueError( diff --git a/src/shapepipe/modules/mccd_package/mccd_interpolation_script.py b/src/shapepipe/modules/mccd_package/mccd_interpolation_script.py index b6183f4e1..0942d93ac 100644 --- a/src/shapepipe/modules/mccd_package/mccd_interpolation_script.py +++ b/src/shapepipe/modules/mccd_package/mccd_interpolation_script.py @@ -442,7 +442,7 @@ def _interpolate_me(self): all_id = np.copy(cat.get_data()["NUMBER"]) key_ne = "N_EPOCH" - if key_ne not in cat.get_data(): + if key_ne not in cat.get_data().dtype.names: raise KeyError( f"Key {key_ne} not found in input galaxy catalogue, needed for" + " PSF interpolation to multi-epoch data; run previous module" diff --git a/workflow/README.md b/workflow/README.md index afdb838e1..ca28a85f0 100644 --- a/workflow/README.md +++ b/workflow/README.md @@ -25,7 +25,8 @@ uv pip install 'snakemake>=9,<10' 'snakemake-executor-plugin-slurm>=2.7,<3' # Edit workflow/config.yaml: tile_list, inputs.tiles/exposures, outputs.run_dir, # outputs.products_dir/index_db, and container. -# `psf_model` is `psfex` or `mccd`; mccd is wired but unvalidated here, while psfex is exercised by smk-g4 through smk-g6. +# `psf_model` is `psfex` or `mccd`. psfex is exercised by smk-g4 through smk-g6; mccd has run the full chain on +# an image-sim star tile (one focal-plane model per exposure, ~1.5 CPU-hours each). # The committed launcher loads apptainer/1.4.5 + the /project venv, so a # fresh shell always has the right state. @@ -45,6 +46,66 @@ Anything other than `run`, `report`, `container`, `cancel` passes straight throu snakemake with the workflow's profile and state dir — the direct command path for `sp --unlock`, `sp --dag`, `sp exp_psf ...`. +## Image simulations + +The same workflow runs the SKiLLS image simulations used to measure the shear +multiplicative bias, so that m calibrates the pipeline that makes the real +catalogue rather than a frozen copy of it. A simulation run sets +`input_type: image_sims`, which points `$SP_CONFIG` at +`config/cfis_image_sims/`. That directory holds real files only for the stages +whose input naming differs (tile Git/Uz/Fe, exposure Gie/Sp) and for the true-PSF +model; everything else is a symlink into `config/cfis/`, so a change to the +real-data chain reaches the simulations with no second edit. Keep the diff of +each overlay file to its `cfis/` original confined to input naming. + +`psf_model: fake` is the simulations' true PSF: the exposure stage runs only +SExtractor (for the background maps the vignets read), and `tile_vignets` runs +`fake_interp_runner`, which writes the `galaxy_psf` product from `psf_dict`. +Simulations that contain stars can run `psfex` or `mccd` exactly as the data do. + +One campaign per shear branch, each with its own run config: + +```bash +SP_PROFILE=candide workflow/bin/sp run -c /path/run_1p2z_grid_1.yaml +``` + +## Run configuration + +A run config passed with `-c/--config-file` is merged on top of +`workflow/config.yaml` and snapshotted with the code. (`-c` is `sp`'s own flag; +pass snakemake's cores as `--cores`/`-j`. `SP_RUN_CONFIG` still works and is what +the jobs read.) `SP_PROFILE` (default `nibi`, or `machine:` in the run config, which must +agree with it) and `input_type:` then select an entry of the `machines:` table, which supplies +`tile_list`, `retrieve` (`symlink` or `vos`), `inputs`, `outputs` and +`container` for any of these the run config leaves unset (`$base_dir` expands +to that machine's `base_dir`, `$run` to the run config's `run:`). A value of `TBD` stops the run at parse time +until it is set. A run config therefore only needs what differs, e.g. for one +SKiLLS shear branch on candide: + +```yaml +machine: candide +input_type: image_sims +psf_model: fake +psf_dict: /home/hervas/fhervas/workdir_skills/input/psf_files/Full_psf_dict.pickle +tile_list: /path/to/tiles.txt +inputs: + tiles: /n09data/hervas/skills_out/1z2z_grid_3/images/SP_tiles + exposures: /n09data/hervas/skills_out/1z2z_grid_3/images/SP_exp +outputs: + run_dir: /path/to/run + products_dir: /path/to/product + index_db: /path/to/run/index.sqlite +``` + +sp_validation's image-simulation workflow drives these campaigns and measures m +from their final catalogues. + +On candide, the node-local tile store (bound from the node's 31 GB `/tmp`) does not +hold several dense image-sim tiles at once. Set `tile_store_root:` in the run +config to a shared directory; `sp run` binds it to `/local/scratch` for that +campaign. The store names carry a per-campaign hash, so all branches can share one +root. + ## The container image `sp container` owns which image the jobs run inside. Two layers, and the second diff --git a/workflow/Snakefile b/workflow/Snakefile index bbdfeefb6..2befdb5ec 100644 --- a/workflow/Snakefile +++ b/workflow/Snakefile @@ -42,41 +42,93 @@ from pathlib import Path from snakemake.exceptions import WorkflowError # Resolved relative to THIS file, not the working directory: snakemake runs with -# --directory on /scratch (bin/sp) so .snakemake/ state never lands on /project -# (group quota is a hard 27/27 TiB — a metadata write mid-run died on it live). +# --directory on /scratch (bin/sp) so .snakemake/ state never lands on /project. +# +# config.yaml is always loaded; SP_RUN_CONFIG (bin/sp) is merged on top, then +# the `machines:` entry for (machine, input_type) fills whatever is still unset +# (scripts/run_config.py, shared with bin/sp and container.py). +sys.path.insert(0, str(Path(workflow.snakefile).parent / "scripts")) +import run_config # noqa: E402 + +_RUN_CONFIG = os.environ.get("SP_RUN_CONFIG") or str( + Path(workflow.snakefile).parent / "config.yaml") configfile: str(Path(workflow.snakefile).parent / "config.yaml") +if os.environ.get("SP_RUN_CONFIG"): + workflow.configfile(os.environ["SP_RUN_CONFIG"]) + +# input_type picks where the pixels come from (and config/cfis vs +# config/cfis_image_sims); everything after ingestion is the same chain. +INPUT_TYPES = {"data", "image_sims"} +INPUT_TYPE = config.get("input_type", "data") +if INPUT_TYPE not in INPUT_TYPES: + raise WorkflowError( + f"Invalid input_type={INPUT_TYPE!r}; expected one of " + f"{sorted(INPUT_TYPES)}.") + +# The machine is SP_PROFILE (bin/sp's profile choice, default nibi). A run +# config may also state `machine:`, and must then agree with it: otherwise +# jobs run under one cluster's SLURM profile against another's paths. +SP_PROFILE = os.environ.get("SP_PROFILE", "nibi") +MACHINE = config.get("machine") or SP_PROFILE +if config.get("machine") is not None and config["machine"] != SP_PROFILE: + raise WorkflowError( + f"machine={MACHINE!r} ({_RUN_CONFIG}) does not match " + f"SP_PROFILE={SP_PROFILE!r}. Set SP_PROFILE={MACHINE}, or fix " + f"`machine:`.") -# Every job's shell runs inside this container (apptainer software-deployment in -# the profile); the user never types apptainer. WHICH image is the one resolution -# order `sp container` exposes — this user's writable sandbox if they built one, -# else their cached SIF if they pulled one, else `container:` above. An empty -# cache therefore lands on exactly the shared /project image the workflow has -# always run, and a package installed into a sandbox reaches the jobs too. -sys.path.insert(0, str(Path(workflow.snakefile).parent / "scripts")) +run_config.apply_machine_defaults(config) +_unresolved = run_config.unresolved(config) +if _unresolved: + raise WorkflowError( + f"Unset or {run_config.PLACEHOLDER!r} for machine={MACHINE!r}, " + f"input_type={INPUT_TYPE!r}: {', '.join(_unresolved)}. Set them in " + f"your run config (SP_RUN_CONFIG).") + +# The image every job runs in: this user's sandbox, else their cached SIF, +# else the run config's `container:`. Checked at parse time, since an empty +# container directive passes a dry run and only fails on a node. import container as _container # noqa: E402 -# Loud at PARSE time, both ways: a broken SP_CONTAINER override, or nothing to -# resolve at all (empty cache AND no `container:` key). The empty string the -# resolver returns for "none" is a valid-looking container directive that passes -# dry-run and only fails once jobs reach a node. try: - _image, _kind = _container.resolve_image() + _image, _kind = _container.resolve_image(config.get("container")) except _container.ContainerError as _exc: raise WorkflowError(str(_exc)) if _kind == "none": raise WorkflowError( - f"No container image resolved: no sandbox, no cached SIF, and no " - f"'{_container.CONFIG_KEY}:' key in {_container.CONFIG_FILE}. " - f"Run `sp container pull`, or restore the key.") + "No container image resolved: no sandbox, no cached SIF, and no " + "`container:` in the run config. Run `sp container pull`, or set it.") container: _image -PSF_MODELS = {"psfex", "mccd"} +# `fake` is the image-simulation true PSF: no exposure PSF fit; the tile's +# galaxy_psf comes from `psf_dict` via fake_interp_runner (read by the +# configs as ${SP_PSF}_interp_runner). Sims with stars may use psfex/mccd. +PSF_MODELS = {"psfex", "mccd", "fake"} PSF_MODEL = config.get("psf_model", "psfex") if PSF_MODEL not in PSF_MODELS: raise WorkflowError( f"Invalid psf_model={PSF_MODEL!r}; expected one of " f"{sorted(PSF_MODELS)}.") +if PSF_MODEL == "fake" and INPUT_TYPE != "image_sims": + raise WorkflowError( + "psf_model=fake is the image-simulation true PSF; it needs " + "input_type=image_sims.") +PSF_DICT = config.get("psf_dict") or "" +if PSF_MODEL == "fake" and not PSF_DICT: + raise WorkflowError("psf_model=fake needs `psf_dict:` (the simulation's " + "pickled PSF dictionary) in the run config.") + +# How images reach the run (exported as SP_RETRIEVE for the Git/Gie configs): +# `symlink` to a local mirror, or `vos` download. Sims are always local. +RETRIEVE_MODES = {"symlink", "vos"} +RETRIEVE_MODE = config.get("retrieve", "symlink") +if RETRIEVE_MODE not in RETRIEVE_MODES: + raise WorkflowError( + f"Invalid retrieve={RETRIEVE_MODE!r}; expected one of " + f"{sorted(RETRIEVE_MODES)}.") +if INPUT_TYPE == "image_sims": + RETRIEVE_MODE = "symlink" + # --- paths ----------------------------------------------------------------- # RUN_DIR is the scratch root: bulk intermediates, sized so a batch finishes @@ -95,8 +147,9 @@ INDEX_DB = Path(OUTPUTS["index_db"]) SCRIPTS = Path(workflow.basedir) / "scripts" # The config chain is the repo's committed directory (D2). The configs and # rules that set their environment variables must be versioned together. There -# is no `config_src` knob. -CONFIG_DIR = Path(workflow.basedir) / "config" / "cfis" +# is no separate `config_src` setting; input_type picks between the two dirs. +CONFIG_DIR = Path(workflow.basedir) / "config" / { + "data": "cfis", "image_sims": "cfis_image_sims"}[INPUT_TYPE] sys.path.insert(0, str(SCRIPTS)) import build_index # noqa: E402 @@ -509,8 +562,20 @@ def unit_pre(stage, unit, *, exp_name=None, forest=None, env=None, the committed config dir. It also exports the configured input roots as ``SP_INPUT_TILES`` and - ``SP_INPUT_EXPOSURES`` and the PSF choice as ``SP_PSF`` for the committed - ini chain. + ``SP_INPUT_EXPOSURES``, the retrieve mode as ``SP_RETRIEVE`` and the PSF + choice as ``SP_PSF`` for the committed ini chain (plus ``PSF_DICT`` for the + image-simulation true PSF). + + Every line here is part of each rule's ``params.pre`` and so of the + ``params`` rerun trigger: a line added for every rule reruns every finished + unit of a campaign on its next ``sp run``. The image-simulation lines are + therefore conditional, and a data run's prologue is byte-identical to what + it was before input_type existed. + + ``tile_numbers.txt`` carries the number in the form the INPUT files are + named with, because get_images substitutes it verbatim into + INPUT_FILE_PATTERN: dot format for the survey tiles (``CFIS.210.282.r``), + dash format for the simulations (``CFIS_simu_image-210-282``). Finally it ``rm -rf``s this stage's own fixed run dir — ShapePipe's FileHandler raises on an existing run dir, and it is how a rerun never sees @@ -529,19 +594,23 @@ def unit_pre(stage, unit, *, exp_name=None, forest=None, env=None, f"export SP_CONFIG='{CONFIG_DIR}'", f"export SP_INPUT_TILES='{INPUTS['tiles']}'", f"export SP_INPUT_EXPOSURES='{INPUTS['exposures']}'", + f"export SP_RETRIEVE='{RETRIEVE_MODE}'", f"export SP_PSF='{PSF_MODEL}'", # Also set via apptainer-args in the profile; kept here so a hand-run of # this same line outside snakemake behaves identically. f"export {_THREAD_CAPS}", 'mkdir -p "$SP_RUN/output" "$SP_RUN/manifests" "$SP_RUN/logs"', ] + if PSF_DICT: + lines.append(f"export PSF_DICT='{PSF_DICT}'") if forest: lines.append(f"export SP_EXP='{forest}'") for k, v in (env or {}).items(): lines.append(f"export {k}='{v}'") if level == "tile": - lines.append(f"printf '%s\\n' '{unit}' > \"$SP_RUN/tile_numbers.txt\"") + number = unit.replace(".", "-") if INPUT_TYPE == "image_sims" else unit + lines.append(f"printf '%s\\n' '{number}' > \"$SP_RUN/tile_numbers.txt\"") else: fe = "$SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output" lines += [f'mkdir -p "{fe}"', @@ -603,9 +672,59 @@ include: "rules/tile.smk" # mask generation.) localrules: all, prepare_all_tiles, clean_exposure, clean_tile +# --- merged catalogue (image sims only) ------------------------------------ +# The sp_validation side starts from ONE hdf5 per branch, not 39 FITS tiles, so +# the campaign is not finished until they are merged. Image sims only: the data +# path's patches are merged separately, on a different naming convention. +# +# Paths are derived from products_dir rather than from `run:`, so they hold +# whatever the run config calls things. create_final_cat.py takes a root to +# scan (-i) and a patch name (-P) that is BOTH the directory to match under it +# and the hdf5 group, so the patch directory is the branch dir and the root is +# its parent -- exactly the `-i .. -P ` the old sp_validation rule ran +# from inside the branch. +MERGE_PATCH_DIR = PRODUCTS_DIR.parent # the branch dir, holding product/tiles +MERGE_ROOT = MERGE_PATCH_DIR.parent # -i +MERGE_PATCH = MERGE_PATCH_DIR.name # -P, and the hdf5 group +MERGED_CAT = PRODUCTS_DIR / f"final_cat_{MERGE_PATCH}.hdf5" +CREATE_FINAL_CAT = Path(workflow.basedir).parent / "scripts" / "python" / "create_final_cat.py" + + +def merge_targets(): + """The merged catalogue, once every ready tile has published.""" + return [str(MERGED_CAT)] if INPUT_TYPE == "image_sims" and TILES_READY else [] + + +rule merge_final_cats: + """Merge the per-tile final catalogues into one hdf5 (image sims). + + Reads the PRODUCTS root, not the run dir: clean_tile deletes the run-dir + copies, so on a reclaimed campaign the published ones are all that is left. + Ordering needs no edge to clean_tile for the same reason. + """ + input: + [final_cat(t) for t in TILES_READY], + output: + cat = str(MERGED_CAT), + n_tiles = str(PRODUCTS_DIR / "n_tiles_final.txt"), + params: + root = str(MERGE_ROOT), + patch = MERGE_PATCH, + param_file = str(CONFIG_DIR / "final_cat.param"), + threads: 1 + resources: + mem_mb = lambda wc, attempt: 8000 * attempt, + runtime = 120 + shell: + f"python {CREATE_FINAL_CAT} -I" + " -m {output.cat} -i {params.root} -p {params.param_file}" + " -P {params.patch} -o {output.n_tiles} -v" + + rule all: input: [final_cat(t) for t in TILES_READY], + merge_targets(), clean_targets(), clean_tile_targets(), diff --git a/workflow/bin/sp b/workflow/bin/sp index fdc3d4f59..7d10b988d 100755 --- a/workflow/bin/sp +++ b/workflow/bin/sp @@ -1,9 +1,10 @@ #!/usr/bin/env bash -# sp — the committed launcher for the ShapePipe Snakemake workflow (PRD #848 D1). +# Main ShapePipe launch script for Snakemake workflow # -# Three verbs, nothing else: +# Usage: # -# sp run [ARGS...] bring the products on disk up to date with the tile list. +# sp run [-c FILE] [ARGS...] +# bring the products on disk up to date with the tile list. # Snapshots the code into the state dir first (see "the # launch code snapshot" below) and runs out of the copy, so # editing the checkout mid-campaign cannot reach the jobs. @@ -11,6 +12,9 @@ # 1. PREPARE snakemake prepare_all_tiles # 2. COMPUTE snakemake all <- its PARSE builds the index # ARGS (--jobs, -n, --forcerun, ...) pass through to BOTH. +# FILE is a user-specified config file that can be used +# to add and/or overwrite entries in the default config +# file workflow/config.yaml (read automatically) # sp report [ARGS...] emit run_report.json now (mid-run is fine). # sp container VERB manage the image every job runs inside: pull, status, # sandbox, exec, resolve. `sp container --help` documents @@ -24,27 +28,108 @@ # executor re-invokes it inside jobs, so it cannot live on a node-local path). # One entry point, so a fresh tmux or a restart after a crash always launches # with the right state. +# +# Typical use, from the repo root: +# +# SP_PROFILE=candide workflow/bin/sp run -c ~/my_run.yaml # a campaign +# SP_PROFILE=candide workflow/bin/sp run -c ~/my_run.yaml -n # dry run +# SP_PROFILE=candide workflow/bin/sp report -c ~/my_run.yaml # status now +# workflow/bin/sp run # config.yaml only +# +# Run config: -c FILE, also --config-file. Read after workflow/config.yaml and +# merged on top; anything still unset comes from the machines: entry for +# SP_PROFILE and input_type (scripts/run_config.py). +# To check a resolved value without running: +# +# python scripts/run_config.py workflow/config.yaml ~/my_run.yaml outputs.run_dir +# +# -c is sp's flag, not snakemake's --cores; pass cores as --cores or -j. +# +# Environment: +# +# SP_PROFILE profiles/ to use (default nibi); also selects the +# machines: entry, and must match `machine:` if set +# SP_SNAKEMAKE_ENV venv to activate; skipped if snakemake is on PATH +# SP_RUN_CONFIG same as -c, and what the jobs read; -c wins +# SP_STATE_DIR snakemake state and code snapshot (default -state) +# SP_CONTAINER image to use, before the config's container: +# SP_PHASE set by sp (prepare/compute/passthrough); do not set set -euo pipefail HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" # workflow/ REPO="$(dirname "$HERE")" VENV="${SP_SNAKEMAKE_ENV:-/project/def-mjhudson/cdaley/snakemake-env}" SCRIPTS="$HERE/scripts" -CONFIG="$HERE/config.yaml" - -module load apptainer/1.4.5 2>/dev/null || true -# shellcheck disable=SC1091 -source "$VENV/bin/activate" - -# Scalar reader for workflow/config.yaml. The venv is active by this point and -# snakemake depends on PyYAML, so this parses the file rather than pattern-matching -# it -- quotes, inline comments and nesting are the parser's problem, not ours. -cfg() { python -c 'import sys, yaml -value = yaml.safe_load(open(sys.argv[1])) -for key in sys.argv[2].split("."): - value = value[key] -print(value or "")' "$CONFIG" "$1"; } +PROFILE="${SP_PROFILE:-nibi}" + +# -c is consumed here, not forwarded: snakemake's own --configfile merges +# differently, and the jobs and the snapshot need it as SP_RUN_CONFIG. The scan +# stops at `--` and at the command after `container exec`, so a -c meant for +# the delegated command (`sp container exec python -c ...`) or for snakemake's +# own escape hatch (`sp -- ...`) is left alone. `--config` is NOT sp's: keeping +# it free lets snakemake's own `--config k=v` (what the Snakefile's flag() +# reads) pass through untouched. +RUN_CONFIG="${SP_RUN_CONFIG:-}" +_sp_args=() +while [ $# -gt 0 ]; do + case "$1" in + --) + _sp_args+=("$@"); break ;; + -c|--config-file) + [ $# -ge 2 ] || { echo "sp: $1 needs a file" >&2; exit 2; } + RUN_CONFIG="$2"; shift 2 ;; + -c=*|--config-file=*) + RUN_CONFIG="${1#*=}"; shift ;; + container) + # Everything from `exec`'s own command on is the delegated command's, + # not sp's: `container.py` reads SP_RUN_CONFIG, so its own -c is never + # sp's run config. + _sp_args+=("$1"); shift + [ "${1:-}" = exec ] && { _sp_args+=("$@"); break; } + ;; + *) _sp_args+=("$1"); shift ;; + esac +done +set -- ${_sp_args[@]+"${_sp_args[@]}"} + +if [ -n "$RUN_CONFIG" ]; then + [ -f "$RUN_CONFIG" ] || { + echo "sp: run config $RUN_CONFIG does not exist" >&2; exit 2; } + # Absolute: jobs and the snapshot run from other directories. + RUN_CONFIG="$(cd "$(dirname "$RUN_CONFIG")" && pwd)/$(basename "$RUN_CONFIG")" + export SP_RUN_CONFIG="$RUN_CONFIG" +fi + +[ -d "$REPO/profiles/$PROFILE" ] || { + echo "sp: no profile $REPO/profiles/$PROFILE (SP_PROFILE=$PROFILE)" >&2; exit 2; } + +# snakemake resolves `apptainer` via PATH at job runtime, so all this has to +# do is put it there. candide already has it in /usr/bin and ships no +# modulefile for it, so probing for one only produces a scary ERROR line; +# skip the module entirely when apptainer is already usable. Both streams are +# dropped because the message arrives on stderr under environment-modules and +# can arrive on stdout under Lmod. +if ! command -v apptainer >/dev/null 2>&1; then + module load apptainer/1.4.5 >/dev/null 2>&1 || true + command -v apptainer >/dev/null 2>&1 || { + echo "sp: apptainer is not on PATH and 'module load apptainer/1.4.5' failed" >&2 + exit 2; } +fi +if [ -f "$VENV/bin/activate" ]; then + # shellcheck disable=SC1091 + source "$VENV/bin/activate" +elif ! command -v snakemake >/dev/null 2>&1; then + echo "sp: no venv at $VENV and no snakemake on PATH (set SP_SNAKEMAKE_ENV)" >&2 + exit 2 +fi + +# Resolved config value (config.yaml + run config + machine defaults), as the +# Snakefile resolves it. +cfg() { python "$SCRIPTS/run_config.py" "$HERE/config.yaml" "${SP_RUN_CONFIG:-}" "$1"; } RUN_DIR="$(cfg outputs.run_dir)"; INDEX_DB="$(cfg outputs.index_db)" +if [ "${1:-}" != container ] && { [ -z "$RUN_DIR" ] || [ "$RUN_DIR" = TBD ]; }; then + echo "sp: outputs.run_dir is unset or TBD for this machine/input_type" >&2; exit 2 +fi # Snakemake state (.snakemake: metadata, locks, incomplete markers) lives NEXT TO # THE RUN on /scratch, never on the persistent root — the one exception to D5's @@ -72,7 +157,8 @@ STATE_DIR="${SP_STATE_DIR:-${RUN_DIR}-state}"; mkdir -p "$STATE_DIR" # WHAT. `sp run` copies the code it is about to launch into $STATE_DIR/code and # runs the campaign entirely out of that copy: the Snakefile, the rules, the # scripts, the ini chain (symlinks DEREFERENCED -- workflow/config/cfis points -# into example/, and the copy must be self-contained), src/, and the profile. +# into example/, and the copy must be self-contained), src/, the profile, and +# the repo's top-level scripts/ (576 KB), which merge_final_cats runs out of. # Every workflow-internal path hangs off `workflow.basedir`, which IS the # snapshot, so they all follow it for free; the profile's PYTHONPATH pin is the # one that cannot (YAML splices nothing) and is rewritten below. @@ -91,10 +177,10 @@ snapshot_code() { mkdir -p "$SNAPSHOT" if command -v rsync >/dev/null 2>&1; then rsync -a --delete --copy-links --exclude '__pycache__' --exclude '*.egg-info' \ - "$HERE" "$REPO/src" "$REPO/profiles" "$SNAPSHOT/" + "$HERE" "$REPO/src" "$REPO/profiles" "$REPO/scripts" "$SNAPSHOT/" else rm -rf "$SNAPSHOT"; mkdir -p "$SNAPSHOT" - cp -rL "$HERE" "$REPO/src" "$REPO/profiles" "$SNAPSHOT/" + cp -rL "$HERE" "$REPO/src" "$REPO/profiles" "$REPO/scripts" "$SNAPSHOT/" find "$SNAPSHOT" -name __pycache__ -type d -prune -exec rm -rf {} + fi @@ -104,7 +190,7 @@ snapshot_code() { # rewritten HERE, in the snapshot's copy, and `sm` points --profile at that copy. # The checked-in profile keeps its literal checkout path and stays the source of # truth for the FLAGS; exactly one path is substituted, by one line of sed-work. - python - "$SNAPSHOT/profiles/nibi/config.yaml" "$SNAPSHOT/src" <<'PY' + python - "$SNAPSHOT/profiles/$PROFILE/config.yaml" "$SNAPSHOT/src" <<'PY' import pathlib, re, sys f, src = pathlib.Path(sys.argv[1]), sys.argv[2] text, n = re.subn(r'(--env PYTHONPATH=)\S+', lambda m: m.group(1) + src, @@ -115,6 +201,41 @@ if not n: f.write_text(text) PY + # `tile_store_root:` in the run config moves the node-local tile store + # (tile.smk's tile_local(): the vignette + WCS stores of each tile_shape group) + # off the profile's `/local/scratch` bind, for this campaign only. It is the + # same substitution as the pin above -- the BIND SOURCE of `/local/scratch` in + # the snapshot's profile -- and deliberately not a path in tile_local(): that + # function's output is a params rerun trigger, so a path there would reschedule + # every finished tile of a resumed campaign (its warning block says how badly), + # while a bind is not a trigger at all (a dry run after changing it schedules + # only the missing tiles). Why a campaign wants it: candide binds the node's + # 31 GB /tmp, which a few dense image-sim tile stores or failed groups' orphans + # fill, after which every later group on that node fails in seconds. A shared + # directory (NFS) trades ngmix read latency for room; the campaign-unique store + # names (LOCAL_TAG) keep concurrent campaigns apart in one root. Profiles with + # no `X:/local/scratch` bind (nibi's `--bind /local`) get one appended. + python - "$SNAPSHOT/profiles/$PROFILE/config.yaml" "$(cfg tile_store_root)" <<'PY' +import os, pathlib, re, sys +f, root = pathlib.Path(sys.argv[1]), sys.argv[2] +if root: + root = str(root) + if not os.path.isabs(root): + sys.exit(f"sp: tile_store_root must be an absolute path, got {root!r}") + os.makedirs(root, exist_ok=True) + bind = f"--bind {root}:/local/scratch" + text, n = re.subn(r'--bind \S+:/local/scratch(?=[\s"])', lambda m: bind, + f.read_text(), count=1) + if not n: + text, n = re.subn(r'(apptainer-args:\s*"[^"]*)"', + lambda m: m.group(1) + " " + bind + '"', text, count=1) + if not n: + sys.exit("sp: tile_store_root is set but the profile has no quoted " + "apptainer-args to bind it with") + f.write_text(text) + print(f"sp: tile store root {root} (bound to /local/scratch)", file=sys.stderr) +PY + python - "$SNAPSHOT/snapshot.json" "$REPO" <<'PY' import json, pathlib, subprocess, sys, time out, repo = pathlib.Path(sys.argv[1]), sys.argv[2] @@ -137,6 +258,11 @@ out.write_text(json.dumps({ "dirty_files": status.splitlines() if status else [], }, indent=2) + "\n") PY + # An external run config is part of what is launched: snapshot it with the + # code, so a job re-parsing hours later reads the config the run started with. + if [ -n "${SP_RUN_CONFIG:-}" ]; then + cp "$SP_RUN_CONFIG" "$SNAPSHOT/run_config.yaml" + fi echo "sp: code snapshot refreshed at $SNAPSHOT" >&2 } @@ -164,9 +290,14 @@ code_root() { [ -f "$SNAPSHOT/workflow/Snakefile" ] && echo "$SNAPSHOT" || echo # index without building it and schedules no side effects. sm() { local root; root="$(code_root)" + # The run config the Snakefile reads: the snapshot's copy when this run has + # one, so every parse of the campaign -- head and jobs -- sees the same file. + if [ -n "${SP_RUN_CONFIG:-}" ] && [ -f "$root/run_config.yaml" ]; then + export SP_RUN_CONFIG="$root/run_config.yaml" + fi SP_PHASE="${SP_PHASE:-passthrough}" \ snakemake --snakefile "$root/workflow/Snakefile" \ - --profile "$root/profiles/nibi" \ + --profile "$root/profiles/$PROFILE" \ --directory "$STATE_DIR" "$@" } diff --git a/workflow/config.yaml b/workflow/config.yaml index 63b29413c..76a7ee3d9 100644 --- a/workflow/config.yaml +++ b/workflow/config.yaml @@ -1,142 +1,93 @@ -# Run configuration for the ShapePipe Snakemake workflow. -# -# A "run" is declared by a tile list plus the paths below. Everything here is -# read at parse time; none of it is a rule input, so editing it (e.g. appending -# tiles) never invalidates completed work — it only changes which jobs exist. - -# The tile list that scopes this run (one "IDra.IDdec" per line). -# The campaign grows by appending to this file — parse-time config, so -# completed work is never invalidated. Current contents: smk-g5, ONE tile -# (186.307), the paired control for the two fixes landed after smk-g4 — the -# node-local WCS sqlite and tile_ngmix's mem_mb 14000 -> 5000. That tile ran -# ALONE as smk-g4 job 20799387 (elapsed 8063 s, chunk median 6813 s, chunk mean -# load 59%, MaxRSS 26.6 GiB), so the baseline carries no concurrency confound -# and the contrast is same tile, same solitude, two commits apart. -# The 34-tile smk-g4 set is preserved at smk-g4/tiles34.txt. -# (The 210/211 quad used earlier is unusable: its tile images are symlinks into -# anaennis' moved processed_tiles tree — ~8.5k of the 10.3k staged tiles are -# broken.) -tile_list: /project/def-mjhudson/cdaley/sp-products/smk-g6/tiles.txt - -# Inputs are the only site-specific data paths; all committed configs consume these -# through SP_INPUT_TILES and SP_INPUT_EXPOSURES. -inputs: - # Pre-staged P3 data (get_images RETRIEVE=symlink). - tiles: /project/def-mjhudson/unions-wl/tiles - exposures: /project/def-mjhudson/unions-wl/exposures - -# The container every job runs inside (apptainer software-deployment in the profile). -container: /project/def-mjhudson/cdaley/containers/shapepipe-develop-runtime.sif - -# PSF model used by the exposure and tile interpolation stages. -psf_model: psfex - -# THE TWO ROOTS (D5). -# -# run_dir is the SCRATCH root and the $SP_RUN every config interpolates: bulk -# intermediates, sized so a batch finishes inside the 60-day purge window. The -# sharded per-unit stores live under it: -# /tiles/<2-char prefix>// and /exp/// -outputs: - run_dir: /scratch/cdaley/shapepipe-output/smk-g6 - -# products_dir is the PERSISTENT root: the durable, low-volume products — the -# final catalogues (/tiles///final_cat-.fits, -# mirroring the scratch tree shard for shard), the index, the report. ~32-46 MB -# per tile, so a full DR6 campaign is a few hundred GB. -# -# The final catalogue is also the tile-finished MARKER that cuts a finished -# tile's exposure edges (D5), which is the second reason it cannot sit on -# scratch: a purge would not merely lose a product, it would make every finished -# tile re-declare inputs against exposure stores reclamation already deleted. -# -# Unset means "one root": products land under run_dir, exactly the pre-D5 -# layout, which is what a fixture or smoke test wants. +# Main configuration for the ShapePipe Snakemake workflow. + +# To run, first set the profile env variable to the desired machine/architecture: + +# SP_PROFILE=nibi|candide + +# Then, run in /path/to/shapepipe: + +# ./workflow/bin/sp run -c /path/to/my_run.yaml # -# Snakemake's own state is the one durable-looking thing that stays on scratch -# (-state; bin/sp explains why). - products_dir: /project/def-mjhudson/cdaley/sp-products/smk-g6 - -# There is no config_src knob: the config chain is workflow/config/cfis, resolved -# relative to the Snakefile. The configs interpolate $SP_RUN / $SP_UNIT_NUM / -# $SP_CONFIG / $SP_EXP / $NGMIX_* and the rules export them -- configs and rules -# are one artefact and must version together, so the dir is fixed by construction. - -# The run index, and — sharing its directory — missing.json and run_report.json. -# On the persistent root with the catalogues (D5): the index is the record of -# which tile reads which exposure, so it is what a post-purge reconstruction -# would otherwise have to rebuild from tile headers. - index_db: /project/def-mjhudson/cdaley/sp-products/smk-g6/index/run_index.sqlite - -# Rolling exposure-store reclamation (D5). When true, the COMPUTE DAG grows one -# `clean_exposure` job per exposure. It fires once every campaign tile that reads -# that exposure has its vignets, deletes the exposure's store AND its manifests, -# and leaves `cleaned.json`, which absorbs the manifests — `sp report` reads them -# back out of the tombstone and reports the exposure as `cleaned`. OFF for smk-g5: -# this is a one-tile control, its ~8 exposure stores cost ~54 GiB against 979 GiB -# free, reclamation is already measured exactly by smk-g4, and keeping the stores -# means a re-measurement re-runs only the shape chain instead of the whole tile. +# The config file my_run.yaml can contain user- or run-specific entries +# that overwrite the main config file workflow/config.yaml . + +# Main keys, global for all machines + +# input_tyle: allowed are data or image_sims +input_type: data + +# psf_model (psfex, mccd, or fake) and psf_dict are set per (machine, +# input_type) in the machines: table below, NOT here: a top-level key +# shadows that table. The Snakefile falls back to psfex if no entry +# supplies one. + +# Run name, available as `$run` in the paths below. +run: smk-g6 + +# Entries per machine (SP_PROFILE, default nibi; implemented: nibi, candide) +# and input_type. A run config may instead state `machine:`, which must +# then agree with SP_PROFILE. +# - base_dir +# - tile_list: text file with tile IDs +# - retrieve: method to get input data, allowed are symlink, vos +# - inputs.tiles,, .exposures: path for input tile and exposure +# - outputs.run_dir: (scratch) run directory, where tmp files will be stored +# - outputs.products_dir: path to final products +# - outputs.index_db: path to bookkeeping index sqlite file +# - container: TBD + +machines: + nibi: + base_dir: /project/def-mjhudson + data: + psf_model: psfex + tile_list: $base_dir/cdaley/sp-products/$run/tiles.txt + retrieve: symlink + inputs: + tiles: $base_dir/unions-wl/tiles + exposures: $base_dir/unions-wl/exposures + outputs: + run_dir: /scratch/cdaley/shapepipe-output/$run + products_dir: $base_dir/cdaley/sp-products/$run + index_db: $base_dir/cdaley/sp-products/$run/index/run_index.sqlite + container: $base_dir/cdaley/containers/shapepipe-develop-runtime.sif + + candide: + base_dir: /n17data/UNIONS/WL + data: + psf_model: psfex + retrieve: vos + container: /n17data/cdaley/containers/shapepipe_develop-runtime-20260718.sif + image_sims: + retrieve: symlink + # True simulation PSF: no exposure PSF fit; fake_interp_runner + # reads psf_dict below. + psf_model: fake + inputs: + # $run is the branch dir (e.g. 1p2z_grid_1), so a run config + # picks the grid by setting `run:` alone. + tiles: /n09data/hervas/skills_out/$run/images/SP_tiles + exposures: /n09data/hervas/skills_out/$run/images/SP_exp + # Dictionary of PSF model stamps + psf_dict: /home/hervas/fhervas/workdir_skills/input/psf_files/Full_psf_dict.pickle + container: /n17data/cdaley/containers/shapepipe_im_sims-runtime.sif + +# Per-exposure store reclamation once every tile reading it has its vignets. clean: true -# Rolling TILE-store reclamation (D5). When true, the COMPUTE DAG grows one -# `clean_tile` job per in-scope tile, ordered after that tile's own final_cat, so -# a tile's scratch store goes as soon as its chain lands rather than at the end -# of the batch. It deletes the tile's whole /tiles/// tree -# except four things another mechanism owns — `cleaned.json`, -# `manifests/tile_vignets.json` (clean_exposure's eligibility currency), -# `manifests/tile_find_exposures.json` (prepare_all_tiles' target) and the Fe -# exposure list under output/ (build_index re-reads it at every compute parse). -# clean_tile.py argues each one. -# -# A SEPARATE SWITCH FROM `clean:` ABOVE, AND NOT AN OVERSIGHT. Exposure -# reclamation is reversible in the only sense that matters: a tile appended later -# rebuilds the exposure chain from VOS, expensively but completely. TILE -# reclamation DESTROYS THE PER-TILE AUDIT TRAIL and nothing rebuilds it. -# TWO tools read it: sp-products/tools/sp_tilecost.py and sp_costmodel.py. Both -# attribute the fused tile_shape group job's cost per tile from that tile's -# scratch store — the SExtractor catalogue (NAXIS2 of the sexcat is the object -# count, the cost model's independent variable; sp_costmodel also reads its -# EPOCH_k extensions for the geometric epoch count) and the eight tile_ngmix -# benchmark TSVs — and that model is how every mem_mb and runtime number in -# tile.smk was derived. The sexcat is ~380 MB per tile and cannot be kept; the -# benchmark ROWS are absorbed into the tombstone, so the measurements survive -# but not at the paths the tools read. -# -# (The third artifact they read, run_sp_tile_ngmix_Ngu/, is NOT lost to this -# flag: tile_ngmix declares its chunk dir temp(), so snakemake reclaims it as -# soon as tile_merge_cats runs. It is already absent from every finished tile, -# clean_tiles or not.) -# -# The honest one-line summary: REQUIRED FOR ANY BATCH OVER ~850 TILES, AND IT -# TURNS OFF PER-TILE COST ATTRIBUTION. The arithmetic: a finished tile leaves -# 1.19 GiB across 137 inodes (measured on 186.307), so a 1 TiB scratch quota is -# full at 859 tiles, and DR6's 23,114 tiles would want 26.9 TiB and 3.17M inodes -# against a 1M quota. Reclaimed, a tile costs 10 inodes and 12.9 KB — 231k inodes -# and ~300 MB for the whole of DR6. -# -# OFF here for the same reason `clean:` is: smk-g5 is a one-tile control, and its -# store is what a re-measurement would read. +# Per-tile store reclamation after the tile's final_cat. Needed above ~850 +# tiles (scratch quota); deletes the per-tile data sp_tilecost/sp_costmodel +# read. Set both false for a small control run you want to re-measure. clean_tiles: true -# Tiles that may NOT pin an exposure store (default: empty). -# -# An exposure is eligible for cleaning only once EVERY consuming tile has its -# vignets. One permanently-failed tile therefore holds all its exposures -# (~8 measured on P3: 7.9 exposures/tile) for the life of the campaign. A tile listed here is dropped from the -# consumer sets, and its exposures become eligible. -# -# READ THIS BEFORE ADDING A TILE. Ignoring a tile is a decision to give up its -# exposures' stores. If you later retry that tile, those exposure chains are -# gone and will be REBUILT from scratch — get_images, split, psf, per -# exposure. That is correct, and expensive. Ignore a tile when you have decided -# it is dead, not while you are still debugging it. +# Failed tiles that should not keep their exposures from being cleaned. +# Their exposures are rebuilt from scratch if the tile is retried. clean_ignore_tiles: [] -# ngmix within-tile chunking: static N chunks (closed ID ranges computed -# per-tile, in-job, from the tile's own sexcat). +# ngmix chunks per tile. ngmix_chunks: 8 -# The container's installed shapepipe is overridden by the prod worktree via -# --env PYTHONPATH in profiles/nibi (settled call 3); no config knob here. -# build_index.py's missing-tile fraction threshold is passed by workflow/bin/sp -# (SP_MISSING_THRESHOLD, default 0.0 = any missing tile is fatal). +# Node-local tile store (vignets, WCS). Unset: the profile's /local/scratch +# bind (nibi NVMe, candide node /tmp). Dense image-sim campaigns on candide +# want a shared directory instead. +# tile_store_root: /n23data1//shapepipe/tilestore diff --git a/workflow/config/cfis/config_exp_Gie.ini b/workflow/config/cfis/config_exp_Gie.ini index b0cce78e8..9f69fbaee 100644 --- a/workflow/config/cfis/config_exp_Gie.ini +++ b/workflow/config/cfis/config_exp_Gie.ini @@ -86,8 +86,8 @@ INPUT_NUMBERING = \d{6} # Output file pattern without number OUTPUT_FILE_PATTERN = image-, weight-, flag- -# Method to retrieve images, one in 'vos', 'symlink' -RETRIEVE = symlink +# 'vos' or 'symlink', from the run config's `retrieve:` +RETRIEVE = $SP_RETRIEVE # If RETRIEVE=vos, number of attempts to download # Optional, default=3 diff --git a/workflow/config/cfis/config_exp_mccd.ini b/workflow/config/cfis/config_exp_mccd.ini index a7d6a3500..5e40a5afa 100644 --- a/workflow/config/cfis/config_exp_mccd.ini +++ b/workflow/config/cfis/config_exp_mccd.ini @@ -50,7 +50,7 @@ OUTPUT_DIR = $SP_RUN/output [JOB] # Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial -SMP_BATCH_SIZE = 1 +SMP_BATCH_SIZE = 8 # Timeout value (optional), default is None, i.e. no timeout limit applied TIMEOUT = 96:00:00 @@ -177,8 +177,24 @@ NUMBERING_SCHEME = -0000000-0 SETOOLS_CONFIG_PATH = $SP_CONFIG/star_selection.setools +# The MCCD chain differs from the PSFEx one only from here on; everything up +# to the star selection is the same, so the two PSF models are fitted on the +# same stars. Every module below chains from the one before it in this run +# (the decorators' input_module), so none needs an INPUT_DIR. +# +# merge_starcat_runner and mccd_plots_runner close the chain with per-exposure +# diagnostics of the validation stars; the tiles need only the fitted model. + [MCCD_PREPROCESSING_RUNNER] +# Serial runner: it receives every CCD's star catalogues of this exposure and +# writes one focal-plane training (80%) and one test (20%) catalogue for it. +FILE_PATTERN = star_split_ratio_80, star_split_ratio_20 + +FILE_EXT = .fits, .fits + +NUMBERING_SCHEME = -0000000-0 + # Path to MCCD config file CONFIG_PATH = $SP_CONFIG/config_MCCD.ini @@ -186,27 +202,22 @@ MODE = FIT_VALIDATION VERBOSE = False -INPUT_DIR = last:setools_runner - -# Input are individual CCDs, thus single-exposure single-HDU images -NUMBERING_SCHEME = -0000000-0 - -FILE_PATTERN = star_split_ratio_80, star_split_ratio_20 - -FILE_EXT = .fits, .fits - [MCCD_FIT_VAL_RUNNER] -# Path to MCCD config file +# One focal-plane model per exposure: fitted_model-.npy, which the tiles' +# mccd_interp_runner reads (config_tile_PiViVi_mccd.ini, ME_DOT_PSF_PATTERN), +# and validation_psf-.fits, the model at the 20% test stars. +NUMBERING_SCHEME = -0000000 + +# Path to MCCD config file (read by mccd_fit_val_runner, together with MODE +# and VERBOSE) CONFIG_PATH = $SP_CONFIG/config_MCCD.ini MODE = FIT_VALIDATION VERBOSE = False -NUMBERING_SCHEME = -0000000 - [MERGE_STARCAT_RUNNER] diff --git a/workflow/config/cfis/config_tile_Git.ini b/workflow/config/cfis/config_tile_Git.ini index cccf0a8b9..8aad4193b 100644 --- a/workflow/config/cfis/config_tile_Git.ini +++ b/workflow/config/cfis/config_tile_Git.ini @@ -80,8 +80,8 @@ INPUT_NUMBERING = \d{3}\.\d{3} # Output file pattern without number OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight- -# Copy/download method, one in 'vos', 'symlink' -RETRIEVE = symlink +# 'vos' or 'symlink', from the run config's `retrieve:` +RETRIEVE = $SP_RETRIEVE # If RETRIEVE=vos, number of attempts to download # Optional, default=3 diff --git a/workflow/config/cfis_image_sims/config_MCCD.ini b/workflow/config/cfis_image_sims/config_MCCD.ini new file mode 120000 index 000000000..3ba64d26e --- /dev/null +++ b/workflow/config/cfis_image_sims/config_MCCD.ini @@ -0,0 +1 @@ +../cfis/config_MCCD.ini \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/config_exp_Gie.ini b/workflow/config/cfis_image_sims/config_exp_Gie.ini new file mode 100644 index 000000000..bbf1b98af --- /dev/null +++ b/workflow/config/cfis_image_sims/config_exp_Gie.ini @@ -0,0 +1,104 @@ +# IMAGE SIMULATIONS overlay of ../cfis/config_exp_Gie.ini: identical except the lines +# marked below. Keep the diff to ../cfis/config_exp_Gie.ini confined to input naming. +# ShapePipe configuration file for: get images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Gie + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = get_images_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get exposures +[GET_IMAGES_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output + +FILE_PATTERN = exp_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + + +# Paths + +# Output path (optional, default is [FILE]:OUTPUT_DIR +# OUTPUT_PATH = input_images + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = $SP_INPUT_EXPOSURES, $SP_INPUT_EXPOSURES, $SP_INPUT_EXPOSURES + +# Input file pattern including tile number as dummy template +# sims +INPUT_FILE_PATTERN = simu_image-0000000, simu_weight-0000000, simu_flag-0000000 + +# Input file extensions +# sims +INPUT_FILE_EXT = .fits, .fits, .fits + +# Input numbering scheme, python regexp +# sims: 7-digit exposure ids +INPUT_NUMBERING = \d{7} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = image-, weight-, flag- + +# 'vos' or 'symlink', from the run config's `retrieve:` +RETRIEVE = $SP_RETRIEVE + +# If RETRIEVE=vos, number of attempts to download +# Optional, default=3 +N_TRY = 3 + +# Retrieve command options, optional +RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem + +#CHECK_EXISTING_DIR = $SP_RUN/output/run_sp_Gie_prev diff --git a/workflow/config/cfis_image_sims/config_exp_Sp.ini b/workflow/config/cfis_image_sims/config_exp_Sp.ini new file mode 100644 index 000000000..9783189f3 --- /dev/null +++ b/workflow/config/cfis_image_sims/config_exp_Sp.ini @@ -0,0 +1,81 @@ +# IMAGE SIMULATIONS overlay of ../cfis/config_exp_Sp.ini: identical except the lines +# marked below. Keep the diff to ../cfis/config_exp_Sp.ini confined to input naming. +# ShapePipe configuration file for single-exposures, +# split images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_Sp + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = split_exp_runner + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed exposure ID (exp split is a "tile-scheme" stage per sp_rule.py). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SPLIT_EXP_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_exp_Gie/get_images_runner/output + +FILE_PATTERN = image, weight, flag + +# Matches compressed single-exposure files +# sims: uncompressed exposures +FILE_EXT = .fits, .fits, .fits + +NUMBERING_SCHEME = -0000000 + +# OUTPUT_SUFFIX, actually file name prefixes. +# Expected keyword "flag" will lead to a behavior where the data are saved as int. +# The code also expects the image data to use the "image" suffix +# (default value in the pipeline). +OUTPUT_SUFFIX = image, weight, flag + +# Number of HDUs/CCDs of mosaic +N_HDU = 40 diff --git a/workflow/config/cfis_image_sims/config_exp_fake.ini b/workflow/config/cfis_image_sims/config_exp_fake.ini new file mode 100644 index 000000000..f09a4054b --- /dev/null +++ b/workflow/config/cfis_image_sims/config_exp_fake.ini @@ -0,0 +1,129 @@ +# ShapePipe configuration file for single-exposures. IMAGE SIMULATIONS, true PSF +# (psf_model: fake). No PSF fit: the simulations' PSF is known, and the tile's +# fake_interp_runner reads it from the PSF dictionary. What remains is the +# SExtractor pass, whose background and background_rms checkimages the tile +# vignets read. Derived from ../cfis/config_exp_psfex.ini: the [SEXTRACTOR_RUNNER] +# section is kept identical to it, and RUN_NAME stays the exp_psf stage's run dir +# (workflow/scripts/completeness.py STAGE_DIR). + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_exp_SxSePsf + +# Add date and time to RUN_NAME, optional, default: True +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = sextractor_runner + +# Run mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names with length matching FILE_PATTERN +INPUT_DIR = $SP_RUN/output + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 8 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[SEXTRACTOR_RUNNER] + +# The split CCDs, and nothing else: ShapePipe generates no masks +INPUT_DIR = $SP_RUN/output/run_sp_exp_Sp/split_exp_runner/output + +# Read the instrument flag image split_exp wrote per CCD +FILE_PATTERN = image, weight, flag + +# Explicit extensions: a 3-entry FILE_PATTERN override must not fall back on +# the decorator's 4-entry FILE_EXT default (length check fails at startup) +FILE_EXT = .fits, .fits, .fits + +NUMBERING_SCHEME = -0000000-0 + +# SExtractor executable path +EXEC_PATH = source-extractor + +# SExtractor configuration files +DOT_SEX_FILE = $SP_CONFIG/default_exp.sex +DOT_PARAM_FILE = $SP_CONFIG//default.param +DOT_CONV_FILE = $SP_CONFIG/default.conv + +# Use input weight image if True +WEIGHT_IMAGE = True + +# Use input flag image if True +FLAG_IMAGE = True + +# Use input PSF file if True +PSF_FILE = False + +# Use distinct image for detection (SExtractor in +# dual-image mode) if True. +DETECTION_IMAGE = False + +# Distinct weight image for detection (SExtractor +# in dual-image mode) +DETECTION_WEIGHT = False + +# True if photometry zero-point is to be read from exposure image header +ZP_FROM_HEADER = True + +# If ZP_FROM_HEADER is True, zero-point key name +ZP_KEY = PHOTZP + +# Background information from image header. +# If BKG_FROM_HEADER is True, background value will be read from header. +# In that case, the value of BACK_TYPE will be set atomatically to MANUAL. +# This is used e.g. for the LSB images. +BKG_FROM_HEADER = False +# LSB images: +# BKG_FROM_HEADER = True + +# If BKG_FROM_HEADER is True, background value key name +# LSB images: +#BKG_KEY = IMMODE + +# Type of image check (optional), default not used, can be a list of +# BACKGROUND, BACKGROUND_RMS, INIBACKGROUND, MINIBACK_RMS, -BACKGROUND, +# FILTERED, OBJECTS, -OBJECTS, SEGMENTATION, APERTURES +CHECKIMAGE = BACKGROUND, BACKGROUND_RMS + +# File name suffix for the output sextractor files (optional) SUFFIX = tile +SUFFIX = sexcat + +## Post-processing + +# Not required for single exposures +MAKE_POST_PROCESS = FALSE diff --git a/workflow/config/cfis_image_sims/config_exp_mccd.ini b/workflow/config/cfis_image_sims/config_exp_mccd.ini new file mode 120000 index 000000000..58166d49c --- /dev/null +++ b/workflow/config/cfis_image_sims/config_exp_mccd.ini @@ -0,0 +1 @@ +../cfis/config_exp_mccd.ini \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/config_exp_psfex.ini b/workflow/config/cfis_image_sims/config_exp_psfex.ini new file mode 120000 index 000000000..ea86cb50f --- /dev/null +++ b/workflow/config/cfis_image_sims/config_exp_psfex.ini @@ -0,0 +1 @@ +../cfis/config_exp_psfex.ini \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/config_tile_Fe.ini b/workflow/config/cfis_image_sims/config_tile_Fe.ini new file mode 100644 index 000000000..2a003ea9c --- /dev/null +++ b/workflow/config/cfis_image_sims/config_tile_Fe.ini @@ -0,0 +1,80 @@ +# IMAGE SIMULATIONS overlay of ../cfis/config_tile_Fe.ini: identical except the lines +# marked below. Keep the diff to ../cfis/config_tile_Fe.ini confined to input naming. +# ShapePipe configuration file for: find exposures + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Fe + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = find_exposures_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get tiles +[FIND_EXPOSURES_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Git/get_images_runner/output + +FILE_PATTERN = CFIS_image + +FILE_EXT = .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Column number of exposure name in FITS header +# sims: HISTORY layout of the simulated tiles +COLNUM = 2 + +# Prefix to remove from exposure name +# sims +EXP_PREFIX = simu_image- + diff --git a/workflow/config/cfis_image_sims/config_tile_Git.ini b/workflow/config/cfis_image_sims/config_tile_Git.ini new file mode 100644 index 000000000..d638ff559 --- /dev/null +++ b/workflow/config/cfis_image_sims/config_tile_Git.ini @@ -0,0 +1,98 @@ +# IMAGE SIMULATIONS overlay of ../cfis/config_tile_Git.ini: identical except the lines +# marked below. Keep the diff to ../cfis/config_tile_Git.ini confined to input naming. +# ShapePipe configuration file for: get tile images + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = False + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Git + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = get_images_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# Input directory, containing input files, single string or list of names +INPUT_DIR = $SP_RUN + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 1 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +# Get tiles +[GET_IMAGES_RUNNER] + +FILE_PATTERN = tile_numbers + +FILE_EXT = .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = + +# Paths + +# Input path where original images are stored. Can be local path or vos url. +# Single string or list of strings +INPUT_PATH = $SP_INPUT_TILES, $SP_INPUT_TILES + +# Input file pattern including tile number as dummy template +# sims +INPUT_FILE_PATTERN = CFIS_simu_image-000-000, CFIS_simu_weight-000-000 + +# Input file extensions +# sims: weight uncompressed +INPUT_FILE_EXT = .fits, .fits + +# Input numbering scheme, python regexp +# sims: dash-numbered tiles +INPUT_NUMBERING = \d{3}-\d{3} + +# Output file pattern without number +OUTPUT_FILE_PATTERN = CFIS_image-, CFIS_weight- + +# 'vos' or 'symlink', from the run config's `retrieve:` +RETRIEVE = $SP_RETRIEVE + +# If RETRIEVE=vos, number of attempts to download +# Optional, default=3 +N_TRY = 3 + +# Copy command options, optional +RETRIEVE_OPTIONS = --certfile=$HOME/.ssl/cadcproxy.pem + +#CHECK_EXISTING_DIR = $SP_RUN/data_tiles diff --git a/workflow/config/cfis_image_sims/config_tile_Mc.ini b/workflow/config/cfis_image_sims/config_tile_Mc.ini new file mode 120000 index 000000000..eeba9f53e --- /dev/null +++ b/workflow/config/cfis_image_sims/config_tile_Mc.ini @@ -0,0 +1 @@ +../cfis/config_tile_Mc.ini \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/config_tile_Mh_exp.ini b/workflow/config/cfis_image_sims/config_tile_Mh_exp.ini new file mode 120000 index 000000000..5e8e91716 --- /dev/null +++ b/workflow/config/cfis_image_sims/config_tile_Mh_exp.ini @@ -0,0 +1 @@ +../cfis/config_tile_Mh_exp.ini \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/config_tile_Ms.ini b/workflow/config/cfis_image_sims/config_tile_Ms.ini new file mode 120000 index 000000000..f301e8673 --- /dev/null +++ b/workflow/config/cfis_image_sims/config_tile_Ms.ini @@ -0,0 +1 @@ +../cfis/config_tile_Ms.ini \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/config_tile_Ng_template.ini b/workflow/config/cfis_image_sims/config_tile_Ng_template.ini new file mode 120000 index 000000000..c00711e7b --- /dev/null +++ b/workflow/config/cfis_image_sims/config_tile_Ng_template.ini @@ -0,0 +1 @@ +../cfis/config_tile_Ng_template.ini \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/config_tile_PiViVi_fake.ini b/workflow/config/cfis_image_sims/config_tile_PiViVi_fake.ini new file mode 100644 index 000000000..4651c4e7c --- /dev/null +++ b/workflow/config/cfis_image_sims/config_tile_PiViVi_fake.ini @@ -0,0 +1,167 @@ +# ShapePipe configuration file for tile, from detection up to shape measurement. +# IMAGE SIMULATIONS, true PSF (psf_model: fake). Derived from +# ../cfis/config_tile_PiViVi_psfex.ini: only the PSF section differs. + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_PiViVi + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +#MODULE = psfex_interp_runner, + +MODULE = ${SP_PSF}_interp_runner, vignetmaker_runner, vignetmaker_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory. +# +# NODE-LOCAL, not $SP_RUN/output. This run produces the tile's ~5.6 GB vignette +# store, which is read only by ngmix and make_cat -- both of which run in the +# same fused group job on the same node (workflow/rules/tile.smk, TILE_GROUP). +# Writing it to the node's NVMe instead of NFS scratch is the whole point: it +# removes the store from the per-tile scratch high-water AND removes ~163 GB of +# small random NFS reads per tile. +# +# $SP_VIGNET_OUT is exported by every member of that group (TILE_LOCAL) as +# $SLURM_TMPDIR/sp-tile/output. Set it to $SP_RUN/output to keep the store on +# shared storage. It is NOT optional: ShapePipe's config expansion is strict +# (pipeline/config.py::_expandvars_strict), so an unset value is a loud error. +# +# The INPUT_DIRs below stay on $SP_RUN -- they are Sx / Mh_exp / Fe products on +# scratch, and every path here is absolute, so nothing resolves through a run +# log that this split would break. +OUTPUT_DIR = $SP_VIGNET_OUT + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options + +[FAKE_INTERP_RUNNER] + +# The tile's multi-epoch SExtractor catalogue: its EPOCH HDUs carry, per object, +# the exposure and CCD each epoch fell on, which key the PSF dictionary. +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output + +FILE_PATTERN = sexcat + +FILE_EXT = .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# The simulation's pickled PSF dictionary ("-" -> PSF), exported by +# the workflow from the run config's `psf_dict:` +PSF_DICT_PATH = $PSF_DICT + + +# Create vignets for tiles weights +[VIGNETMAKER_RUNNER_RUN_1] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Uz/uncompress_fits_runner/output + +FILE_PATTERN = sexcat, CFIS_weight + +FILE_EXT = .fits, .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +MASKING = False +MASK_VALUE = 0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = CLASSIC + +# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) +COORD = PIX +POSITION_PARAMS = XWIN_IMAGE,YWIN_IMAGE + +# Vignet size in pixels +STAMP_SIZE = 51 + +# Output file name prefix, file name is _vignet.fits +PREFIX = weight + + +[VIGNETMAKER_RUNNER_RUN_2] + +# Create multi-epoch vignets for tiles corresponding to +# positions on single-exposures + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Sx/sextractor_runner/output, $SP_RUN/output/run_sp_tile_Mh_exp/merge_headers_runner/output, $SP_RUN/output/run_sp_tile_Fe/find_exposures_runner/output + +FILE_PATTERN = sexcat, log_exp_headers, exp_numbers + +FILE_EXT = .fits, .sqlite, .txt + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +MASKING = False +MASK_VALUE = 0 + +# Run mode for psfex interpolation: +# CLASSIC: 'classical' run, interpolate to object positions +# MULTI-EPOCH: interpolate for multi-epoch images +# VALIDATION: validation for single-epoch images +MODE = MULTI-EPOCH + +# Coordinate frame type, one in PIX (pixel frame), SPHE (spherical coordinates) +COORD = SPHE +POSITION_PARAMS = XWIN_WORLD,YWIN_WORLD + +# Vignet size in pixels +STAMP_SIZE = 51 + +# Output file name prefix, file name is vignet.fits +PREFIX = + +# Additional parameters for path and file pattern corresponding to single-exposure +# run outputs. ME_IMAGE_EXP_DIR/ME_IMAGE_EXP_RUNNERS replace ME_IMAGE_DIR for +# the v2.0 per-exposure pipeline; output dirs are discovered by scanning $SP_EXP. +ME_IMAGE_EXP_DIR = $SP_EXP +ME_IMAGE_EXP_RUNNERS = split_exp_runner, split_exp_runner, split_exp_runner, sextractor_runner, sextractor_runner +ME_IMAGE_PATTERN = flag, image, weight, background, background_rms diff --git a/workflow/config/cfis_image_sims/config_tile_PiViVi_mccd.ini b/workflow/config/cfis_image_sims/config_tile_PiViVi_mccd.ini new file mode 120000 index 000000000..6271fb312 --- /dev/null +++ b/workflow/config/cfis_image_sims/config_tile_PiViVi_mccd.ini @@ -0,0 +1 @@ +../cfis/config_tile_PiViVi_mccd.ini \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/config_tile_PiViVi_psfex.ini b/workflow/config/cfis_image_sims/config_tile_PiViVi_psfex.ini new file mode 120000 index 000000000..5f120bca9 --- /dev/null +++ b/workflow/config/cfis_image_sims/config_tile_PiViVi_psfex.ini @@ -0,0 +1 @@ +../cfis/config_tile_PiViVi_psfex.ini \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/config_tile_Sx.ini b/workflow/config/cfis_image_sims/config_tile_Sx.ini new file mode 120000 index 000000000..81637de66 --- /dev/null +++ b/workflow/config/cfis_image_sims/config_tile_Sx.ini @@ -0,0 +1 @@ +../cfis/config_tile_Sx.ini \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/config_tile_Uz.ini b/workflow/config/cfis_image_sims/config_tile_Uz.ini new file mode 100644 index 000000000..18a13e6eb --- /dev/null +++ b/workflow/config/cfis_image_sims/config_tile_Uz.ini @@ -0,0 +1,77 @@ +# IMAGE SIMULATIONS overlay of ../cfis/config_tile_Uz.ini: identical except the lines +# marked below. Keep the diff to ../cfis/config_tile_Uz.ini confined to input naming. +# ShapePipe configuration file for: uncompress FITS image + + +## Default ShapePipe options +[DEFAULT] + +# verbose mode (optional), default: True, print messages on terminal +VERBOSE = True + +# Name of run (optional) default: shapepipe_run +RUN_NAME = run_sp_tile_Uz + +# Add date and time to RUN_NAME, optional, default: False +RUN_DATETIME = False + + +## ShapePipe execution options +[EXECUTION] + +# Module name, single string or comma-separated list of valid module runner names +MODULE = uncompress_fits_runner + +# Parallel processing mode, SMP or MPI +MODE = SMP + + +## ShapePipe file handling options +[FILE] + +# Log file master name, optional, default: shapepipe +LOG_NAME = log_sp + +# Runner log file name, optional, default: shapepipe_runs +RUN_LOG_NAME = log_run_sp + +# NUMBER_LIST selects this unit; the workflow sets SP_UNIT_NUM to the +# dashed tile ID (e.g. -210-282). +NUMBER_LIST = $SP_UNIT_NUM + +# Input directory, containing input files, single string or list of names +INPUT_DIR = . + +# Output directory +OUTPUT_DIR = $SP_RUN/output + + +## ShapePipe job handling options +[JOB] + +# Batch size of parallel processing (optional), default is 1, i.e. run all jobs in serial +SMP_BATCH_SIZE = 16 + +# Timeout value (optional), default is None, i.e. no timeout limit applied +TIMEOUT = 96:00:00 + + +## Module options +[UNCOMPRESS_FITS_RUNNER] + +INPUT_DIR = $SP_RUN/output/run_sp_tile_Git/get_images_runner/output + +FILE_PATTERN = CFIS_weight + +# sims: the weight is not compressed +FILE_EXT = .fits + +# NUMBERING_SCHEME (optional) string with numbering pattern for input files +NUMBERING_SCHEME = -000-000 + +# Input HDU of image data, optional, default=0 +# sims: plain FITS, data in the primary HDU +HDU_DATA = 0 + +# Output file pattern +OUTPUT_PATTERN = CFIS_weight diff --git a/workflow/config/cfis_image_sims/default.conv b/workflow/config/cfis_image_sims/default.conv new file mode 120000 index 000000000..bd71df850 --- /dev/null +++ b/workflow/config/cfis_image_sims/default.conv @@ -0,0 +1 @@ +../cfis/default.conv \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/default.param b/workflow/config/cfis_image_sims/default.param new file mode 120000 index 000000000..49e000314 --- /dev/null +++ b/workflow/config/cfis_image_sims/default.param @@ -0,0 +1 @@ +../cfis/default.param \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/default.psfex b/workflow/config/cfis_image_sims/default.psfex new file mode 120000 index 000000000..65dd716d1 --- /dev/null +++ b/workflow/config/cfis_image_sims/default.psfex @@ -0,0 +1 @@ +../cfis/default.psfex \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/default_exp.sex b/workflow/config/cfis_image_sims/default_exp.sex new file mode 120000 index 000000000..4c83dea8e --- /dev/null +++ b/workflow/config/cfis_image_sims/default_exp.sex @@ -0,0 +1 @@ +../cfis/default_exp.sex \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/default_noimaflags.param b/workflow/config/cfis_image_sims/default_noimaflags.param new file mode 120000 index 000000000..75451801e --- /dev/null +++ b/workflow/config/cfis_image_sims/default_noimaflags.param @@ -0,0 +1 @@ +../cfis/default_noimaflags.param \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/default_tile.sex b/workflow/config/cfis_image_sims/default_tile.sex new file mode 120000 index 000000000..8770da87e --- /dev/null +++ b/workflow/config/cfis_image_sims/default_tile.sex @@ -0,0 +1 @@ +../cfis/default_tile.sex \ No newline at end of file diff --git a/workflow/config/cfis_image_sims/final_cat.param b/workflow/config/cfis_image_sims/final_cat.param new file mode 100644 index 000000000..5bb0bdd17 --- /dev/null +++ b/workflow/config/cfis_image_sims/final_cat.param @@ -0,0 +1,113 @@ +# Final-catalogue column selection for the image simulations. +# +# create_final_cat.py -I selects exactly these columns from each simulated +# tile's make_cat output (tiles///output/run_sp_tile_Mc/...) into +# final_cat_{sim}.hdf5, and sp_validation's extract step reads the same list; +# every entry must exist in that catalogue. +# +# A real file in this overlay dir, not a symlink into ../cfis/, because the +# real-data list names columns make_cat does not write for the simulations: +# - IMAFLAGS_ISO : the tile SExtractor runs without a flag image +# (config_tile_Sx.ini: FLAG_IMAGE = False) +# - NGMIX_MOM_FAIL : make_cat writes it as NGMIX_MCAL_TYPES_FAIL +# and it omits two that sp_validation's image-sim extract step reads: NUMBER and +# NGMIX_MCAL_TYPES_FAIL (the metacal moments-failure cut). + +# coordinates +XWIN_WORLD +YWIN_WORLD + +# tile ID, for plot of tile-dependent additive bias. +TILE_ID + +# SExtractor number +NUMBER + +# flags +FLAGS +NGMIX_MCAL_FLAGS +NGMIX_MCAL_TYPES_FAIL +NGMIX_NEIGHBOUR_FLAG + +# PSF ellipticity (original image PSF) +NGMIX_G1_PSF_ORIG_NOSHEAR +NGMIX_G2_PSF_ORIG_NOSHEAR + +# Number of epochs (exposures) +N_EPOCH +NGMIX_N_EPOCH + +## Shape measurement outputs +## Ngmix: model fitting + +# galaxy ellipticity +NGMIX_G1_1M +NGMIX_G2_1M +NGMIX_G1_1P +NGMIX_G2_1P +NGMIX_G1_2M +NGMIX_G2_2M +NGMIX_G1_2P +NGMIX_G2_2P +NGMIX_G1_NOSHEAR +NGMIX_G2_NOSHEAR +NGMIX_G1_ERR_NOSHEAR +NGMIX_G2_ERR_NOSHEAR + +# flags +NGMIX_FLAGS_1M +NGMIX_FLAGS_1P +NGMIX_FLAGS_2M +NGMIX_FLAGS_2P +NGMIX_FLAGS_NOSHEAR + +# size and error +NGMIX_T_1M +NGMIX_T_1P +NGMIX_T_2M +NGMIX_T_2P +NGMIX_T_NOSHEAR +NGMIX_T_ERR_1M +NGMIX_T_ERR_1P +NGMIX_T_ERR_2M +NGMIX_T_ERR_2P +NGMIX_T_ERR_NOSHEAR + +# reconvolved PSF size (used by the metacal size cut / Tpsf) +NGMIX_T_PSF_RECONV_1M +NGMIX_T_PSF_RECONV_1P +NGMIX_T_PSF_RECONV_2M +NGMIX_T_PSF_RECONV_2P +NGMIX_T_PSF_RECONV_NOSHEAR + +# flux and error +NGMIX_FLUX_1M +NGMIX_FLUX_1P +NGMIX_FLUX_2M +NGMIX_FLUX_2P +NGMIX_FLUX_NOSHEAR +NGMIX_FLUX_ERR_1M +NGMIX_FLUX_ERR_1P +NGMIX_FLUX_ERR_2M +NGMIX_FLUX_ERR_2P +NGMIX_FLUX_ERR_NOSHEAR + +# magnitudes +MAG_AUTO +MAGERR_AUTO +MAG_WIN +MAGERR_WIN +FLUX_AUTO +FLUXERR_AUTO +FLUX_APER +FLUXERR_APER +FLUX_RADIUS + +# SNR from SExtractor +SNR_WIN + +FWHM_IMAGE +FWHM_WORLD + +# PSF size measured on original image +NGMIX_T_PSF_ORIG_NOSHEAR diff --git a/workflow/config/cfis_image_sims/star_selection.setools b/workflow/config/cfis_image_sims/star_selection.setools new file mode 120000 index 000000000..11a570187 --- /dev/null +++ b/workflow/config/cfis_image_sims/star_selection.setools @@ -0,0 +1 @@ +../cfis/star_selection.setools \ No newline at end of file diff --git a/workflow/rules/exposure.smk b/workflow/rules/exposure.smk index 7f3802c29..637c1dc15 100644 --- a/workflow/rules/exposure.smk +++ b/workflow/rules/exposure.smk @@ -83,6 +83,13 @@ rule exp_split: # -> psfex_interp, per CCD. Under SP_PSF=psfex, setools_runner is mandatory # (expect=80) and only psfex_interp_runner is :warn; under SP_PSF=mccd every # runner in the chain is :warn (see completeness.py's COMPLETENESS table). +# +# MCCD instead fits ONE focal-plane model per exposure after the per-CCD +# stages: ~85 min single-threaded for ~2500 stars (SKiLLS star sim, 8.3 GB), +# and only 1.75x faster on 8 BLAS threads (which the thread caps forbid anyway). +# With 8 cores reserved the fit would idle 7 of them for its whole length, so +# the mccd chain takes 2: the per-CCD stages run 2 wide (minutes), the fit is +# unchanged, and the exposure reserves a quarter of the core-hours. rule exp_psf: input: rules.exp_split.output.manifest @@ -93,7 +100,7 @@ rule exp_psf: params: pre = lambda wc: unit_pre("exp_psf", wc.exp), script_hash = SCRIPT_HASH - threads: 8 + threads: 2 if PSF_MODEL == "mccd" else 8 retries: 2 benchmark: # BESIDE manifests/, not inside it: clean_exposure deletes manifests/ diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index 68d6da9ba..ccbc83fbf 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -91,7 +91,9 @@ is what downstream selections cut on. # # HOW THE PATH GETS IN HERE. Not through the environment: profiles/nibi passes # --bind /local and tile_local() below DERIVES the path from the tile wildcard. -# Why nothing can be communicated instead is on that profile line. +# Why nothing can be communicated instead is on that profile line. A campaign +# that needs the store elsewhere (candide's 31 GB /tmp) moves the BIND, via the +# run config's `tile_store_root` (bin/sp), never the path here -- see below. # # THE COST WE ACCEPT: a failure anywhere in the tile re-runs the WHOLE tile, # not one chunk, because the store dies with the job. At ~1 h per fused tile @@ -130,6 +132,11 @@ is what downstream selections cut on. # `--rerun-triggers mtime code software-env`, accepting that clean_exposure's # consumer-set staleness detection (which rides on params) is off for that # invocation. +# Per-campaign prefix of the node-local store name; see tile_local(). +LOCAL_TAG = (hashlib.sha1(str(RUN_DIR).encode()).hexdigest()[:8] + "-" + if INPUT_TYPE == "image_sims" else "") + + def tile_local(tile): """The node-local prologue, as bash, for one tile. @@ -144,7 +151,15 @@ def tile_local(tile): the container (probe job 20798618) -- and the tile id is a wildcard snakemake substitutes at DAG time, so the shell string carries a concrete path with no `$` left for anything to escape. One group job per tile means - the name cannot collide; the sticky bit means nobody else can remove it. + the name cannot collide within a campaign; the sticky bit means nobody else + can remove it. Across campaigns it can: the image-simulation shear branches + are concurrent campaigns over the SAME tile IDs, and on candide + `/local/scratch` is the node's shared `/tmp`. Two branches' fused jobs on one + node then share one store -- a second `tile_vignets` wipes and rewrites it + under the first branch's ngmix, and the first `tile_make_cat`'s EXIT trap + deletes it under the second (seen on candide n09). So image_sims prefixes + the name with LOCAL_TAG, a hash of the run dir. Data campaigns keep the + bare tile name, so their shell commands -- a rerun trigger -- are unchanged. What we give up is Slurm's own cleanup of `$SLURM_TMPDIR`. TILE_VIGNET_FRESH reclaims a stale directory on the next attempt for the same tile, the trap @@ -201,7 +216,7 @@ if [ ! -d /local/scratch ]; then echo " profiles/nibi/config.yaml apptainer-args must carry --bind /local" >&2 exit 1 fi -export SP_LOCAL="/local/scratch/sp-{tile}" +export SP_LOCAL="/local/scratch/sp-{LOCAL_TAG}{tile}" export SP_VIGNET_OUT="$SP_LOCAL/output" export NGMIX_VIGNET_DIR="$SP_LOCAL/output/run_sp_tile_PiViVi" export SP_WCS_DIR="$SP_LOCAL/wcs" diff --git a/workflow/run_template.yaml b/workflow/run_template.yaml new file mode 100644 index 000000000..5516aa58f --- /dev/null +++ b/workflow/run_template.yaml @@ -0,0 +1,18 @@ +# Run config template file, can be specified along with main config.yaml for +# ShapePipe run. + +input_type: image_sims + +# Run name: the SKiLLS grid to use. Available as $run in the paths below. +run: 1z2z_grid_1 + +tile_list: /tiles.txt + +inputs: + tiles: /n09data/hervas/skills_out/$run/images/SP_tiles + exposures: /n09data/hervas/skills_out/$run/images/SP_exp + +outputs: + run_dir: /$run/run + products_dir: /$run/product + index_db: /$run/run/index.sqlite diff --git a/workflow/scripts/completeness.py b/workflow/scripts/completeness.py index 66e3b6a94..3b4cf3743 100644 --- a/workflow/scripts/completeness.py +++ b/workflow/scripts/completeness.py @@ -102,10 +102,12 @@ "psfex_runner": dict(expect=80), "psfex_interp_runner": dict(expect=40, warn=True), }, - # MCCD counts are derived from config_exp_mccd.ini and its per-CCD - # runners, but this chain has not been exercised through this workflow. - # Keep the expected counts visible while making the unverified branch - # warning-only until a real campaign validates its counts. + # MCCD shares the chain up to setools with PSFEx, then fits one + # focal-plane model per exposure. Preprocessing is a serial runner that + # merges the 40 CCDs' split catalogues into one training and one test + # catalogue; fit_val writes the model (fitted_model-.npy, what the + # tiles interpolate) and its validation catalogue; merge_starcat and + # mccd_plots add per-exposure diagnostics. Every runner here is :warn. "mccd": { "sextractor_runner": dict(expect=120, warn=True), "mask_query_runner": dict(expect=40, warn=True), @@ -121,6 +123,13 @@ # config_exp_mccd enables the ten meanshape and six histogram plots. "mccd_plots_runner": dict(expect=16, warn=True), }, + # Image simulations with the true PSF (psf_model: fake): no PSF fit on + # the exposures, only the SExtractor pass whose background/background_rms + # checkimages the tile vignets read (config_exp_fake.ini in + # config/cfis_image_sims). Same 40 CCDs x (sexcat, background, rms). + "fake": { + "sextractor_runner": dict(expect=120), + }, }, # --- tile post --- @@ -134,12 +143,19 @@ # v2.0's 4 was the canfar flavor. every vignette feeds ngmix, so the expected count is all-or-nothing. "vignetmaker_runner_run_2": dict(expect=5), }, - # MCCD is wired but unvalidated here; retain the expected runner names - # and counts as warnings until a workflow campaign exercises them. + # As psfex: mccd_interp writes the tile's galaxy_psf store from the + # exposures' focal-plane models (SKiLLS star sim 1z2z_1, 233.293). "mccd": { - "mccd_interp_runner": dict(expect=1, warn=True), - "vignetmaker_runner_run_1": dict(expect=1, warn=True), - "vignetmaker_runner_run_2": dict(expect=5, warn=True), + "mccd_interp_runner": dict(expect=1), + "vignetmaker_runner_run_1": dict(expect=1), + "vignetmaker_runner_run_2": dict(expect=5), + }, + # Image simulations: fake_interp_runner writes the same galaxy_psf + # sqlite psfex_interp_runner writes, from the simulation's PSF dictionary. + "fake": { + "fake_interp_runner": dict(expect=1), + "vignetmaker_runner_run_1": dict(expect=1), + "vignetmaker_runner_run_2": dict(expect=5), }, }, # One check runs inside run_sp_tile_ngmix_Ng${SP_NGMIX_CHUNK}u per chunk, @@ -191,7 +207,7 @@ def check_counts(stage, run_dir): table = table[psf_model] except KeyError as exc: raise ValueError( - f"Invalid SP_PSF={psf_model!r}; expected one of psfex, mccd." + f"Invalid SP_PSF={psf_model!r}; expected one of {sorted(table)}." ) from exc details, ok = [], True for runner, spec in table.items(): @@ -345,10 +361,20 @@ def build_manifest(stage, run_dir, unit, stage_subdir=None): def write_if_changed(path: Path, text: str) -> None: - """Write only when the bytes differ — see the module docstring on mtime.""" + """Write only when the bytes differ (see the module docstring on mtime). + + Atomic (temp file + os.replace): a reader such as run_report.py must never + see the file truncated mid-write. + """ path.parent.mkdir(parents=True, exist_ok=True) if not path.exists() or path.read_text() != text: - path.write_text(text) + tmp = path.with_name(f".{path.name}.tmp{os.getpid()}") + try: + tmp.write_text(text) + os.replace(tmp, path) + except BaseException: + tmp.unlink(missing_ok=True) + raise def _unit_from_run_dir(run_dir): diff --git a/workflow/scripts/container.py b/workflow/scripts/container.py index 88b879f0d..00467aa59 100644 --- a/workflow/scripts/container.py +++ b/workflow/scripts/container.py @@ -56,17 +56,19 @@ class ContainerError(Exception): # same image CI tested. CONTAINER_URI = "docker://ghcr.io/cosmostat/shapepipe:develop" -# The single source of truth for the fallback image: the workflow's own -# `container:` key, which is also what the Snakefile reads. Written down once, -# here, so the CLI and the workflow cannot disagree about the default. -CONFIG_FILE = Path(__file__).resolve().parents[1] / "config.yaml" +# The fallback image is the run config's resolved `container:` (config.yaml, +# SP_RUN_CONFIG on top, then the machine default; see run_config.py). +BASE_CONFIG = Path(__file__).resolve().parents[1] / "config.yaml" +CONFIG_FILE = Path(os.environ.get("SP_RUN_CONFIG") or BASE_CONFIG) CONFIG_KEY = "container" # The profile whose `apptainer-args:` every workflow job runs under. `exec` reads # it at runtime rather than restating it, so a one-off `sp container exec` and a # job see the same environment (the PYTHONPATH pin above all: a divergence there # means the one-off imports a different src/ than the workflow does). -PROFILE_FILE = Path(__file__).resolve().parents[2] / "profiles" / "nibi" / "config.yaml" +# SP_PROFILE names it (profiles//, default nibi), as it does for bin/sp. +PROFILE_FILE = (Path(__file__).resolve().parents[2] / "profiles" + / os.environ.get("SP_PROFILE", "nibi") / "config.yaml") # ~/.cache/shapepipe by default; SP_CACHE_DIR moves the whole cache (e.g. onto # a filesystem with room), XDG_CACHE_HOME moves it with everything else. @@ -88,17 +90,22 @@ class ContainerError(Exception): def configured_default(): - """Return the ``container:`` path from workflow/config.yaml, or ``None``. + """Return the resolved ``container:`` of the run config, or ``None``. - A deliberately minimal scalar read (the same one bin/sp does in sed): this - module is stdlib-only, so there is no yaml to import. + Via run_config.py when PyYAML is available; on a bare host without it, + a top-level ``container:`` line in the run config file is still read. """ try: - text = CONFIG_FILE.read_text() - except OSError: - return None - match = re.search(rf"^{CONFIG_KEY}:[ \t]*(\S+)", text, re.MULTILINE) - return match.group(1) if match else None + import run_config + except ImportError: + try: + text = CONFIG_FILE.read_text() + except OSError: + return None + match = re.search(rf"^{CONFIG_KEY}:[ \t]*(\S+)", text, re.MULTILINE) + return match.group(1) if match else None + run = os.environ.get("SP_RUN_CONFIG") + return run_config.load(BASE_CONFIG, run).get(CONFIG_KEY) or None def profile_apptainer_args(): @@ -127,9 +134,12 @@ def local_sandbox(): return (Path(override) if override else DEFAULT_SANDBOX).expanduser() -def resolve_image(): +def resolve_image(configured=None): """Return ``(path, kind)`` for the image everything should run. + ``configured`` is the run config's resolved ``container:`` when the + caller already has it (the Snakefile); otherwise it is looked up. + ``kind`` is ``"sandbox"``, ``"sif"``, ``"configured"`` (the shared /project image named in config.yaml -- the default when the cache is empty) or ``"none"``. @@ -148,7 +158,7 @@ def resolve_image(): f"SP_CONTAINER={os.environ['SP_CONTAINER']} does not exist " f"(resolved to {sif}). Unset it, or point it at an image that does." ) - default = configured_default() + default = configured or configured_default() if default: return default, "configured" return "", "none" diff --git a/workflow/scripts/run_config.py b/workflow/scripts/run_config.py new file mode 100644 index 000000000..50cb9a417 --- /dev/null +++ b/workflow/scripts/run_config.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python3 +"""Resolve a run config: config.yaml, then SP_RUN_CONFIG merged on top, then +the `machines:` entry for (machine, input_type) filling whatever is still +unset. One definition shared by the Snakefile, bin/sp and container.py. + +CLI (used by bin/sp): run_config.py CONFIG_YAML RUN_CONFIG KEY[.SUBKEY] +prints the resolved value, or an empty line if unset. RUN_CONFIG may be "". +""" + +import os +import re +import sys + +import yaml + +PLACEHOLDER = "TBD" +# Keys the machines: table may default, per (machine, input_type). +# psf_model/psf_dict belong here because they are per-input_type facts, +# not per-run ones: psf_model=fake is only legal with +# input_type=image_sims, and psf_dict is the sim PSF it reads. A key +# also present at the TOP level of config.yaml shadows the table (the +# setdefault below only fires when the key is absent), so a key listed +# here must not carry a top-level default as well. +MACHINE_KEYS = ("tile_list", "retrieve", "container", "inputs", "outputs", + "psf_model", "psf_dict") +REQUIRED = ("tile_list", "inputs.tiles", "inputs.exposures", + "outputs.run_dir", "outputs.index_db") + + +def merge(base, over): + """Recursive dict merge; `over` wins (as snakemake's update_config).""" + out = dict(base) + for key, value in over.items(): + if isinstance(value, dict) and isinstance(out.get(key), dict): + out[key] = merge(out[key], value) + else: + out[key] = value + return out + + +# $name or ${name}. The brace form is not decoration: `_` is a word character, +# so `$shear_$grid` parses its first name as `shear_` and silently fails to +# match a `shear` key. Anything adjacent to more word characters needs braces. +_VAR_RE = re.compile(r"\$\{(\w+)\}|\$(\w+)") + + +def _expand(value, variables): + """Replace $name / ${name} for each set name in `variables`. + + Names come from every top-level scalar in the config plus `base_dir` from + the machines: entry, so a run config can name its own shorthands -- a long + output root written once and reused, rather than repeated per path. An + unknown name is left as-is, which is what makes the "unresolved" check + below able to see it. + """ + if isinstance(value, str): + return _VAR_RE.sub( + lambda m: str(variables.get(m.group(1) or m.group(2)) + or m.group(0)), + value, + ) + if isinstance(value, dict): + return {k: _expand(v, variables) for k, v in value.items()} + return value + + +def apply_machine_defaults(config): + """Fill unset MACHINE_KEYS from machines[machine][input_type], in place. + + The machine is `machine:` when the run config states one, else SP_PROFILE + (default nibi) -- the same value bin/sp picks the SLURM profile with. + + A key already in `config` wins; for `inputs`/`outputs` the merge is per + sub-key. In all of these, `$base_dir` expands to machines[machine].base_dir + and `$run` to the top-level `run:`. + """ + machine = config.get("machine") or os.environ.get("SP_PROFILE", "nibi") + entry = (config.get("machines") or {}).get(machine) or {} + defaults = entry.get(config.get("input_type", "data")) or {} + # Every top-level scalar is a variable, so a run config can define its own + # shorthands. They are resolved AGAINST EACH OTHER first, to a fixpoint, so + # one shorthand may be written in terms of another + # (grid: grid_2 / base: .../${grid} / run: ${shear}_${grid}). One pass is + # not enough: re.sub does not rescan what it substitutes, so a nested name + # would survive into the paths and only surface at the unresolved check. + # The loop is capped, so a self-reference (a: $b, b: $a) stops rather than + # spinning -- the leftover $ is then caught by unresolved() by design. + variables = {k: v for k, v in config.items() + if isinstance(v, (str, int, float))} + variables["base_dir"] = entry.get("base_dir") + for _ in range(10): + resolved = {k: (_expand(v, variables) if isinstance(v, str) else v) + for k, v in variables.items()} + if resolved == variables: + break + variables = resolved + # `run` is consumed downstream (paths, the hdf5 group name), so the + # resolved value has to go back into the config, not just the table. + if isinstance(config.get("run"), str): + config["run"] = variables.get("run", config["run"]) + for key in MACHINE_KEYS: + default = defaults.get(key) + if isinstance(default, dict): + config[key] = merge(default, config.get(key) or {}) + elif default is not None: + config.setdefault(key, default) + if key in config: + config[key] = _expand(config[key], variables) + return config + + +def get(config, dotted): + value = config + for part in dotted.split("."): + value = value.get(part) if isinstance(value, dict) else None + return value + + +def unresolved(config): + """REQUIRED keys that are unset, the placeholder, or hold an unexpanded + $variable (e.g. `$run` with no `run:` set).""" + return [k for k in REQUIRED + if get(config, k) in (None, "", PLACEHOLDER) or "$" in str(get(config, k))] + + +def load(config_yaml, run_config=None): + with open(config_yaml) as f: + config = yaml.safe_load(f) or {} + if run_config: + with open(run_config) as f: + config = merge(config, yaml.safe_load(f) or {}) + return apply_machine_defaults(config) + + +if __name__ == "__main__": + config_yaml, run_config, key = sys.argv[1:4] + value = get(load(config_yaml, run_config or None), key) + print("" if value is None else value)