diff --git a/tests/workflow/README.md b/tests/workflow/README.md index b26309227..509670be3 100644 --- a/tests/workflow/README.md +++ b/tests/workflow/README.md @@ -77,6 +77,7 @@ Apply mutations only to a disposable checkout, run the named test without `--upd | `test_final_cat_merge_reads_every_ready_tile` | Drop one ready tile; append an out-of-scope tile. | | `test_products_use_products_dir_and_run_name` | Rename either merged catalogue or the persist manifest; route products to scratch; derive `CAMPAIGN` from the products directory's basename. | | `test_tile_store_is_unique_per_campaign` | Make `LOCAL_TAG` conditional on `image_sims` (or constant); give one tile_shape member a different store path. | +| `test_module_logs_outlive_their_run_dirs` | Drop `post=` from tile_vignets or tile_ngmix; point the copy at `$SP_LOCAL` or `$SP_RUN/output`; move it after `exit $rc`. | | `test_missing_run_fails_during_parse` | Remove `run` from `run_config.REQUIRED`; literal paths must still receive the required-key diagnostic, not a later `KeyError`. | | `test_unit_pre_changes_at_campaign_boundary` | Append a line to `unit_pre`; change one rule's `params.pre`; change one shell; change a rendered thread count; make `LOCAL_TAG` empty for data. | | `test_params_pin_ignores_fixture_root` | Remove fixture-root or run-dir-hash normalization. | diff --git a/tests/workflow/params_pin.json b/tests/workflow/params_pin.json index 2284a7222..2e6ff6fe9 100644 --- a/tests/workflow/params_pin.json +++ b/tests/workflow/params_pin.json @@ -19,12 +19,12 @@ "tile_make_cat": "1558e38ee60bf9d1e794c303567442b1d298898ab8d56bf1b30aa63974d212dd", "tile_merge_cats": "0783c93deeb7f90bdb9772bbc4117851dff1a671518c81f67edd4c7c79cc7e6a", "tile_merge_headers": "3e674f961fd5bb93ce1f3b0e2bb072a25a421f2828315a3db45a960c44a42111", - "tile_ngmix": "87618cc2318729e5c09d8f2915361f79a3fd0773d3eedd93cb9f94dcb3d97d5b", + "tile_ngmix": "5b83650fe8fdfaf24b6f1045cd418e816f2c5a9851e3e35852864772298ea26a", "tile_uncompress": "4a013cdd04bc3e3c1082e477161d2b03e0eefe280b1aabb9af25c149a977f55b", - "tile_vignets": "89d95bd9107c124e86dfb029d5ba2a2fc1871311a71a8fe18466302ef2213b14" + "tile_vignets": "1d434b4a6a4bcdeb8f1e5e9a5e4f228b67ade400245b41b033a2c883fb6a3267" }, "schema": 1, - "sha256": "eab6930e0fd1044b9ce7a99172dbf9cbdcdec21291f8fa640d6e60c2efd3dd03", + "sha256": "782565d7614ad050cedec22e63b46d0913e8f14d931012d28343e76ba682af37", "unit_pre": { "exp_get_images": "dac6685a207dae3ea81d296ce53e9ba75d4636cf1f81e6399f2dda452680c992", "exp_psf": "87fc8ea1153a709ab7ba56542cfacc041f5f8b7f6cf2f38a9bc9d8004d1cecb4", diff --git a/tests/workflow/test_dag.py b/tests/workflow/test_dag.py index b23493ecc..35c06688e 100644 --- a/tests/workflow/test_dag.py +++ b/tests/workflow/test_dag.py @@ -145,6 +145,29 @@ def test_tile_store_is_unique_per_campaign(campaign, tmp_path, resolve_dag): assert all(first[tile] != second[tile] for tile in first), (first, second) +def test_module_logs_outlive_their_run_dirs(campaign, dag): + """tile_vignets (node-local) and every ngmix chunk (temp()) copy their + module logs into the tile's logs/modules/ on the shared root, after + shapepipe_run and before the rule exits, whatever its rc.""" + sources = {"tile_vignets": lambda job: '"$NGMIX_VIGNET_DIR"', + "tile_ngmix": lambda job: f'"{job.output.chunkdir}"'} + for rule, source in sources.items(): + for job in dag.jobs_for(rule): + name = ("run_sp_tile_PiViVi" if rule == "tile_vignets" else + f"run_sp_tile_ngmix_Ng{job.wildcards.chunk}u") + lines = job.shellcmd.splitlines() + keep = [i for i, line in enumerate(lines) + if f'"$SP_RUN/logs/modules/{name}"' in line] + assert len(keep) == 1, (rule, job.wildcards_dict) + line = lines[keep[0]] + assert f"cd {source(job)}" in line + assert '-path "*/logs/*"' in line and line.rstrip().endswith(">&2") + run = next(i for i, l in enumerate(lines) + if l.startswith("shapepipe_run ")) + assert run < keep[0] < len(lines) - 1 + assert lines[-1] == "exit $rc" + + def test_missing_run_fails_during_parse(campaign, resolve_dag): """Explicit paths cannot bypass the required campaign name diagnostic.""" campaign.omit_run() diff --git a/workflow/rules/tile.smk b/workflow/rules/tile.smk index f607e3617..15a7bd3cc 100644 --- a/workflow/rules/tile.smk +++ b/workflow/rules/tile.smk @@ -346,6 +346,25 @@ fi +# THE MODULE LOGS OUTLIVE THE RUN DIRS THAT HOLD THEM. A chunk's ShapePipe run +# dir is temp() (tile_merge_cats is its last reader) and tile_vignets' lives in +# the node-local store, so both kinds of log went with their dirs: the +# run's own logs/ and each module's logs/process-*.log, where ngmix writes its +# per-chunk `epoch cuts:` line and every per-object "ngmix failed" message. +# The rule copies them to $SP_RUN/logs/modules// (the tile dir on the +# shared root) before it exits, whatever its rc, so a failed chunk keeps its +# evidence too. A few KB per chunk; clean_tile reclaims them with the rest of +# logs/. The copy never changes the rule's exit status. +def keep_module_logs(run_root, run_name): + dest = f'"$SP_RUN/logs/modules/{run_name}"' + return ( + f'rm -rf {dest} && mkdir -p {dest} && ' + f'(cd "{run_root}" && find . -path "*/logs/*" -type f ' + f'-exec cp -p --parents -t {dest} {{{{}}}} +) || ' + f'echo "could not keep the module logs of {run_root}" >&2\n' + ) + + def tile_exp(wc): return tile_exposures(wc.tile) @@ -612,7 +631,8 @@ rule tile_vignets: # The completeness check is pointed at the NODE-LOCAL run root; see # sp_shell's check_args for what the two flags do. sp_shell("tile_vignets", f"config_tile_PiViVi_{PSF_MODEL}.ini", - check_args=' --run-dir "$SP_LOCAL" --unit {wildcards.tile}') + check_args=' --run-dir "$SP_LOCAL" --unit {wildcards.tile}', + post=keep_module_logs("$NGMIX_VIGNET_DIR", "run_sp_tile_PiViVi")) # ngmix shape measurement — N chunks per tile (D4). Each chunk LOOKS UP its own # CLOSED catalogue-row range in the file tile_vignets materialised at the top of this @@ -828,7 +848,9 @@ rule tile_ngmix: runtime = lambda wc, attempt: 120 * attempt, slurm_extra = TILE_SLURM_EXTRA shell: - sp_shell("tile_ngmix", "config_tile_Ng_template.ini") + sp_shell("tile_ngmix", "config_tile_Ng_template.ini", + post=keep_module_logs("{output.chunkdir}", + "run_sp_tile_ngmix_Ng{wildcards.chunk}u")) def ngmix_manifests(wc):