From 3061a27f9b0d08e9b3eaae3373886cb83195efd5 Mon Sep 17 00:00:00 2001 From: Henry Priest Date: Mon, 21 Sep 2026 17:27:58 +0000 Subject: [PATCH 1/3] first pass of enabling per-step slurm dispatch, with per-step configuration of method and param string --- .github/ci/example_1b_apptainer_config.yaml | 5 + .github/ci/example_1b_conda_config.yaml | 5 + .github/ci/example_2a_apptainer_config.yaml | 5 + .github/ci/example_2a_conda_config.yaml | 5 + .github/ci/example_3_apptainer_config.yaml | 17 +++ .github/ci/example_3_conda_config.yaml | 17 +++ .../example_user_config.yaml | 8 ++ examples/2a_grass/example_user_config.yaml | 8 ++ examples/3_rowcrop/example_user_config.yaml | 20 ++++ magic-ensemble | 92 +++++++++++++-- magic-ensemble-DEVELOPERS.md | 109 +++++++++++++----- workflow/workflow_manifest.yaml | 29 ++++- 12 files changed, 282 insertions(+), 38 deletions(-) diff --git a/.github/ci/example_1b_apptainer_config.yaml b/.github/ci/example_1b_apptainer_config.yaml index 09381dc..2ef131a 100644 --- a/.github/ci/example_1b_apptainer_config.yaml +++ b/.github/ci/example_1b_apptainer_config.yaml @@ -11,6 +11,11 @@ pecan_parallelism_mode: slurm-sbatch sbatch_additional_arguments: "-p node" workflow_parallelism_mode: slurm-srun srun_additional_arguments: "-p node --mem=0" +step_dispatch_configuration: + build-ic: + slurm: true + method: srun + additional_arguments: "-p node --mem=0" aws_profile: magic external_paths: copy_to_rundir: diff --git a/.github/ci/example_1b_conda_config.yaml b/.github/ci/example_1b_conda_config.yaml index e1d3203..a7ad3fc 100644 --- a/.github/ci/example_1b_conda_config.yaml +++ b/.github/ci/example_1b_conda_config.yaml @@ -11,6 +11,11 @@ pecan_parallelism_mode: slurm-sbatch sbatch_additional_arguments: "-p node" workflow_parallelism_mode: slurm-srun srun_additional_arguments: "-p node --mem=0" +step_dispatch_configuration: + build-ic: + slurm: true + method: srun + additional_arguments: "-p node --mem=0" aws_profile: magic external_paths: copy_to_rundir: diff --git a/.github/ci/example_2a_apptainer_config.yaml b/.github/ci/example_2a_apptainer_config.yaml index c6ec9c5..e9932a6 100644 --- a/.github/ci/example_2a_apptainer_config.yaml +++ b/.github/ci/example_2a_apptainer_config.yaml @@ -11,6 +11,11 @@ pecan_parallelism_mode: slurm-sbatch sbatch_additional_arguments: "-p node" workflow_parallelism_mode: slurm-srun srun_additional_arguments: "-p node --mem=0" +step_dispatch_configuration: + build-ic: + slurm: true + method: srun + additional_arguments: "-p node --mem=0" aws_profile: magic external_paths: copy_to_rundir: diff --git a/.github/ci/example_2a_conda_config.yaml b/.github/ci/example_2a_conda_config.yaml index e186f32..cdc44dc 100644 --- a/.github/ci/example_2a_conda_config.yaml +++ b/.github/ci/example_2a_conda_config.yaml @@ -11,6 +11,11 @@ pecan_parallelism_mode: slurm-sbatch sbatch_additional_arguments: "-p node" workflow_parallelism_mode: slurm-srun srun_additional_arguments: "-p node --mem=0" +step_dispatch_configuration: + build-ic: + slurm: true + method: srun + additional_arguments: "-p node --mem=0" aws_profile: magic external_paths: copy_to_rundir: diff --git a/.github/ci/example_3_apptainer_config.yaml b/.github/ci/example_3_apptainer_config.yaml index 02ef6df..98a3597 100644 --- a/.github/ci/example_3_apptainer_config.yaml +++ b/.github/ci/example_3_apptainer_config.yaml @@ -12,6 +12,23 @@ pecan_parallelism_mode: slurm-sbatch sbatch_additional_arguments: "-p node" workflow_parallelism_mode: slurm-srun srun_additional_arguments: "-p node --mem=0" +step_dispatch_configuration: + convert-clim: + slurm: true + method: srun + additional_arguments: "-p node --mem=0" + build-ic: + slurm: true + method: srun + additional_arguments: "-p node --mem=0" + build-events: + slurm: true + method: srun + additional_arguments: "-p node --mem=0" + build-xml: + slurm: true + method: srun + additional_arguments: "-p node --mem=0" aws_profile: magic external_paths: copy_to_rundir: diff --git a/.github/ci/example_3_conda_config.yaml b/.github/ci/example_3_conda_config.yaml index 6c240d1..2b2a508 100644 --- a/.github/ci/example_3_conda_config.yaml +++ b/.github/ci/example_3_conda_config.yaml @@ -12,6 +12,23 @@ pecan_parallelism_mode: slurm-sbatch sbatch_additional_arguments: "-p node" workflow_parallelism_mode: slurm-srun srun_additional_arguments: "-p node --mem=0" +step_dispatch_configuration: + convert-clim: + slurm: true + method: srun + additional_arguments: "-p node --mem=0" + build-ic: + slurm: true + method: srun + additional_arguments: "-p node --mem=0" + build-events: + slurm: true + method: srun + additional_arguments: "-p node --mem=0" + build-xml: + slurm: true + method: srun + additional_arguments: "-p node --mem=0" aws_profile: magic external_paths: copy_to_rundir: diff --git a/examples/1b_statewide_woody/example_user_config.yaml b/examples/1b_statewide_woody/example_user_config.yaml index 0c37f02..3668e17 100644 --- a/examples/1b_statewide_woody/example_user_config.yaml +++ b/examples/1b_statewide_woody/example_user_config.yaml @@ -30,6 +30,14 @@ workflow_parallelism_mode: slurm-srun # separate slurm_partition key -- it's just another srun flag). srun_additional_arguments: "--mem=0" +# Steps the manifest marks slurm: true (here: build-ic) must have a matching +# entry below or the CLI fails fast. +step_dispatch_configuration: + build-ic: + slurm: true + method: srun + additional_arguments: "--mem=0" + # AWS CLI named profile for S3 access (get-demo-data command). aws_profile: magic diff --git a/examples/2a_grass/example_user_config.yaml b/examples/2a_grass/example_user_config.yaml index 5f652af..00ad592 100644 --- a/examples/2a_grass/example_user_config.yaml +++ b/examples/2a_grass/example_user_config.yaml @@ -56,5 +56,13 @@ workflow_parallelism_mode: slurm-srun # separate slurm_partition key -- it's just another srun flag). srun_additional_arguments: "--mem=0" +# Steps the manifest marks slurm: true (here: build-ic) must have a matching +# entry below or the CLI fails fast. +step_dispatch_configuration: + build-ic: + slurm: true + method: srun + additional_arguments: "--mem=0" + # AWS CLI named profile for S3 access (get-demo-data command). aws_profile: magic diff --git a/examples/3_rowcrop/example_user_config.yaml b/examples/3_rowcrop/example_user_config.yaml index a4fb4f7..628f64e 100644 --- a/examples/3_rowcrop/example_user_config.yaml +++ b/examples/3_rowcrop/example_user_config.yaml @@ -36,6 +36,26 @@ workflow_parallelism_mode: slurm-srun # separate slurm_partition key -- pass -p here if your cluster needs one. srun_additional_arguments: "--mem=0" +# Steps the manifest marks slurm: true for prepare-example-3 (convert-clim, +# build-ic, build-events, build-xml) must each have a matching entry below +# or the CLI fails fast. +step_dispatch_configuration: + convert-clim: + slurm: true + method: srun + additional_arguments: "--mem=0" + build-ic: + slurm: true + method: srun + additional_arguments: "--mem=0" + build-events: + slurm: true + method: srun + additional_arguments: "--mem=0" + build-xml: + slurm: true + method: srun + additional_arguments: "--mem=0" # Pecan parallelism pecan_parallelism_mode: slurm-sbatch diff --git a/magic-ensemble b/magic-ensemble index 4ecded0..81aad2b 100755 --- a/magic-ensemble +++ b/magic-ensemble @@ -296,13 +296,14 @@ check_slurm_available() { # for srun). --- build_srun_prefix() { local job_name="$1" + local extra_args="$2" SRUN_PREFIX=(srun) if [[ -n "$job_name" ]]; then SRUN_PREFIX+=(--job-name "Magic-${job_name}") fi - if [[ -n "$srun_additional_arguments" && "$srun_additional_arguments" != "null" ]]; then + if [[ -n "$extra_args" && "$extra_args" != "null" ]]; then local -a extra - read -r -a extra <<< "$srun_additional_arguments" + read -r -a extra <<< "$extra_args" SRUN_PREFIX+=("${extra[@]}") fi } @@ -393,6 +394,69 @@ validate_slurm_steps() { done } +# --- Validate and resolve the user config's step_dispatch_configuration map +# against the manifest's per-step 'slurm' flags. A step marked slurm: true +# in the manifest requires a matching entry here, or the CLI fails fast +# (before any step runs); a step marked slurm: false ignores any entry +# present, but warns, since it indicates user intent that won't take +# effect. Populates STEP_SLURM_USER_ENABLED[] (the entry's own opt-out) and +# STEP_SLURM_PARAMS[] (resolved additional_arguments, falling back to the +# global srun_additional_arguments -- with a warning -- when the entry +# doesn't supply its own), both index-aligned with STEPS/STEP_NAMES/STEP_SLURM. --- +validate_step_dispatch_configuration() { + STEP_SLURM_USER_ENABLED=() + STEP_SLURM_PARAMS=() + local i name entry enabled method params + for i in "${!STEPS[@]}"; do + name="${STEP_NAMES[i]}" + entry=$(yq eval ".step_dispatch_configuration[\"${name}\"]" "$CONFIG_FILE" 2>/dev/null) + + if [[ "${STEP_SLURM[i]}" != "true" ]]; then + if [[ -n "$entry" && "$entry" != "null" ]]; then + echo "magic-ensemble: WARNING: step '${name}' has a step_dispatch_configuration entry in your config, but the manifest marks it slurm: false -- running it without Slurm, as the manifest requires." >&2 + fi + STEP_SLURM_USER_ENABLED+=("false") + STEP_SLURM_PARAMS+=("") + continue + fi + + if [[ -z "$entry" || "$entry" == "null" ]]; then + echo "magic-ensemble: step '${name}' is marked slurm: true in the manifest but has no step_dispatch_configuration entry in your config ($CONFIG_FILE). Add one (see workflow_manifest.yaml / magic-ensemble-DEVELOPERS.md) or the manifest cannot dispatch this step." >&2 + exit 1 + fi + + enabled=$(yq eval ".step_dispatch_configuration[\"${name}\"].slurm" "$CONFIG_FILE" 2>/dev/null) + [[ -z "$enabled" || "$enabled" == "null" ]] && enabled="true" + STEP_SLURM_USER_ENABLED+=("$enabled") + + if [[ "$enabled" != "true" ]]; then + STEP_SLURM_PARAMS+=("") + continue + fi + + method=$(yq eval ".step_dispatch_configuration[\"${name}\"].method" "$CONFIG_FILE" 2>/dev/null) + [[ -z "$method" || "$method" == "null" ]] && method="srun" + case "$method" in + srun) ;; + sbatch) + echo "magic-ensemble: step '${name}' requests step_dispatch_configuration method 'sbatch', but sbatch step-wrapping is not yet implemented -- only 'srun' is supported. Use method: srun or remove the method key." >&2 + exit 1 + ;; + *) + echo "magic-ensemble: step '${name}' has unknown step_dispatch_configuration method '${method}'. Valid options: srun" >&2 + exit 1 + ;; + esac + + params=$(yq eval ".step_dispatch_configuration[\"${name}\"].additional_arguments" "$CONFIG_FILE" 2>/dev/null) + if [[ -z "$params" || "$params" == "null" ]]; then + echo "magic-ensemble: WARNING: step '${name}' has no additional_arguments in its step_dispatch_configuration entry -- falling back to the global srun_additional_arguments." >&2 + params="$srun_additional_arguments" + fi + STEP_SLURM_PARAMS+=("$params") + done +} + # --- Decide whether step i of the current command should be srun-wrapped: # manifest opts in, user config hasn't force-disabled it, and srun is # actually available. Sets STEP_SLURM_ARGS to () or ("--slurm" "") @@ -412,6 +476,11 @@ compute_slurm_arg() { [[ "${STEP_SLURM[i]}" != "true" ]] && return + if [[ "${STEP_SLURM_USER_ENABLED[i]}" != "true" ]]; then + log_info "step '${STEP_NAMES[i]}': slurm disabled via step_dispatch_configuration in your config -- running it unwrapped, as configured." + return + fi + if [[ "$workflow_parallelism_mode" == "local" ]]; then echo "magic-ensemble: WARNING: step '${STEP_NAMES[i]}' is marked slurm: true in the manifest, but workflow_parallelism_mode is set to 'local' in your config -- running it unwrapped, as configured." >&2 return @@ -422,7 +491,7 @@ compute_slurm_arg() { return fi - STEP_SLURM_ARGS=(--slurm "${STEP_NAMES[i]}") + STEP_SLURM_ARGS=(--slurm "${STEP_NAMES[i]}" "${STEP_SLURM_PARAMS[i]}") } # --- R library check for step at index i (reads r_libraries from manifest step; skip if empty or .sh) --- @@ -463,19 +532,21 @@ run_script() { local use_apptainer=0 local use_slurm=0 local slurm_job_name="" + local slurm_params="" local script_cwd="$REPO_ROOT" while [[ $# -gt 0 ]]; do case "${1:-}" in '') shift ;; --apptainer) use_apptainer=1; shift ;; --slurm) - if [[ $# -lt 2 ]]; then - echo "magic-ensemble: run_script --slurm requires STEP_NAME." >&2 + if [[ $# -lt 3 ]]; then + echo "magic-ensemble: run_script --slurm requires STEP_NAME PARAMS." >&2 exit 1 fi use_slurm=1 slurm_job_name="$2" - shift 2 + slurm_params="$3" + shift 3 ;; --cwd) if [[ $# -lt 2 ]]; then @@ -498,7 +569,7 @@ run_script() { local -a SRUN_PREFIX=() if [[ $use_slurm -eq 1 ]]; then - build_srun_prefix "$slurm_job_name" + build_srun_prefix "$slurm_job_name" "$slurm_params" fi if [[ $use_apptainer -eq 1 ]]; then @@ -535,9 +606,11 @@ run_shell_script() { local step_index="${2:-0}" local use_slurm=0 local slurm_job_name="" + local slurm_params="" if [[ "${3:-}" == "--slurm" ]]; then use_slurm=1 slurm_job_name="${4:-}" + slurm_params="${5:-}" fi local script_path="${REPO_ROOT}/${script}" if [[ ! -f "$script_path" ]]; then @@ -547,7 +620,7 @@ run_shell_script() { local -a SRUN_PREFIX=() if [[ $use_slurm -eq 1 ]]; then - build_srun_prefix "$slurm_job_name" + build_srun_prefix "$slurm_job_name" "$slurm_params" fi local script_basename="${script##*/}" @@ -737,6 +810,7 @@ run_manifest_command() { { # block for output redirection get_steps_array validate_slurm_steps + validate_step_dispatch_configuration validate_pecan_parallelism_mode local apptainer_arg="" if [[ $use_apptainer -eq 1 ]]; then @@ -790,6 +864,7 @@ run_manifest_command() { run_get_demo_data() { get_steps_array validate_slurm_steps + validate_step_dispatch_configuration check_aws for i in "${!STEPS[@]}"; do check_r_libs_for_step "$i" @@ -809,6 +884,7 @@ run_run_ensembles() { { # block for output redirection get_steps_array validate_slurm_steps + validate_step_dispatch_configuration if [[ $use_apptainer -eq 1 ]]; then ensure_apptainer_available diff --git a/magic-ensemble-DEVELOPERS.md b/magic-ensemble-DEVELOPERS.md index 5d62117..f0f3396 100644 --- a/magic-ensemble-DEVELOPERS.md +++ b/magic-ensemble-DEVELOPERS.md @@ -273,12 +273,53 @@ srun availability. It does not touch `pecan_parallelism_mode: slurm-sbatch`. Any value other than `local`/`slurm-srun` is a hard error at startup. -There's deliberately no per-step config override (no `slurm_steps: {name: -bool}`-style dict) — the user config has no existing notion of addressing -individual steps, and inventing one would mean every user/CI config needs -updating whenever a step is added/renamed in the manifest. The manifest's -per-step flag is the single source of truth for *which* steps are eligible; -the config only controls whether wrapping happens *at all* this run. +### Config: `step_dispatch_configuration` (per-step override) + +Users need more than a single global on/off switch: a specific step may +need its own dispatch method and its own resource-flag string (e.g. +`build-ic` needs a bigmem partition that other steps don't). The user +config's top-level `step_dispatch_configuration` map, keyed by step `name`, +provides that: + +```yaml +step_dispatch_configuration: + build-ic: + slurm: true # optional, default true when the entry exists + method: srun # optional, default "srun"; "sbatch" is accepted but errors (not yet implemented) + additional_arguments: "-p bigmem --mem=0" # optional; falls back to the global srun_additional_arguments (with a warning) if omitted + build-xml: + slurm: false # opt out for this run even though the manifest marks it slurm: true +``` + +The manifest's per-step `slurm:` flag remains the single source of truth +for *which* steps are eligible at all; `step_dispatch_configuration` only +governs *how* an eligible step is dispatched, or whether it's dispatched +this run. The two are cross-validated by `validate_step_dispatch_configuration()` +before any step runs: + +- `slurm: true` in the manifest, no matching `step_dispatch_configuration.` + entry → hard error, exit 1. Every step the manifest marks eligible must be + explicitly configured. +- `slurm: true` in the manifest, entry present with its own `slurm: false` + → honored silently as a deliberate per-run opt-out (the entry itself + satisfies "configured"; no error, no warning). +- `slurm: false` in the manifest, entry present anyway → warns (the entry + can't take effect; the manifest wins) and runs unwrapped. +- An entry's `additional_arguments` is optional; if omitted, the CLI warns + and falls back to the global `srun_additional_arguments`. +- `method: sbatch` is accepted by the schema but is a hard, fail-fast error + today — no sbatch-based step-wrapping code exists yet (see "Availability" + below for the wholly separate existing `sbatch` usage via + `pecan_parallelism_mode: slurm-sbatch`). + +This reverses an earlier design choice (this section used to say there was +deliberately no per-step override, since the user config had no existing +notion of addressing individual steps, and inventing one meant every +user/CI config would need updating whenever a step is added/renamed in the +manifest). That maintenance cost is now accepted deliberately, in exchange +for per-step Slurm control — every `examples/*/example_user_config.yaml` +and `.github/ci/example_*_config.yaml` fixture was updated alongside this +change to add the now-required entries. `srun_additional_arguments` (default `""`) is a free-form string appended verbatim to every `srun` invocation (e.g. `"--mem=64G --time=02:00:00 @@ -328,27 +369,43 @@ tools are reachable at all. A user running interactively from within an - `get_steps_array()` populates a third array, `STEP_SLURM`, index-aligned with `STEPS`/`STEP_NAMES`. +- `validate_step_dispatch_configuration()` runs right after + `validate_slurm_steps()` (in all three command handlers) and cross-checks + every step against the user config's `step_dispatch_configuration` map, + per the rules above. It fails fast (exit 1) on a missing entry for an + eligible step, an unknown `method`, or `method: sbatch`. It populates two + new index-aligned arrays: `STEP_SLURM_USER_ENABLED` (the entry's own + `slurm:` opt-out, default `true`) and `STEP_SLURM_PARAMS` (resolved + `additional_arguments`, falling back to the global + `srun_additional_arguments` with a warning when the entry omits one). - `compute_slurm_arg ` resolves the effective decision for step `i` - (manifest flag AND `workflow_parallelism_mode != local` AND `check_slurm_available`) - into `STEP_SLURM_ARGS` (`()` or `(--slurm "")`) for direct use - as `run_script`/`run_shell_script` arguments via - `"${STEP_SLURM_ARGS[@]}"`. The step name travels alongside the flag so the - eventual `srun` call can be tagged with a job name. Whenever a step's - manifest `slurm: true` can't be honored as requested, it warns on stderr - before falling back rather than downgrading silently: once if the step - conflicts with a config-level `workflow_parallelism_mode: local`, once if srun - simply isn't available on the host. A step with `slurm: false` never - triggers either warning. -- `run_script()` and `run_shell_script()` both accept `--slurm STEP_NAME` - (a value, like `--cwd DIR`) and, when set, prefix their actual invocation - with `srun` (built by `build_srun_prefix "$slurm_job_name"`) — including - the Apptainer branch of `run_script()`, so `srun apptainer run ...` lands - the container itself on the allocated node rather than wrapping - `apptainer` around an `srun` that stays on the invocation host. - `build_srun_prefix` sets `--job-name "Magic-"` first, before - `srun_additional_arguments` — so if a user's `srun_additional_arguments` - happens to include its own `--job-name`/`-J`, that later flag wins - (`srun` uses last-flag-wins for repeated options). + (manifest flag AND `STEP_SLURM_USER_ENABLED[i]` AND + `workflow_parallelism_mode != local` AND `check_slurm_available`) into + `STEP_SLURM_ARGS` (`()` or `(--slurm "" "")`) for + direct use as `run_script`/`run_shell_script` arguments via + `"${STEP_SLURM_ARGS[@]}"`. The step name and the resolved + `STEP_SLURM_PARAMS[i]` string travel alongside the flag, so the eventual + `srun` call can be tagged with a job name and given its own resolved + parameter string. Whenever a step's manifest `slurm: true` can't be + honored as requested, it warns/logs before falling back rather than + downgrading silently: `log_info` (not a warning — an expected, deliberate + choice) if the step's own `step_dispatch_configuration` entry sets + `slurm: false`; a `WARNING` if the step conflicts with a config-level + `workflow_parallelism_mode: local`; a `WARNING` if srun simply isn't + available on the host. A step with manifest `slurm: false` never triggers + either warning (though `validate_step_dispatch_configuration()` warns + earlier if it has a `step_dispatch_configuration` entry anyway). +- `run_script()` and `run_shell_script()` both accept `--slurm STEP_NAME + PARAMS` (two values, like `--cwd DIR` takes one) and, when set, prefix + their actual invocation with `srun` (built by `build_srun_prefix + "$slurm_job_name" "$slurm_params"`) — including the Apptainer branch of + `run_script()`, so `srun apptainer run ...` lands the container itself on + the allocated node rather than wrapping `apptainer` around an `srun` that + stays on the invocation host. `build_srun_prefix` sets `--job-name + "Magic-"` first, before the resolved params string — so if a + step's `additional_arguments` (or the global `srun_additional_arguments` + fallback) happens to include its own `--job-name`/`-J`, that later flag + wins (`srun` uses last-flag-wins for repeated options). --- diff --git a/workflow/workflow_manifest.yaml b/workflow/workflow_manifest.yaml index ee8a907..947af47 100644 --- a/workflow/workflow_manifest.yaml +++ b/workflow/workflow_manifest.yaml @@ -15,7 +15,9 @@ # below). Defaults to false; must never be true for the # 00_fetch_s3_and_prepare_run_dir.sh / 00_stage_external_inputs.sh # staging steps (fetch/copy, not compute) -- the CLI fails -# fast if it is. +# fast if it is. A step marked slurm: true here REQUIRES a +# matching step_dispatch_configuration. entry in the +# user's config -- the CLI fails fast if one is missing. # r_libraries: R packages to check before running this script # inputs: Dict of {cli_flag: path_key} for path-valued flags this script reads # outputs: Dict of {cli_flag: path_key} for path-valued flags this script @@ -25,9 +27,22 @@ # falling back to the manifest's own top-level value) # # Slurm (srun) step wrapping: a step marked slurm: true is prepended with -# `srun` (plus any config-supplied srun_additional_arguments, e.g. "-p -# node") whenever srun is available on the invoking host and the user -# config's workflow_parallelism_mode is not "local". +# `srun` whenever srun is available on the invoking host and the user +# config's workflow_parallelism_mode is not "local". The user's config MUST +# also configure the step under a top-level step_dispatch_configuration map, +# keyed by the step's `name` (e.g. build-ic), or the CLI fails fast before +# running anything: +# +# step_dispatch_configuration: +# build-ic: +# slurm: true # optional, default true; set false to opt this step out for this run +# method: srun # optional, default "srun" (only method implemented so far) +# additional_arguments: "-p node --mem=0" # optional; falls back (with a warning) to the global srun_additional_arguments below +# +# A step marked slurm: false in the manifest ignores any +# step_dispatch_configuration entry for it (just warns that the entry has +# no effect) -- the manifest is the source of truth for step eligibility; +# the config only controls whether/how wrapping happens for eligible steps. # This is a wholly separate mechanism from pecan_parallelism_mode/host_xml # below, which governs PEcAn's own sbatch-based submission of ensemble # members and is unaffected by workflow_parallelism_mode. @@ -526,6 +541,9 @@ workflow_parallelism_mode: "slurm-srun" # Free-form extra flags appended to every srun invocation (e.g. "-p node # --mem=64G --time=02:00:00 --cpus-per-task=4"). No structured schema; # passed through verbatim, including the partition (-p) if one is wanted. +# This is now the FALLBACK: a per-step step_dispatch_configuration. +# entry with its own additional_arguments takes precedence; a configured +# step without one falls back here (and warns that it's doing so). srun_additional_arguments: "" # Free-form extra flags appended to every sbatch invocation issued by # pecan_parallelism_mode: slurm-sbatch for ensemble-member submission (e.g. @@ -533,6 +551,9 @@ srun_additional_arguments: "" # is wanted. Independent of srun_additional_arguments -- sbatch and srun # don't share an identical flag surface, and the two mechanisms have very # different resource/blast-radius profiles (see magic-ensemble-DEVELOPERS.md). +# Not affected by step_dispatch_configuration: step_dispatch_configuration's +# method: sbatch (step-wrapping via sbatch, distinct from this ensemble-member +# submission mechanism) is not implemented yet. sbatch_additional_arguments: "" From 383906e70211fa032ea88b80ffd461c72719c209 Mon Sep 17 00:00:00 2001 From: Henry Priest Date: Mon, 21 Sep 2026 20:37:23 +0000 Subject: [PATCH 2/3] in example3, moved paths from link-in-place to simple paths, as they are placed there during demo data step --- examples/3_rowcrop/example_user_config.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/examples/3_rowcrop/example_user_config.yaml b/examples/3_rowcrop/example_user_config.yaml index 628f64e..7dfcc5e 100644 --- a/examples/3_rowcrop/example_user_config.yaml +++ b/examples/3_rowcrop/example_user_config.yaml @@ -70,8 +70,8 @@ external_paths: template_file: "examples/3_rowcrop/template.xml" pft_map: "examples/3_rowcrop/crop2pft.csv" link_in_place: - site_era5_path: "data_raw/ERA5_CA_nc" - landtrendr_agb_file: "data_raw/IC/landtrendr_2016_biomass_by_dwr_parcel.parquet" + # site_era5_path: "data_raw/ERA5_CA_nc" + # landtrendr_agb_file: "data_raw/IC/landtrendr_2016_biomass_by_dwr_parcel.parquet" paths: phenology_info_path: "data_raw/management/phenology/v1.0/" @@ -81,3 +81,5 @@ paths: irrigation_info_path: "data_raw/management/irrigation/v1.1/" fertilization_info_path: "data_raw/management/fertilization/v1.0/" ncc_info_path: "data_raw/management/ncc/v1.0/" + site_era5_path: "data_raw/ERA5_CA_nc" + landtrendr_agb_file: "data_raw/IC/landtrendr_2016_biomass_by_dwr_parcel.parquet" From 246bd99add710390a625ddc41ccb4045a4e329d9 Mon Sep 17 00:00:00 2001 From: Henry Priest Date: Mon, 21 Sep 2026 20:40:00 +0000 Subject: [PATCH 3/3] add paths to CI --- .github/ci/example_3_conda_config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/ci/example_3_conda_config.yaml b/.github/ci/example_3_conda_config.yaml index 2b2a508..2daaf98 100644 --- a/.github/ci/example_3_conda_config.yaml +++ b/.github/ci/example_3_conda_config.yaml @@ -46,3 +46,5 @@ paths: irrigation_info_path: "data_raw/management/irrigation/v1.1/" fertilization_info_path: "data_raw/management/fertilization/v1.0/" ncc_info_path: "data_raw/management/ncc/v1.0/" + site_era5_path: "data_raw/ERA5_CA_nc" + landtrendr_agb_file: "data_raw/IC/landtrendr_2016_biomass_by_dwr_parcel.parquet"