diff --git a/.github/workflows/ci_test.yml b/.github/workflows/ci_test.yml index dabcd7f..dc2d61c 100644 --- a/.github/workflows/ci_test.yml +++ b/.github/workflows/ci_test.yml @@ -171,14 +171,19 @@ jobs: --build-dir="$GITHUB_WORKSPACE/build/$variant" \ --prefix="$GITHUB_WORKSPACE/install/$variant" done - - name: Run the selected CTest suite and compare executions + - name: Run PR and restart CTest suites and compare executions run: | OPTIONS=() for variant in serial omp mpi hybrid; do OPTIONS+=(--build-dir "$GITHUB_WORKSPACE/build/$variant") done - python -B tests/regression/regression.py suite "${OPTIONS[@]}" \ - --suite pr --run-root "$GITHUB_WORKSPACE/build/results" + # Collect both suites even if one fails; either failure fails CI. + STATUS=0 + for suite in pr restart; do + python -B tests/regression/regression.py suite "${OPTIONS[@]}" \ + --suite "$suite" --run-root "$GITHUB_WORKSPACE/build/results" || STATUS=1 + done + exit "$STATUS" - name: Save an unapproved reference from the passing suite if: ${{ inputs.create_reference }} run: | diff --git a/compile.sh b/compile.sh index aebb866..e66bf73 100755 --- a/compile.sh +++ b/compile.sh @@ -31,7 +31,7 @@ usage () { printf " build with verbose output\n" printf " --test[=TEST_NAME], -t[=TEST_NAME]\n" printf " run one exact test name, or the quick suite\n" - printf " --test-suite=quick|pr|full|unit\n" + printf " --test-suite=quick|pr|full|restart|unit\n" printf " --reference=DIRECTORY\n" printf " override the reference; strict defaults to the bundled PR baseline\n" printf " --no-reference\n" diff --git a/tests/AGENTS.md b/tests/AGENTS.md new file mode 100644 index 0000000..f863c51 --- /dev/null +++ b/tests/AGENTS.md @@ -0,0 +1,67 @@ +# Instructions for tests + +These instructions apply to all tests below this directory, including Fortran +unit tests in `unit/` and the Python regression harness in `regression/`. + +## Unit-test comments + +Each new or modified unit test must include an informative comment explaining +the behavior it checks, why the setup exercises that behavior, and the expected +result or rejection. For files covering several scenarios, summarize the shared +purpose near the top and explain distinct cases where they are defined. +Include relevant scientific assumptions, units, timing, or boundary conditions. +Explain what a failure would mean; avoid comments that merely repeat assertions +or depend on internal planning labels or PR numbers. + +## User-facing documentation + +Write user-facing test documentation so readers can understand it without +knowing the development history. Describe the scientific behavior, configuration, +or compatibility condition being tested. Do not use internal planning labels +such as P00/P02 or PR numbers as substitutes for those descriptions. + +For example, use "restart-loading probes for moisture initialization" instead +of "probes for P02", and "legacy restart metadata" instead of "PR #39 metadata". +Explain which metadata is absent and why it matters when discussing legacy +compatibility; "restart metadata" alone does not identify that distinction. + +Keep PR references, planning labels, revisions, and historical validation results +in validation records such as regression/doc/validation.md or session notes. +Preserve that provenance when revising user-facing guides. + +## Regression harness + +Read [regression/README.md](regression/README.md) before editing the harness. +Use the Google-based Python formatting in `regression/.style.yapf`, with +four-space indentation and an 80-column target. Run the pinned YAPF check +documented in that README. Keep the formatter out of runtime dependencies. + +Use descriptive snake_case names, pathlib paths, and public-function type hints. +Separate logical operations with blank lines. Explain scientific choices and +synthetic test substitutions in comments. A formatter pass does not replace +review for readability. + +Do not add personal filesystem paths or require a developer's Conda installation. +Use the module-provided environment documented in `regression/README.md` on NCAR +systems. `CFBM_TEST_TMP` is required for direct Python tests; CTest supplies a +build-tree artifact directory. Keep generated files in the repository's ignored +`build/` directory or outside the checkout, and retain failure evidence. Use PBS +for model/MPI runs and for the shared MPI-enabled Python environment. + +Python test filenames end in `_test.py`. Use unittest discovery with +`-p '*_test.py'` and confirm that both Python test files are collected after any +rename. Keep the repository's `compile.sh` responsible for build options, CTest +responsible for test invocation, and `regression/cases.yaml` responsible for the +scientific cases and executions. Do not change scientific tolerances or legacy +references to conceal failures. + +Documentation lives in `regression/doc/`. Keep `regression/README.md` as the entry +point. Update the case figures and comparison table when scientific settings or +outputs change. Read [contributing.md](regression/doc/contributing.md) and +[pr-checklist.md](regression/doc/pr-checklist.md) before extending the cases. +Keep historical fixtures in `legacy/` and preserve their reference data. + +Use exact namelist names in `regression/cases.yaml`. Cases own explicit +configurations; do not add generic option-category expansion. Compare only +matching case, configuration, and scale. No creation or assistance header is +required. diff --git a/tests/regression/AGENTS.md b/tests/regression/AGENTS.md index 3d55c2a..913dc9d 100644 --- a/tests/regression/AGENTS.md +++ b/tests/regression/AGENTS.md @@ -1,33 +1,4 @@ -# Instructions for regression harness changes +# Test instructions -Read README.md here before editing. Use the Google-based Python formatting in -.style.yapf, with four-space indentation and an 80-column target. Run the pinned -YAPF check documented in README.md. Keep the formatter out of runtime dependencies. - -Use descriptive snake_case names, pathlib paths, and public-function type hints. -Separate logical operations with blank lines. Explain scientific choices and -synthetic test substitutions in comments. A formatter pass does not replace -review for readability. - -Do not add personal filesystem paths or require a developer's Conda installation. -Use the module-provided environment documented in README.md on NCAR systems. -CFBM_TEST_TMP is required for direct Python tests; CTest supplies a build-tree -artifact directory. Keep generated files in the ignored build/ directory or -outside the checkout, and retain -failure evidence. Use PBS for model/MPI runs and for the shared MPI-enabled -Python environment. - -Python test filenames end in _test.py. Use unittest discovery with -p '*_test.py' -and confirm that both Python test files are collected after any rename. -Keep compile.sh responsible for build options, CTest responsible for test -invocation, and cases.yaml responsible for the scientific cases and executions. -Do not change scientific tolerances or legacy references to conceal failures. - -Documentation lives in doc/. Keep README.md as the entry point. Update the -case figures and comparison table when scientific settings or outputs change. -Read doc/contributing.md and doc/pr-checklist.md before extending the cases. -Keep historical fixtures in ../legacy/ and preserve their reference data. - -Use exact namelist names in cases.yaml. Cases own explicit configurations; do -not add generic option-category expansion. Compare only matching case, -configuration, and scale. No creation or assistance header is required. +The shared test instructions, including regression-harness guidance, are in +[tests/AGENTS.md](../AGENTS.md). diff --git a/tests/regression/CMakeLists.txt b/tests/regression/CMakeLists.txt index 97c5827..b97b317 100644 --- a/tests/regression/CMakeLists.txt +++ b/tests/regression/CMakeLists.txt @@ -109,5 +109,5 @@ endif() add_test(NAME regression_python COMMAND "${Python3_EXECUTABLE}" -B -m unittest discover -s "${CMAKE_CURRENT_SOURCE_DIR}/tests" -p "*_test.py" -v) set_tests_properties(regression_python PROPERTIES - LABELS "unit;quick;pr;full" + LABELS "unit;quick;pr;full;restart" ENVIRONMENT "CFBM_TEST_TMP=${PROJECT_BINARY_DIR}/python-test-artifacts") diff --git a/tests/regression/README.md b/tests/regression/README.md index 96dc80c..3bdab1d 100644 --- a/tests/regression/README.md +++ b/tests/regression/README.md @@ -42,6 +42,8 @@ difference between one `compile.sh` call and the complete four-build matrix. | Understand terrain, fuel, ignition, winds, and timing | [Case configurations and figures](doc/cases.md) | | Follow the build/run flow and connections between files | [Execution overview and file relationships](doc/structure.md) | | Find every compared field and its acceptance rule | [Variable comparison table](doc/comparison.md) | +| Check continuous versus restarted integrations | [Restart continuity](doc/restart.md) | +| Distinguish CTest invocations, model runs, and checkpoint checks | [How test counts are reported](#how-test-counts-are-reported) | | Understand improvements and remaining legacy coverage | [New and legacy tests](doc/advantages.md) | | Find outputs or prepare an unapproved reference | [Results and references](doc/references.md) | | Add or extend a case, namelist option, or output | [Contributor guide](doc/contributing.md) | @@ -61,6 +63,11 @@ configuration names, and the `large` scale. GNU `Strict` builds select the shared `github-pr` baseline by default. Use `--reference` to override it or `--no-reference` to disable historical comparison deliberately. Configuration warns when no reference is selected. + +GitHub CI runs both `pr` and `restart` using the same four builds; either suite +failing makes the regression check fail. The restart suite compares uninterrupted +and segmented runs from the same revision and needs no stored baseline. Both +suites' reports and model files are retained in the test-evidence artifact. The [comparison table](doc/comparison.md) separates bitwise checks from numerical tolerance. Passing without an approved reference establishes the configured behavior checks and agreement among selected executions, not historical @@ -80,6 +87,79 @@ investigations, use `compile_legacy.sh` with separate build/install directories. Coverage limits and postponed investigations are recorded in [deferred work](doc/deferred.md). +## How test counts are reported + +The counts below describe the complete four-build `pr` and `restart` selections +validated at `5de15ed` in [CI run 39970138677](https://github.com/NCAR/fire_behavior/actions/runs/39970138677). +They count different levels of work: CTest invocations, model integrations, +individual checkpoint-file checks, and comparisons between files. Selecting +fewer builds or filtering cases changes these totals. When registrations or +schedules change, update this breakdown from the suite summaries, per-build +CTest/JUnit reports, and each case's `restart.json`. + +### 450 checkpoint-validity checks + +These are checks of **450 checkpoint files** across 15 restart cases. Each case +writes checkpoints every 4 s, from 4 through 60 s: + +- 15 files from the uninterrupted run. +- 15 files produced across the segmented run. +- Total: **15 cases × 30 files = 450 checkpoint-file checks**. + +Each file is checked for required fields, finite/unmasked values, REAL64 fuel +storage, correct dates and timestep count, and the expected perimeter-activation +flag. The final checkpoint also checks for precision that would be lost if +fuel state were reduced to REAL32. Evolving-moisture cases additionally check +initialization status, update cadence and times, five-class dimensions, and +moisture bounds appropriate to these dry-forcing cases. The class-moisture +range `(0,1)` is specific to these cases, not a general bound on fuel moisture. +The checks are implemented in [check_checkpoint](restart.py). + +These checks test each file against the expected checkpoint contract. The +separately reported **320 file comparisons** compare continuous and restarted +results: 90 regular-output comparisons, 225 checkpoint comparisons, and five +accepted-legacy checkpoint comparisons. Each checkpoint comparison requires +exact physical-state agreement. The 450 validity checks run inside the 15 +restart cases; they are not 450 additional CTest invocations. See +[restart continuity](doc/restart.md) for field applicability and comparison rules. + +### 238 PR CTest invocations + +The `pr` suite executes the following registered CTests across four builds: + +| Build | Fortran unit checks | Python harness invocation | Model runs | Total | +| --- | ---: | ---: | ---: | ---: | +| Serial | 49 | 1 | 4 | 54 | +| OpenMP | 49 | 1 | 8 | 58 | +| MPI | 50 | 1 | 16 | 67 | +| Hybrid | 50 | 1 | 8 | 59 | +| **Total** | **198** | **4** | **36** | **238** | + +The [Fortran unit checks](../unit/CMakeLists.txt) cover namelist validation, +timing, tiles, perimeter activation, numerical precision, wind interpolation, +and moisture initialization. MPI and hybrid add the namelist-broadcast check. + +Each `regression_python` invocation runs **45 Python tests**, but CTest counts +that invocation as one entry. Many unit checks are repeated across builds to +exercise the corresponding compiled configuration. Thus 238 is a count of +CTest invocations, not 238 distinct scientific scenarios. The 40 cross-execution +comparisons and 36 approved-reference comparisons are reported separately from +these invocation counts. + +### 67 restart CTest invocations + +The `restart` suite is separate from the 238 PR invocations. It contains 15 +restart cases, 48 moisture-initialization unit invocations (12 in each build), +and four Python harness invocations: **15 + 48 + 4 = 67**. The unit and Python +checks overlap with `pr` and run again when both suites are selected. + +Each restart case invokes the model several times. Together they run 15 +uninterrupted integrations, 50 chain segments, and 30 restart-loading probes: +**95 model invocations** inside the 15 case-level CTests. The loading probes +include five accepted legacy files and 25 expected rejections of ambiguous or +damaged checkpoints. An expected rejection passes only when its diagnostic +and absence of regular output match the test requirements. + ## Python style Use Google-based YAPF formatting, four-space indentation, an 80-column target, diff --git a/tests/regression/cases.yaml b/tests/regression/cases.yaml index 709b801..650f9aa 100644 --- a/tests/regression/cases.yaml +++ b/tests/regression/cases.yaml @@ -115,6 +115,12 @@ cases: base_elevation_m: 0.0 configurations: base: {} + restart_prescribed: + restart_splits: [28] + namelist: + time: + restart_interval: 4 + interval_output: 12 godunov: namelist: fire: @@ -202,6 +208,31 @@ cases: namelist: fire: wind_vinterp_opt: 1 + restart_time: + # Updates at 4, 20, 36, 52 s; splits bracket perimeter activation + # and include times between moisture updates and forcing records. + restart_splits: [4, 12, 28] + namelist: + time: + restart_interval: 4 + interval_output: 12 + atm: + interval_atm: 12 + fire: + fmoist_freq: 0 + fmoist_dt: 16.0 + restart_step: + # Initialize at 4 s; updates at 12, 24, ... s retain the original phase. + restart_splits: [4, 12, 28] + namelist: + time: + restart_interval: 4 + interval_output: 12 + atm: + interval_atm: 12 + fire: + fmoist_freq: 3 + fmoist_dt: 12.0 u3d: namelist: fire: @@ -269,6 +300,15 @@ executions: timeout_seconds: 1800 build: mpi suites: + restart: + scale: small + cases: + circle: + configurations: [restart_prescribed] + executions: [serial, omp4, mpi1, mpi4, hybrid4] + terrain: + configurations: [restart_time, restart_step] + executions: [serial, omp4, mpi1, mpi4, hybrid4] quick: scale: small cases: diff --git a/tests/regression/compare_outputs.py b/tests/regression/compare_outputs.py index ab469cf..8bdc602 100644 --- a/tests/regression/compare_outputs.py +++ b/tests/regression/compare_outputs.py @@ -242,8 +242,9 @@ def compare_file( test_path: Path, static_fields: set[str], volatile_global_attributes: set[str] | None = None, + required_fields: set[str] | None = None, ) -> dict[str, Any]: - """Compare two NetCDF files and every field under the fixed policy.""" + """Compare full schemas and all fields, or an explicit required-field subset.""" result: dict[str, Any] = { "reference": str(reference_path), @@ -280,7 +281,15 @@ def compare_file( if _attributes(reference, volatile_global_attributes) != _attributes( test, volatile_global_attributes): result["reasons"].append("global metadata differs") - for name in sorted(set(reference.variables) & set(test.variables)): + shared = set(reference.variables) & set(test.variables) + if required_fields is not None: + result["uncompared_fields"] = sorted(shared - required_fields) + if required_fields - shared: + result["reasons"].append( + f"required fields missing: {sorted(required_fields - shared)}" + ) + shared &= required_fields + for name in sorted(shared): result["fields"].append( compare_variable(reference.variables[name], test.variables[name], name in static_fields)) diff --git a/tests/regression/config.py b/tests/regression/config.py index 9351fab..8eea422 100644 --- a/tests/regression/config.py +++ b/tests/regression/config.py @@ -19,6 +19,7 @@ "time": { "dt": float, "interval_output": float, + "restart_interval": int, "num_tiles": int, "tile_strategy": int }, @@ -89,7 +90,9 @@ "v_profile_m_s", }, } -SCIENCE_KEYS = {"start", "duration_seconds", "namelist", "inputs"} +SCIENCE_KEYS = { + "start", "duration_seconds", "namelist", "inputs", "restart_splits" +} DRIVERS = {"standalone", "nuopc", "esmx"} BUILDS = {"serial", "omp", "mpi", "hybrid"} ALLOWED_METHOD_PAIRS = {(9, 4), (2, 4)} @@ -284,9 +287,12 @@ def registrations(document: dict[str, Any], variant: str, if layout["build"] != variant: continue # Validate scientific combinations while configuring CMake. - resolve_spec(document, case, scale, configuration, - execution) + spec = resolve_spec(document, case, scale, configuration, + execution) for driver in drivers: + if spec.get( + "restart_splits") and driver != "standalone": + continue if driver not in document["cases"][case]["drivers"]: continue name = f"{driver}_{case}_{configuration}_{scale}_{execution}" @@ -328,6 +334,18 @@ def validate_spec(spec: dict[str, Any]) -> None: if abs(duration / interval - round(duration / interval)) > 1.0e-10: raise ValueError( "Integration end must align with output and forcing records") + if "restart_splits" in spec: + splits = spec["restart_splits"] + interval = _require_number(time, "restart_interval", step, "time") + if interval / step != round(interval / step): + raise ValueError( + "restart_interval must align with the fire timestep") + if (not isinstance(splits, list) or not splits or any( + type(value) is not int or not 0 < value < duration or value % + interval for value in splits) or + splits != sorted(set(splits)) or duration % interval): + raise ValueError("restart_splits must increase within the run and " + "align with restart_interval, including the end") start = dt.datetime.strptime(spec["start"], "%Y-%m-%d_%H:%M:%S") spec["end"] = (start + dt.timedelta(seconds=duration)).strftime("%Y-%m-%d_%H:%M:%S") diff --git a/tests/regression/doc/cases.md b/tests/regression/doc/cases.md index abdc7c5..393ba3c 100644 --- a/tests/regression/doc/cases.md +++ b/tests/regression/doc/cases.md @@ -13,6 +13,12 @@ scientific options within each case; see [configuration.md](configuration.md). | `fuels` | Line ignition, twelve fuel categories, nearest interpolation | | `terrain` / `u10m` | Terrain, twelve fuel strips, delayed observed perimeter, varying moisture, terrain-dependent 10 m winds | | `terrain` / `u3d` | Same terrain experiment with a terrain-dependent vertical wind profile | +| `circle` / `restart_prescribed` | Same circle geometry with prescribed moisture and a restart at 28 s | +| `terrain` / `restart_time`, `restart_step` | Same terrain geometry and 10 m winds with restart chains bracketing perimeter activation and moisture updates | + +The restart configurations use different output, atmospheric, and moisture +cadences. Their [schedule table](restart.md) supplements the geometry figures +below. Existing `quick`, `pr`, and `full` selections retain their settings. The shared configuration uses a 72 x 72 fire grid, 100 m spacing, 4 s timesteps, and 60 s integrations. NUOPC and ESMX run both terrain wind configurations with generated @@ -43,10 +49,9 @@ between mass levels, where unscaled (U, V) = (12, 8) m/s. For 3D winds, both drivers first remap the wind and geopotential profiles horizontally to the fire grid, then sample vertically using mapped roughness. -PR #57 applies this order to standalone; previously it sampled vertically on -the atmospheric grid first. The logarithmic surface calculation is nonlinear, -so reversing these operations can change the result. Matching the order does -not make the two horizontal interpolation implementations bitwise identical. +The logarithmic surface calculation is nonlinear, so reversing these operations +can change the result. Matching the order does not make the two horizontal +interpolation implementations bitwise identical. Both wind paths retain their final values in the fire-state `uf`/`vf` arrays. The 10 m path maps `u10`/`v10` directly before applying WAF; it no longer diff --git a/tests/regression/doc/comparison.md b/tests/regression/doc/comparison.md index 973260e..bc5c26e 100644 --- a/tests/regression/doc/comparison.md +++ b/tests/regression/doc/comparison.md @@ -2,6 +2,10 @@ [Documentation index](../README.md) +The separate [restart suite](restart.md) also compares physical checkpoint +state bitwise, validates REAL64 fuel storage, and checks clock and activation +metadata. The regular-output acceptance rules below remain unchanged. + The standard inventory contains 19 variables. Each expected output file must contain exactly the fields listed in `expected_output_fields` in [cases.yaml](../cases.yaml). The comparator checks all variables, not a selected diff --git a/tests/regression/doc/configuration.md b/tests/regression/doc/configuration.md index 33a952f..38d4f2f 100644 --- a/tests/regression/doc/configuration.md +++ b/tests/regression/doc/configuration.md @@ -2,6 +2,10 @@ [Documentation index](../README.md) +Restart configurations additionally specify `restart_splits` and the exact +`namelist.time.restart_interval`; the runner derives `time.restart` and segment +dates. See [restart continuity](restart.md) for schedule and driver constraints. + Use one [cases.yaml](../cases.yaml) with exact Fortran block and option names under `namelist`. For example, `namelist.fire.wind_vinterp_opt: 0` writes `wind_vinterp_opt=0` in `&fire`. Input-generator settings live under `inputs`; @@ -12,17 +16,22 @@ they describe synthetic fields and are not additional model namelist options. | Selection | Meaning | Examples | | --- | --- | --- | | Case | Shared physical setup and its purpose | `circle`, `fuels`, `terrain` | -| Configuration | Explicit set of scientific option/input overrides within a case | `base`, `u10m`, `u3d`, `godunov3d` | +| Configuration | Explicit set of scientific option/input overrides within a case | `base`, `u10m`, `u3d`, `godunov3d`, `restart_time` | | Scale | Grid dimensions, spacing, integration duration, and intervals | `small`, `large` | | Driver | Executable entry point | `standalone`, `nuopc`, `esmx` | | Execution | Compiled build, MPI ranks, OpenMP threads, timeout | `serial`, `omp4`, `mpi4`, `hybrid4` | -| Suite | Explicit selections of cases, configurations, scale, and executions | `quick`, `pr`, `full` | +| Suite | Explicit selections of cases, configurations, scale, and executions | `quick`, `pr`, `full`, `restart` | `unit` and `legacy` are additional CTest selections, not generated experiments. The suite `full` uses scale `large`. Scale is more than domain size: it also changes the integration schedule. The name `small` replaces `standard`, and `large` replaces the old scale name `full`. +Suite membership is explicit: selecting `pr` or `full` does not automatically +include the `restart` configurations. GitHub CI invokes both `pr` and `restart` +using the same four builds. The `restart` suite selects standalone runs even +when the parent case also supports NUOPC and ESMX. + Resolution is **defaults → case → configuration → scale**. Nested mappings merge; scalar values and lists replace earlier values. Execution settings are kept separately and cannot override science. If `case --configuration` is @@ -100,6 +109,12 @@ cases: godunov3d: namelist: fire: {wind_vinterp_opt: 0, fire_upwinding: 2} + restart_time: + restart_splits: [4, 12, 28] + namelist: + time: {restart_interval: 4, interval_output: 12} + atm: {interval_atm: 12} + fire: {fmoist_freq: 0, fmoist_dt: 16.0} scales: small: {} @@ -116,6 +131,12 @@ executions: mpi4: {build: mpi, ranks: 4, threads: 1, timeout_seconds: 900} suites: + restart: + scale: small + cases: + terrain: + configurations: [restart_time] + executions: [serial, mpi4] pr: scale: small cases: @@ -124,6 +145,16 @@ suites: executions: [serial, mpi4] ``` +The `restart_time` example inherits `dt=4` and a 60 s duration. It writes +checkpoints every 4 s and regular output every 12 s. For each selected execution, +the harness runs one uninterrupted integration from 0 to 60 s and a chain of +segments spanning 0-4, 4-12, 12-28, and 28-60 s. The 28 s split falls between +regular outputs at 24 and 36 s; the first output after resuming is still at +36 s. `restart_splits` controls the harness, while `restart_interval` is a +Fortran namelist option. The complete production selection also includes +prescribed and step-based moisture cases across five execution layouts; see +[restart continuity](restart.md). + ## Add options without adding a category system A configuration can set any supported option under its real namelist block. @@ -157,3 +188,10 @@ example `nuopc_terrain_u3d_small_mpi4`. Only runs sharing the same case, configuration, and scale are compared. Wind choices and numerical schemes are separate experiments, not numerical references for one another. Resolved settings are saved in each `result.json`. + +Restart configurations compare each resumed segment with the uninterrupted run +from the same revision and execution layout. Physical checkpoint state must +match exactly; regular output follows the existing comparison policy. These +cases do not use the stored `github-pr` baseline. Running `restart` alongside +`pr` in CI therefore requires no baseline update: `pr` keeps its approved +reference, and reference generation continues to use only the `pr` results. diff --git a/tests/regression/doc/deferred.md b/tests/regression/doc/deferred.md index 90c99c1..03d0266 100644 --- a/tests/regression/doc/deferred.md +++ b/tests/regression/doc/deferred.md @@ -11,9 +11,9 @@ coverage limits. | --- | --- | | Intermediate-time 3D standalone/coupled agreement | Earlier diagnostics exceeded tolerance between saved PR endpoints. The bilinear-precision experiment passed an every-4-second GitHub comparison through 60 s. These are diagnostic results, not an extension of the PR acceptance schedule. See [validation history](validation.md). | | Longer standalone/coupled agreement | The 600 s terrain diagnostics still exceed the short-run comparison tolerance, including at 6.096 m with double-precision bilinear arithmetic. A longer-duration acceptance policy would need separate scientific review; it is not required for this PR. See [validation history](validation.md). | -| Humidity interpretation | Scientific review associated with PR #46 is independent of reproducibility; current moisture remains enabled. | -| Restart | Restart continuity, restart inputs, and segmented-run comparisons. | -| PR #39 changes and method `(4,5)` | Separate scientific review and suitable comparison cases before matrix inclusion. | +| Humidity interpretation | Scientific review of the humidity convention is independent of reproducibility; current moisture remains enabled. | +| Restart | Standalone same-revision chains are covered by the [restart suite](restart.md). Coupled/WRF-host continuity and cross-revision compatibility remain untested. Known production failures remain failures. | +| Method `(4,5)` | Separate scientific review and suitable comparison cases before matrix inclusion. | | ESMX_Data feedback | The generated coupled cases use WRF-data forcing. Legacy `testx` still exercises a distinct feedback fixture. | | Coupled OpenMP-only | MPI and hybrid NUOPC/ESMX are included in the PR matrix. Coupled runs using an OpenMP-only build remain outside this selection. | | Atmospheric hosts | WRF/UFS host integrations, including UFS mass-centred winds and any future staggered interpolation option. | diff --git a/tests/regression/doc/references.md b/tests/regression/doc/references.md index 57f336c..dd7ee90 100644 --- a/tests/regression/doc/references.md +++ b/tests/regression/doc/references.md @@ -47,17 +47,22 @@ history](validation.md). ## GitHub review and activation -CI uses the fixed `pr` suite with all four GNU Strict builds: serial, OpenMP, -MPI and hybrid, both including NUOPC/ESMX. Its 36 generated runs produce 40 -cross-execution comparisons. Unit checks run in each build. PRs targeting -prerequisite branches are included, so a merge into `develop` is not needed. +CI runs `pr` and `restart` using all four GNU Strict builds: serial, OpenMP, +MPI and hybrid, with NUOPC/ESMX in the latter two. The `pr` suite's 36 generated +runs produce 40 cross-execution comparisons. The `restart` suite compares +continuous and segmented standalone integrations from the same revision and +does not use a stored baseline. Both suites must pass. Unit checks run in each +build. PRs targeting prerequisite branches are included, so a merge into +`develop` is not needed. Normal Strict checks default to the [approved GitHub reference](../baselines/github-pr/README.md) committed for the complete PR suite. A missing or unapproved reference fails the check. To generate a future candidate, select **Run workflow** for **Fire Behavior CI Test**, choose the PR branch, and enable `create_reference`. -That run checks the same complete PR selection without a reference, then -uploads an unapproved baseline only if all checks and comparisons pass. +That run checks both suites without a stored reference, then creates a candidate +from only the `pr` results. It uploads the unapproved baseline only if all +checks and comparisons pass. Adding restart coverage does not change the +approved baseline or include restart checkpoints in a reference candidate. Download and extract both `.tar.gz` artifacts (candidate and test evidence). Archiving preserves the colons in NetCDF timestamp filenames, which GitHub's diff --git a/tests/regression/doc/restart.md b/tests/regression/doc/restart.md new file mode 100644 index 0000000..71964e5 --- /dev/null +++ b/tests/regression/doc/restart.md @@ -0,0 +1,99 @@ +# Restart continuity + +[Documentation index](../README.md) + +Run `regression.py suite --suite restart --build-dir=BUILD --run-root=RESULTS` +inside a PBS allocation on NCAR systems. Repeat `--build-dir` for serial, +OpenMP, MPI, and hybrid builds. `compile.sh` still owns compilation and CTest +owns invocation. No production physics is changed by these tests. + +The suite uses the existing small circle and terrain geometries, fuel maps, +forcing endpoints, and propagation methods. Their existing configuration +figures remain applicable. This table specifies the different event schedules; +all times are seconds from the original 2020-01-01 00:00:00 origin. + +| Configuration | Moisture | Atmospheric records | Restart splits | Perimeter | +| --- | --- | --- | --- | --- | +| circle/restart_prescribed | Prescribed 0.08 kg kg-1 | Idealized, no input file | 28 | Line/point ignition | +| terrain/restart_time | `fmoist_freq=0`, `fmoist_dt=16` | Every 12 s | 4, 12, 28 | Observed perimeter at 8 s | +| terrain/restart_step | `fmoist_freq=3`, `fmoist_dt=12` | Every 12 s | 4, 12, 28 | Observed perimeter at 8 s | + +All integrations use `dt=4`, run through 60 s, write regular output every 12 s, +and write checkpoints every 4 s. Time-based moisture updates occur at 4, 20, +36, and 52 s. The split at 4 s precedes perimeter activation; 12 and 28 s +follow it. The 28 s split is between moisture updates +and atmospheric records. Step-based moisture initializes at 4 s and first +advances at 12 s using an 8 s interval, then advances every 12 s. These cases +verify initialization before the first scheduled step-based moisture update +and restoration of that state on restart. +A failure remains a failing CTest; the suite has no expected-failure conversion. + +Each case runs one uninterrupted integration and a separate cold first segment, +then resumes only from the preceding segment's checkpoint. Generated input files +are copied unchanged to every segment. In particular, shortening a segment does +not regenerate the forcing or change its temperature/humidity trajectory. Input +and incoming-checkpoint hashes, namelists, original/resolved settings, command, +process status, and model logs are retained. Schedule inventories derive from +the original origin, including output after a split between output events. + +| Compared content | Acceptance | +| --- | --- | +| Regular output at every scheduled time | Existing static-field bitwise policy and dynamic `rtol=1e-4`, `atol=0`; unchanged schema/metadata checks | +| Physical checkpoint fields in `restart.STATE_FIELDS`, current forcing, previous forcing and five moisture classes where applicable | Bitwise equality for the same executable and execution layout | +| Checkpoint dimensions, full variable inventory, and global metadata | Exact equality, including clock origin, step count, perimeter activation, and moisture times | +| Fuel fraction and fuel fraction burned in the current timestep | REAL64 storage and a non-REAL32 signal at the final checkpoint; catches quantization even when stored as float64 | +| Independent checkpoint validity | Required fields, finite/unmasked owned cells, expected clock and activation state, five moisture-class dimensions, initialized moisture at 4 s, expected cadence and last/next times, and moisture in (0,1) for these dry-forcing cases | + +The restart writer also saves method work arrays `lfn_0/1/2`, `lfn_s0/1/2/3` +and unused crown fluxes `fcanhfx/fcanqfx`. Their values are not all initialized +for these methods. They remain in the schema comparison but are explicitly +listed as uncompared fields. Physical fields are not excluded to obtain a pass. +`lfn_hist` is checked only for observed perimeters, whose initialization sets +it; `fz0` and atmospheric forcing are checked only for real cases. Idealized +runs do not use those atmospheric fields. +Regular-output tolerances do not weaken the exact checkpoint-state comparison. + +The complete selection contains 15 cases across `serial`, `omp4`, `mpi1`, +`mpi4`, and `hybrid4`: 65 model invocations, including the 15 continuous runs, +and 315 file comparisons (90 output and 225 checkpoint comparisons). Counts +of attempted/completed segments and failed comparisons are recorded separately +in `summary.json`; complete field metrics and checkpoint checks are in each +case's `restart.json`. Failed cases are excluded from cross-layout comparisons +and remain failures in the summary. Cross-layout comparisons use continuous +outputs and the existing comparison policy. + +The new configurations use same-revision comparisons and do not select the +historical `github-pr` reference, which does not contain their keys. The +existing `pr` suite remains unchanged and checks its approved reference. +No reference candidate is created or approved by this suite. + +The suite additionally runs 30 restart-loading probes for moisture initialization +across the five layouts. Each time-based case resumes a checkpoint copy with +legacy restart metadata and requires an exact checkpoint match. The legacy copy +omits five attributes recording the initialization flag and time, whether +moisture evolution is enabled, and the step-based and time-based update cadences. +Four additional probes must reject current checkpoints with an unset +initialization flag, changed cadence, future last-update time, or invalid previous +pressure before writing output. Each step-based case must reject the ambiguous +legacy schema. These probes add 30 model invocations and five checkpoint +comparisons to the continuity counts above. Reports keep +expected rejection diagnostics separate from failed continuity comparisons. +Synthetic legacy files exercise schema compatibility; they do not establish +cross-revision equivalence. + +Restart experiments currently require the standalone driver. NUOPC/ESMX and +WRF-host restart continuity are not established here. Existing coupled PR +tests exercise ordinary integrations only. Cross-layout checkpoint transfers, +legacy-file compatibility across revisions, fractional timesteps, and coupled +flux accumulation/reset are outside this suite's coverage and require separate +validation. + +## Extend a restart experiment + +Add `restart_splits` (an increasing list of whole seconds) and the exact +`namelist.time.restart_interval` to a named configuration in `cases.yaml`. +Splits and the final time must align with checkpoint writes; checkpoint writes +must align with the fire timestep. The segment runner derives `time.restart` +and the start/end dates. Add the configuration and executions explicitly to +the `restart` suite. Preserve original forcing and compare every expected +checkpoint and regular output, including the first resumed event. diff --git a/tests/regression/doc/running.md b/tests/regression/doc/running.md index 05a8056..58fcdd0 100644 --- a/tests/regression/doc/running.md +++ b/tests/regression/doc/running.md @@ -130,7 +130,7 @@ python -B tests/regression/regression.py suite --suite pr \ selection accepts regex syntax. Internally, Python reads CTest's JSON listing and supplies a numeric list of selected tests to CTest. -`--suite` accepts **five values**. `--test` accepts **any exact CTest name +`--suite` accepts **six values**. `--test` accepts **any exact CTest name registered in the selected build**, so its available names depend on the build configuration. @@ -142,12 +142,20 @@ The suite options are defined in [regression.py](../regression.py): | `pr` | Unit tests plus all four default case/configuration combinations across `serial`, `omp1`, `omp4`, `mpi1`, `mpi4`, and `hybrid4`, where supported. | | `full` | Unit tests plus the larger experiments: 320 × 320 grids, 3600 s integrations, both method pairs `(9,4)` and `(2,4)`, and the additional `mpi8` execution. Requires `CFBM_FULL_TESTS=ON`. | | `unit` | Unit Fortran tests and the Python harness tests. No generated fire-case integrations. | +| `restart` | Moisture initialization unit checks, Python harness checks, and standalone continuous/restarted cases in serial, OpenMP, MPI, and hybrid layouts. See [restart continuity](restart.md) for schedules, counts, and coverage limits. | | `legacy` | Original tests registered by the legacy build route. | These selections follow [cases.yaml](../cases.yaml). Each build contributes only the executions it supports, such as serial or MPI. **`full` selects only the larger experiments.** +GitHub CI runs `pr` and `restart` consecutively using the same serial, OpenMP, +MPI and hybrid builds. Both suites run even if one fails, and either failure +fails the regression check. Their separate `summary.json` files, JUnit reports, +model logs and outputs are included in the test-evidence artifact. Restart +comparisons use the uninterrupted run from the same revision; the existing +`github-pr` baseline remains specific to `pr`. + Enable their registration in an appropriately sized allocation: ```bash @@ -210,7 +218,7 @@ namelist_broadcast_mpi Here `*_unit` describes a naming pattern, not a literal selector. Copy the complete name from the CTest listing when using `--test`. `regression_python` tests **the Python regression harness**: input generation, configuration, output -checks, comparisons, and reporting. It currently contains **40 Python tests** +checks, comparisons, and reporting. It currently contains **45 Python tests** across two files and does not launch the Fortran model. | Area | What it checks | @@ -223,9 +231,10 @@ across two files and does not launch the Fortran model. | Numerical comparisons | Exercises relative tolerance, zero references, tolerance boundaries, bitwise static-field checks, NaNs, infinities, missing-data masks, and empty valid-data sets. | | Comparison policy | Checks cross-driver roughness tolerance, exact fuel categories, comparisons across execution layouts, and rejection of incompatible scientific settings. | | Failure detection and reporting | Checks rejection of corrupted or unapproved references, extra output files, and a successful process exit with no outputs; also exercises JSON reporting. | +| Restart chains | Checks checkpoint precision, state and metadata validity, segment schedules, and preservation of forcing across segments. | The implementation is in -[python_harness_test.py](../tests/python_harness_test.py) (23 tests) and +[python_harness_test.py](../tests/python_harness_test.py) (28 tests) and [generator_comparator_test.py](../tests/generator_comparator_test.py) (17 tests). @@ -346,7 +355,7 @@ The build loop calls `compile.sh` once per configuration; each call builds only that configuration. After all four builds finish, one suite invocation selects their tests and compares the outputs. Settings are grouped near the top of the template. -`SUITES=(quick pr)` can explicitly run both when separate reports are needed. +Set `SUITES=(pr restart)` to run both CI suites and retain separate reports. The template runs only the generated regression harness. Legacy investigations use `compile_legacy.sh` separately with their own build/install directories. diff --git a/tests/regression/doc/structure.md b/tests/regression/doc/structure.md index 6f0b070..10c2a7c 100644 --- a/tests/regression/doc/structure.md +++ b/tests/regression/doc/structure.md @@ -43,6 +43,8 @@ flowchart TD CTEST -->|runs| UNIT CTEST -->|runs each model case through| CASE CASE -->|stages and launches| MODEL + CASE -->|for restart configurations| RESTART["restart.py
Cold first segment and chained restarts"] + RESTART -->|checks each segment and checkpoint| RESULT MODEL -->|produces fields and logs for| CHECK BASELINE -->|supplies accepted fields to| CHECK CHECK -->|records findings in| RESULT @@ -62,6 +64,12 @@ Strict selects the shared baseline automatically. `--reference` overrides it; cross-execution checks remain active. See [running tests](running.md) for commands and [comparison rules](comparison.md) for acceptance criteria. +The separately selected [restart suite](restart.md) uses `run_case.run_model` +for the continuous run and every segment. `restart.py` copies the original +forcing, derives segment namelists, and calls the shared comparison routines. +Its results enter the same case and suite reports; it does not build or launch +tests through a second framework. + ## File relationships inside the harness `regression.py` is the command interface. CMake uses its `registrations` command diff --git a/tests/regression/doc/validation.md b/tests/regression/doc/validation.md index e40a7f0..21b688a 100644 --- a/tests/regression/doc/validation.md +++ b/tests/regression/doc/validation.md @@ -5,6 +5,106 @@ These records describe specific revisions. They do not automatically validate later source, documentation examples, renamed registrations, or new cases. +## P00 on merged P02/develop (2026-10-22) + +After PR #39 and P02/PR #62 were squash-merged, the review base became +`develop` at `622e191`. P00 merges that revision into its published history, +retaining the merged moisture component and restart reader. The 19-file review +diff contains only tests and documentation; production code and P02's focused +unit tests match `622e191` exactly. The harness now includes the initialization +metadata checks and 30 restart-loading probes from the earlier P00/P02 test +integration. + +Derecho PBS `7915414.desched1` tests candidate `9a64864`. The published-history +merge `ce551ee` has the identical complete file tree; subsequent edits only +update documentation. All four GNU Strict builds pass: serial, OpenMP, MPI, +and hybrid, with NUOPC/ESMX enabled in the latter two. The environment is GNU +12.2.0, Cray MPICH 8.1.25, NetCDF 4.9.2, ESMF 8.5.0, and shared NPL Python. + +All 67 restart CTests pass, comprising 15 restart cases, 48 P02 lifecycle +checks, and four Python harness registrations. All five formerly failing +step-based moisture cases now pass. The suite completes 65 continuity model +invocations and 30 loading probes. Its 315 continuity file comparisons and +five accepted-legacy checkpoint comparisons are bitwise equal, with zero +maximum absolute error. All 450 checkpoint validity checks, 30 loading probes, +and 12 cross-layout comparisons pass; cross-layout error is also zero. + +The same job passes all 238 existing PR CTests: 202 unit registrations and +36 model runs. All 40 execution comparisons and 36 approved-reference +comparisons pass. The reference error is zero; the maximum cross-driver error +is 0.125 W m-2 in `fgrnhfx`, within the unchanged tolerance. The PBS job exits +zero. The 45 Python tests also pass in the serial-NetCDF development environment; +YAPF 0.43.0, shell syntax, and whitespace checks pass. + +Evidence is preserved under `cfbm-P00-develop-20261022T230324-0600/` in the +validation submitter's Derecho scratch `tmp/`. The historical P00 failures +below remain preserved. This validation establishes same-revision standalone +restart continuity, including P02's delayed initialization correction. It does +not establish NUOPC/ESMX or WRF-host restart continuity, cross-layout checkpoint +transfer, or combined P01/P04 behavior. + +## P00 restart foundation (2026-10-20) + +The original review base was local merge `bc205bc`, combining upstream +`26d57b6` with restart PR #39 at `59b9081`. Remote heads were checked on +2026-10-20 and still matched that snapshot. P00 changes only the harness and +its documentation. +At that time, its feature branch required an explicit stacked review base; +targeting upstream develop would also have included the unmerged restart work. + +Derecho PBS `7889837.desched1` built this base in serial, OpenMP, MPI, and +hybrid GNU Strict configurations. All 190 PR CTests passed: 154 unit CTests +and 36 model runs. The 40 cross-execution comparisons and all 36 approved +`github-pr` reference comparisons passed. The largest cross-execution absolute +difference was 0.125 W m-2 in `fgrnhfx`; the applicable relative tolerance was +unchanged. This establishes the base's existing regression behavior, not +restart continuity. + +The environment was GNU 12.2.0, Cray MPICH 8.1.25, NetCDF 4.9.2, ESMF 8.5.0, +and the shared `npl-2026a` Python interpreter. P00's 45 Python tests also passed +in a serial-NetCDF development environment; YAPF 0.43.0 and Python compilation +checks passed. Both existing Python test modules remain in discovery. + +The first harness attempt, `1f61101` in PBS `7889874.desched1`, passed all five +time-based moisture restart cases and the unchanged PR matrix. Its prescribed +cases exposed a checker applicability error: idealized `fz0` is deliberately +unavailable. `a863938` corrects that check and restricts `lfn_hist` checks to +observed perimeters, whose initialization sets it. It also records executable +hashes and execution environment. The original failed evidence is retained. + +The step-based configuration exposes the pre-existing initialization defect +assigned to P02. In `physics/fmc_wrffire_mod.F90`, `Advance_fmc_model` gates the +call on update cadence, but `Advance_moisture_classes` initializes only when +`itimestep == 1`. With `fmoist_freq=3`, that first call is missed. Results include +invalid class moisture, nonfinite moisture/heat fluxes, differing restart state, +and MPI aborts reporting equilibrium-moisture NaNs. The tests remain failures; +P00 does not change the production implementation or claim scientific acceptance +of this configuration. + +The corrected harness at `a863938`, PBS `7889890.desched1`, passes all ten +prescribed/time-based cases across serial, OpenMP, one-rank MPI, four-rank MPI, +and hybrid layouts. Their 210 output/checkpoint comparisons have zero absolute +error, including REAL64 fuel state. All eight cross-layout comparisons pass. +All five step-based cases fail on the P02 defect described above. There are +61 attempted model invocations rather than the scheduled 65: the four-rank +MPI and hybrid chains cannot continue after their first resumed segment aborts. +The report contains 283 file-comparison attempts, including four missing-file +failures; 279 comparisons read both files. Agreement in an invalid step-based +case does not establish restart acceptance. The overall restart suite exits +nonzero, as intended for an unresolved production defect. + +The same job also passed all 190 existing PR CTests, all 40 cross-execution +comparisons, and all 36 approved-reference comparisons. Maximum cross-driver +absolute error remained 0.125 W m-2 in `fgrnhfx`. The PBS job exits nonzero +because of the restart suite's five P02 cases, not because the existing PR +suite failed. Subsequent P00 edits only document these results and list the +new suite in `compile.sh --help`; runtime validation applies to `a863938`. + +Evidence is preserved under `cfbm-P00-20261021T024500/` in the validation +submitter's Derecho scratch `tmp/`, with separate `base`, `attempt-01`, and +`attempt-02` source/build/run trees. See [restart continuity](restart.md) for +the comparison policy, scheduled counts, and intentionally untested paths. + ## Coupled hybrid baseline extension (2026-10-14) At Maria Frediani's request, revision `4b6535d` enables NUOPC and ESMX in diff --git a/tests/regression/regression.py b/tests/regression/regression.py index 356f0ac..db49c74 100644 --- a/tests/regression/regression.py +++ b/tests/regression/regression.py @@ -42,7 +42,8 @@ def build_parser() -> argparse.ArgumentParser: help="Select and run registered CTests") suite.add_argument("--build-dir", type=Path, action="append", required=True) suite.add_argument("--suite", - choices=("quick", "pr", "full", "unit", "legacy"), + choices=("quick", "pr", "full", "restart", "unit", + "legacy"), default="quick") suite.add_argument("--case") suite.add_argument("--driver", choices=("standalone", "nuopc", "esmx")) diff --git a/tests/regression/restart.py b/tests/regression/restart.py new file mode 100644 index 0000000..434ff48 --- /dev/null +++ b/tests/regression/restart.py @@ -0,0 +1,379 @@ +#!/usr/bin/env python3 +# Copyright 2026 Research Applications Laboratory (RAL), +# National Center for Atmospheric Research (NCAR), +# University Corporation for Atmospheric Research (UCAR) +#-------------------------------------------------------------------------------- +# Created by Maria Frediani (frediani@ucar.edu) on 2026-10-20 +# run python -B tests/regression/regression.py suite --suite restart --help +"""Check same-executable restart chains against an uninterrupted integration.""" +from __future__ import annotations + +#-------------------------------------------------------------------------------- +# Python modules and checkpoint contract +#-------------------------------------------------------------------------------- +import copy +import datetime as dt +from pathlib import Path +import shutil +import subprocess +from typing import Any + +import netCDF4 +import numpy as np + +from compare_outputs import compare_file +from generate_inputs import DATE_FORMAT, sha256_file +from render_namelist import render_namelist +from reports import write_result + +# Compare persistent physical state. Method work arrays lfn_0/1/2 and lfn_s0/1/2/3, +# and unused crown fluxes fcanhfx/fcanqfx, are written by PR #39 but are not all +# initialized for the selected methods. Their names remain in the schema check. +STATE_FIELDS = { + "lfn", "lfn_out", "tign_g", "fuel_frac", "fuel_frac_burnt_dt", "fire_area", + "fgrnhfx", "fgrnqfx", "flame_length", "ros", "ros_front", "emis_smoke", + "fmc_g", "fuel_load_g", "fuel_time", "zsf", "dzdxf", "dzdyf", "nfuel_cat", + "uf", "vf" +} +FORCING_FIELDS = {"fire_t2", "fire_q2", "fire_psfc", "fire_rain"} +REAL64_FIELDS = {"fuel_frac", "fuel_frac_burnt_dt"} +MOISTURE_TIMES = ("fmc_fmoist_lasttime", "fmc_fmoist_nexttime") + +#-------------------------------------------------------------------------------- +# Original-origin schedules and checkpoint validation +#-------------------------------------------------------------------------------- + + +def timestamp(spec: dict[str, Any], seconds: int) -> str: + """Translate elapsed seconds from the original origin into a file timestamp.""" + return (dt.datetime.strptime(spec["start"], DATE_FORMAT) + + dt.timedelta(seconds=seconds)).strftime(DATE_FORMAT) + + +def segment_names(spec: dict[str, Any], + begin: int, + end: int, + checkpoint: bool = False) -> list[str]: + """Enumerate writes independently of a resumed segment's local start time.""" + key = "restart_interval" if checkpoint else "interval_output" + interval = int(spec["namelist"]["time"][key]) + seconds = list(range((begin // interval + 1) * interval, end + 1, interval)) + if begin == 0 and not checkpoint: + seconds.insert(0, 0) + prefix = "fire_restart" if checkpoint else "fire_output" + return [f"{prefix}_{timestamp(spec, second)}.nc" for second in seconds] + + +def checkpoint_fields(spec: dict[str, Any]) -> set[str]: + """Select physical fields whose owned cells must survive a restart exactly.""" + fields = set(STATE_FIELDS) + fire = spec["namelist"]["fire"] + # Init_fire_perimeter alone initializes lfn_hist; ideal runs leave fz0 + # unavailable. Neither is consumed by the corresponding excluded path. + if fire["fire_is_real_perim"]: + fields.add("lfn_hist") + if fire["ideal_opt"] == 0: + fields |= FORCING_FIELDS | {"fz0"} + if fire["fmoist_run"]: + fields |= {"fmc_gc"} | {f"{name}_old" for name in FORCING_FIELDS} + return fields + + +def check_checkpoint(path: Path, spec: dict[str, Any], + seconds: int) -> dict[str, Any]: + """Require valid clock, perimeter, class state, and double-precision fuel data.""" + result = {"path": str(path), "pass": False, "reasons": []} + if not path.is_file(): + result["reasons"].append("Checkpoint is missing") + return result + reasons = result["reasons"] + fire = spec["namelist"]["fire"] + with netCDF4.Dataset(path) as dataset: + required = checkpoint_fields(spec) + missing = required - set(dataset.variables) + if missing: + reasons.append(f"Checkpoint fields missing: {sorted(missing)}") + for name in required & set(dataset.variables): + variable = dataset[name] + values = np.ma.asarray(variable[:]) + if (not values.size or np.ma.count_masked(values) or + not np.isfinite(values).all()): + reasons.append( + f"Unavailable or nonfinite checkpoint state: {name}") + if name in REAL64_FIELDS and variable.dtype != np.dtype("float64"): + reasons.append(f"{name} must retain REAL64 storage") + if (name in REAL64_FIELDS and + seconds == spec["duration_seconds"] and not np.any( + values != values.astype("float32").astype("float64"))): + reasons.append(f"{name} lacks a non-REAL32 precision signal") + # Check dates and step count independently of the comparison file. + for prefix, date in (("start", spec["start"]), + ("restart", timestamp(spec, seconds))): + parsed = dt.datetime.strptime(date, DATE_FORMAT) + for part in ("year", "month", "day", "hour", "minute", "second"): + name = f"{prefix}_{part}" + if getattr(dataset, name, None) != getattr(parsed, part): + reasons.append(f"Incorrect checkpoint clock: {name}") + if getattr(dataset, "itimestep", + None) != seconds / spec["namelist"]["time"]["dt"]: + reasons.append("Incorrect checkpoint itimestep") + ignited = int(fire["fire_is_real_perim"] and + seconds >= fire["fire_ignition_start_time1"]) + if getattr(dataset, "fire_perimeter_ignited", None) != ignited: + reasons.append("Incorrect observed-perimeter activation state") + if fire["fmoist_run"]: + if getattr(dataset, "fmc_initialized", None) != 1: + reasons.append("Moisture initialization flag is not set") + step = spec["namelist"]["time"]["dt"] + if getattr(dataset, "fmc_initialization_time", + None) != np.float32(step): + reasons.append( + "Incorrect effective moisture initialization time") + for name in ("fmoist_freq", "fmoist_dt"): + if getattr(dataset, name, None) != np.float32(fire[name]): + reasons.append(f"Incorrect moisture cadence: {name}") + for name in MOISTURE_TIMES: + value = getattr(dataset, name, None) + if value is None or not np.isfinite(value) or value < 0: + reasons.append(f"Missing or invalid moisture time: {name}") + if fire["fmoist_freq"]: + # Check the component's default-real timestamp convention; + # the separate fire clock may normalize a rounded input dt. + count = int(dataset.itimestep) + last_step = max( + 1, count // fire["fmoist_freq"] * fire["fmoist_freq"]) + last = float(np.float32(last_step) * np.float32(step)) + next_time = 0.0 + else: + interval = fire["fmoist_dt"] + last = step + (seconds - step) // interval * interval + next_time = last + interval + for name, expected in zip(MOISTURE_TIMES, (last, next_time)): + if getattr(dataset, name, None) != expected: + reasons.append(f"Incorrect scheduled moisture time: {name}") + if "fmc_gc" in dataset.variables: + variable = dataset["fmc_gc"] + if (variable.dimensions != ("ny", "moisture_class", "nx") or + variable.shape[1] != 5): + reasons.append("Incorrect five-class moisture dimensions") + # Positive, finite moisture is required for these nonzero initial + # values and dry forcing. Zero-filled uninitialized storage fails. + if np.any(variable[:] <= 0) or np.any(variable[:] >= 1): + reasons.append( + "Class moisture is outside this case's physical range (0,1)" + ) + result["pass"] = not reasons + return result + + +def compare_checkpoint(reference: Path, test: Path, + spec: dict[str, Any]) -> dict[str, Any]: + """Require bitwise physical state and identical metadata for the same layout.""" + fields = checkpoint_fields(spec) + result = compare_file(reference, test, fields, required_fields=fields) + result["comparison_kind"] = "restart_checkpoint" + return result + + +#-------------------------------------------------------------------------------- +# Segmented integrations using the existing case launcher +#-------------------------------------------------------------------------------- + + +def check_moisture_restart_loading(spec: dict[str, Any], directory: Path, + command: list[str], + environment: dict[str, str]) -> list[dict]: + """Exercise legacy inference and reject damaged lifecycle state before use.""" + from run_case import run_model + + fire = spec["namelist"]["fire"] + if spec["identity"]["configuration"] not in ("restart_time", + "restart_step"): + return [] + # These schema probes resume after initialization and before the first + # time-based update. No atmospheric refresh can hide a damaged history. + probes = {"legacy": None} + if fire["fmoist_freq"]: + probes[ + "legacy"] = "Legacy restart cannot establish moisture initialization" + else: + probes.update({ + "uninitialized": + "Restart moisture state is not initialized", + "cadence": + "Restart metadata mismatch: fmoist_freq", + "future_time": + "Restart moisture times are inconsistent", + "history": + "Restart moisture forcing history is unavailable or invalid" + }) + records = [] + begin = spec["namelist"]["time"]["dt"] + end = 2 * begin + incoming = f"fire_restart_{timestamp(spec, begin)}.nc" + for probe, diagnostic in probes.items(): + target = directory / f"loading-{probe}" + target.mkdir() + for name in ("wrf.nc", "geo_em.d01.nc", incoming): + shutil.copy2(directory / name, target / name) + with netCDF4.Dataset(target / incoming, "a") as dataset: + if probe == "legacy": + for name in ("fmc_initialized", "fmc_initialization_time", + "fmoist_run", "fmoist_freq", "fmoist_dt"): + dataset.delncattr(name) + elif probe == "uninitialized": + dataset.fmc_initialized = 0 + elif probe == "cadence": + dataset.fmoist_freq = fire["fmoist_freq"] + 1 + elif probe == "future_time": + dataset.fmc_fmoist_lasttime = 1000.0 + elif probe == "history": + dataset["fire_psfc_old"][0, 0] = 0.0 + resumed = copy.deepcopy(spec) + resumed.update(start=timestamp(spec, begin), + end=timestamp(spec, end), + duration_seconds=end - begin) + resumed["namelist"]["time"]["restart"] = True + (target / "namelist.fire").write_text(render_namelist(resumed)) + write_result( + target / "settings.json", { + "spec": resumed, + "probe": probe, + "expected_diagnostic": diagnostic, + "restart_sha256": sha256_file(target / incoming) + }) + record = run_model(command, target, environment, + spec["execution"]["timeout_seconds"], "standalone") + record.update(probe=probe, directory=str(target)) + if diagnostic: + log = (target / "model.log").read_text(errors="replace") + record["pass"] = diagnostic in log and not any( + target.glob("fire_output_*.nc")) + else: + outgoing = f"fire_restart_{timestamp(spec, end)}.nc" + comparison = compare_checkpoint(directory / outgoing, + target / outgoing, spec) + record["comparison"] = comparison + record["pass"] = not record["reasons"] and comparison["pass"] + write_result(target / "execution.json", record) + records.append(record) + return records + + +def run_restarts(spec: dict[str, Any], directory: Path, command: list[str], + environment: dict[str, str], + static_fields: set[str]) -> dict[str, Any]: + """Run a cold first segment and chained restarts against unchanged forcing.""" + from run_case import run_model + + result = { + "pass": False, + "reasons": [], + "segments": [], + "comparisons": [], + "checkpoint_checks": [] + } + duration = int(spec["duration_seconds"]) + splits = [0, *spec["restart_splits"], duration] + inputs = [ + path for name in ("wrf.nc", "geo_em.d01.nc") + if (path := directory / name).is_file() + ] + result["input_sha256"] = {path.name: sha256_file(path) for path in inputs} + expected = set(segment_names(spec, 0, duration, checkpoint=True)) + if {path.name for path in directory.glob("fire_restart_*.nc")} != expected: + result["reasons"].append("Continuous checkpoint inventory differs") + # Validate every continuous checkpoint, including times before first ignition. + for seconds in range(spec["namelist"]["time"]["restart_interval"], + duration + 1, + spec["namelist"]["time"]["restart_interval"]): + result["checkpoint_checks"].append( + check_checkpoint( + directory / f"fire_restart_{timestamp(spec, seconds)}.nc", spec, + seconds)) + previous = None + for index, (begin, end) in enumerate(zip(splits, splits[1:])): + segment = directory / f"segment-{index:02d}" + segment.mkdir() + segment_spec = copy.deepcopy(spec) + segment_spec.update(start=timestamp(spec, begin), + end=timestamp(spec, end), + duration_seconds=end - begin) + segment_spec["namelist"]["time"]["restart"] = begin > 0 + for path in inputs: + shutil.copy2(path, segment / path.name) + incoming = None + if begin: + incoming = f"fire_restart_{timestamp(spec, begin)}.nc" + source = previous / incoming + if not source.is_file(): + result["reasons"].append(f"Cannot resume: missing {source}") + break + shutil.copy2(source, segment / incoming) + (segment / "namelist.fire").write_text(render_namelist(segment_spec)) + write_result( + segment / "settings.json", { + "spec": + segment_spec, + "original_spec": + spec, + "input_sha256": + result["input_sha256"], + "restart_source": + str(previous / incoming) if incoming else None, + "restart_sha256": + sha256_file(segment / incoming) if incoming else None + }) + try: + record = run_model(command, segment, environment, + spec["execution"]["timeout_seconds"], + "standalone") + except (OSError, RuntimeError, subprocess.SubprocessError) as error: + record = {"reasons": [f"{type(error).__name__}: {error}"]} + record.update(directory=str(segment), + begin_seconds=begin, + end_seconds=end) + for checkpoint in (False, True): + expected = segment_names(spec, begin, end, checkpoint) + prefix = "fire_restart" if checkpoint else "fire_output" + actual = {path.name for path in segment.glob(f"{prefix}_*.nc")} + inventory = set(expected) | ({incoming} + if checkpoint and incoming else set()) + if actual != inventory: + record["reasons"].append( + f"{prefix} inventory differs: expected={sorted(inventory)}, actual={sorted(actual)}" + ) + for name in expected: + if checkpoint: + comparison = compare_checkpoint(directory / name, + segment / name, spec) + seconds = int( + (dt.datetime.strptime(name[13:-3], DATE_FORMAT) - + dt.datetime.strptime(spec["start"], + DATE_FORMAT)).total_seconds()) + result["checkpoint_checks"].append( + check_checkpoint(segment / name, spec, seconds)) + else: + comparison = compare_file(directory / name, segment / name, + static_fields) + comparison["comparison_kind"] = "restart_output" + result["comparisons"].append(comparison) + record["pass"] = not record["reasons"] + write_result(segment / "execution.json", record) + result["segments"].append(record) + previous = segment + if len(result["segments"]) != len(splits) - 1 or any( + not record["pass"] for record in result["segments"]): + result["reasons"].append( + "One or more restart segments did not complete") + if any(not item["pass"] for item in result["checkpoint_checks"]): + result["reasons"].append("Checkpoint contract failed") + if any(not item["pass"] for item in result["comparisons"]): + result["reasons"].append("Continuous/restarted comparison failed") + result["loading_checks"] = check_moisture_restart_loading( + spec, directory, command, environment) + if any(not item["pass"] for item in result["loading_checks"]): + result["reasons"].append("Moisture restart loading checks failed") + result["pass"] = not result["reasons"] + write_result(directory / "restart.json", result) + return result diff --git a/tests/regression/run_case.py b/tests/regression/run_case.py index 8c03287..fbb526a 100644 --- a/tests/regression/run_case.py +++ b/tests/regression/run_case.py @@ -13,6 +13,7 @@ from pathlib import Path import shutil import subprocess +import sys import tempfile import time from typing import Any @@ -21,7 +22,7 @@ from check_outputs import expected_output_names, validate_outputs from config import load_yaml, resolve_spec -from generate_inputs import generate_inputs +from generate_inputs import generate_inputs, sha256_file from render_namelist import render_namelist from reports import print_result, write_result @@ -79,6 +80,44 @@ def write_esmx_config(path: Path, spec: dict[str, Any]) -> None: path.write_text(yaml.safe_dump(config, sort_keys=False), encoding="utf-8") +def run_model(command: list[str], directory: Path, environment: dict[str, str], + timeout_seconds: int, driver: str) -> dict[str, Any]: + """Launch one integration and retain process and fatal-log evidence.""" + result = { + "reasons": [], + "command": command, + "started_utc": dt.datetime.now(dt.timezone.utc).isoformat() + } + with (directory / "model.log").open("w") as log: + process = subprocess.run(command, + cwd=directory, + env=environment, + stdout=log, + stderr=subprocess.STDOUT, + timeout=timeout_seconds, + check=False) + result["exit_status"] = process.returncode + if process.returncode: + result["reasons"].append( + f"Model exited with status {process.returncode}") + logs = [directory / "model.log", *directory.glob("PET*.ESMF_LogFile")] + for log in logs: + lines = log.read_text(errors="replace").splitlines() + result["reasons"].extend( + line.strip() + for line in lines + if line.strip().startswith(FATAL_PREFIXES) or " ERROR " in line) + if driver != "standalone": + marker = ("esmApp FINISHED" if driver == "nuopc" else + "ESMX (Earth System Model eXecutable) FINISHED") + completed = any( + marker in log.read_text(errors="replace") for log in logs) + result["coupled_finalize_observed"] = completed + if not completed: + result["reasons"].append("Coupled finalization was not observed") + return result + + def run_case(args: Any) -> dict[str, Any]: """Stage a case, launch the model, and reject incomplete integrations.""" if Path("/glade").exists() and not os.environ.get("PBS_JOBID"): @@ -89,6 +128,8 @@ def run_case(args: Any) -> dict[str, Any]: args.execution) if args.driver not in document["cases"][args.case]["drivers"]: raise ValueError(f"No coupled configuration for {args.case}") + if spec.get("restart_splits") and args.driver != "standalone": + raise ValueError("Restart experiments currently require standalone") executable = args.executable.resolve(strict=True) root = Path(os.environ.get("CFBM_RUN_ROOT", str(args.run_root))).resolve() if root.is_relative_to(SOURCE_ROOT) and not root.is_relative_to( @@ -105,7 +146,14 @@ def run_case(args: Any) -> dict[str, Any]: "driver": args.driver, "pass": False, "reasons": [], - "source": source_identity() + "source": source_identity(), + "executable": str(executable), + "executable_sha256": sha256_file(executable), + "environment": { + "python": sys.executable, + "PBS_JOBID": os.environ.get("PBS_JOBID"), + "LOADEDMODULES": os.environ.get("LOADEDMODULES") + } } started = time.monotonic() try: @@ -135,37 +183,20 @@ def run_case(args: Any) -> dict[str, Any]: result.update(command=command, started_utc=dt.datetime.now(dt.timezone.utc).isoformat(), threads=execution["threads"]) - with (directory / "model.log").open("w") as log: - process = subprocess.run(command, - cwd=directory, - env=environment, - stdout=log, - stderr=subprocess.STDOUT, - timeout=execution["timeout_seconds"], - check=False) - result["exit_status"] = process.returncode - if process.returncode: - result["reasons"].append( - f"Model exited with status {process.returncode}") - logs = [directory / "model.log", *directory.glob("PET*.ESMF_LogFile")] - for log in logs: - lines = log.read_text(errors="replace").splitlines() - result["reasons"].extend( - line.strip() - for line in lines - if line.strip().startswith(FATAL_PREFIXES) or " ERROR " in line) - if args.driver != "standalone": - marker = ("esmApp FINISHED" if args.driver == "nuopc" else - "ESMX (Earth System Model eXecutable) FINISHED") - completed = any( - marker in log.read_text(errors="replace") for log in logs) - result["coupled_finalize_observed"] = completed - if not completed: - result["reasons"].append( - "Coupled finalization was not observed") + result.update( + run_model(command, directory, environment, + execution["timeout_seconds"], args.driver)) result["validation"] = validate_outputs(directory, result) result["reasons"].extend(result["validation"]["reasons"]) - if args.reference: + if spec.get("restart_splits"): + from restart import run_restarts + result["restart"] = run_restarts(spec, directory, command, + environment, + set(document["static_fields"])) + result["reference_policy"] = "same-revision continuous/restarted" + if not result["restart"]["pass"]: + result["reasons"].append("Restart continuity checks failed") + if args.reference and not spec.get("restart_splits"): from reference import compare_reference result["reference"] = compare_reference(args.reference, result, document) diff --git a/tests/regression/run_suite.py b/tests/regression/run_suite.py index 37b30f8..2a2b771 100644 --- a/tests/regression/run_suite.py +++ b/tests/regression/run_suite.py @@ -155,7 +155,8 @@ def run_suite(args: Any) -> dict[str, Any]: json.loads(path.read_text()) for path in sorted(root.glob("*/result.json")) ] - if args.suite in ("quick", "pr", "full") and not args.test and not results: + if args.suite in ("quick", "pr", "full", + "restart") and not args.test and not results: summary["reasons"].append( "No generated cases ran; full requires CFBM_FULL_TESTS=ON") if any(not result["pass"] for result in results): @@ -165,6 +166,27 @@ def run_suite(args: Any) -> dict[str, Any]: "pass": result["pass"], "directory": result["directory"] } for result in results] + summary["restart_cases"] = [{ + "name": + result["name"], + "pass": + result["restart"]["pass"], + "segments": + len(result["restart"]["segments"]), + "completed_segments": + sum(item["pass"] for item in result["restart"]["segments"]), + "model_invocations": + 1 + len(result["restart"]["segments"]) + + len(result["restart"].get("loading_checks", [])), + "loading_checks": + len(result["restart"].get("loading_checks", [])), + "comparisons": + len(result["restart"]["comparisons"]), + "failed_comparisons": + sum(not item["pass"] for item in result["restart"]["comparisons"]), + "report": + str(Path(result["directory"]) / "restart.json") + } for result in results if "restart" in result] comparisons = compare_executions( results, set(load_yaml(args.config)["static_fields"])) summary["comparisons"] = comparisons diff --git a/tests/regression/tests/python_harness_test.py b/tests/regression/tests/python_harness_test.py index 15e4213..3ebff60 100644 --- a/tests/regression/tests/python_harness_test.py +++ b/tests/regression/tests/python_harness_test.py @@ -14,8 +14,10 @@ from __future__ import annotations import json +import datetime as dt import os from pathlib import Path +import shutil import sys import tempfile from types import SimpleNamespace @@ -35,6 +37,8 @@ from render_namelist import render_namelist from run_case import run_case, write_esmx_config from run_suite import select_tests +from restart import (check_checkpoint, checkpoint_fields, compare_checkpoint, + run_restarts, segment_names) # CTest supplies a directory under the build tree. Direct Python invocations # must supply CFBM_TEST_TMP, so artifacts never go to a developer's home/scratch. @@ -382,6 +386,187 @@ def test_selection_uses_exact_names(self) -> None: args.test = "terrain.*" self.assertEqual(select_tests(tests, args), []) + #------------------------------------------------------------------------ + # Restart configuration, precision, and segmentation + #------------------------------------------------------------------------ + + def test_restart_registration_and_phase(self) -> None: + """Select standalone restart cases without changing existing PR matrices.""" + records = registrations(self.document, "mpi", + ["standalone", "nuopc", "esmx"]) + selected = [r for r in records if "restart" in r["labels"]] + self.assertEqual(len(selected), 6) + self.assertEqual({r["driver"] for r in selected}, {"standalone"}) + self.assertTrue(all("pr" not in r["labels"] for r in selected)) + spec = resolve_spec(self.document, + "terrain", + configuration="restart_time") + self.assertEqual(segment_names(spec, 28, 60), [ + "fire_output_2020-01-01_00:00:36.nc", + "fire_output_2020-01-01_00:00:48.nc", + "fire_output_2020-01-01_00:01:00.nc" + ]) + self.assertEqual(segment_names(spec, 0, 4), + ["fire_output_2020-01-01_00:00:00.nc"]) + + def test_invalid_restart_splits_fail(self) -> None: + """Reject absent checkpoints, unordered splits, and invalid endpoints.""" + for values in ([], [0], [60], [5], [12, 4], [4, 4], [True], [4.0]): + with self.subTest(values=values): + self.document["cases"]["terrain"]["configurations"][ + "restart_time"]["restart_splits"] = values + with self.assertRaises(ValueError): + resolve_spec(self.document, + "terrain", + configuration="restart_time") + + def _checkpoint(self, + path: Path, + spec: dict, + seconds: int, + fuel_dtype: str = "f8") -> None: + """Write independent small physical-state fixtures with REAL64 signals.""" + with netCDF4.Dataset(path, "w") as dataset: + for name, size in (("nx", 2), ("ny", 2), ("moisture_class", 5)): + dataset.createDimension(name, size) + for name in checkpoint_fields(spec): + dimensions = ("ny", "moisture_class", + "nx") if name == "fmc_gc" else ("ny", "nx") + kind = fuel_dtype if name in { + "fuel_frac", "fuel_frac_burnt_dt" + } else "f4" + dataset.createVariable(name, kind, + dimensions)[:] = 0.123456789012345 + origin = dt.datetime(2020, 1, 1) + for prefix, date in (("start", origin), + ("restart", + origin + dt.timedelta(seconds=seconds))): + for part in ("year", "month", "day", "hour", "minute", + "second"): + setattr(dataset, f"{prefix}_{part}", getattr(date, part)) + dataset.itimestep = seconds // 4 + dataset.fire_perimeter_ignited = int( + spec["namelist"]["fire"]["fire_is_real_perim"] and seconds >= 8) + fire = spec["namelist"]["fire"] + dataset.fmc_initialized = 1 + dataset.fmc_initialization_time = 4.0 + dataset.fmoist_freq = fire["fmoist_freq"] + dataset.fmoist_dt = fire["fmoist_dt"] + if fire["fmoist_freq"]: + interval = fire["fmoist_freq"] * 4 + dataset.fmc_fmoist_lasttime = max( + 4, seconds // interval * interval) + dataset.fmc_fmoist_nexttime = 0.0 + else: + last = 4 + (seconds - + 4) // fire["fmoist_dt"] * fire["fmoist_dt"] + dataset.fmc_fmoist_lasttime = last + dataset.fmc_fmoist_nexttime = last + fire["fmoist_dt"] + + def test_checkpoint_detects_real64_loss(self) -> None: + """Detect both float32 storage and quantization hidden in float64 storage.""" + spec = resolve_spec(self.document, + "circle", + configuration="restart_prescribed") + for kind in ("f4", "f8"): + path = self.root / f"{kind}.nc" + self._checkpoint(path, spec, 60, kind) + check = check_checkpoint(path, spec, 60) + self.assertEqual(check["pass"], kind == "f8", check) + path = self.root / "f8.nc" + with netCDF4.Dataset(path, "a") as dataset: + dataset["fuel_frac"][:] = dataset["fuel_frac"][:].astype("f4") + self.assertFalse(check_checkpoint(path, spec, 60)["pass"]) + + def test_checkpoint_rejects_state_and_metadata_damage(self) -> None: + """Require class state, clock origin, activation flag, and exact fuel bits.""" + spec = resolve_spec(self.document, + "terrain", + configuration="restart_time") + reference = self.root / "reference.nc" + self._checkpoint(reference, spec, 12) + self.assertTrue(check_checkpoint(reference, spec, 12)["pass"]) + for defect in ("fuel_bit", "perimeter", "clock", "class_zero", + "class_nan", "moisture_time", "uninitialized", + "init_time"): + test = self.root / f"{defect}.nc" + shutil.copy2(reference, test) + with netCDF4.Dataset(test, "a") as dataset: + if defect == "fuel_bit": + value = dataset["fuel_frac"][0, 0] + dataset["fuel_frac"][0, 0] = np.nextafter(value, np.inf) + elif defect == "perimeter": + dataset.fire_perimeter_ignited = 0 + elif defect == "clock": + dataset.start_second = 4 + elif defect == "class_zero": + dataset["fmc_gc"][:] = 0 + elif defect == "class_nan": + dataset["fmc_gc"][:] = np.nan + elif defect == "uninitialized": + dataset.fmc_initialized = 0 + elif defect == "init_time": + dataset.fmc_initialization_time = 0.0 + else: + dataset.fmc_fmoist_nexttime = 999.0 + self.assertFalse( + compare_checkpoint(reference, test, spec)["pass"], defect) + if defect != "fuel_bit": + self.assertFalse( + check_checkpoint(test, spec, 12)["pass"], defect) + self.assertFalse( + compare_checkpoint(reference, self.root / "absent.nc", + spec)["pass"]) + + def test_restart_chain_preserves_forcing_and_uses_previous_segment( + self) -> None: + """Exercise staging with a synthetic launcher and reject an extra output.""" + spec = resolve_spec(self.document, + "terrain", + configuration="restart_time") + for extra in (False, True): + root = self.root / str(extra) + root.mkdir() + (root / "wrf.nc").write_bytes(b"original full-duration forcing") + for seconds in range(4, 61, 4): + name = f"fire_restart_2020-01-01_00:{seconds // 60:02d}:{seconds % 60:02d}.nc" + self._checkpoint(root / name, spec, seconds) + for name in segment_names(spec, 0, 60): + with netCDF4.Dataset(root / name, "w") as dataset: + dataset.createDimension("nx", 1) + dataset.createVariable("field", "f4", ("nx",))[:] = 1.0 + calls = [] + + def launch(command: list, directory: Path, environment: dict, + timeout: int, driver: str) -> dict: + """Emulate only file writes; no model process is launched.""" + begin, end = [(0, 4), (4, 12), (12, 28), (28, 60)][len(calls)] + settings = json.loads((directory / "settings.json").read_text()) + self.assertEqual((directory / "wrf.nc").read_bytes(), + (root / "wrf.nc").read_bytes()) + if calls: + self.assertEqual( + Path(settings["restart_source"]).parent, calls[-1]) + self.assertIn("restart=.true.", + (directory / "namelist.fire").read_text()) + for checkpoint in (False, True): + for name in segment_names(spec, begin, end, checkpoint): + shutil.copy2(root / name, directory / name) + if extra: + (directory / + "fire_output_unexpected.nc").write_bytes(b"unexpected") + calls.append(directory) + return {"reasons": [], "exit_status": 0} + + # Loader rejection probes are exercised by model CTests, separately + # from this synthetic segment-staging check. + with patch("run_case.run_model", side_effect=launch), patch( + "restart.check_moisture_restart_loading", return_value=[]): + result = run_restarts(spec, root, ["fixture"], {}, set()) + self.assertEqual(result["pass"], not extra, result["reasons"]) + self.assertEqual(len(calls), 4) + self.assertEqual(len(result["comparisons"]), 21) + #------------------------------------------------------------------------ # Stored reference integrity and approval #------------------------------------------------------------------------