From d0f673fc3c5e3dbb84c3791089df82dfd6674a0c Mon Sep 17 00:00:00 2001 From: masih Date: Wed, 26 Aug 2026 11:54:49 -0600 Subject: [PATCH 01/19] Add restart capability to the fire behavior model Lets a simulation write its dynamic fire state to a NetCDF restart file at a fixed interval and resume from one of those files, for standalone runs (serial and MPI) and for runs coupled to WRF. Namelist (time block): restart - resume from fire_restart_.nc restart_interval - restart write interval in seconds, -1 disables state_mod: Read_restart and Write_restart type-bound methods, plus Build_restart_file_name and the MPI helpers they need. A restart read validates the grid metadata stored in the file against the current configuration before touching any state, so a mismatched restart file fails early instead of producing a silently wrong run. netcdf_mod: Attribute writers (Add_netcdf_att) and MPI-aware readers and writers for distributed 2D and 3D fields. initialize_mod: Init_fire_state_from_restart takes the real-case restart path, so the existing geogrid initialization is left as is. The WRF-coupled path reads the restart after the fire components are initialized. fire_behavior: Reads the restart state at startup instead of Save_state, and writes a restart every restart_interval seconds. restart_interval must map to a whole number of time steps or the run stops with a message. fmc_wrffire_mod: Drops the unused state_mod and namelist_mod imports. state_mod now uses fmc_wrffire_mod for the moisture restart fields, so these imports would otherwise form a circular module dependency. The code was created with assistance from Codex GPT-5.5 and Claude Opus 4.7. Co-authored-by: mefrediani --- driver/fire_behavior.F90 | 40 +- driver/initialize_mod.F90 | 38 +- io/namelist_mod.F90 | 18 +- io/netcdf_mod.F90 | 313 +++++++++++- physics/fmc_wrffire_mod.F90 | 2 - state/state_mod.F90 | 919 +++++++++++++++++++++++++++++++++++- 6 files changed, 1311 insertions(+), 19 deletions(-) diff --git a/driver/fire_behavior.F90 b/driver/fire_behavior.F90 index 0510bbf6..ed1d2ab2 100644 --- a/driver/fire_behavior.F90 +++ b/driver/fire_behavior.F90 @@ -8,15 +8,22 @@ program fire_behavior use initialize_mod, only : Init_fire_state, Init_atm_state use advance_mod, only : Advance_state use wrfdata_mod, only : wrfdata_t + use datetime_mod, only : datetime_t + use stderrout_mod, only : Stop_simulation use, intrinsic :: iso_fortran_env, only : ERROR_UNIT, OUTPUT_UNIT implicit none integer :: ierr, rank, mpi_comm_cfbm + integer :: restart_step_interval + real :: restart_interval_seconds, restart_interval_error type (state_fire_t) :: grid type (wrfdata_t) :: atm_state type (namelist_t) :: config_flags + type (datetime_t) :: datetime_check + character (len = 256) :: msg logical, parameter :: DEBUG_LOCAL = .false. + real, parameter :: RESTART_INTERVAL_TOL = 1.0e-6 if (DEBUG_LOCAL) write (OUTPUT_UNIT, *) 'Running fire_behavior...' @@ -71,14 +78,43 @@ program fire_behavior end select - if (DEBUG_LOCAL) write (OUTPUT_UNIT, *) ' Saving fire state...' - call grid%Save_state () + if (config_flags%restart) then + if (DEBUG_LOCAL) write (OUTPUT_UNIT, *) ' Reading restart state...' + call grid%Read_restart (config_flags) + + else + if (DEBUG_LOCAL) write (OUTPUT_UNIT, *) ' Saving fire state...' + call grid%Save_state () + end if + + restart_step_interval = -1 + if (config_flags%restart_interval > 0) then + restart_step_interval = nint (real (config_flags%restart_interval) / grid%dt) + restart_interval_seconds = restart_step_interval * grid%dt + restart_interval_error = abs (restart_interval_seconds - real (config_flags%restart_interval)) + + if (restart_step_interval <= 0 .or. & + restart_interval_error > max (RESTART_INTERVAL_TOL, abs (real (config_flags%restart_interval)) * RESTART_INTERVAL_TOL)) then + write (msg, '(a, i0, a, f12.6)') 'restart_interval must map to an integer number of time steps: restart_interval = ', & + config_flags%restart_interval, ', dt = ', grid%dt + call Stop_simulation (msg) + end if + end if if (DEBUG_LOCAL) write (OUTPUT_UNIT, *) ' Starting temporal loop...' do while (grid%datetime_now < grid%datetime_end) call Advance_state (grid, config_flags) + + datetime_check = grid%datetime_start + call datetime_check%Add_seconds (grid%itimestep * grid%dt) + if (datetime_check /= grid%datetime_now) call Stop_simulation ('Model clock is inconsistent with itimestep and dt') + call grid%Handle_output (config_flags) if (config_flags%ideal_opt == 0) call grid%Handle_wrfdata_update (atm_state, config_flags) + + if (restart_step_interval > 0) then + if (mod (grid%itimestep, restart_step_interval) == 0) call grid%Write_restart (config_flags) + end if end do if (DEBUG_LOCAL) write (OUTPUT_UNIT, *) ' Completed temporal loop' diff --git a/driver/initialize_mod.F90 b/driver/initialize_mod.F90 index 9feaa95d..23869f43 100644 --- a/driver/initialize_mod.F90 +++ b/driver/initialize_mod.F90 @@ -1,10 +1,11 @@ module initialize_mod - use state_mod, only : state_fire_t + use state_mod, only : state_fire_t, Build_restart_file_name use namelist_mod, only : namelist_t use geogrid_mod, only : geogrid_t use wrfdata_mod, only : wrfdata_t use fire_driver_mod, only : Init_fire_components + use datetime_mod, only : datetime_t use stderrout_mod, only: Print_message, Stop_simulation #ifdef DM_PARALLEL use mpi_mod, only : Convert_mpi_comm_to_f08 @@ -59,7 +60,11 @@ subroutine Init_fire_state (grid, config_flags, wrf) if (DEBUG_LOCAL) call Print_message (' Entering subroutine Init_state') if (DEBUG_LOCAL) call Print_message (' Initialization...') - if (config_flags%ideal_opt == 0) then + if (config_flags%ideal_opt == 0 .and. config_flags%restart) then + ! Real world, restart run: the grid metadata comes from the restart file + call Init_fire_state_from_restart (grid, config_flags) + + else if (config_flags%ideal_opt == 0) then ! Real world if (DEBUG_LOCAL) call Print_message (' Reading geogrid file') #ifdef DM_PARALLEL @@ -110,7 +115,7 @@ subroutine Init_fire_state (grid, config_flags, wrf) call Init_fire_components (grid, config_flags) - if (present (wrf)) then + if (present (wrf) .and. .not. config_flags%restart) then if (DEBUG_LOCAL) call Print_message (' Initializing atmospheric state') call grid%Handle_wrfdata_update (wrf, config_flags) end if @@ -143,6 +148,31 @@ subroutine Init_fire_state (grid, config_flags, wrf) end subroutine Init_fire_state + subroutine Init_fire_state_from_restart (grid, config_flags) + + implicit none + + type (state_fire_t), intent (in out) :: grid + type (namelist_t), intent (in) :: config_flags + + type (datetime_t) :: datetime_restart + character (len = :), allocatable :: file_restart + logical, parameter :: DEBUG_LOCAL = .false. + + + if (DEBUG_LOCAL) call Print_message (' Entering subroutine Init_fire_state_from_restart') + + datetime_restart = datetime_t (config_flags%start_year, config_flags%start_month, config_flags%start_day, & + config_flags%start_hour, config_flags%start_minute, config_flags%start_second) + file_restart = Build_restart_file_name (datetime_restart%datetime) + + if (DEBUG_LOCAL) call Print_message (' Initializing fire state from restart metadata') + call grid%Initialization (config_flags, restart_file = file_restart) + + if (DEBUG_LOCAL) call Print_message (' Leaving subroutine Init_fire_state_from_restart') + + end subroutine Init_fire_state_from_restart + subroutine Init_fire_state_within_wrf (state, config_flags, & ifds, ifde, ifms, ifme, ifps, ifpe, & jfds, jfde, jfms, jfme, jfps, jfpe, & @@ -185,6 +215,8 @@ subroutine Init_fire_state_within_wrf (state, config_flags, & call Init_fire_components (state, config_flags) + if (config_flags%restart) call state%Read_restart (config_flags) + if (DEBUG_LOCAL) call Print_message (' Leaving subroutine Init_fire_state_within_wrf...') end subroutine Init_fire_state_within_wrf diff --git a/io/namelist_mod.F90 b/io/namelist_mod.F90 index 05b23905..9da188b8 100644 --- a/io/namelist_mod.F90 +++ b/io/namelist_mod.F90 @@ -38,6 +38,10 @@ module namelist_mod integer :: interval_output = -1 ! Frequency to save the output [s] real :: dt = 2.0 ! Time step of the fire model [s] + ! Restart controls + logical :: restart = .false. ! Read initial dynamic fire state from fire_restart_.nc + integer :: restart_interval = -1 ! Restart output interval [s]; -1 disables restart writes + integer :: num_tiles = 1 ! Number of tiles in each patch integer :: tile_strategy = 0 ! Strategy for the tile decomposition: 0) ... @@ -215,6 +219,8 @@ subroutine Broadcast_nml (this, mpi_comm_cfbm) call Broadcast_integer (this%ideal_opt) call Broadcast_integer (this%devel_opt) + call Broadcast_logical (this%restart) + call Broadcast_integer (this%restart_interval) call Broadcast_integer (this%fuel_opt) call Broadcast_integer (this%ros_opt) call Broadcast_integer (this%emis_opt) @@ -383,6 +389,9 @@ subroutine Check_nml (this) if (this%ideal_opt /= 0 .and. this%fmoist_run) & call Stop_simulation ('ideal runs do not support a FMC model') + if (this%restart_interval /= -1 .and. this%restart_interval <= 0) & + call Stop_simulation ('restart_interval must be positive or -1') + end subroutine Check_nml subroutine Init_atm_block (this, file_name) @@ -765,15 +774,16 @@ subroutine Init_time_block (this, file_name) integer :: start_year, start_month, start_day, start_hour, start_minute, start_second, & end_year, end_month, end_day, end_hour, end_minute, end_second, interval_output, & - num_tiles, tile_strategy + restart_interval, num_tiles, tile_strategy real :: dt + logical :: restart character (len = :), allocatable :: msg integer :: unit_nml, io_stat namelist /time/ start_year, start_month, start_day, start_hour, start_minute, start_second, & end_year, end_month, end_day, end_hour, end_minute, end_second, dt, interval_output, & - num_tiles + restart, restart_interval, num_tiles ! Set default values @@ -791,6 +801,8 @@ subroutine Init_time_block (this, file_name) end_second = this%end_second dt = this%dt interval_output = this%interval_output + restart = this%restart + restart_interval = this%restart_interval num_tiles = this%num_tiles tile_strategy = this%tile_strategy @@ -822,6 +834,8 @@ subroutine Init_time_block (this, file_name) this%end_second = end_second this%dt = dt this%interval_output = interval_output + this%restart = restart + this%restart_interval = restart_interval this%num_tiles = num_tiles this%tile_strategy = tile_strategy diff --git a/io/netcdf_mod.F90 b/io/netcdf_mod.F90 index 50a73a66..1bb38e39 100644 --- a/io/netcdf_mod.F90 +++ b/io/netcdf_mod.F90 @@ -8,8 +8,13 @@ module netcdf_mod private character (len = 2), parameter :: NAME_DIM_X = 'nx', NAME_DIM_Y = 'ny' - public :: Get_netcdf_var, Get_netcdf_att, Get_netcdf_dim, Create_netcdf_file, Add_netcdf_dim, Add_netcdf_var, & - Is_netcdf_file_present, Is_netcdf_var_present, Add_netcdf_var_mpi, NAME_DIM_X, NAME_DIM_Y + public :: Get_netcdf_var, Get_netcdf_att, Get_netcdf_dim, Create_netcdf_file, Add_netcdf_dim, Add_netcdf_var, Add_netcdf_att, & + Is_netcdf_file_present, Is_netcdf_var_present, Add_netcdf_var_mpi, Get_netcdf_var_mpi, NAME_DIM_X, NAME_DIM_Y + + interface Add_netcdf_att + module procedure Add_netcdf_att_int32 + module procedure Add_netcdf_att_real32 + end interface Add_netcdf_att interface Add_netcdf_var module procedure Add_netcdf_var_real32_2d @@ -18,6 +23,7 @@ module netcdf_mod interface Add_netcdf_var_mpi module procedure Add_netcdf_var_real32_2d_mpi + module procedure Add_netcdf_var_real32_3d_mpi end interface Add_netcdf_var_mpi interface Get_netcdf_var @@ -28,6 +34,11 @@ module netcdf_mod module procedure Get_netcdf_var_real32_4d end interface Get_netcdf_var + interface Get_netcdf_var_mpi + module procedure Get_netcdf_var_real32_2d_mpi + module procedure Get_netcdf_var_real32_3d_mpi + end interface Get_netcdf_var_mpi + interface Get_netcdf_att module procedure Get_netcdf_att_int32 module procedure Get_netcdf_att_real32 @@ -35,6 +46,84 @@ module netcdf_mod contains + subroutine Add_netcdf_att_int32 (file_name, var_name, att_name, att_value) + + use netcdf + + use, intrinsic :: iso_fortran_env, only : INT32 + + implicit none + + character (len = *), intent(in) :: file_name, var_name, att_name + integer (kind = INT32), intent(in) :: att_value + + integer :: status, ncid, varid + + + status = nf90_open (trim(file_name), NF90_WRITE, ncid) + call Check_status (status) + + status = nf90_redef (ncid) + call Check_status (status) + + if (var_name == 'global') then + status = nf90_put_att (ncid, NF90_GLOBAL, att_name, att_value) + call Check_status (status) + else + status = nf90_inq_varid (ncid, var_name, varid) + call Check_status (status) + + status = nf90_put_att (ncid, varid, att_name, att_value) + call Check_status (status) + end if + + status = nf90_enddef (ncid) + call Check_status (status) + + status = nf90_close (ncid) + call Check_status (status) + + end subroutine Add_netcdf_att_int32 + + subroutine Add_netcdf_att_real32 (file_name, var_name, att_name, att_value) + + use netcdf + + use, intrinsic :: iso_fortran_env, only : REAL32 + + implicit none + + character (len = *), intent(in) :: file_name, var_name, att_name + real (kind = REAL32), intent(in) :: att_value + + integer :: status, ncid, varid + + + status = nf90_open (trim(file_name), NF90_WRITE, ncid) + call Check_status (status) + + status = nf90_redef (ncid) + call Check_status (status) + + if (var_name == 'global') then + status = nf90_put_att (ncid, NF90_GLOBAL, att_name, att_value) + call Check_status (status) + else + status = nf90_inq_varid (ncid, var_name, varid) + call Check_status (status) + + status = nf90_put_att (ncid, varid, att_name, att_value) + call Check_status (status) + end if + + status = nf90_enddef (ncid) + call Check_status (status) + + status = nf90_close (ncid) + call Check_status (status) + + end subroutine Add_netcdf_att_real32 + subroutine Add_netcdf_dim (file_name, name_dim, val_dim) use netcdf @@ -147,6 +236,128 @@ subroutine Add_netcdf_var_real32_2d_mpi (file_name, cfbm_comm, nx, ny, ifps, ifp end subroutine Add_netcdf_var_real32_2d_mpi + subroutine Get_netcdf_var_real32_2d_mpi (file_name, cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, var_name, var2d_local) + +#ifdef DM_PARALLEL + use mpi +#endif + implicit none + + character (len = *), intent (in) :: file_name, var_name + integer, intent (in) :: cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe + real, dimension(ifps:ifpe, jfps:jfpe), intent (out) :: var2d_local + + real, dimension(:, :), allocatable :: var2d_restart + character (len = :), allocatable :: msg + integer :: rank, ierr + logical, parameter :: DEBUG_LOCAL = .false. + + + if (DEBUG_LOCAL) call Print_message ('Entering Get_netcdf_var_real32_2d_mpi...') + +#ifdef DM_PARALLEL + call Mpi_comm_rank (cfbm_comm, rank, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_comm_rank ') + + if (rank == 0) then + call Get_netcdf_var (file_name, var_name, var2d_restart) + if (size (var2d_restart, 1) /= nx .or. size (var2d_restart, 2) /= ny) then + msg = 'Restart variable has unexpected dimensions: '//trim (var_name) + call Stop_simulation (msg) + end if + end if + + call Distribute_global_var2d (cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, var2d_restart, var2d_local) +#else + call Get_netcdf_var (file_name, var_name, var2d_restart) + if (size (var2d_restart, 1) /= nx .or. size (var2d_restart, 2) /= ny) then + msg = 'Restart variable has unexpected dimensions: '//trim (var_name) + call Stop_simulation (msg) + end if + var2d_local(1:nx, 1:ny) = var2d_restart(1:nx, 1:ny) +#endif + + if (allocated (var2d_restart)) deallocate (var2d_restart) + + if (DEBUG_LOCAL) call Print_message ('Leaving Get_netcdf_var_real32_2d_mpi...') + + end subroutine Get_netcdf_var_real32_2d_mpi + + subroutine Distribute_global_var2d (cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, var2d_global, var2d_local) + +#ifdef DM_PARALLEL + use mpi +#endif + implicit none + + integer, intent (in) :: cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe + real, dimension(:, :), allocatable, intent (in out) :: var2d_global + real, dimension(ifps:ifpe, jfps:jfpe), intent (out) :: var2d_local + + integer :: i, ierr, j, kbuf, nprocs, nx_local, ny_local, r, rank + integer, dimension(:), allocatable :: all_ifps, all_ifpe, all_jfps, all_jfpe, displs, sendcounts + real, dimension(:), allocatable :: sendbuf + + +#ifdef DM_PARALLEL + call Mpi_comm_rank (cfbm_comm, rank, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_comm_rank ') + call Mpi_comm_size (cfbm_comm, nprocs, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems getting the number of MPI tasks') + + nx_local = ifpe - ifps + 1 + ny_local = jfpe - jfps + 1 + + if (rank == 0) then + allocate (all_ifps(nprocs), all_ifpe(nprocs), all_jfps(nprocs), all_jfpe(nprocs)) + else + allocate (all_ifps(1), all_ifpe(1), all_jfps(1), all_jfpe(1)) + end if + + call MPI_Gather (ifps, 1, MPI_INTEGER, all_ifps, 1, MPI_INTEGER, 0, cfbm_comm, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with MPI_Gather for restart ifps') + call MPI_Gather (ifpe, 1, MPI_INTEGER, all_ifpe, 1, MPI_INTEGER, 0, cfbm_comm, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with MPI_Gather for restart ifpe') + call MPI_Gather (jfps, 1, MPI_INTEGER, all_jfps, 1, MPI_INTEGER, 0, cfbm_comm, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with MPI_Gather for restart jfps') + call MPI_Gather (jfpe, 1, MPI_INTEGER, all_jfpe, 1, MPI_INTEGER, 0, cfbm_comm, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with MPI_Gather for restart jfpe') + + if (rank == 0) then + allocate (sendcounts(nprocs), displs(nprocs)) + do r = 1, nprocs + sendcounts(r) = (all_ifpe(r) - all_ifps(r) + 1) * (all_jfpe(r) - all_jfps(r) + 1) + end do + + displs(1) = 0 + do r = 2, nprocs + displs(r) = displs(r - 1) + sendcounts(r - 1) + end do + + allocate (sendbuf(sum (sendcounts))) + kbuf = 0 + do r = 1, nprocs + do j = all_jfps(r), all_jfpe(r) + do i = all_ifps(r), all_ifpe(r) + kbuf = kbuf + 1 + sendbuf(kbuf) = var2d_global(i, j) + end do + end do + end do + else + allocate (sendcounts(1), displs(1), sendbuf(1)) + end if + + call MPI_Scatterv (sendbuf, sendcounts, displs, MPI_REAL, var2d_local, nx_local * ny_local, MPI_REAL, 0, cfbm_comm, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with MPI_Scatterv for restart variable') + + deallocate (all_ifps, all_ifpe, all_jfps, all_jfpe, sendcounts, displs, sendbuf) +#else + var2d_local(1:nx, 1:ny) = var2d_global(1:nx, 1:ny) +#endif + + end subroutine Distribute_global_var2d + subroutine Add_netcdf_var_real32_3d (file_name, name_dims, varname, var) use netcdf @@ -190,6 +401,104 @@ subroutine Add_netcdf_var_real32_3d (file_name, name_dims, varname, var) end subroutine Add_netcdf_var_real32_3d + subroutine Add_netcdf_var_real32_3d_mpi (file_name, name_dims, cfbm_comm, nx, ny, n2, ifps, ifpe, jfps, jfpe, varname, var3d_local) + +#ifdef DM_PARALLEL + use mpi +#endif + implicit none + + character (len = *), intent (in) :: file_name, varname + character (len = *), dimension(:), intent (in) :: name_dims + integer, intent (in) :: cfbm_comm, nx, ny, n2, ifps, ifpe, jfps, jfpe + real, dimension(ifps:ifpe, n2, jfps:jfpe), intent (in) :: var3d_local + + real, dimension(ifps:ifpe, jfps:jfpe) :: var2d_local + real, dimension(nx, ny) :: var2d_global + real, dimension(:, :, :), allocatable :: var3d_global + integer :: rank, ierr, k + logical, parameter :: DEBUG_LOCAL = .false. + + + if (DEBUG_LOCAL) call Print_message ('Entering Add_netcdf_var_real32_3d_mpi...') + +#ifdef DM_PARALLEL + call Mpi_comm_rank (cfbm_comm, rank, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_comm_rank ') + + if (rank == 0) allocate (var3d_global(nx, n2, ny)) + do k = 1, n2 + var2d_local(ifps:ifpe, jfps:jfpe) = var3d_local(ifps:ifpe, k, jfps:jfpe) + call Gather_var2d (cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, var2d_local, var2d_global) + if (rank == 0) var3d_global(1:nx, k, 1:ny) = var2d_global(1:nx, 1:ny) + end do + + if (rank == 0) call Add_netcdf_var (file_name, name_dims, varname, var3d_global) +#else + call Add_netcdf_var (file_name, name_dims, varname, var3d_local(1:nx, 1:n2, 1:ny)) +#endif + + if (allocated (var3d_global)) deallocate (var3d_global) + + if (DEBUG_LOCAL) call Print_message ('Leaving Add_netcdf_var_real32_3d_mpi...') + + end subroutine Add_netcdf_var_real32_3d_mpi + + subroutine Get_netcdf_var_real32_3d_mpi (file_name, cfbm_comm, nx, ny, n2, ifps, ifpe, jfps, jfpe, var_name, var3d_local) + +#ifdef DM_PARALLEL + use mpi +#endif + implicit none + + character (len = *), intent (in) :: file_name, var_name + integer, intent (in) :: cfbm_comm, nx, ny, n2, ifps, ifpe, jfps, jfpe + real, dimension(ifps:ifpe, n2, jfps:jfpe), intent (out) :: var3d_local + + real, dimension(:, :), allocatable :: var2d_restart + real, dimension(:, :, :), allocatable :: var3d_restart + character (len = :), allocatable :: msg + integer :: rank, ierr, k + logical, parameter :: DEBUG_LOCAL = .false. + + + if (DEBUG_LOCAL) call Print_message ('Entering Get_netcdf_var_real32_3d_mpi...') + +#ifdef DM_PARALLEL + call Mpi_comm_rank (cfbm_comm, rank, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_comm_rank ') + + if (rank == 0) then + call Get_netcdf_var (file_name, var_name, var3d_restart) + if (size (var3d_restart, 1) /= nx .or. size (var3d_restart, 2) /= n2 .or. size (var3d_restart, 3) /= ny) then + msg = 'Restart variable has unexpected dimensions: '//trim (var_name) + call Stop_simulation (msg) + end if + end if + + do k = 1, n2 + if (rank == 0) then + allocate (var2d_restart(nx, ny)) + var2d_restart(1:nx, 1:ny) = var3d_restart(1:nx, k, 1:ny) + end if + call Distribute_global_var2d (cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, var2d_restart, var3d_local(ifps:ifpe, k, jfps:jfpe)) + if (allocated (var2d_restart)) deallocate (var2d_restart) + end do +#else + call Get_netcdf_var (file_name, var_name, var3d_restart) + if (size (var3d_restart, 1) /= nx .or. size (var3d_restart, 2) /= n2 .or. size (var3d_restart, 3) /= ny) then + msg = 'Restart variable has unexpected dimensions: '//trim (var_name) + call Stop_simulation (msg) + end if + var3d_local(1:nx, 1:n2, 1:ny) = var3d_restart(1:nx, 1:n2, 1:ny) +#endif + + if (allocated (var3d_restart)) deallocate (var3d_restart) + + if (DEBUG_LOCAL) call Print_message ('Leaving Get_netcdf_var_real32_3d_mpi...') + + end subroutine Get_netcdf_var_real32_3d_mpi + subroutine Check_status (status) diff --git a/physics/fmc_wrffire_mod.F90 b/physics/fmc_wrffire_mod.F90 index 7110ac94..252f0c50 100644 --- a/physics/fmc_wrffire_mod.F90 +++ b/physics/fmc_wrffire_mod.F90 @@ -15,8 +15,6 @@ module fmc_wrffire_mod use stderrout_mod, only : Stop_simulation use ros_mod, only : ros_t - use state_mod, only: state_fire_t - use namelist_mod, only : namelist_t use fmc_mod, only : fmc_t use fuel_mod, only : fuel_t diff --git a/state/state_mod.F90 b/state/state_mod.F90 index 1738359c..c81ca6cb 100644 --- a/state/state_mod.F90 +++ b/state/state_mod.F90 @@ -6,25 +6,34 @@ module state_mod use constants_mod, only : PI use datetime_mod, only : datetime_t use fmc_mod, only : fmc_t + use fmc_wrffire_mod, only : fmc_wrffire_t use fuel_mod, only : fuel_t, FUEL_ANDERSON, Crosswalk_from_scottburgan_to_anderson use geogrid_mod, only : geogrid_t use ignition_line_mod, only : ignition_line_t use namelist_mod, only : namelist_t - use netcdf_mod, only : Create_netcdf_file, Add_netcdf_dim, Add_netcdf_var, Add_netcdf_var_mpi, NAME_DIM_X, NAME_DIM_Y + use netcdf_mod, only : Create_netcdf_file, Add_netcdf_att, Add_netcdf_dim, Add_netcdf_var_mpi, Get_netcdf_att, & + Get_netcdf_var_mpi, Is_netcdf_file_present, NAME_DIM_X, NAME_DIM_Y use proj_lc_mod, only : proj_lc_t use ros_mod, only : ros_t use stderrout_mod, only : Stop_simulation, Print_message use tiles_mod, only : Calc_tiles_dims use wrfdata_mod, only : wrfdata_t, G, RERADIUS - use mpi_mod, only : Calc_tasks_in_x_and_y, Calc_patch_dims, Distribute_var2d, Print_cart_info, topology_dim_order + use mpi_mod, only : Calc_tasks_in_x_and_y, Calc_patch_dims, Distribute_var2d, Do_halo_exchange_with_corners, & + Print_cart_info, topology_dim_order + use, intrinsic :: iso_fortran_env, only : INT32, REAL32 implicit none private - public :: state_fire_t, N_POINTS_IN_HALO + public :: state_fire_t, N_POINTS_IN_HALO, Build_restart_file_name integer, parameter :: N_POINTS_IN_HALO = 5, N_DIMS = 2 + integer, parameter :: RESTART_IO_ROOT = 0 + character (len = *), parameter :: NAME_DIM_MOISTURE_CLASS = 'moisture_class' + character (len = *), parameter :: NAME_VAR_FMC_GC = 'fmc_gc' + character (len = *), parameter :: NAME_ATT_FMOIST_LASTTIME = 'fmc_fmoist_lasttime' + character (len = *), parameter :: NAME_ATT_FMOIST_NEXTTIME = 'fmc_fmoist_nexttime' logical, dimension(2), parameter :: PERIODS = [ .false., .false. ] logical, parameter :: REORDER = .true. ! Allow MPI recording tasks for performance @@ -129,10 +138,12 @@ module state_mod procedure :: Interpolate_vars_atm_to_fire => Interpolate_vars_atm_to_fire procedure, public :: Print => Print_domain ! private procedure, public :: Print_tiles => Print_tiles + procedure, public :: Read_restart => Read_restart procedure, public :: Save_state => Save_state procedure, public :: Set_vars_to_default => Set_vars_to_default procedure, public :: Set_mpi_comm_cfbm => Set_mpi_comm_cfbm procedure, public :: Set_time_stamps => Set_time_stamps + procedure, public :: Write_restart => Write_restart end type state_fire_t contains @@ -302,7 +313,7 @@ subroutine Init_domain (this, config_flags, geogrid, & kfds, kfde, kfms, kfme, kfps, kfpe, & kfts, kfte, ide, jde, & cen_lat, cen_lon, truelat1, truelat2, stand_lon, & - dx, dy, sr_x, sr_y, nfuel_cat, zsf, dzdxf, dzdyf) + dx, dy, sr_x, sr_y, nfuel_cat, zsf, dzdxf, dzdyf, restart_file) implicit none @@ -310,6 +321,7 @@ subroutine Init_domain (this, config_flags, geogrid, & class (state_fire_t), intent(in out) :: this type (namelist_t), intent (in) :: config_flags type (geogrid_t), intent (in out), optional :: geogrid + character (len = *), intent (in), optional :: restart_file integer, intent (in), optional :: ifds, ifde, ifms, ifme, ifps, ifpe, & jfds, jfde, jfms, jfme, jfps, jfpe, & kfds, kfde, kfms, kfme, kfps, kfpe, & @@ -317,10 +329,15 @@ subroutine Init_domain (this, config_flags, geogrid, & real, intent (in), optional :: cen_lat, cen_lon, truelat1, truelat2, stand_lon, dx, dy real, dimension(:, :), intent (in), optional :: nfuel_cat, zsf, dzdxf, dzdyf - integer, parameter :: INIT_MODE_NONE = 0, INIT_MODE_GEOGRID = 1, INIT_MODE_WRF = 2, INIT_MODE_IDEAL = 3 + integer, parameter :: INIT_MODE_NONE = 0, INIT_MODE_GEOGRID = 1, INIT_MODE_WRF = 2, INIT_MODE_IDEAL = 3, INIT_MODE_RESTART = 4 type (proj_lc_t) :: proj logical, parameter :: DEBUG_LOCAL = .false. - integer :: ids0, ide0, jds0, jde0, i, j, init_mode, px, py, ntasks, ierr, cart_comm, rank, ips, ipe, jps, jpe, is_lfn_init_allocated + integer :: ids0, ide0, jds0, jde0, i, j, init_mode, px, py, ntasks, ierr, cart_comm, rank, ips, ipe, jps, jpe, & + is_lfn_init_allocated, restart_nx, restart_ny + integer :: restart_map_proj, restart_sr_x, restart_sr_y + integer (kind = INT32) :: att_int32 + real (kind = REAL32) :: att_real32 + real :: restart_dx, restart_dy, restart_cen_lat, restart_cen_lon, restart_stand_lon, restart_true_lat_1, restart_true_lat_2 integer, dimension(2) :: coords character (len = 300) :: msg @@ -330,6 +347,7 @@ subroutine Init_domain (this, config_flags, geogrid, & init_mode = INIT_MODE_NONE if (config_flags%ideal_opt == 1) init_mode = INIT_MODE_IDEAL if (present (geogrid)) init_mode = INIT_MODE_GEOGRID + if (present (restart_file)) init_mode = INIT_MODE_RESTART if (present (ifds) .and. present (ifde) .and. present (ifms) .and. present (ifme) .and. present (ifps) .and. present (ifpe) .and. & present (jfds) .and. present (jfde) .and. present (jfms) .and. present (jfme) .and. present (jfps) .and. present (jfpe) .and. & present (kfds) .and. present (kfde) .and. present (kfms) .and. present (kfme) .and. present (kfps) .and. present (kfpe) .and. & @@ -345,7 +363,7 @@ subroutine Init_domain (this, config_flags, geogrid, & ! Set dimensions if (DEBUG_LOCAL) call Print_message (' Setting dimensions...') Set_dims: select case (init_mode) - case (INIT_MODE_GEOGRID, INIT_MODE_IDEAL) + case (INIT_MODE_GEOGRID, INIT_MODE_IDEAL, INIT_MODE_RESTART) if (init_mode == INIT_MODE_GEOGRID) then @@ -434,6 +452,54 @@ subroutine Init_domain (this, config_flags, geogrid, & call Calc_patch_dims (config_flags%nx, config_flags%ny, this%px, this%py, coords, ips, ipe, jps, jpe) +#else + ips = ids0 + ipe = ide0 + jps = jds0 + jpe = jde0 +#endif + else if (init_mode == INIT_MODE_RESTART) then + + restart_nx = 0 + restart_ny = 0 + if (Is_restart_io_root (this)) then + call Get_netcdf_att (trim (restart_file), 'global', 'nx', att_int32) + restart_nx = att_int32 + call Get_netcdf_att (trim (restart_file), 'global', 'ny', att_int32) + restart_ny = att_int32 + end if + call Broadcast_restart_integer (this, restart_nx) + call Broadcast_restart_integer (this, restart_ny) + + ids0 = 1 + ide0 = restart_nx + jds0 = 1 + jde0 = restart_ny + +#ifdef DM_PARALLEL + if (.not. this%is_cfbm_comm_set) call Stop_simulation ('The MPI CFBM communicator has not been set') + + call Mpi_comm_size (this%cfbm_comm, ntasks, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems getting the number of MPI tasks') + this%ntasks = ntasks + + call Calc_tasks_in_x_and_y (this%ntasks, restart_nx, restart_ny, px, py) + this%px = px + this%py = py + write (msg, '(a25, 2(1x, i5))') 'MPI TASKS in x and y =', this%px, this%py + call Print_message (msg) + + call Mpi_cart_create (this%cfbm_comm, N_DIMS, [this%px, this%py], PERIODS, REORDER, cart_comm, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_cart_create') + this%cart_comm = cart_comm + + call Mpi_comm_rank (this%cart_comm, rank, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_comm_rank ') + + call Mpi_cart_coords (this%cart_comm, rank, N_DIMS, coords, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_cart_coords') + + call Calc_patch_dims (restart_nx, restart_ny, this%px, this%py, coords, ips, ipe, jps, jpe) #else ips = ids0 ipe = ide0 @@ -563,6 +629,73 @@ subroutine Init_domain (this, config_flags, geogrid, & call this%Init_latlons (proj) + case (INIT_MODE_RESTART) + restart_dx = 0.0 + restart_dy = 0.0 + restart_map_proj = 0 + restart_sr_x = 0 + restart_sr_y = 0 + restart_cen_lat = 0.0 + restart_cen_lon = 0.0 + restart_stand_lon = 0.0 + restart_true_lat_1 = 0.0 + restart_true_lat_2 = 0.0 + + if (Is_restart_io_root (this)) then + call Get_netcdf_att (trim (restart_file), 'global', 'dx', att_real32) + restart_dx = att_real32 + call Get_netcdf_att (trim (restart_file), 'global', 'dy', att_real32) + restart_dy = att_real32 + call Get_netcdf_att (trim (restart_file), 'global', 'map_proj', att_int32) + restart_map_proj = att_int32 + call Get_netcdf_att (trim (restart_file), 'global', 'sr_x', att_int32) + restart_sr_x = att_int32 + call Get_netcdf_att (trim (restart_file), 'global', 'sr_y', att_int32) + restart_sr_y = att_int32 + call Get_netcdf_att (trim (restart_file), 'global', 'cen_lat', att_real32) + restart_cen_lat = att_real32 + call Get_netcdf_att (trim (restart_file), 'global', 'cen_lon', att_real32) + restart_cen_lon = att_real32 + call Get_netcdf_att (trim (restart_file), 'global', 'stand_lon', att_real32) + restart_stand_lon = att_real32 + call Get_netcdf_att (trim (restart_file), 'global', 'true_lat_1', att_real32) + restart_true_lat_1 = att_real32 + call Get_netcdf_att (trim (restart_file), 'global', 'true_lat_2', att_real32) + restart_true_lat_2 = att_real32 + end if + + call Broadcast_restart_real (this, restart_dx) + call Broadcast_restart_real (this, restart_dy) + call Broadcast_restart_integer (this, restart_map_proj) + call Broadcast_restart_integer (this, restart_sr_x) + call Broadcast_restart_integer (this, restart_sr_y) + call Broadcast_restart_real (this, restart_cen_lat) + call Broadcast_restart_real (this, restart_cen_lon) + call Broadcast_restart_real (this, restart_stand_lon) + call Broadcast_restart_real (this, restart_true_lat_1) + call Broadcast_restart_real (this, restart_true_lat_2) + + this%dx = restart_dx + this%dy = restart_dy + + if (restart_map_proj /= 1) call Stop_simulation ('Restart map projection is not supported') + + if (restart_sr_x <= 0 .or. restart_sr_y <= 0) call Stop_simulation ('Restart subgrid ratios must be positive') + if (mod (this%nx, restart_sr_x) /= 0 .or. mod (this%ny, restart_sr_y) /= 0) & + call Stop_simulation ('Restart subgrid ratios do not divide fire-grid dimensions') + if (this%nx / restart_sr_x <= 1 .or. this%ny / restart_sr_y <= 1) & + call Stop_simulation ('Restart subgrid ratios imply invalid atmospheric dimensions') + + this%cen_lat = restart_cen_lat + this%cen_lon = restart_cen_lon + + proj = proj_lc_t (cen_lat = this%cen_lat , cen_lon = this%cen_lon, dx = this%dx * restart_sr_x, & + dy = this%dy * restart_sr_y, & + standard_lon = restart_stand_lon, true_lat_1 = restart_true_lat_1, & + true_lat_2 = restart_true_lat_2, nx = this%nx / restart_sr_x - 1, ny = this%ny / restart_sr_y - 1) + + call this%Init_latlons (proj, srx = restart_sr_x, sry = restart_sr_y) + case default call Stop_simulation ('Not ready to complete fire state initialization 2') @@ -613,12 +746,24 @@ subroutine Init_domain (this, config_flags, geogrid, & if (config_flags%fire_is_real_perim) & call Stop_simulation ('Not ready to initialize from fire perimeter in idealized mode') + case (INIT_MODE_RESTART) + call Read_restart_field_2d (trim (restart_file), 'zsf', this%nx, this%ny, & + this%ifms, this%ifme, this%jfms, this%jfme, this%ifps, this%ifpe, this%jfps, this%jfpe, this%cfbm_comm, this%zsf) + call Read_restart_field_2d (trim (restart_file), 'dzdxf', this%nx, this%ny, & + this%ifms, this%ifme, this%jfms, this%jfme, this%ifps, this%ifpe, this%jfps, this%jfpe, this%cfbm_comm, this%dzdxf) + call Read_restart_field_2d (trim (restart_file), 'dzdyf', this%nx, this%ny, & + this%ifms, this%ifme, this%jfms, this%jfme, this%ifps, this%ifpe, this%jfps, this%jfpe, this%cfbm_comm, this%dzdyf) + call Read_restart_field_2d (trim (restart_file), 'nfuel_cat', this%nx, this%ny, & + this%ifms, this%ifme, this%jfms, this%jfme, this%ifps, this%ifpe, this%jfps, this%jfpe, this%cfbm_comm, this%nfuel_cat) + call Read_restart_field_2d (trim (restart_file), 'fz0', this%nx, this%ny, & + this%ifms, this%ifme, this%jfms, this%jfme, this%ifps, this%ifpe, this%jfps, this%jfpe, this%cfbm_comm, this%fz0) + case default call Stop_simulation ('Not ready to complete fire state initialization 3') end select Set_topo_fuels - if (config_flags%fuel_opt == FUEL_ANDERSON) call this%Convert_sb_to_ander () + if (config_flags%fuel_opt == FUEL_ANDERSON .and. init_mode /= INIT_MODE_RESTART) call this%Convert_sb_to_ander () ! Set clock if (DEBUG_LOCAL) call Print_message (' Setting clock...') @@ -894,6 +1039,764 @@ subroutine Print_tiles (this) end subroutine Print_tiles + function Build_restart_file_name (restart_datetime) result (file_restart) + + implicit none + + character (len = *), intent (in) :: restart_datetime + + character (len = :), allocatable :: file_restart + + + file_restart = 'fire_restart_'//trim (restart_datetime)//'.nc' + + end function Build_restart_file_name + + subroutine Require_restart_mpi_comm (this) + + implicit none + + class (state_fire_t), intent (in) :: this + + +#ifdef DM_PARALLEL + if (.not. this%is_cfbm_comm_set) call Stop_simulation ('The MPI CFBM communicator has not been set') +#endif + + end subroutine Require_restart_mpi_comm + + function Is_restart_io_root (this) result (am_root) + + implicit none + + class (state_fire_t), intent (in) :: this + logical :: am_root + + integer :: rank, ierr + + +#ifdef DM_PARALLEL + call Require_restart_mpi_comm (this) + call Mpi_comm_rank (this%cfbm_comm, rank, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_comm_rank ') + am_root = rank == RESTART_IO_ROOT +#else + am_root = .true. +#endif + + end function Is_restart_io_root + + subroutine Restart_io_barrier (this) + + implicit none + + class (state_fire_t), intent (in) :: this + + integer :: ierr + + +#ifdef DM_PARALLEL + call Require_restart_mpi_comm (this) + call MPI_Barrier (this%cfbm_comm, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with MPI_Barrier ') +#endif + + end subroutine Restart_io_barrier + + subroutine Broadcast_restart_integer (this, value) + + implicit none + + class (state_fire_t), intent (in) :: this + integer, intent (in out) :: value + + integer :: ierr + + +#ifdef DM_PARALLEL + call Require_restart_mpi_comm (this) + call MPI_Bcast (value, 1, MPI_INTEGER, RESTART_IO_ROOT, this%cfbm_comm, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with MPI_Bcast for restart integer metadata') +#endif + + end subroutine Broadcast_restart_integer + + subroutine Broadcast_restart_real (this, value) + + implicit none + + class (state_fire_t), intent (in) :: this + real, intent (in out) :: value + + integer :: packed_value + + +#ifdef DM_PARALLEL + packed_value = 0 + if (Is_restart_io_root (this)) packed_value = transfer (value, packed_value) + call Broadcast_restart_integer (this, packed_value) + value = transfer (packed_value, value) +#endif + + end subroutine Broadcast_restart_real + + subroutine Broadcast_restart_logical (this, value) + + implicit none + + class (state_fire_t), intent (in) :: this + logical, intent (in out) :: value + + integer :: flag + + +#ifdef DM_PARALLEL + flag = 0 + if (Is_restart_io_root (this)) then + if (value) then + flag = 1 + else + flag = 0 + end if + end if + call Broadcast_restart_integer (this, flag) + value = flag /= 0 +#endif + + end subroutine Broadcast_restart_logical + + subroutine Broadcast_restart_failure (this, is_valid, failure_code, failure_msg) + + implicit none + + class (state_fire_t), intent (in) :: this + logical, intent (in out) :: is_valid + integer, intent (in out) :: failure_code + character (len = *), intent (in out) :: failure_msg + + integer :: ierr, n, msg_len + integer, dimension(:), allocatable :: msg_codes + + + call Broadcast_restart_logical (this, is_valid) + call Broadcast_restart_integer (this, failure_code) +#ifdef DM_PARALLEL + call Require_restart_mpi_comm (this) + msg_len = len (failure_msg) + allocate (msg_codes(msg_len)) + if (Is_restart_io_root (this)) then + do n = 1, msg_len + msg_codes(n) = iachar (failure_msg(n:n)) + end do + end if + call MPI_Bcast (msg_codes(1), msg_len, MPI_INTEGER, RESTART_IO_ROOT, this%cfbm_comm, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with MPI_Bcast for restart failure metadata') + if (.not. Is_restart_io_root (this)) then + do n = 1, msg_len + failure_msg(n:n) = achar (msg_codes(n)) + end do + end if + deallocate (msg_codes) +#endif + + end subroutine Broadcast_restart_failure + + subroutine Read_restart (this, config_flags) + + implicit none + + class (state_fire_t), intent (in out) :: this + type (namelist_t), intent (in) :: config_flags + + type (datetime_t) :: datetime_restart, datetime_check + character (len = :), allocatable :: file_restart + integer (kind = INT32) :: att_int, restart_year, restart_month, restart_day, restart_hour, restart_minute, restart_second, & + start_year, start_month, start_day, start_hour, start_minute, start_second + integer :: ij, expected_sr_x, expected_sr_y, ierr, rank + logical, parameter :: DEBUG_LOCAL = .false. + + + if (DEBUG_LOCAL) call Print_message ('Entering Read_restart...') + +#ifdef DM_PARALLEL + call Require_restart_mpi_comm (this) + call Mpi_comm_rank (this%cfbm_comm, rank, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_comm_rank ') +#else + rank = 0 +#endif + + if (config_flags%ideal_opt /= 0 .and. config_flags%ideal_opt /= 1) & + call Stop_simulation ('Read_restart is implemented for standalone idealized and real runs only') + + datetime_restart = datetime_t (config_flags%start_year, config_flags%start_month, config_flags%start_day, & + config_flags%start_hour, config_flags%start_minute, config_flags%start_second) + file_restart = Build_restart_file_name (datetime_restart%datetime) + + if (rank == 0) then + call Is_netcdf_file_present (file_restart) + + call Validate_restart_integer (file_restart, 'restart_year', config_flags%start_year) + call Validate_restart_integer (file_restart, 'restart_month', config_flags%start_month) + call Validate_restart_integer (file_restart, 'restart_day', config_flags%start_day) + call Validate_restart_integer (file_restart, 'restart_hour', config_flags%start_hour) + call Validate_restart_integer (file_restart, 'restart_minute', config_flags%start_minute) + call Validate_restart_integer (file_restart, 'restart_second', config_flags%start_second) + + call Validate_restart_integer (file_restart, 'nx', this%nx) + call Validate_restart_integer (file_restart, 'ny', this%ny) + call Validate_restart_real (file_restart, 'dt', this%dt) + call Validate_restart_real (file_restart, 'dx', this%dx) + call Validate_restart_real (file_restart, 'dy', this%dy) + call Validate_restart_real (file_restart, 'cen_lat', this%cen_lat) + call Validate_restart_real (file_restart, 'cen_lon', this%cen_lon) + expected_sr_x = nint (this%proj%dx / this%dx) + expected_sr_y = nint (this%proj%dy / this%dy) + call Validate_restart_integer (file_restart, 'map_proj', 1) + call Validate_restart_integer (file_restart, 'sr_x', expected_sr_x) + call Validate_restart_integer (file_restart, 'sr_y', expected_sr_y) + call Validate_restart_integer (file_restart, 'ideal_opt', config_flags%ideal_opt) + call Validate_restart_integer (file_restart, 'fuel_opt', config_flags%fuel_opt) + call Validate_restart_integer (file_restart, 'ros_opt', config_flags%ros_opt) + call Validate_restart_integer (file_restart, 'fmc_opt', config_flags%fmc_opt) + call Validate_restart_integer (file_restart, 'emis_opt', config_flags%emis_opt) + call Validate_restart_integer (file_restart, 'fire_upwinding', config_flags%fire_upwinding) + call Validate_restart_integer (file_restart, 'fire_upwinding_reinit', config_flags%fire_upwinding_reinit) + call Validate_restart_integer (file_restart, 'fire_lsm_reinit_iter', config_flags%fire_lsm_reinit_iter) + call Validate_restart_real (file_restart, 'fire_viscosity', config_flags%fire_viscosity) + call Validate_restart_real (file_restart, 'fire_viscosity_bg', config_flags%fire_viscosity_bg) + call Validate_restart_integer (file_restart, 'fire_viscosity_ngp', config_flags%fire_viscosity_ngp) + call Validate_restart_real (file_restart, 'fire_viscosity_band', config_flags%fire_viscosity_band) + call Validate_restart_real (file_restart, 'reinit_pseudot_coef', config_flags%reinit_pseudot_coef) + + select case (config_flags%ideal_opt) + case (0) + call Validate_restart_real (file_restart, 'stand_lon', this%proj%standard_lon) + call Validate_restart_real (file_restart, 'true_lat_1', this%proj%true_lat_1) + call Validate_restart_real (file_restart, 'true_lat_2', this%proj%true_lat_2) + + case (1) + call Validate_restart_real (file_restart, 'stand_lon', config_flags%stand_lon) + call Validate_restart_real (file_restart, 'true_lat_1', config_flags%true_lat_1) + call Validate_restart_real (file_restart, 'true_lat_2', config_flags%true_lat_2) + end select + + call Get_netcdf_att (file_restart, 'global', 'start_year', start_year) + call Get_netcdf_att (file_restart, 'global', 'start_month', start_month) + call Get_netcdf_att (file_restart, 'global', 'start_day', start_day) + call Get_netcdf_att (file_restart, 'global', 'start_hour', start_hour) + call Get_netcdf_att (file_restart, 'global', 'start_minute', start_minute) + call Get_netcdf_att (file_restart, 'global', 'start_second', start_second) + + call Get_netcdf_att (file_restart, 'global', 'restart_year', restart_year) + call Get_netcdf_att (file_restart, 'global', 'restart_month', restart_month) + call Get_netcdf_att (file_restart, 'global', 'restart_day', restart_day) + call Get_netcdf_att (file_restart, 'global', 'restart_hour', restart_hour) + call Get_netcdf_att (file_restart, 'global', 'restart_minute', restart_minute) + call Get_netcdf_att (file_restart, 'global', 'restart_second', restart_second) + + call Get_netcdf_att (file_restart, 'global', 'itimestep', att_int) + end if + + call Broadcast_restart_integer (this, start_year) + call Broadcast_restart_integer (this, start_month) + call Broadcast_restart_integer (this, start_day) + call Broadcast_restart_integer (this, start_hour) + call Broadcast_restart_integer (this, start_minute) + call Broadcast_restart_integer (this, start_second) + call Broadcast_restart_integer (this, restart_year) + call Broadcast_restart_integer (this, restart_month) + call Broadcast_restart_integer (this, restart_day) + call Broadcast_restart_integer (this, restart_hour) + call Broadcast_restart_integer (this, restart_minute) + call Broadcast_restart_integer (this, restart_second) + call Broadcast_restart_integer (this, att_int) + this%itimestep = att_int + + this%datetime_start = datetime_t (start_year, start_month, start_day, start_hour, start_minute, start_second) + this%datetime_now = datetime_t (restart_year, restart_month, restart_day, restart_hour, restart_minute, restart_second) + + datetime_check = this%datetime_start + call datetime_check%Add_seconds (this%itimestep * this%dt) + if (datetime_check /= this%datetime_now) call Stop_simulation ('Restart clock is inconsistent with itimestep and dt') + + call Set_next_datetime_after_restart (this%datetime_next_output, config_flags%interval_output) + call Set_next_datetime_after_restart (this%datetime_next_atm_update, config_flags%interval_atm) + + call Read_restart_field (file_restart, 'lfn', this%lfn) + call Read_restart_field (file_restart, 'lfn_hist', this%lfn_hist) + call Read_restart_field (file_restart, 'lfn_0', this%lfn_0) + call Read_restart_field (file_restart, 'lfn_1', this%lfn_1) + call Read_restart_field (file_restart, 'lfn_2', this%lfn_2) + call Read_restart_field (file_restart, 'lfn_s0', this%lfn_s0) + call Read_restart_field (file_restart, 'lfn_s1', this%lfn_s1) + call Read_restart_field (file_restart, 'lfn_s2', this%lfn_s2) + call Read_restart_field (file_restart, 'lfn_s3', this%lfn_s3) + call Read_restart_field (file_restart, 'lfn_out', this%lfn_out) + call Read_restart_field (file_restart, 'tign_g', this%tign_g) + call Read_restart_field (file_restart, 'fuel_frac', this%fuel_frac) + call Read_restart_field (file_restart, 'fire_area', this%fire_area) + call Read_restart_field (file_restart, 'fuel_frac_burnt_dt', this%fuel_frac_burnt_dt) + call Read_restart_field (file_restart, 'fgrnhfx', this%fgrnhfx) + call Read_restart_field (file_restart, 'fgrnqfx', this%fgrnqfx) + call Read_restart_field (file_restart, 'fcanhfx', this%fcanhfx) + call Read_restart_field (file_restart, 'fcanqfx', this%fcanqfx) + call Read_restart_field (file_restart, 'flame_length', this%flame_length) + call Read_restart_field (file_restart, 'ros', this%ros) + call Read_restart_field (file_restart, 'ros_front', this%ros_front) + call Read_restart_field (file_restart, 'emis_smoke', this%emis_smoke) + call Read_restart_field (file_restart, 'fmc_g', this%fmc_g) + call Read_restart_field (file_restart, 'fuel_load_g', this%fuel_load_g) + call Read_restart_field (file_restart, 'fuel_time', this%fuel_time) + call Read_restart_field (file_restart, 'zsf', this%zsf) + call Read_restart_field (file_restart, 'dzdxf', this%dzdxf) + call Read_restart_field (file_restart, 'dzdyf', this%dzdyf) + call Read_restart_field (file_restart, 'nfuel_cat', this%nfuel_cat) + call Read_restart_field (file_restart, 'uf', this%uf) + call Read_restart_field (file_restart, 'vf', this%vf) + call Read_restart_field (file_restart, 'fz0', this%fz0) + + if (config_flags%ideal_opt == 0) then + call Read_restart_field (file_restart, 'fire_t2', this%fire_t2) + call Read_restart_field (file_restart, 'fire_q2', this%fire_q2) + call Read_restart_field (file_restart, 'fire_psfc', this%fire_psfc) + call Read_restart_field (file_restart, 'fire_rain', this%fire_rain) + if (config_flags%fmoist_run) then + call Read_restart_field (file_restart, 'fire_t2_old', this%fire_t2_old) + call Read_restart_field (file_restart, 'fire_q2_old', this%fire_q2_old) + call Read_restart_field (file_restart, 'fire_psfc_old', this%fire_psfc_old) + call Read_restart_field (file_restart, 'fire_rain_old', this%fire_rain_old) + end if + end if + if (config_flags%fmoist_run) call Read_restart_fmc (file_restart) + + call Exchange_restart_halos () + + if (allocated (this%ros_param) .and. allocated (this%fuels)) then + do ij = 1, this%num_tiles + call this%ros_param%Set_params (this%ifms, this%ifme, this%jfms, this%jfme, this%i_start(ij), this%i_end(ij), & + this%j_start(ij), this%j_end(ij), this%fuels, this%nfuel_cat, this%fmc_g) + end do + end if + + if (DEBUG_LOCAL) call Print_message ('Leaving Read_restart...') + + contains + + subroutine Exchange_restart_halos () + + implicit none + + +#ifdef DM_PARALLEL + call Exchange_restart_field (this%lfn) + call Exchange_restart_field (this%lfn_hist) + call Exchange_restart_field (this%lfn_0) + call Exchange_restart_field (this%lfn_1) + call Exchange_restart_field (this%lfn_2) + call Exchange_restart_field (this%lfn_s0) + call Exchange_restart_field (this%lfn_s1) + call Exchange_restart_field (this%lfn_s2) + call Exchange_restart_field (this%lfn_s3) + call Exchange_restart_field (this%lfn_out) + call Exchange_restart_field (this%tign_g) + call Exchange_restart_field (this%fuel_frac) + call Exchange_restart_field (this%fire_area) + call Exchange_restart_field (this%fuel_frac_burnt_dt) + call Exchange_restart_field (this%fgrnhfx) + call Exchange_restart_field (this%fgrnqfx) + call Exchange_restart_field (this%fcanhfx) + call Exchange_restart_field (this%fcanqfx) + call Exchange_restart_field (this%flame_length) + call Exchange_restart_field (this%ros) + call Exchange_restart_field (this%ros_front) + call Exchange_restart_field (this%emis_smoke) + call Exchange_restart_field (this%fmc_g) + call Exchange_restart_field (this%fuel_load_g) + call Exchange_restart_field (this%fuel_time) + call Exchange_restart_field (this%zsf) + call Exchange_restart_field (this%dzdxf) + call Exchange_restart_field (this%dzdyf) + call Exchange_restart_field (this%nfuel_cat) + call Exchange_restart_field (this%uf) + call Exchange_restart_field (this%vf) + call Exchange_restart_field (this%fz0) + + if (config_flags%ideal_opt == 0) then + call Exchange_restart_field (this%fire_t2) + call Exchange_restart_field (this%fire_q2) + call Exchange_restart_field (this%fire_psfc) + call Exchange_restart_field (this%fire_rain) + if (config_flags%fmoist_run) then + call Exchange_restart_field (this%fire_t2_old) + call Exchange_restart_field (this%fire_q2_old) + call Exchange_restart_field (this%fire_psfc_old) + call Exchange_restart_field (this%fire_rain_old) + end if + end if + + if (config_flags%fmoist_run) call Exchange_restart_fmc_halos () +#endif + + end subroutine Exchange_restart_halos + + subroutine Exchange_restart_field (var) + + implicit none + + real, dimension(this%ifms:this%ifme, this%jfms:this%jfme), intent (in out) :: var + + +#ifdef DM_PARALLEL + call Do_halo_exchange_with_corners (var, this%ifms, this%ifme, this%jfms, this%jfme, & + this%ifps, this%ifpe, this%jfps, this%jfpe, N_POINTS_IN_HALO, this%cart_comm) +#endif + + end subroutine Exchange_restart_field + + subroutine Exchange_restart_fmc_halos () + + implicit none + + integer :: k, n_moisture_classes + + +#ifdef DM_PARALLEL + if (.not. allocated (this%fmc_param)) call Stop_simulation ('FMC restart halo exchange requires allocated fmc_param') + + select type (fmc_param => this%fmc_param) + type is (fmc_wrffire_t) + if (.not. allocated (fmc_param%fmc_gc)) call Stop_simulation ('FMC restart halo exchange requires allocated fmc_gc') + + n_moisture_classes = size (fmc_param%fmc_gc, 2) + do k = 1, n_moisture_classes + call Do_halo_exchange_with_corners (fmc_param%fmc_gc(this%ifms:this%ifme, k, this%jfms:this%jfme), & + this%ifms, this%ifme, this%jfms, this%jfme, & + this%ifps, this%ifpe, this%jfps, this%jfpe, N_POINTS_IN_HALO, this%cart_comm) + end do + + class default + call Stop_simulation ('Restart halo exchange is not implemented for selected FMC component') + end select +#endif + + end subroutine Exchange_restart_fmc_halos + + subroutine Read_restart_field (file_name, var_name, var) + + implicit none + + character (len = *), intent (in) :: file_name, var_name + real, dimension(this%ifms:this%ifme, this%jfms:this%jfme), intent (in out) :: var + + + call Get_netcdf_var_mpi (file_name, this%cfbm_comm, this%nx, this%ny, & + this%ifps, this%ifpe, this%jfps, this%jfpe, var_name, var(this%ifps:this%ifpe, this%jfps:this%jfpe)) + + end subroutine Read_restart_field + + subroutine Read_restart_fmc (file_name) + + implicit none + + character (len = *), intent (in) :: file_name + + real (kind = REAL32) :: att_real32 + real :: fmoist_lasttime, fmoist_nexttime + integer :: n_moisture_classes + + + if (.not. allocated (this%fmc_param)) call Stop_simulation ('FMC restart read requires allocated fmc_param') + + select type (fmc_param => this%fmc_param) + type is (fmc_wrffire_t) + if (.not. allocated (fmc_param%fmc_gc)) call Stop_simulation ('FMC restart read requires allocated fmc_gc') + + n_moisture_classes = size (fmc_param%fmc_gc, 2) + call Get_netcdf_var_mpi (file_name, this%cfbm_comm, this%nx, this%ny, n_moisture_classes, & + this%ifps, this%ifpe, this%jfps, this%jfpe, NAME_VAR_FMC_GC, & + fmc_param%fmc_gc(this%ifps:this%ifpe, 1:n_moisture_classes, this%jfps:this%jfpe)) + + fmoist_lasttime = 0.0 + fmoist_nexttime = 0.0 + if (rank == 0) then + call Get_netcdf_att (file_name, 'global', NAME_ATT_FMOIST_LASTTIME, att_real32) + fmoist_lasttime = att_real32 + call Get_netcdf_att (file_name, 'global', NAME_ATT_FMOIST_NEXTTIME, att_real32) + fmoist_nexttime = att_real32 + end if + call Broadcast_restart_real (this, fmoist_lasttime) + call Broadcast_restart_real (this, fmoist_nexttime) + fmc_param%fmoist_lasttime = fmoist_lasttime + fmc_param%fmoist_nexttime = fmoist_nexttime + + class default + call Stop_simulation ('Restart read is not implemented for selected FMC component') + end select + + end subroutine Read_restart_fmc + + subroutine Set_next_datetime_after_restart (datetime_next, interval_seconds) + + implicit none + + type (datetime_t), intent (in out) :: datetime_next + integer, intent (in) :: interval_seconds + + datetime_next = this%datetime_now + if (interval_seconds > 0) then + datetime_next = this%datetime_start + do while (datetime_next <= this%datetime_now) + call datetime_next%Add_seconds (interval_seconds) + end do + end if + + end subroutine Set_next_datetime_after_restart + + end subroutine Read_restart + + subroutine Read_restart_field_2d (file_name, var_name, nx, ny, ifms, ifme, jfms, jfme, ifps, ifpe, jfps, jfpe, cfbm_comm, var) + + implicit none + + character (len = *), intent (in) :: file_name, var_name + integer, intent (in) :: nx, ny, ifms, ifme, jfms, jfme, ifps, ifpe, jfps, jfpe, cfbm_comm + real, dimension(ifms:ifme, jfms:jfme), intent (in out) :: var + + + call Get_netcdf_var_mpi (file_name, cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, & + var_name, var(ifps:ifpe, jfps:jfpe)) + + end subroutine Read_restart_field_2d + + subroutine Validate_restart_integer (file_name, att_name, expected_value) + + implicit none + + character (len = *), intent (in) :: file_name, att_name + integer, intent (in) :: expected_value + + integer (kind = INT32) :: att_value + character (len = :), allocatable :: msg + + + call Get_netcdf_att (file_name, 'global', att_name, att_value) + if (att_value /= expected_value) then + msg = 'Restart metadata mismatch: '//trim (att_name) + call Stop_simulation (msg) + end if + + end subroutine Validate_restart_integer + + subroutine Validate_restart_real (file_name, att_name, expected_value) + + implicit none + + character (len = *), intent (in) :: file_name, att_name + real, intent (in) :: expected_value + + real (kind = REAL32) :: att_value, tolerance + character (len = :), allocatable :: msg + + + call Get_netcdf_att (file_name, 'global', att_name, att_value) + tolerance = max (1.0e-6_REAL32, abs (real (expected_value, kind = REAL32)) * 1.0e-6_REAL32) + if (abs (att_value - real (expected_value, kind = REAL32)) > tolerance) then + msg = 'Restart metadata mismatch: '//trim (att_name) + call Stop_simulation (msg) + end if + + end subroutine Validate_restart_real + + subroutine Write_restart (this, config_flags) + + implicit none + + class (state_fire_t), intent (in) :: this + type (namelist_t), intent (in) :: config_flags + + character (len = :), allocatable :: file_restart + integer :: start_year, start_month, start_day, start_hour, start_minute, start_second, & + restart_year, restart_month, restart_day, restart_hour, restart_minute, restart_second + integer :: restart_sr_x, restart_sr_y, rank, ierr + logical, parameter :: DEBUG_LOCAL = .false. + + + if (DEBUG_LOCAL) call Print_message ('Entering Write_restart...') + +#ifdef DM_PARALLEL + if (.not. this%is_cfbm_comm_set) call Stop_simulation ('The MPI CFBM communicator has not been set') + call Mpi_comm_rank (this%cfbm_comm, rank, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_comm_rank ') +#else + rank = 0 +#endif + + if (config_flags%ideal_opt /= 0 .and. config_flags%ideal_opt /= 1) & + call Stop_simulation ('Write_restart is implemented for standalone idealized and real runs only') + + file_restart = Build_restart_file_name (this%datetime_now%datetime) + + call this%datetime_start%Get_datetime_as_ints (start_year, start_month, start_day, start_hour, start_minute, start_second) + call this%datetime_now%Get_datetime_as_ints (restart_year, restart_month, restart_day, restart_hour, restart_minute, restart_second) + restart_sr_x = nint (this%proj%dx / this%dx) + restart_sr_y = nint (this%proj%dy / this%dy) + if (restart_sr_x <= 0 .or. restart_sr_y <= 0) call Stop_simulation ('Restart subgrid ratios must be positive') + + if (rank == 0) then + call Create_netcdf_file (file_name = file_restart) + call Add_netcdf_dim (file_restart, NAME_DIM_X, this%nx) + call Add_netcdf_dim (file_restart, NAME_DIM_Y, this%ny) + + call Add_netcdf_att (file_restart, 'global', 'start_year', int (start_year, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'start_month', int (start_month, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'start_day', int (start_day, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'start_hour', int (start_hour, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'start_minute', int (start_minute, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'start_second', int (start_second, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'restart_year', int (restart_year, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'restart_month', int (restart_month, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'restart_day', int (restart_day, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'restart_hour', int (restart_hour, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'restart_minute', int (restart_minute, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'restart_second', int (restart_second, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'itimestep', int (this%itimestep, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'nx', int (this%nx, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'ny', int (this%ny, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'dt', real (this%dt, kind = REAL32)) + call Add_netcdf_att (file_restart, 'global', 'dx', real (this%dx, kind = REAL32)) + call Add_netcdf_att (file_restart, 'global', 'dy', real (this%dy, kind = REAL32)) + call Add_netcdf_att (file_restart, 'global', 'cen_lat', real (this%cen_lat, kind = REAL32)) + call Add_netcdf_att (file_restart, 'global', 'cen_lon', real (this%cen_lon, kind = REAL32)) + call Add_netcdf_att (file_restart, 'global', 'map_proj', int (1, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'sr_x', int (restart_sr_x, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'sr_y', int (restart_sr_y, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'stand_lon', real (this%proj%standard_lon, kind = REAL32)) + call Add_netcdf_att (file_restart, 'global', 'true_lat_1', real (this%proj%true_lat_1, kind = REAL32)) + call Add_netcdf_att (file_restart, 'global', 'true_lat_2', real (this%proj%true_lat_2, kind = REAL32)) + call Add_netcdf_att (file_restart, 'global', 'ideal_opt', int (config_flags%ideal_opt, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'fuel_opt', int (config_flags%fuel_opt, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'ros_opt', int (config_flags%ros_opt, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'fmc_opt', int (config_flags%fmc_opt, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'emis_opt', int (config_flags%emis_opt, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'fire_upwinding', int (config_flags%fire_upwinding, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'fire_upwinding_reinit', int (config_flags%fire_upwinding_reinit, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'fire_lsm_reinit_iter', int (config_flags%fire_lsm_reinit_iter, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'fire_viscosity', real (config_flags%fire_viscosity, kind = REAL32)) + call Add_netcdf_att (file_restart, 'global', 'fire_viscosity_bg', real (config_flags%fire_viscosity_bg, kind = REAL32)) + call Add_netcdf_att (file_restart, 'global', 'fire_viscosity_ngp', int (config_flags%fire_viscosity_ngp, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'fire_viscosity_band', real (config_flags%fire_viscosity_band, kind = REAL32)) + call Add_netcdf_att (file_restart, 'global', 'reinit_pseudot_coef', real (config_flags%reinit_pseudot_coef, kind = REAL32)) + end if + + call Restart_io_barrier (this) + + call Add_restart_field ('lfn', this%lfn) + call Add_restart_field ('lfn_hist', this%lfn_hist) + call Add_restart_field ('lfn_0', this%lfn_0) + call Add_restart_field ('lfn_1', this%lfn_1) + call Add_restart_field ('lfn_2', this%lfn_2) + call Add_restart_field ('lfn_s0', this%lfn_s0) + call Add_restart_field ('lfn_s1', this%lfn_s1) + call Add_restart_field ('lfn_s2', this%lfn_s2) + call Add_restart_field ('lfn_s3', this%lfn_s3) + call Add_restart_field ('lfn_out', this%lfn_out) + call Add_restart_field ('tign_g', this%tign_g) + call Add_restart_field ('fuel_frac', this%fuel_frac) + call Add_restart_field ('fire_area', this%fire_area) + call Add_restart_field ('fuel_frac_burnt_dt', this%fuel_frac_burnt_dt) + call Add_restart_field ('fgrnhfx', this%fgrnhfx) + call Add_restart_field ('fgrnqfx', this%fgrnqfx) + call Add_restart_field ('fcanhfx', this%fcanhfx) + call Add_restart_field ('fcanqfx', this%fcanqfx) + call Add_restart_field ('flame_length', this%flame_length) + call Add_restart_field ('ros', this%ros) + call Add_restart_field ('ros_front', this%ros_front) + call Add_restart_field ('emis_smoke', this%emis_smoke) + call Add_restart_field ('fmc_g', this%fmc_g) + call Add_restart_field ('fuel_load_g', this%fuel_load_g) + call Add_restart_field ('fuel_time', this%fuel_time) + call Add_restart_field ('zsf', this%zsf) + call Add_restart_field ('dzdxf', this%dzdxf) + call Add_restart_field ('dzdyf', this%dzdyf) + call Add_restart_field ('nfuel_cat', this%nfuel_cat) + call Add_restart_field ('uf', this%uf) + call Add_restart_field ('vf', this%vf) + call Add_restart_field ('fz0', this%fz0) + + if (config_flags%ideal_opt == 0) then + call Add_restart_field ('fire_t2', this%fire_t2) + call Add_restart_field ('fire_q2', this%fire_q2) + call Add_restart_field ('fire_psfc', this%fire_psfc) + call Add_restart_field ('fire_rain', this%fire_rain) + if (config_flags%fmoist_run) then + call Add_restart_field ('fire_t2_old', this%fire_t2_old) + call Add_restart_field ('fire_q2_old', this%fire_q2_old) + call Add_restart_field ('fire_psfc_old', this%fire_psfc_old) + call Add_restart_field ('fire_rain_old', this%fire_rain_old) + end if + end if + if (config_flags%fmoist_run) call Write_restart_fmc () + + call Restart_io_barrier (this) + + if (DEBUG_LOCAL) call Print_message ('Leaving Write_restart...') + + contains + + subroutine Add_restart_field (var_name, var) + + implicit none + + character (len = *), intent (in) :: var_name + real, dimension(this%ifms:this%ifme, this%jfms:this%jfme), intent (in) :: var + + + call Add_netcdf_var_mpi (file_restart, this%cfbm_comm, this%nx, this%ny, this%ifps, this%ifpe, this%jfps, this%jfpe, & + var_name, var(this%ifps:this%ifpe, this%jfps:this%jfpe)) + + end subroutine Add_restart_field + + subroutine Write_restart_fmc () + + implicit none + + character (len = 32), dimension(3) :: dim_names_fmc_gc + integer :: n_moisture_classes + + + if (.not. allocated (this%fmc_param)) call Stop_simulation ('FMC restart write requires allocated fmc_param') + + select type (fmc_param => this%fmc_param) + type is (fmc_wrffire_t) + if (.not. allocated (fmc_param%fmc_gc)) call Stop_simulation ('FMC restart write requires allocated fmc_gc') + + n_moisture_classes = size (fmc_param%fmc_gc, 2) + if (rank == 0) call Add_netcdf_dim (file_restart, NAME_DIM_MOISTURE_CLASS, n_moisture_classes) + call Restart_io_barrier (this) + + dim_names_fmc_gc(1) = NAME_DIM_X + dim_names_fmc_gc(2) = NAME_DIM_MOISTURE_CLASS + dim_names_fmc_gc(3) = NAME_DIM_Y + call Add_netcdf_var_mpi (file_restart, dim_names_fmc_gc, this%cfbm_comm, this%nx, this%ny, n_moisture_classes, & + this%ifps, this%ifpe, this%jfps, this%jfpe, NAME_VAR_FMC_GC, & + fmc_param%fmc_gc(this%ifps:this%ifpe, 1:n_moisture_classes, this%jfps:this%jfpe)) + + if (rank == 0) then + call Add_netcdf_att (file_restart, 'global', NAME_ATT_FMOIST_LASTTIME, & + real (fmc_param%fmoist_lasttime, kind = REAL32)) + call Add_netcdf_att (file_restart, 'global', NAME_ATT_FMOIST_NEXTTIME, & + real (fmc_param%fmoist_nexttime, kind = REAL32)) + end if + + class default + call Stop_simulation ('Restart write is not implemented for selected FMC component') + end select + + end subroutine Write_restart_fmc + + end subroutine Write_restart + subroutine Save_state (this) implicit none From cf7696cbc90125096255dcb39a338314a17bc77b Mon Sep 17 00:00:00 2001 From: masih Date: Thu, 24 Sep 2026 15:44:53 -0600 Subject: [PATCH 02/19] Refactor restart code into state submodule Move restart-specific state code and helpers into a new state_restart_submod.F90 submodule. --- CMakeLists.txt | 1 + state/state_mod.F90 | 930 ++------------------------------- state/state_restart_submod.F90 | 848 ++++++++++++++++++++++++++++++ 3 files changed, 887 insertions(+), 892 deletions(-) create mode 100644 state/state_restart_submod.F90 diff --git a/CMakeLists.txt b/CMakeLists.txt index 3307e8bd..34ed67b7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -59,6 +59,7 @@ endif() # State list(APPEND _state_files state/state_mod.F90 + state/state_restart_submod.F90 state/ignition_line_mod.F90 state/tiles_mod.F90) diff --git a/state/state_mod.F90 b/state/state_mod.F90 index c81ca6cb..fbc6efb9 100644 --- a/state/state_mod.F90 +++ b/state/state_mod.F90 @@ -6,7 +6,6 @@ module state_mod use constants_mod, only : PI use datetime_mod, only : datetime_t use fmc_mod, only : fmc_t - use fmc_wrffire_mod, only : fmc_wrffire_t use fuel_mod, only : fuel_t, FUEL_ANDERSON, Crosswalk_from_scottburgan_to_anderson use geogrid_mod, only : geogrid_t use ignition_line_mod, only : ignition_line_t @@ -20,7 +19,6 @@ module state_mod use wrfdata_mod, only : wrfdata_t, G, RERADIUS use mpi_mod, only : Calc_tasks_in_x_and_y, Calc_patch_dims, Distribute_var2d, Do_halo_exchange_with_corners, & Print_cart_info, topology_dim_order - use, intrinsic :: iso_fortran_env, only : INT32, REAL32 implicit none @@ -29,11 +27,6 @@ module state_mod public :: state_fire_t, N_POINTS_IN_HALO, Build_restart_file_name integer, parameter :: N_POINTS_IN_HALO = 5, N_DIMS = 2 - integer, parameter :: RESTART_IO_ROOT = 0 - character (len = *), parameter :: NAME_DIM_MOISTURE_CLASS = 'moisture_class' - character (len = *), parameter :: NAME_VAR_FMC_GC = 'fmc_gc' - character (len = *), parameter :: NAME_ATT_FMOIST_LASTTIME = 'fmc_fmoist_lasttime' - character (len = *), parameter :: NAME_ATT_FMOIST_NEXTTIME = 'fmc_fmoist_nexttime' logical, dimension(2), parameter :: PERIODS = [ .false., .false. ] logical, parameter :: REORDER = .true. ! Allow MPI recording tasks for performance @@ -146,6 +139,40 @@ module state_mod procedure, public :: Write_restart => Write_restart end type state_fire_t + interface + module function Build_restart_file_name (restart_datetime) result (file_restart) + character (len = *), intent (in) :: restart_datetime + character (len = :), allocatable :: file_restart + end function Build_restart_file_name + + module subroutine Init_restart_dimensions (this, restart_file, ids0, ide0, jds0, jde0, ips, ipe, jps, jpe) + class (state_fire_t), intent (in out) :: this + character (len = *), intent (in) :: restart_file + integer, intent (out) :: ids0, ide0, jds0, jde0, ips, ipe, jps, jpe + end subroutine Init_restart_dimensions + + module subroutine Init_restart_projection (this, restart_file, proj) + class (state_fire_t), intent (in out) :: this + character (len = *), intent (in) :: restart_file + type (proj_lc_t), intent (out) :: proj + end subroutine Init_restart_projection + + module subroutine Read_restart_static_fields (this, restart_file) + class (state_fire_t), intent (in out) :: this + character (len = *), intent (in) :: restart_file + end subroutine Read_restart_static_fields + + module subroutine Read_restart (this, config_flags) + class (state_fire_t), intent (in out) :: this + type (namelist_t), intent (in) :: config_flags + end subroutine Read_restart + + module subroutine Write_restart (this, config_flags) + class (state_fire_t), intent (in) :: this + type (namelist_t), intent (in) :: config_flags + end subroutine Write_restart + end interface + contains subroutine Allocate_vars (this, ifms, ifme, jfms, jfme) @@ -333,11 +360,7 @@ subroutine Init_domain (this, config_flags, geogrid, & type (proj_lc_t) :: proj logical, parameter :: DEBUG_LOCAL = .false. integer :: ids0, ide0, jds0, jde0, i, j, init_mode, px, py, ntasks, ierr, cart_comm, rank, ips, ipe, jps, jpe, & - is_lfn_init_allocated, restart_nx, restart_ny - integer :: restart_map_proj, restart_sr_x, restart_sr_y - integer (kind = INT32) :: att_int32 - real (kind = REAL32) :: att_real32 - real :: restart_dx, restart_dy, restart_cen_lat, restart_cen_lon, restart_stand_lon, restart_true_lat_1, restart_true_lat_2 + is_lfn_init_allocated integer, dimension(2) :: coords character (len = 300) :: msg @@ -460,52 +483,7 @@ subroutine Init_domain (this, config_flags, geogrid, & #endif else if (init_mode == INIT_MODE_RESTART) then - restart_nx = 0 - restart_ny = 0 - if (Is_restart_io_root (this)) then - call Get_netcdf_att (trim (restart_file), 'global', 'nx', att_int32) - restart_nx = att_int32 - call Get_netcdf_att (trim (restart_file), 'global', 'ny', att_int32) - restart_ny = att_int32 - end if - call Broadcast_restart_integer (this, restart_nx) - call Broadcast_restart_integer (this, restart_ny) - - ids0 = 1 - ide0 = restart_nx - jds0 = 1 - jde0 = restart_ny - -#ifdef DM_PARALLEL - if (.not. this%is_cfbm_comm_set) call Stop_simulation ('The MPI CFBM communicator has not been set') - - call Mpi_comm_size (this%cfbm_comm, ntasks, ierr) - if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems getting the number of MPI tasks') - this%ntasks = ntasks - - call Calc_tasks_in_x_and_y (this%ntasks, restart_nx, restart_ny, px, py) - this%px = px - this%py = py - write (msg, '(a25, 2(1x, i5))') 'MPI TASKS in x and y =', this%px, this%py - call Print_message (msg) - - call Mpi_cart_create (this%cfbm_comm, N_DIMS, [this%px, this%py], PERIODS, REORDER, cart_comm, ierr) - if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_cart_create') - this%cart_comm = cart_comm - - call Mpi_comm_rank (this%cart_comm, rank, ierr) - if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_comm_rank ') - - call Mpi_cart_coords (this%cart_comm, rank, N_DIMS, coords, ierr) - if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_cart_coords') - - call Calc_patch_dims (restart_nx, restart_ny, this%px, this%py, coords, ips, ipe, jps, jpe) -#else - ips = ids0 - ipe = ide0 - jps = jds0 - jpe = jde0 -#endif + call Init_restart_dimensions (this, trim (restart_file), ids0, ide0, jds0, jde0, ips, ipe, jps, jpe) end if this%ifds = ids0 @@ -630,71 +608,7 @@ subroutine Init_domain (this, config_flags, geogrid, & call this%Init_latlons (proj) case (INIT_MODE_RESTART) - restart_dx = 0.0 - restart_dy = 0.0 - restart_map_proj = 0 - restart_sr_x = 0 - restart_sr_y = 0 - restart_cen_lat = 0.0 - restart_cen_lon = 0.0 - restart_stand_lon = 0.0 - restart_true_lat_1 = 0.0 - restart_true_lat_2 = 0.0 - - if (Is_restart_io_root (this)) then - call Get_netcdf_att (trim (restart_file), 'global', 'dx', att_real32) - restart_dx = att_real32 - call Get_netcdf_att (trim (restart_file), 'global', 'dy', att_real32) - restart_dy = att_real32 - call Get_netcdf_att (trim (restart_file), 'global', 'map_proj', att_int32) - restart_map_proj = att_int32 - call Get_netcdf_att (trim (restart_file), 'global', 'sr_x', att_int32) - restart_sr_x = att_int32 - call Get_netcdf_att (trim (restart_file), 'global', 'sr_y', att_int32) - restart_sr_y = att_int32 - call Get_netcdf_att (trim (restart_file), 'global', 'cen_lat', att_real32) - restart_cen_lat = att_real32 - call Get_netcdf_att (trim (restart_file), 'global', 'cen_lon', att_real32) - restart_cen_lon = att_real32 - call Get_netcdf_att (trim (restart_file), 'global', 'stand_lon', att_real32) - restart_stand_lon = att_real32 - call Get_netcdf_att (trim (restart_file), 'global', 'true_lat_1', att_real32) - restart_true_lat_1 = att_real32 - call Get_netcdf_att (trim (restart_file), 'global', 'true_lat_2', att_real32) - restart_true_lat_2 = att_real32 - end if - - call Broadcast_restart_real (this, restart_dx) - call Broadcast_restart_real (this, restart_dy) - call Broadcast_restart_integer (this, restart_map_proj) - call Broadcast_restart_integer (this, restart_sr_x) - call Broadcast_restart_integer (this, restart_sr_y) - call Broadcast_restart_real (this, restart_cen_lat) - call Broadcast_restart_real (this, restart_cen_lon) - call Broadcast_restart_real (this, restart_stand_lon) - call Broadcast_restart_real (this, restart_true_lat_1) - call Broadcast_restart_real (this, restart_true_lat_2) - - this%dx = restart_dx - this%dy = restart_dy - - if (restart_map_proj /= 1) call Stop_simulation ('Restart map projection is not supported') - - if (restart_sr_x <= 0 .or. restart_sr_y <= 0) call Stop_simulation ('Restart subgrid ratios must be positive') - if (mod (this%nx, restart_sr_x) /= 0 .or. mod (this%ny, restart_sr_y) /= 0) & - call Stop_simulation ('Restart subgrid ratios do not divide fire-grid dimensions') - if (this%nx / restart_sr_x <= 1 .or. this%ny / restart_sr_y <= 1) & - call Stop_simulation ('Restart subgrid ratios imply invalid atmospheric dimensions') - - this%cen_lat = restart_cen_lat - this%cen_lon = restart_cen_lon - - proj = proj_lc_t (cen_lat = this%cen_lat , cen_lon = this%cen_lon, dx = this%dx * restart_sr_x, & - dy = this%dy * restart_sr_y, & - standard_lon = restart_stand_lon, true_lat_1 = restart_true_lat_1, & - true_lat_2 = restart_true_lat_2, nx = this%nx / restart_sr_x - 1, ny = this%ny / restart_sr_y - 1) - - call this%Init_latlons (proj, srx = restart_sr_x, sry = restart_sr_y) + call Init_restart_projection (this, trim (restart_file), proj) case default call Stop_simulation ('Not ready to complete fire state initialization 2') @@ -747,16 +661,7 @@ subroutine Init_domain (this, config_flags, geogrid, & call Stop_simulation ('Not ready to initialize from fire perimeter in idealized mode') case (INIT_MODE_RESTART) - call Read_restart_field_2d (trim (restart_file), 'zsf', this%nx, this%ny, & - this%ifms, this%ifme, this%jfms, this%jfme, this%ifps, this%ifpe, this%jfps, this%jfpe, this%cfbm_comm, this%zsf) - call Read_restart_field_2d (trim (restart_file), 'dzdxf', this%nx, this%ny, & - this%ifms, this%ifme, this%jfms, this%jfme, this%ifps, this%ifpe, this%jfps, this%jfpe, this%cfbm_comm, this%dzdxf) - call Read_restart_field_2d (trim (restart_file), 'dzdyf', this%nx, this%ny, & - this%ifms, this%ifme, this%jfms, this%jfme, this%ifps, this%ifpe, this%jfps, this%jfpe, this%cfbm_comm, this%dzdyf) - call Read_restart_field_2d (trim (restart_file), 'nfuel_cat', this%nx, this%ny, & - this%ifms, this%ifme, this%jfms, this%jfme, this%ifps, this%ifpe, this%jfps, this%jfpe, this%cfbm_comm, this%nfuel_cat) - call Read_restart_field_2d (trim (restart_file), 'fz0', this%nx, this%ny, & - this%ifms, this%ifme, this%jfms, this%jfme, this%ifps, this%ifpe, this%jfps, this%jfpe, this%cfbm_comm, this%fz0) + call Read_restart_static_fields (this, trim (restart_file)) case default call Stop_simulation ('Not ready to complete fire state initialization 3') @@ -1039,764 +944,6 @@ subroutine Print_tiles (this) end subroutine Print_tiles - function Build_restart_file_name (restart_datetime) result (file_restart) - - implicit none - - character (len = *), intent (in) :: restart_datetime - - character (len = :), allocatable :: file_restart - - - file_restart = 'fire_restart_'//trim (restart_datetime)//'.nc' - - end function Build_restart_file_name - - subroutine Require_restart_mpi_comm (this) - - implicit none - - class (state_fire_t), intent (in) :: this - - -#ifdef DM_PARALLEL - if (.not. this%is_cfbm_comm_set) call Stop_simulation ('The MPI CFBM communicator has not been set') -#endif - - end subroutine Require_restart_mpi_comm - - function Is_restart_io_root (this) result (am_root) - - implicit none - - class (state_fire_t), intent (in) :: this - logical :: am_root - - integer :: rank, ierr - - -#ifdef DM_PARALLEL - call Require_restart_mpi_comm (this) - call Mpi_comm_rank (this%cfbm_comm, rank, ierr) - if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_comm_rank ') - am_root = rank == RESTART_IO_ROOT -#else - am_root = .true. -#endif - - end function Is_restart_io_root - - subroutine Restart_io_barrier (this) - - implicit none - - class (state_fire_t), intent (in) :: this - - integer :: ierr - - -#ifdef DM_PARALLEL - call Require_restart_mpi_comm (this) - call MPI_Barrier (this%cfbm_comm, ierr) - if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with MPI_Barrier ') -#endif - - end subroutine Restart_io_barrier - - subroutine Broadcast_restart_integer (this, value) - - implicit none - - class (state_fire_t), intent (in) :: this - integer, intent (in out) :: value - - integer :: ierr - - -#ifdef DM_PARALLEL - call Require_restart_mpi_comm (this) - call MPI_Bcast (value, 1, MPI_INTEGER, RESTART_IO_ROOT, this%cfbm_comm, ierr) - if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with MPI_Bcast for restart integer metadata') -#endif - - end subroutine Broadcast_restart_integer - - subroutine Broadcast_restart_real (this, value) - - implicit none - - class (state_fire_t), intent (in) :: this - real, intent (in out) :: value - - integer :: packed_value - - -#ifdef DM_PARALLEL - packed_value = 0 - if (Is_restart_io_root (this)) packed_value = transfer (value, packed_value) - call Broadcast_restart_integer (this, packed_value) - value = transfer (packed_value, value) -#endif - - end subroutine Broadcast_restart_real - - subroutine Broadcast_restart_logical (this, value) - - implicit none - - class (state_fire_t), intent (in) :: this - logical, intent (in out) :: value - - integer :: flag - - -#ifdef DM_PARALLEL - flag = 0 - if (Is_restart_io_root (this)) then - if (value) then - flag = 1 - else - flag = 0 - end if - end if - call Broadcast_restart_integer (this, flag) - value = flag /= 0 -#endif - - end subroutine Broadcast_restart_logical - - subroutine Broadcast_restart_failure (this, is_valid, failure_code, failure_msg) - - implicit none - - class (state_fire_t), intent (in) :: this - logical, intent (in out) :: is_valid - integer, intent (in out) :: failure_code - character (len = *), intent (in out) :: failure_msg - - integer :: ierr, n, msg_len - integer, dimension(:), allocatable :: msg_codes - - - call Broadcast_restart_logical (this, is_valid) - call Broadcast_restart_integer (this, failure_code) -#ifdef DM_PARALLEL - call Require_restart_mpi_comm (this) - msg_len = len (failure_msg) - allocate (msg_codes(msg_len)) - if (Is_restart_io_root (this)) then - do n = 1, msg_len - msg_codes(n) = iachar (failure_msg(n:n)) - end do - end if - call MPI_Bcast (msg_codes(1), msg_len, MPI_INTEGER, RESTART_IO_ROOT, this%cfbm_comm, ierr) - if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with MPI_Bcast for restart failure metadata') - if (.not. Is_restart_io_root (this)) then - do n = 1, msg_len - failure_msg(n:n) = achar (msg_codes(n)) - end do - end if - deallocate (msg_codes) -#endif - - end subroutine Broadcast_restart_failure - - subroutine Read_restart (this, config_flags) - - implicit none - - class (state_fire_t), intent (in out) :: this - type (namelist_t), intent (in) :: config_flags - - type (datetime_t) :: datetime_restart, datetime_check - character (len = :), allocatable :: file_restart - integer (kind = INT32) :: att_int, restart_year, restart_month, restart_day, restart_hour, restart_minute, restart_second, & - start_year, start_month, start_day, start_hour, start_minute, start_second - integer :: ij, expected_sr_x, expected_sr_y, ierr, rank - logical, parameter :: DEBUG_LOCAL = .false. - - - if (DEBUG_LOCAL) call Print_message ('Entering Read_restart...') - -#ifdef DM_PARALLEL - call Require_restart_mpi_comm (this) - call Mpi_comm_rank (this%cfbm_comm, rank, ierr) - if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_comm_rank ') -#else - rank = 0 -#endif - - if (config_flags%ideal_opt /= 0 .and. config_flags%ideal_opt /= 1) & - call Stop_simulation ('Read_restart is implemented for standalone idealized and real runs only') - - datetime_restart = datetime_t (config_flags%start_year, config_flags%start_month, config_flags%start_day, & - config_flags%start_hour, config_flags%start_minute, config_flags%start_second) - file_restart = Build_restart_file_name (datetime_restart%datetime) - - if (rank == 0) then - call Is_netcdf_file_present (file_restart) - - call Validate_restart_integer (file_restart, 'restart_year', config_flags%start_year) - call Validate_restart_integer (file_restart, 'restart_month', config_flags%start_month) - call Validate_restart_integer (file_restart, 'restart_day', config_flags%start_day) - call Validate_restart_integer (file_restart, 'restart_hour', config_flags%start_hour) - call Validate_restart_integer (file_restart, 'restart_minute', config_flags%start_minute) - call Validate_restart_integer (file_restart, 'restart_second', config_flags%start_second) - - call Validate_restart_integer (file_restart, 'nx', this%nx) - call Validate_restart_integer (file_restart, 'ny', this%ny) - call Validate_restart_real (file_restart, 'dt', this%dt) - call Validate_restart_real (file_restart, 'dx', this%dx) - call Validate_restart_real (file_restart, 'dy', this%dy) - call Validate_restart_real (file_restart, 'cen_lat', this%cen_lat) - call Validate_restart_real (file_restart, 'cen_lon', this%cen_lon) - expected_sr_x = nint (this%proj%dx / this%dx) - expected_sr_y = nint (this%proj%dy / this%dy) - call Validate_restart_integer (file_restart, 'map_proj', 1) - call Validate_restart_integer (file_restart, 'sr_x', expected_sr_x) - call Validate_restart_integer (file_restart, 'sr_y', expected_sr_y) - call Validate_restart_integer (file_restart, 'ideal_opt', config_flags%ideal_opt) - call Validate_restart_integer (file_restart, 'fuel_opt', config_flags%fuel_opt) - call Validate_restart_integer (file_restart, 'ros_opt', config_flags%ros_opt) - call Validate_restart_integer (file_restart, 'fmc_opt', config_flags%fmc_opt) - call Validate_restart_integer (file_restart, 'emis_opt', config_flags%emis_opt) - call Validate_restart_integer (file_restart, 'fire_upwinding', config_flags%fire_upwinding) - call Validate_restart_integer (file_restart, 'fire_upwinding_reinit', config_flags%fire_upwinding_reinit) - call Validate_restart_integer (file_restart, 'fire_lsm_reinit_iter', config_flags%fire_lsm_reinit_iter) - call Validate_restart_real (file_restart, 'fire_viscosity', config_flags%fire_viscosity) - call Validate_restart_real (file_restart, 'fire_viscosity_bg', config_flags%fire_viscosity_bg) - call Validate_restart_integer (file_restart, 'fire_viscosity_ngp', config_flags%fire_viscosity_ngp) - call Validate_restart_real (file_restart, 'fire_viscosity_band', config_flags%fire_viscosity_band) - call Validate_restart_real (file_restart, 'reinit_pseudot_coef', config_flags%reinit_pseudot_coef) - - select case (config_flags%ideal_opt) - case (0) - call Validate_restart_real (file_restart, 'stand_lon', this%proj%standard_lon) - call Validate_restart_real (file_restart, 'true_lat_1', this%proj%true_lat_1) - call Validate_restart_real (file_restart, 'true_lat_2', this%proj%true_lat_2) - - case (1) - call Validate_restart_real (file_restart, 'stand_lon', config_flags%stand_lon) - call Validate_restart_real (file_restart, 'true_lat_1', config_flags%true_lat_1) - call Validate_restart_real (file_restart, 'true_lat_2', config_flags%true_lat_2) - end select - - call Get_netcdf_att (file_restart, 'global', 'start_year', start_year) - call Get_netcdf_att (file_restart, 'global', 'start_month', start_month) - call Get_netcdf_att (file_restart, 'global', 'start_day', start_day) - call Get_netcdf_att (file_restart, 'global', 'start_hour', start_hour) - call Get_netcdf_att (file_restart, 'global', 'start_minute', start_minute) - call Get_netcdf_att (file_restart, 'global', 'start_second', start_second) - - call Get_netcdf_att (file_restart, 'global', 'restart_year', restart_year) - call Get_netcdf_att (file_restart, 'global', 'restart_month', restart_month) - call Get_netcdf_att (file_restart, 'global', 'restart_day', restart_day) - call Get_netcdf_att (file_restart, 'global', 'restart_hour', restart_hour) - call Get_netcdf_att (file_restart, 'global', 'restart_minute', restart_minute) - call Get_netcdf_att (file_restart, 'global', 'restart_second', restart_second) - - call Get_netcdf_att (file_restart, 'global', 'itimestep', att_int) - end if - - call Broadcast_restart_integer (this, start_year) - call Broadcast_restart_integer (this, start_month) - call Broadcast_restart_integer (this, start_day) - call Broadcast_restart_integer (this, start_hour) - call Broadcast_restart_integer (this, start_minute) - call Broadcast_restart_integer (this, start_second) - call Broadcast_restart_integer (this, restart_year) - call Broadcast_restart_integer (this, restart_month) - call Broadcast_restart_integer (this, restart_day) - call Broadcast_restart_integer (this, restart_hour) - call Broadcast_restart_integer (this, restart_minute) - call Broadcast_restart_integer (this, restart_second) - call Broadcast_restart_integer (this, att_int) - this%itimestep = att_int - - this%datetime_start = datetime_t (start_year, start_month, start_day, start_hour, start_minute, start_second) - this%datetime_now = datetime_t (restart_year, restart_month, restart_day, restart_hour, restart_minute, restart_second) - - datetime_check = this%datetime_start - call datetime_check%Add_seconds (this%itimestep * this%dt) - if (datetime_check /= this%datetime_now) call Stop_simulation ('Restart clock is inconsistent with itimestep and dt') - - call Set_next_datetime_after_restart (this%datetime_next_output, config_flags%interval_output) - call Set_next_datetime_after_restart (this%datetime_next_atm_update, config_flags%interval_atm) - - call Read_restart_field (file_restart, 'lfn', this%lfn) - call Read_restart_field (file_restart, 'lfn_hist', this%lfn_hist) - call Read_restart_field (file_restart, 'lfn_0', this%lfn_0) - call Read_restart_field (file_restart, 'lfn_1', this%lfn_1) - call Read_restart_field (file_restart, 'lfn_2', this%lfn_2) - call Read_restart_field (file_restart, 'lfn_s0', this%lfn_s0) - call Read_restart_field (file_restart, 'lfn_s1', this%lfn_s1) - call Read_restart_field (file_restart, 'lfn_s2', this%lfn_s2) - call Read_restart_field (file_restart, 'lfn_s3', this%lfn_s3) - call Read_restart_field (file_restart, 'lfn_out', this%lfn_out) - call Read_restart_field (file_restart, 'tign_g', this%tign_g) - call Read_restart_field (file_restart, 'fuel_frac', this%fuel_frac) - call Read_restart_field (file_restart, 'fire_area', this%fire_area) - call Read_restart_field (file_restart, 'fuel_frac_burnt_dt', this%fuel_frac_burnt_dt) - call Read_restart_field (file_restart, 'fgrnhfx', this%fgrnhfx) - call Read_restart_field (file_restart, 'fgrnqfx', this%fgrnqfx) - call Read_restart_field (file_restart, 'fcanhfx', this%fcanhfx) - call Read_restart_field (file_restart, 'fcanqfx', this%fcanqfx) - call Read_restart_field (file_restart, 'flame_length', this%flame_length) - call Read_restart_field (file_restart, 'ros', this%ros) - call Read_restart_field (file_restart, 'ros_front', this%ros_front) - call Read_restart_field (file_restart, 'emis_smoke', this%emis_smoke) - call Read_restart_field (file_restart, 'fmc_g', this%fmc_g) - call Read_restart_field (file_restart, 'fuel_load_g', this%fuel_load_g) - call Read_restart_field (file_restart, 'fuel_time', this%fuel_time) - call Read_restart_field (file_restart, 'zsf', this%zsf) - call Read_restart_field (file_restart, 'dzdxf', this%dzdxf) - call Read_restart_field (file_restart, 'dzdyf', this%dzdyf) - call Read_restart_field (file_restart, 'nfuel_cat', this%nfuel_cat) - call Read_restart_field (file_restart, 'uf', this%uf) - call Read_restart_field (file_restart, 'vf', this%vf) - call Read_restart_field (file_restart, 'fz0', this%fz0) - - if (config_flags%ideal_opt == 0) then - call Read_restart_field (file_restart, 'fire_t2', this%fire_t2) - call Read_restart_field (file_restart, 'fire_q2', this%fire_q2) - call Read_restart_field (file_restart, 'fire_psfc', this%fire_psfc) - call Read_restart_field (file_restart, 'fire_rain', this%fire_rain) - if (config_flags%fmoist_run) then - call Read_restart_field (file_restart, 'fire_t2_old', this%fire_t2_old) - call Read_restart_field (file_restart, 'fire_q2_old', this%fire_q2_old) - call Read_restart_field (file_restart, 'fire_psfc_old', this%fire_psfc_old) - call Read_restart_field (file_restart, 'fire_rain_old', this%fire_rain_old) - end if - end if - if (config_flags%fmoist_run) call Read_restart_fmc (file_restart) - - call Exchange_restart_halos () - - if (allocated (this%ros_param) .and. allocated (this%fuels)) then - do ij = 1, this%num_tiles - call this%ros_param%Set_params (this%ifms, this%ifme, this%jfms, this%jfme, this%i_start(ij), this%i_end(ij), & - this%j_start(ij), this%j_end(ij), this%fuels, this%nfuel_cat, this%fmc_g) - end do - end if - - if (DEBUG_LOCAL) call Print_message ('Leaving Read_restart...') - - contains - - subroutine Exchange_restart_halos () - - implicit none - - -#ifdef DM_PARALLEL - call Exchange_restart_field (this%lfn) - call Exchange_restart_field (this%lfn_hist) - call Exchange_restart_field (this%lfn_0) - call Exchange_restart_field (this%lfn_1) - call Exchange_restart_field (this%lfn_2) - call Exchange_restart_field (this%lfn_s0) - call Exchange_restart_field (this%lfn_s1) - call Exchange_restart_field (this%lfn_s2) - call Exchange_restart_field (this%lfn_s3) - call Exchange_restart_field (this%lfn_out) - call Exchange_restart_field (this%tign_g) - call Exchange_restart_field (this%fuel_frac) - call Exchange_restart_field (this%fire_area) - call Exchange_restart_field (this%fuel_frac_burnt_dt) - call Exchange_restart_field (this%fgrnhfx) - call Exchange_restart_field (this%fgrnqfx) - call Exchange_restart_field (this%fcanhfx) - call Exchange_restart_field (this%fcanqfx) - call Exchange_restart_field (this%flame_length) - call Exchange_restart_field (this%ros) - call Exchange_restart_field (this%ros_front) - call Exchange_restart_field (this%emis_smoke) - call Exchange_restart_field (this%fmc_g) - call Exchange_restart_field (this%fuel_load_g) - call Exchange_restart_field (this%fuel_time) - call Exchange_restart_field (this%zsf) - call Exchange_restart_field (this%dzdxf) - call Exchange_restart_field (this%dzdyf) - call Exchange_restart_field (this%nfuel_cat) - call Exchange_restart_field (this%uf) - call Exchange_restart_field (this%vf) - call Exchange_restart_field (this%fz0) - - if (config_flags%ideal_opt == 0) then - call Exchange_restart_field (this%fire_t2) - call Exchange_restart_field (this%fire_q2) - call Exchange_restart_field (this%fire_psfc) - call Exchange_restart_field (this%fire_rain) - if (config_flags%fmoist_run) then - call Exchange_restart_field (this%fire_t2_old) - call Exchange_restart_field (this%fire_q2_old) - call Exchange_restart_field (this%fire_psfc_old) - call Exchange_restart_field (this%fire_rain_old) - end if - end if - - if (config_flags%fmoist_run) call Exchange_restart_fmc_halos () -#endif - - end subroutine Exchange_restart_halos - - subroutine Exchange_restart_field (var) - - implicit none - - real, dimension(this%ifms:this%ifme, this%jfms:this%jfme), intent (in out) :: var - - -#ifdef DM_PARALLEL - call Do_halo_exchange_with_corners (var, this%ifms, this%ifme, this%jfms, this%jfme, & - this%ifps, this%ifpe, this%jfps, this%jfpe, N_POINTS_IN_HALO, this%cart_comm) -#endif - - end subroutine Exchange_restart_field - - subroutine Exchange_restart_fmc_halos () - - implicit none - - integer :: k, n_moisture_classes - - -#ifdef DM_PARALLEL - if (.not. allocated (this%fmc_param)) call Stop_simulation ('FMC restart halo exchange requires allocated fmc_param') - - select type (fmc_param => this%fmc_param) - type is (fmc_wrffire_t) - if (.not. allocated (fmc_param%fmc_gc)) call Stop_simulation ('FMC restart halo exchange requires allocated fmc_gc') - - n_moisture_classes = size (fmc_param%fmc_gc, 2) - do k = 1, n_moisture_classes - call Do_halo_exchange_with_corners (fmc_param%fmc_gc(this%ifms:this%ifme, k, this%jfms:this%jfme), & - this%ifms, this%ifme, this%jfms, this%jfme, & - this%ifps, this%ifpe, this%jfps, this%jfpe, N_POINTS_IN_HALO, this%cart_comm) - end do - - class default - call Stop_simulation ('Restart halo exchange is not implemented for selected FMC component') - end select -#endif - - end subroutine Exchange_restart_fmc_halos - - subroutine Read_restart_field (file_name, var_name, var) - - implicit none - - character (len = *), intent (in) :: file_name, var_name - real, dimension(this%ifms:this%ifme, this%jfms:this%jfme), intent (in out) :: var - - - call Get_netcdf_var_mpi (file_name, this%cfbm_comm, this%nx, this%ny, & - this%ifps, this%ifpe, this%jfps, this%jfpe, var_name, var(this%ifps:this%ifpe, this%jfps:this%jfpe)) - - end subroutine Read_restart_field - - subroutine Read_restart_fmc (file_name) - - implicit none - - character (len = *), intent (in) :: file_name - - real (kind = REAL32) :: att_real32 - real :: fmoist_lasttime, fmoist_nexttime - integer :: n_moisture_classes - - - if (.not. allocated (this%fmc_param)) call Stop_simulation ('FMC restart read requires allocated fmc_param') - - select type (fmc_param => this%fmc_param) - type is (fmc_wrffire_t) - if (.not. allocated (fmc_param%fmc_gc)) call Stop_simulation ('FMC restart read requires allocated fmc_gc') - - n_moisture_classes = size (fmc_param%fmc_gc, 2) - call Get_netcdf_var_mpi (file_name, this%cfbm_comm, this%nx, this%ny, n_moisture_classes, & - this%ifps, this%ifpe, this%jfps, this%jfpe, NAME_VAR_FMC_GC, & - fmc_param%fmc_gc(this%ifps:this%ifpe, 1:n_moisture_classes, this%jfps:this%jfpe)) - - fmoist_lasttime = 0.0 - fmoist_nexttime = 0.0 - if (rank == 0) then - call Get_netcdf_att (file_name, 'global', NAME_ATT_FMOIST_LASTTIME, att_real32) - fmoist_lasttime = att_real32 - call Get_netcdf_att (file_name, 'global', NAME_ATT_FMOIST_NEXTTIME, att_real32) - fmoist_nexttime = att_real32 - end if - call Broadcast_restart_real (this, fmoist_lasttime) - call Broadcast_restart_real (this, fmoist_nexttime) - fmc_param%fmoist_lasttime = fmoist_lasttime - fmc_param%fmoist_nexttime = fmoist_nexttime - - class default - call Stop_simulation ('Restart read is not implemented for selected FMC component') - end select - - end subroutine Read_restart_fmc - - subroutine Set_next_datetime_after_restart (datetime_next, interval_seconds) - - implicit none - - type (datetime_t), intent (in out) :: datetime_next - integer, intent (in) :: interval_seconds - - datetime_next = this%datetime_now - if (interval_seconds > 0) then - datetime_next = this%datetime_start - do while (datetime_next <= this%datetime_now) - call datetime_next%Add_seconds (interval_seconds) - end do - end if - - end subroutine Set_next_datetime_after_restart - - end subroutine Read_restart - - subroutine Read_restart_field_2d (file_name, var_name, nx, ny, ifms, ifme, jfms, jfme, ifps, ifpe, jfps, jfpe, cfbm_comm, var) - - implicit none - - character (len = *), intent (in) :: file_name, var_name - integer, intent (in) :: nx, ny, ifms, ifme, jfms, jfme, ifps, ifpe, jfps, jfpe, cfbm_comm - real, dimension(ifms:ifme, jfms:jfme), intent (in out) :: var - - - call Get_netcdf_var_mpi (file_name, cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, & - var_name, var(ifps:ifpe, jfps:jfpe)) - - end subroutine Read_restart_field_2d - - subroutine Validate_restart_integer (file_name, att_name, expected_value) - - implicit none - - character (len = *), intent (in) :: file_name, att_name - integer, intent (in) :: expected_value - - integer (kind = INT32) :: att_value - character (len = :), allocatable :: msg - - - call Get_netcdf_att (file_name, 'global', att_name, att_value) - if (att_value /= expected_value) then - msg = 'Restart metadata mismatch: '//trim (att_name) - call Stop_simulation (msg) - end if - - end subroutine Validate_restart_integer - - subroutine Validate_restart_real (file_name, att_name, expected_value) - - implicit none - - character (len = *), intent (in) :: file_name, att_name - real, intent (in) :: expected_value - - real (kind = REAL32) :: att_value, tolerance - character (len = :), allocatable :: msg - - - call Get_netcdf_att (file_name, 'global', att_name, att_value) - tolerance = max (1.0e-6_REAL32, abs (real (expected_value, kind = REAL32)) * 1.0e-6_REAL32) - if (abs (att_value - real (expected_value, kind = REAL32)) > tolerance) then - msg = 'Restart metadata mismatch: '//trim (att_name) - call Stop_simulation (msg) - end if - - end subroutine Validate_restart_real - - subroutine Write_restart (this, config_flags) - - implicit none - - class (state_fire_t), intent (in) :: this - type (namelist_t), intent (in) :: config_flags - - character (len = :), allocatable :: file_restart - integer :: start_year, start_month, start_day, start_hour, start_minute, start_second, & - restart_year, restart_month, restart_day, restart_hour, restart_minute, restart_second - integer :: restart_sr_x, restart_sr_y, rank, ierr - logical, parameter :: DEBUG_LOCAL = .false. - - - if (DEBUG_LOCAL) call Print_message ('Entering Write_restart...') - -#ifdef DM_PARALLEL - if (.not. this%is_cfbm_comm_set) call Stop_simulation ('The MPI CFBM communicator has not been set') - call Mpi_comm_rank (this%cfbm_comm, rank, ierr) - if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_comm_rank ') -#else - rank = 0 -#endif - - if (config_flags%ideal_opt /= 0 .and. config_flags%ideal_opt /= 1) & - call Stop_simulation ('Write_restart is implemented for standalone idealized and real runs only') - - file_restart = Build_restart_file_name (this%datetime_now%datetime) - - call this%datetime_start%Get_datetime_as_ints (start_year, start_month, start_day, start_hour, start_minute, start_second) - call this%datetime_now%Get_datetime_as_ints (restart_year, restart_month, restart_day, restart_hour, restart_minute, restart_second) - restart_sr_x = nint (this%proj%dx / this%dx) - restart_sr_y = nint (this%proj%dy / this%dy) - if (restart_sr_x <= 0 .or. restart_sr_y <= 0) call Stop_simulation ('Restart subgrid ratios must be positive') - - if (rank == 0) then - call Create_netcdf_file (file_name = file_restart) - call Add_netcdf_dim (file_restart, NAME_DIM_X, this%nx) - call Add_netcdf_dim (file_restart, NAME_DIM_Y, this%ny) - - call Add_netcdf_att (file_restart, 'global', 'start_year', int (start_year, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'start_month', int (start_month, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'start_day', int (start_day, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'start_hour', int (start_hour, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'start_minute', int (start_minute, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'start_second', int (start_second, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'restart_year', int (restart_year, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'restart_month', int (restart_month, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'restart_day', int (restart_day, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'restart_hour', int (restart_hour, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'restart_minute', int (restart_minute, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'restart_second', int (restart_second, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'itimestep', int (this%itimestep, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'nx', int (this%nx, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'ny', int (this%ny, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'dt', real (this%dt, kind = REAL32)) - call Add_netcdf_att (file_restart, 'global', 'dx', real (this%dx, kind = REAL32)) - call Add_netcdf_att (file_restart, 'global', 'dy', real (this%dy, kind = REAL32)) - call Add_netcdf_att (file_restart, 'global', 'cen_lat', real (this%cen_lat, kind = REAL32)) - call Add_netcdf_att (file_restart, 'global', 'cen_lon', real (this%cen_lon, kind = REAL32)) - call Add_netcdf_att (file_restart, 'global', 'map_proj', int (1, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'sr_x', int (restart_sr_x, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'sr_y', int (restart_sr_y, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'stand_lon', real (this%proj%standard_lon, kind = REAL32)) - call Add_netcdf_att (file_restart, 'global', 'true_lat_1', real (this%proj%true_lat_1, kind = REAL32)) - call Add_netcdf_att (file_restart, 'global', 'true_lat_2', real (this%proj%true_lat_2, kind = REAL32)) - call Add_netcdf_att (file_restart, 'global', 'ideal_opt', int (config_flags%ideal_opt, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'fuel_opt', int (config_flags%fuel_opt, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'ros_opt', int (config_flags%ros_opt, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'fmc_opt', int (config_flags%fmc_opt, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'emis_opt', int (config_flags%emis_opt, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'fire_upwinding', int (config_flags%fire_upwinding, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'fire_upwinding_reinit', int (config_flags%fire_upwinding_reinit, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'fire_lsm_reinit_iter', int (config_flags%fire_lsm_reinit_iter, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'fire_viscosity', real (config_flags%fire_viscosity, kind = REAL32)) - call Add_netcdf_att (file_restart, 'global', 'fire_viscosity_bg', real (config_flags%fire_viscosity_bg, kind = REAL32)) - call Add_netcdf_att (file_restart, 'global', 'fire_viscosity_ngp', int (config_flags%fire_viscosity_ngp, kind = INT32)) - call Add_netcdf_att (file_restart, 'global', 'fire_viscosity_band', real (config_flags%fire_viscosity_band, kind = REAL32)) - call Add_netcdf_att (file_restart, 'global', 'reinit_pseudot_coef', real (config_flags%reinit_pseudot_coef, kind = REAL32)) - end if - - call Restart_io_barrier (this) - - call Add_restart_field ('lfn', this%lfn) - call Add_restart_field ('lfn_hist', this%lfn_hist) - call Add_restart_field ('lfn_0', this%lfn_0) - call Add_restart_field ('lfn_1', this%lfn_1) - call Add_restart_field ('lfn_2', this%lfn_2) - call Add_restart_field ('lfn_s0', this%lfn_s0) - call Add_restart_field ('lfn_s1', this%lfn_s1) - call Add_restart_field ('lfn_s2', this%lfn_s2) - call Add_restart_field ('lfn_s3', this%lfn_s3) - call Add_restart_field ('lfn_out', this%lfn_out) - call Add_restart_field ('tign_g', this%tign_g) - call Add_restart_field ('fuel_frac', this%fuel_frac) - call Add_restart_field ('fire_area', this%fire_area) - call Add_restart_field ('fuel_frac_burnt_dt', this%fuel_frac_burnt_dt) - call Add_restart_field ('fgrnhfx', this%fgrnhfx) - call Add_restart_field ('fgrnqfx', this%fgrnqfx) - call Add_restart_field ('fcanhfx', this%fcanhfx) - call Add_restart_field ('fcanqfx', this%fcanqfx) - call Add_restart_field ('flame_length', this%flame_length) - call Add_restart_field ('ros', this%ros) - call Add_restart_field ('ros_front', this%ros_front) - call Add_restart_field ('emis_smoke', this%emis_smoke) - call Add_restart_field ('fmc_g', this%fmc_g) - call Add_restart_field ('fuel_load_g', this%fuel_load_g) - call Add_restart_field ('fuel_time', this%fuel_time) - call Add_restart_field ('zsf', this%zsf) - call Add_restart_field ('dzdxf', this%dzdxf) - call Add_restart_field ('dzdyf', this%dzdyf) - call Add_restart_field ('nfuel_cat', this%nfuel_cat) - call Add_restart_field ('uf', this%uf) - call Add_restart_field ('vf', this%vf) - call Add_restart_field ('fz0', this%fz0) - - if (config_flags%ideal_opt == 0) then - call Add_restart_field ('fire_t2', this%fire_t2) - call Add_restart_field ('fire_q2', this%fire_q2) - call Add_restart_field ('fire_psfc', this%fire_psfc) - call Add_restart_field ('fire_rain', this%fire_rain) - if (config_flags%fmoist_run) then - call Add_restart_field ('fire_t2_old', this%fire_t2_old) - call Add_restart_field ('fire_q2_old', this%fire_q2_old) - call Add_restart_field ('fire_psfc_old', this%fire_psfc_old) - call Add_restart_field ('fire_rain_old', this%fire_rain_old) - end if - end if - if (config_flags%fmoist_run) call Write_restart_fmc () - - call Restart_io_barrier (this) - - if (DEBUG_LOCAL) call Print_message ('Leaving Write_restart...') - - contains - - subroutine Add_restart_field (var_name, var) - - implicit none - - character (len = *), intent (in) :: var_name - real, dimension(this%ifms:this%ifme, this%jfms:this%jfme), intent (in) :: var - - - call Add_netcdf_var_mpi (file_restart, this%cfbm_comm, this%nx, this%ny, this%ifps, this%ifpe, this%jfps, this%jfpe, & - var_name, var(this%ifps:this%ifpe, this%jfps:this%jfpe)) - - end subroutine Add_restart_field - - subroutine Write_restart_fmc () - - implicit none - - character (len = 32), dimension(3) :: dim_names_fmc_gc - integer :: n_moisture_classes - - - if (.not. allocated (this%fmc_param)) call Stop_simulation ('FMC restart write requires allocated fmc_param') - - select type (fmc_param => this%fmc_param) - type is (fmc_wrffire_t) - if (.not. allocated (fmc_param%fmc_gc)) call Stop_simulation ('FMC restart write requires allocated fmc_gc') - - n_moisture_classes = size (fmc_param%fmc_gc, 2) - if (rank == 0) call Add_netcdf_dim (file_restart, NAME_DIM_MOISTURE_CLASS, n_moisture_classes) - call Restart_io_barrier (this) - - dim_names_fmc_gc(1) = NAME_DIM_X - dim_names_fmc_gc(2) = NAME_DIM_MOISTURE_CLASS - dim_names_fmc_gc(3) = NAME_DIM_Y - call Add_netcdf_var_mpi (file_restart, dim_names_fmc_gc, this%cfbm_comm, this%nx, this%ny, n_moisture_classes, & - this%ifps, this%ifpe, this%jfps, this%jfpe, NAME_VAR_FMC_GC, & - fmc_param%fmc_gc(this%ifps:this%ifpe, 1:n_moisture_classes, this%jfps:this%jfpe)) - - if (rank == 0) then - call Add_netcdf_att (file_restart, 'global', NAME_ATT_FMOIST_LASTTIME, & - real (fmc_param%fmoist_lasttime, kind = REAL32)) - call Add_netcdf_att (file_restart, 'global', NAME_ATT_FMOIST_NEXTTIME, & - real (fmc_param%fmoist_nexttime, kind = REAL32)) - end if - - class default - call Stop_simulation ('Restart write is not implemented for selected FMC component') - end select - - end subroutine Write_restart_fmc - - end subroutine Write_restart - subroutine Save_state (this) implicit none @@ -1963,4 +1110,3 @@ subroutine Set_vars_to_default (this, config_flags) end subroutine Set_vars_to_default end module state_mod - diff --git a/state/state_restart_submod.F90 b/state/state_restart_submod.F90 new file mode 100644 index 00000000..65a706f2 --- /dev/null +++ b/state/state_restart_submod.F90 @@ -0,0 +1,848 @@ + submodule (state_mod) state_restart_submod + +#ifdef DM_PARALLEL + use mpi +#endif + use fmc_wrffire_mod, only : fmc_wrffire_t + use, intrinsic :: iso_fortran_env, only : INT32, REAL32 + + implicit none + + integer, parameter :: RESTART_IO_ROOT = 0 + character (len = *), parameter :: NAME_DIM_MOISTURE_CLASS = 'moisture_class' + character (len = *), parameter :: NAME_VAR_FMC_GC = 'fmc_gc' + character (len = *), parameter :: NAME_ATT_FMOIST_LASTTIME = 'fmc_fmoist_lasttime' + character (len = *), parameter :: NAME_ATT_FMOIST_NEXTTIME = 'fmc_fmoist_nexttime' + + contains + + module procedure Build_restart_file_name + + file_restart = 'fire_restart_'//trim (restart_datetime)//'.nc' + + end procedure Build_restart_file_name + + module procedure Init_restart_dimensions + + integer :: restart_nx, restart_ny + integer (kind = INT32) :: att_int32 + integer :: ntasks, px, py, ierr, cart_comm, rank + integer, dimension(2) :: coords + character (len = 300) :: msg + + + restart_nx = 0 + restart_ny = 0 + if (Is_restart_io_root (this)) then + call Get_netcdf_att (trim (restart_file), 'global', 'nx', att_int32) + restart_nx = att_int32 + call Get_netcdf_att (trim (restart_file), 'global', 'ny', att_int32) + restart_ny = att_int32 + end if + call Broadcast_restart_integer (this, restart_nx) + call Broadcast_restart_integer (this, restart_ny) + + ids0 = 1 + ide0 = restart_nx + jds0 = 1 + jde0 = restart_ny + +#ifdef DM_PARALLEL + if (.not. this%is_cfbm_comm_set) call Stop_simulation ('The MPI CFBM communicator has not been set') + + call Mpi_comm_size (this%cfbm_comm, ntasks, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems getting the number of MPI tasks') + this%ntasks = ntasks + + call Calc_tasks_in_x_and_y (this%ntasks, restart_nx, restart_ny, px, py) + this%px = px + this%py = py + write (msg, '(a25, 2(1x, i5))') 'MPI TASKS in x and y =', this%px, this%py + call Print_message (msg) + + call Mpi_cart_create (this%cfbm_comm, N_DIMS, [this%px, this%py], PERIODS, REORDER, cart_comm, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_cart_create') + this%cart_comm = cart_comm + + call Mpi_comm_rank (this%cart_comm, rank, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_comm_rank ') + + call Mpi_cart_coords (this%cart_comm, rank, N_DIMS, coords, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_cart_coords') + + call Calc_patch_dims (restart_nx, restart_ny, this%px, this%py, coords, ips, ipe, jps, jpe) +#else + ips = ids0 + ipe = ide0 + jps = jds0 + jpe = jde0 +#endif + + end procedure Init_restart_dimensions + + module procedure Init_restart_projection + + integer :: restart_map_proj, restart_sr_x, restart_sr_y + integer (kind = INT32) :: att_int32 + real (kind = REAL32) :: att_real32 + real :: restart_dx, restart_dy, restart_cen_lat, restart_cen_lon, restart_stand_lon, restart_true_lat_1, restart_true_lat_2 + + + restart_dx = 0.0 + restart_dy = 0.0 + restart_map_proj = 0 + restart_sr_x = 0 + restart_sr_y = 0 + restart_cen_lat = 0.0 + restart_cen_lon = 0.0 + restart_stand_lon = 0.0 + restart_true_lat_1 = 0.0 + restart_true_lat_2 = 0.0 + + if (Is_restart_io_root (this)) then + call Get_netcdf_att (trim (restart_file), 'global', 'dx', att_real32) + restart_dx = att_real32 + call Get_netcdf_att (trim (restart_file), 'global', 'dy', att_real32) + restart_dy = att_real32 + call Get_netcdf_att (trim (restart_file), 'global', 'map_proj', att_int32) + restart_map_proj = att_int32 + call Get_netcdf_att (trim (restart_file), 'global', 'sr_x', att_int32) + restart_sr_x = att_int32 + call Get_netcdf_att (trim (restart_file), 'global', 'sr_y', att_int32) + restart_sr_y = att_int32 + call Get_netcdf_att (trim (restart_file), 'global', 'cen_lat', att_real32) + restart_cen_lat = att_real32 + call Get_netcdf_att (trim (restart_file), 'global', 'cen_lon', att_real32) + restart_cen_lon = att_real32 + call Get_netcdf_att (trim (restart_file), 'global', 'stand_lon', att_real32) + restart_stand_lon = att_real32 + call Get_netcdf_att (trim (restart_file), 'global', 'true_lat_1', att_real32) + restart_true_lat_1 = att_real32 + call Get_netcdf_att (trim (restart_file), 'global', 'true_lat_2', att_real32) + restart_true_lat_2 = att_real32 + end if + + call Broadcast_restart_real (this, restart_dx) + call Broadcast_restart_real (this, restart_dy) + call Broadcast_restart_integer (this, restart_map_proj) + call Broadcast_restart_integer (this, restart_sr_x) + call Broadcast_restart_integer (this, restart_sr_y) + call Broadcast_restart_real (this, restart_cen_lat) + call Broadcast_restart_real (this, restart_cen_lon) + call Broadcast_restart_real (this, restart_stand_lon) + call Broadcast_restart_real (this, restart_true_lat_1) + call Broadcast_restart_real (this, restart_true_lat_2) + + this%dx = restart_dx + this%dy = restart_dy + + if (restart_map_proj /= 1) call Stop_simulation ('Restart map projection is not supported') + + if (restart_sr_x <= 0 .or. restart_sr_y <= 0) call Stop_simulation ('Restart subgrid ratios must be positive') + if (mod (this%nx, restart_sr_x) /= 0 .or. mod (this%ny, restart_sr_y) /= 0) & + call Stop_simulation ('Restart subgrid ratios do not divide fire-grid dimensions') + if (this%nx / restart_sr_x <= 1 .or. this%ny / restart_sr_y <= 1) & + call Stop_simulation ('Restart subgrid ratios imply invalid atmospheric dimensions') + + this%cen_lat = restart_cen_lat + this%cen_lon = restart_cen_lon + + proj = proj_lc_t (cen_lat = this%cen_lat , cen_lon = this%cen_lon, dx = this%dx * restart_sr_x, & + dy = this%dy * restart_sr_y, & + standard_lon = restart_stand_lon, true_lat_1 = restart_true_lat_1, & + true_lat_2 = restart_true_lat_2, nx = this%nx / restart_sr_x - 1, ny = this%ny / restart_sr_y - 1) + + call this%Init_latlons (proj, srx = restart_sr_x, sry = restart_sr_y) + + end procedure Init_restart_projection + + module procedure Read_restart_static_fields + + call Read_restart_field_2d (trim (restart_file), 'zsf', this%nx, this%ny, & + this%ifms, this%ifme, this%jfms, this%jfme, this%ifps, this%ifpe, this%jfps, this%jfpe, this%cfbm_comm, this%zsf) + call Read_restart_field_2d (trim (restart_file), 'dzdxf', this%nx, this%ny, & + this%ifms, this%ifme, this%jfms, this%jfme, this%ifps, this%ifpe, this%jfps, this%jfpe, this%cfbm_comm, this%dzdxf) + call Read_restart_field_2d (trim (restart_file), 'dzdyf', this%nx, this%ny, & + this%ifms, this%ifme, this%jfms, this%jfme, this%ifps, this%ifpe, this%jfps, this%jfpe, this%cfbm_comm, this%dzdyf) + call Read_restart_field_2d (trim (restart_file), 'nfuel_cat', this%nx, this%ny, & + this%ifms, this%ifme, this%jfms, this%jfme, this%ifps, this%ifpe, this%jfps, this%jfpe, this%cfbm_comm, this%nfuel_cat) + call Read_restart_field_2d (trim (restart_file), 'fz0', this%nx, this%ny, & + this%ifms, this%ifme, this%jfms, this%jfme, this%ifps, this%ifpe, this%jfps, this%jfpe, this%cfbm_comm, this%fz0) + + end procedure Read_restart_static_fields + + subroutine Require_restart_mpi_comm (this) + + implicit none + + class (state_fire_t), intent (in) :: this + + +#ifdef DM_PARALLEL + if (.not. this%is_cfbm_comm_set) call Stop_simulation ('The MPI CFBM communicator has not been set') +#endif + + end subroutine Require_restart_mpi_comm + + function Is_restart_io_root (this) result (am_root) + + implicit none + + class (state_fire_t), intent (in) :: this + logical :: am_root + + integer :: rank, ierr + + +#ifdef DM_PARALLEL + call Require_restart_mpi_comm (this) + call Mpi_comm_rank (this%cfbm_comm, rank, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_comm_rank ') + am_root = rank == RESTART_IO_ROOT +#else + am_root = .true. +#endif + + end function Is_restart_io_root + + subroutine Restart_io_barrier (this) + + implicit none + + class (state_fire_t), intent (in) :: this + + integer :: ierr + + +#ifdef DM_PARALLEL + call Require_restart_mpi_comm (this) + call MPI_Barrier (this%cfbm_comm, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with MPI_Barrier ') +#endif + + end subroutine Restart_io_barrier + + subroutine Broadcast_restart_integer (this, value) + + implicit none + + class (state_fire_t), intent (in) :: this + integer, intent (in out) :: value + + integer :: ierr + + +#ifdef DM_PARALLEL + call Require_restart_mpi_comm (this) + call MPI_Bcast (value, 1, MPI_INTEGER, RESTART_IO_ROOT, this%cfbm_comm, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with MPI_Bcast for restart integer metadata') +#endif + + end subroutine Broadcast_restart_integer + + subroutine Broadcast_restart_real (this, value) + + implicit none + + class (state_fire_t), intent (in) :: this + real, intent (in out) :: value + + integer :: packed_value + + +#ifdef DM_PARALLEL + packed_value = 0 + if (Is_restart_io_root (this)) packed_value = transfer (value, packed_value) + call Broadcast_restart_integer (this, packed_value) + value = transfer (packed_value, value) +#endif + + end subroutine Broadcast_restart_real + + module procedure Read_restart + + type (datetime_t) :: datetime_restart, datetime_check + character (len = :), allocatable :: file_restart + integer (kind = INT32) :: att_int, restart_year, restart_month, restart_day, restart_hour, restart_minute, restart_second, & + start_year, start_month, start_day, start_hour, start_minute, start_second + integer :: ij, expected_sr_x, expected_sr_y, ierr, rank + logical, parameter :: DEBUG_LOCAL = .false. + + + if (DEBUG_LOCAL) call Print_message ('Entering Read_restart...') + +#ifdef DM_PARALLEL + call Require_restart_mpi_comm (this) + call Mpi_comm_rank (this%cfbm_comm, rank, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_comm_rank ') +#else + rank = 0 +#endif + + if (config_flags%ideal_opt /= 0 .and. config_flags%ideal_opt /= 1) & + call Stop_simulation ('Read_restart is implemented for standalone idealized and real runs only') + + datetime_restart = datetime_t (config_flags%start_year, config_flags%start_month, config_flags%start_day, & + config_flags%start_hour, config_flags%start_minute, config_flags%start_second) + file_restart = Build_restart_file_name (datetime_restart%datetime) + + if (rank == 0) then + call Is_netcdf_file_present (file_restart) + + call Validate_restart_integer (file_restart, 'restart_year', config_flags%start_year) + call Validate_restart_integer (file_restart, 'restart_month', config_flags%start_month) + call Validate_restart_integer (file_restart, 'restart_day', config_flags%start_day) + call Validate_restart_integer (file_restart, 'restart_hour', config_flags%start_hour) + call Validate_restart_integer (file_restart, 'restart_minute', config_flags%start_minute) + call Validate_restart_integer (file_restart, 'restart_second', config_flags%start_second) + + call Validate_restart_integer (file_restart, 'nx', this%nx) + call Validate_restart_integer (file_restart, 'ny', this%ny) + call Validate_restart_real (file_restart, 'dt', this%dt) + call Validate_restart_real (file_restart, 'dx', this%dx) + call Validate_restart_real (file_restart, 'dy', this%dy) + call Validate_restart_real (file_restart, 'cen_lat', this%cen_lat) + call Validate_restart_real (file_restart, 'cen_lon', this%cen_lon) + expected_sr_x = nint (this%proj%dx / this%dx) + expected_sr_y = nint (this%proj%dy / this%dy) + call Validate_restart_integer (file_restart, 'map_proj', 1) + call Validate_restart_integer (file_restart, 'sr_x', expected_sr_x) + call Validate_restart_integer (file_restart, 'sr_y', expected_sr_y) + call Validate_restart_integer (file_restart, 'ideal_opt', config_flags%ideal_opt) + call Validate_restart_integer (file_restart, 'fuel_opt', config_flags%fuel_opt) + call Validate_restart_integer (file_restart, 'ros_opt', config_flags%ros_opt) + call Validate_restart_integer (file_restart, 'fmc_opt', config_flags%fmc_opt) + call Validate_restart_integer (file_restart, 'emis_opt', config_flags%emis_opt) + call Validate_restart_integer (file_restart, 'fire_upwinding', config_flags%fire_upwinding) + call Validate_restart_integer (file_restart, 'fire_upwinding_reinit', config_flags%fire_upwinding_reinit) + call Validate_restart_integer (file_restart, 'fire_lsm_reinit_iter', config_flags%fire_lsm_reinit_iter) + call Validate_restart_real (file_restart, 'fire_viscosity', config_flags%fire_viscosity) + call Validate_restart_real (file_restart, 'fire_viscosity_bg', config_flags%fire_viscosity_bg) + call Validate_restart_integer (file_restart, 'fire_viscosity_ngp', config_flags%fire_viscosity_ngp) + call Validate_restart_real (file_restart, 'fire_viscosity_band', config_flags%fire_viscosity_band) + call Validate_restart_real (file_restart, 'reinit_pseudot_coef', config_flags%reinit_pseudot_coef) + + select case (config_flags%ideal_opt) + case (0) + call Validate_restart_real (file_restart, 'stand_lon', this%proj%standard_lon) + call Validate_restart_real (file_restart, 'true_lat_1', this%proj%true_lat_1) + call Validate_restart_real (file_restart, 'true_lat_2', this%proj%true_lat_2) + + case (1) + call Validate_restart_real (file_restart, 'stand_lon', config_flags%stand_lon) + call Validate_restart_real (file_restart, 'true_lat_1', config_flags%true_lat_1) + call Validate_restart_real (file_restart, 'true_lat_2', config_flags%true_lat_2) + end select + + call Get_netcdf_att (file_restart, 'global', 'start_year', start_year) + call Get_netcdf_att (file_restart, 'global', 'start_month', start_month) + call Get_netcdf_att (file_restart, 'global', 'start_day', start_day) + call Get_netcdf_att (file_restart, 'global', 'start_hour', start_hour) + call Get_netcdf_att (file_restart, 'global', 'start_minute', start_minute) + call Get_netcdf_att (file_restart, 'global', 'start_second', start_second) + + call Get_netcdf_att (file_restart, 'global', 'restart_year', restart_year) + call Get_netcdf_att (file_restart, 'global', 'restart_month', restart_month) + call Get_netcdf_att (file_restart, 'global', 'restart_day', restart_day) + call Get_netcdf_att (file_restart, 'global', 'restart_hour', restart_hour) + call Get_netcdf_att (file_restart, 'global', 'restart_minute', restart_minute) + call Get_netcdf_att (file_restart, 'global', 'restart_second', restart_second) + + call Get_netcdf_att (file_restart, 'global', 'itimestep', att_int) + end if + + call Broadcast_restart_integer (this, start_year) + call Broadcast_restart_integer (this, start_month) + call Broadcast_restart_integer (this, start_day) + call Broadcast_restart_integer (this, start_hour) + call Broadcast_restart_integer (this, start_minute) + call Broadcast_restart_integer (this, start_second) + call Broadcast_restart_integer (this, restart_year) + call Broadcast_restart_integer (this, restart_month) + call Broadcast_restart_integer (this, restart_day) + call Broadcast_restart_integer (this, restart_hour) + call Broadcast_restart_integer (this, restart_minute) + call Broadcast_restart_integer (this, restart_second) + call Broadcast_restart_integer (this, att_int) + this%itimestep = att_int + + this%datetime_start = datetime_t (start_year, start_month, start_day, start_hour, start_minute, start_second) + this%datetime_now = datetime_t (restart_year, restart_month, restart_day, restart_hour, restart_minute, restart_second) + + datetime_check = this%datetime_start + call datetime_check%Add_seconds (this%itimestep * this%dt) + if (datetime_check /= this%datetime_now) call Stop_simulation ('Restart clock is inconsistent with itimestep and dt') + + call Set_next_datetime_after_restart (this%datetime_next_output, config_flags%interval_output) + call Set_next_datetime_after_restart (this%datetime_next_atm_update, config_flags%interval_atm) + + call Read_restart_field (file_restart, 'lfn', this%lfn) + call Read_restart_field (file_restart, 'lfn_hist', this%lfn_hist) + call Read_restart_field (file_restart, 'lfn_0', this%lfn_0) + call Read_restart_field (file_restart, 'lfn_1', this%lfn_1) + call Read_restart_field (file_restart, 'lfn_2', this%lfn_2) + call Read_restart_field (file_restart, 'lfn_s0', this%lfn_s0) + call Read_restart_field (file_restart, 'lfn_s1', this%lfn_s1) + call Read_restart_field (file_restart, 'lfn_s2', this%lfn_s2) + call Read_restart_field (file_restart, 'lfn_s3', this%lfn_s3) + call Read_restart_field (file_restart, 'lfn_out', this%lfn_out) + call Read_restart_field (file_restart, 'tign_g', this%tign_g) + call Read_restart_field (file_restart, 'fuel_frac', this%fuel_frac) + call Read_restart_field (file_restart, 'fire_area', this%fire_area) + call Read_restart_field (file_restart, 'fuel_frac_burnt_dt', this%fuel_frac_burnt_dt) + call Read_restart_field (file_restart, 'fgrnhfx', this%fgrnhfx) + call Read_restart_field (file_restart, 'fgrnqfx', this%fgrnqfx) + call Read_restart_field (file_restart, 'fcanhfx', this%fcanhfx) + call Read_restart_field (file_restart, 'fcanqfx', this%fcanqfx) + call Read_restart_field (file_restart, 'flame_length', this%flame_length) + call Read_restart_field (file_restart, 'ros', this%ros) + call Read_restart_field (file_restart, 'ros_front', this%ros_front) + call Read_restart_field (file_restart, 'emis_smoke', this%emis_smoke) + call Read_restart_field (file_restart, 'fmc_g', this%fmc_g) + call Read_restart_field (file_restart, 'fuel_load_g', this%fuel_load_g) + call Read_restart_field (file_restart, 'fuel_time', this%fuel_time) + call Read_restart_field (file_restart, 'zsf', this%zsf) + call Read_restart_field (file_restart, 'dzdxf', this%dzdxf) + call Read_restart_field (file_restart, 'dzdyf', this%dzdyf) + call Read_restart_field (file_restart, 'nfuel_cat', this%nfuel_cat) + call Read_restart_field (file_restart, 'uf', this%uf) + call Read_restart_field (file_restart, 'vf', this%vf) + call Read_restart_field (file_restart, 'fz0', this%fz0) + + if (config_flags%ideal_opt == 0) then + call Read_restart_field (file_restart, 'fire_t2', this%fire_t2) + call Read_restart_field (file_restart, 'fire_q2', this%fire_q2) + call Read_restart_field (file_restart, 'fire_psfc', this%fire_psfc) + call Read_restart_field (file_restart, 'fire_rain', this%fire_rain) + if (config_flags%fmoist_run) then + call Read_restart_field (file_restart, 'fire_t2_old', this%fire_t2_old) + call Read_restart_field (file_restart, 'fire_q2_old', this%fire_q2_old) + call Read_restart_field (file_restart, 'fire_psfc_old', this%fire_psfc_old) + call Read_restart_field (file_restart, 'fire_rain_old', this%fire_rain_old) + end if + end if + if (config_flags%fmoist_run) call Read_restart_fmc (file_restart) + + call Exchange_restart_halos () + + if (allocated (this%ros_param) .and. allocated (this%fuels)) then + do ij = 1, this%num_tiles + call this%ros_param%Set_params (this%ifms, this%ifme, this%jfms, this%jfme, this%i_start(ij), this%i_end(ij), & + this%j_start(ij), this%j_end(ij), this%fuels, this%nfuel_cat, this%fmc_g) + end do + end if + + if (DEBUG_LOCAL) call Print_message ('Leaving Read_restart...') + + contains + + subroutine Exchange_restart_halos () + + implicit none + + +#ifdef DM_PARALLEL + call Exchange_restart_field (this%lfn) + call Exchange_restart_field (this%lfn_hist) + call Exchange_restart_field (this%lfn_0) + call Exchange_restart_field (this%lfn_1) + call Exchange_restart_field (this%lfn_2) + call Exchange_restart_field (this%lfn_s0) + call Exchange_restart_field (this%lfn_s1) + call Exchange_restart_field (this%lfn_s2) + call Exchange_restart_field (this%lfn_s3) + call Exchange_restart_field (this%lfn_out) + call Exchange_restart_field (this%tign_g) + call Exchange_restart_field (this%fuel_frac) + call Exchange_restart_field (this%fire_area) + call Exchange_restart_field (this%fuel_frac_burnt_dt) + call Exchange_restart_field (this%fgrnhfx) + call Exchange_restart_field (this%fgrnqfx) + call Exchange_restart_field (this%fcanhfx) + call Exchange_restart_field (this%fcanqfx) + call Exchange_restart_field (this%flame_length) + call Exchange_restart_field (this%ros) + call Exchange_restart_field (this%ros_front) + call Exchange_restart_field (this%emis_smoke) + call Exchange_restart_field (this%fmc_g) + call Exchange_restart_field (this%fuel_load_g) + call Exchange_restart_field (this%fuel_time) + call Exchange_restart_field (this%zsf) + call Exchange_restart_field (this%dzdxf) + call Exchange_restart_field (this%dzdyf) + call Exchange_restart_field (this%nfuel_cat) + call Exchange_restart_field (this%uf) + call Exchange_restart_field (this%vf) + call Exchange_restart_field (this%fz0) + + if (config_flags%ideal_opt == 0) then + call Exchange_restart_field (this%fire_t2) + call Exchange_restart_field (this%fire_q2) + call Exchange_restart_field (this%fire_psfc) + call Exchange_restart_field (this%fire_rain) + if (config_flags%fmoist_run) then + call Exchange_restart_field (this%fire_t2_old) + call Exchange_restart_field (this%fire_q2_old) + call Exchange_restart_field (this%fire_psfc_old) + call Exchange_restart_field (this%fire_rain_old) + end if + end if + + if (config_flags%fmoist_run) call Exchange_restart_fmc_halos () +#endif + + end subroutine Exchange_restart_halos + + subroutine Exchange_restart_field (var) + + implicit none + + real, dimension(this%ifms:this%ifme, this%jfms:this%jfme), intent (in out) :: var + + +#ifdef DM_PARALLEL + call Do_halo_exchange_with_corners (var, this%ifms, this%ifme, this%jfms, this%jfme, & + this%ifps, this%ifpe, this%jfps, this%jfpe, N_POINTS_IN_HALO, this%cart_comm) +#endif + + end subroutine Exchange_restart_field + + subroutine Exchange_restart_fmc_halos () + + implicit none + + integer :: k, n_moisture_classes + + +#ifdef DM_PARALLEL + if (.not. allocated (this%fmc_param)) call Stop_simulation ('FMC restart halo exchange requires allocated fmc_param') + + select type (fmc_param => this%fmc_param) + type is (fmc_wrffire_t) + if (.not. allocated (fmc_param%fmc_gc)) call Stop_simulation ('FMC restart halo exchange requires allocated fmc_gc') + + n_moisture_classes = size (fmc_param%fmc_gc, 2) + do k = 1, n_moisture_classes + call Do_halo_exchange_with_corners (fmc_param%fmc_gc(this%ifms:this%ifme, k, this%jfms:this%jfme), & + this%ifms, this%ifme, this%jfms, this%jfme, & + this%ifps, this%ifpe, this%jfps, this%jfpe, N_POINTS_IN_HALO, this%cart_comm) + end do + + class default + call Stop_simulation ('Restart halo exchange is not implemented for selected FMC component') + end select +#endif + + end subroutine Exchange_restart_fmc_halos + + subroutine Read_restart_field (file_name, var_name, var) + + implicit none + + character (len = *), intent (in) :: file_name, var_name + real, dimension(this%ifms:this%ifme, this%jfms:this%jfme), intent (in out) :: var + + + call Get_netcdf_var_mpi (file_name, this%cfbm_comm, this%nx, this%ny, & + this%ifps, this%ifpe, this%jfps, this%jfpe, var_name, var(this%ifps:this%ifpe, this%jfps:this%jfpe)) + + end subroutine Read_restart_field + + subroutine Read_restart_fmc (file_name) + + implicit none + + character (len = *), intent (in) :: file_name + + real (kind = REAL32) :: att_real32 + real :: fmoist_lasttime, fmoist_nexttime + integer :: n_moisture_classes + + + if (.not. allocated (this%fmc_param)) call Stop_simulation ('FMC restart read requires allocated fmc_param') + + select type (fmc_param => this%fmc_param) + type is (fmc_wrffire_t) + if (.not. allocated (fmc_param%fmc_gc)) call Stop_simulation ('FMC restart read requires allocated fmc_gc') + + n_moisture_classes = size (fmc_param%fmc_gc, 2) + call Get_netcdf_var_mpi (file_name, this%cfbm_comm, this%nx, this%ny, n_moisture_classes, & + this%ifps, this%ifpe, this%jfps, this%jfpe, NAME_VAR_FMC_GC, & + fmc_param%fmc_gc(this%ifps:this%ifpe, 1:n_moisture_classes, this%jfps:this%jfpe)) + + fmoist_lasttime = 0.0 + fmoist_nexttime = 0.0 + if (rank == 0) then + call Get_netcdf_att (file_name, 'global', NAME_ATT_FMOIST_LASTTIME, att_real32) + fmoist_lasttime = att_real32 + call Get_netcdf_att (file_name, 'global', NAME_ATT_FMOIST_NEXTTIME, att_real32) + fmoist_nexttime = att_real32 + end if + call Broadcast_restart_real (this, fmoist_lasttime) + call Broadcast_restart_real (this, fmoist_nexttime) + fmc_param%fmoist_lasttime = fmoist_lasttime + fmc_param%fmoist_nexttime = fmoist_nexttime + + class default + call Stop_simulation ('Restart read is not implemented for selected FMC component') + end select + + end subroutine Read_restart_fmc + + subroutine Set_next_datetime_after_restart (datetime_next, interval_seconds) + + implicit none + + type (datetime_t), intent (in out) :: datetime_next + integer, intent (in) :: interval_seconds + + datetime_next = this%datetime_now + if (interval_seconds > 0) then + datetime_next = this%datetime_start + do while (datetime_next <= this%datetime_now) + call datetime_next%Add_seconds (interval_seconds) + end do + end if + + end subroutine Set_next_datetime_after_restart + + end procedure Read_restart + + subroutine Read_restart_field_2d (file_name, var_name, nx, ny, ifms, ifme, jfms, jfme, ifps, ifpe, jfps, jfpe, cfbm_comm, var) + + implicit none + + character (len = *), intent (in) :: file_name, var_name + integer, intent (in) :: nx, ny, ifms, ifme, jfms, jfme, ifps, ifpe, jfps, jfpe, cfbm_comm + real, dimension(ifms:ifme, jfms:jfme), intent (in out) :: var + + + call Get_netcdf_var_mpi (file_name, cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, & + var_name, var(ifps:ifpe, jfps:jfpe)) + + end subroutine Read_restart_field_2d + + subroutine Validate_restart_integer (file_name, att_name, expected_value) + + implicit none + + character (len = *), intent (in) :: file_name, att_name + integer, intent (in) :: expected_value + + integer (kind = INT32) :: att_value + character (len = :), allocatable :: msg + + + call Get_netcdf_att (file_name, 'global', att_name, att_value) + if (att_value /= expected_value) then + msg = 'Restart metadata mismatch: '//trim (att_name) + call Stop_simulation (msg) + end if + + end subroutine Validate_restart_integer + + subroutine Validate_restart_real (file_name, att_name, expected_value) + + implicit none + + character (len = *), intent (in) :: file_name, att_name + real, intent (in) :: expected_value + + real (kind = REAL32) :: att_value, tolerance + character (len = :), allocatable :: msg + + + call Get_netcdf_att (file_name, 'global', att_name, att_value) + tolerance = max (1.0e-6_REAL32, abs (real (expected_value, kind = REAL32)) * 1.0e-6_REAL32) + if (abs (att_value - real (expected_value, kind = REAL32)) > tolerance) then + msg = 'Restart metadata mismatch: '//trim (att_name) + call Stop_simulation (msg) + end if + + end subroutine Validate_restart_real + + module procedure Write_restart + + character (len = :), allocatable :: file_restart + integer :: start_year, start_month, start_day, start_hour, start_minute, start_second, & + restart_year, restart_month, restart_day, restart_hour, restart_minute, restart_second + integer :: restart_sr_x, restart_sr_y, rank, ierr + logical, parameter :: DEBUG_LOCAL = .false. + + + if (DEBUG_LOCAL) call Print_message ('Entering Write_restart...') + +#ifdef DM_PARALLEL + if (.not. this%is_cfbm_comm_set) call Stop_simulation ('The MPI CFBM communicator has not been set') + call Mpi_comm_rank (this%cfbm_comm, rank, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_comm_rank ') +#else + rank = 0 +#endif + + if (config_flags%ideal_opt /= 0 .and. config_flags%ideal_opt /= 1) & + call Stop_simulation ('Write_restart is implemented for standalone idealized and real runs only') + + file_restart = Build_restart_file_name (this%datetime_now%datetime) + + call this%datetime_start%Get_datetime_as_ints (start_year, start_month, start_day, start_hour, start_minute, start_second) + call this%datetime_now%Get_datetime_as_ints (restart_year, restart_month, restart_day, restart_hour, restart_minute, restart_second) + restart_sr_x = nint (this%proj%dx / this%dx) + restart_sr_y = nint (this%proj%dy / this%dy) + if (restart_sr_x <= 0 .or. restart_sr_y <= 0) call Stop_simulation ('Restart subgrid ratios must be positive') + + if (rank == 0) then + call Create_netcdf_file (file_name = file_restart) + call Add_netcdf_dim (file_restart, NAME_DIM_X, this%nx) + call Add_netcdf_dim (file_restart, NAME_DIM_Y, this%ny) + + call Add_netcdf_att (file_restart, 'global', 'start_year', int (start_year, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'start_month', int (start_month, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'start_day', int (start_day, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'start_hour', int (start_hour, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'start_minute', int (start_minute, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'start_second', int (start_second, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'restart_year', int (restart_year, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'restart_month', int (restart_month, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'restart_day', int (restart_day, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'restart_hour', int (restart_hour, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'restart_minute', int (restart_minute, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'restart_second', int (restart_second, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'itimestep', int (this%itimestep, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'nx', int (this%nx, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'ny', int (this%ny, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'dt', real (this%dt, kind = REAL32)) + call Add_netcdf_att (file_restart, 'global', 'dx', real (this%dx, kind = REAL32)) + call Add_netcdf_att (file_restart, 'global', 'dy', real (this%dy, kind = REAL32)) + call Add_netcdf_att (file_restart, 'global', 'cen_lat', real (this%cen_lat, kind = REAL32)) + call Add_netcdf_att (file_restart, 'global', 'cen_lon', real (this%cen_lon, kind = REAL32)) + call Add_netcdf_att (file_restart, 'global', 'map_proj', int (1, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'sr_x', int (restart_sr_x, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'sr_y', int (restart_sr_y, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'stand_lon', real (this%proj%standard_lon, kind = REAL32)) + call Add_netcdf_att (file_restart, 'global', 'true_lat_1', real (this%proj%true_lat_1, kind = REAL32)) + call Add_netcdf_att (file_restart, 'global', 'true_lat_2', real (this%proj%true_lat_2, kind = REAL32)) + call Add_netcdf_att (file_restart, 'global', 'ideal_opt', int (config_flags%ideal_opt, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'fuel_opt', int (config_flags%fuel_opt, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'ros_opt', int (config_flags%ros_opt, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'fmc_opt', int (config_flags%fmc_opt, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'emis_opt', int (config_flags%emis_opt, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'fire_upwinding', int (config_flags%fire_upwinding, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'fire_upwinding_reinit', int (config_flags%fire_upwinding_reinit, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'fire_lsm_reinit_iter', int (config_flags%fire_lsm_reinit_iter, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'fire_viscosity', real (config_flags%fire_viscosity, kind = REAL32)) + call Add_netcdf_att (file_restart, 'global', 'fire_viscosity_bg', real (config_flags%fire_viscosity_bg, kind = REAL32)) + call Add_netcdf_att (file_restart, 'global', 'fire_viscosity_ngp', int (config_flags%fire_viscosity_ngp, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'fire_viscosity_band', real (config_flags%fire_viscosity_band, kind = REAL32)) + call Add_netcdf_att (file_restart, 'global', 'reinit_pseudot_coef', real (config_flags%reinit_pseudot_coef, kind = REAL32)) + end if + + call Restart_io_barrier (this) + + call Add_restart_field ('lfn', this%lfn) + call Add_restart_field ('lfn_hist', this%lfn_hist) + call Add_restart_field ('lfn_0', this%lfn_0) + call Add_restart_field ('lfn_1', this%lfn_1) + call Add_restart_field ('lfn_2', this%lfn_2) + call Add_restart_field ('lfn_s0', this%lfn_s0) + call Add_restart_field ('lfn_s1', this%lfn_s1) + call Add_restart_field ('lfn_s2', this%lfn_s2) + call Add_restart_field ('lfn_s3', this%lfn_s3) + call Add_restart_field ('lfn_out', this%lfn_out) + call Add_restart_field ('tign_g', this%tign_g) + call Add_restart_field ('fuel_frac', this%fuel_frac) + call Add_restart_field ('fire_area', this%fire_area) + call Add_restart_field ('fuel_frac_burnt_dt', this%fuel_frac_burnt_dt) + call Add_restart_field ('fgrnhfx', this%fgrnhfx) + call Add_restart_field ('fgrnqfx', this%fgrnqfx) + call Add_restart_field ('fcanhfx', this%fcanhfx) + call Add_restart_field ('fcanqfx', this%fcanqfx) + call Add_restart_field ('flame_length', this%flame_length) + call Add_restart_field ('ros', this%ros) + call Add_restart_field ('ros_front', this%ros_front) + call Add_restart_field ('emis_smoke', this%emis_smoke) + call Add_restart_field ('fmc_g', this%fmc_g) + call Add_restart_field ('fuel_load_g', this%fuel_load_g) + call Add_restart_field ('fuel_time', this%fuel_time) + call Add_restart_field ('zsf', this%zsf) + call Add_restart_field ('dzdxf', this%dzdxf) + call Add_restart_field ('dzdyf', this%dzdyf) + call Add_restart_field ('nfuel_cat', this%nfuel_cat) + call Add_restart_field ('uf', this%uf) + call Add_restart_field ('vf', this%vf) + call Add_restart_field ('fz0', this%fz0) + + if (config_flags%ideal_opt == 0) then + call Add_restart_field ('fire_t2', this%fire_t2) + call Add_restart_field ('fire_q2', this%fire_q2) + call Add_restart_field ('fire_psfc', this%fire_psfc) + call Add_restart_field ('fire_rain', this%fire_rain) + if (config_flags%fmoist_run) then + call Add_restart_field ('fire_t2_old', this%fire_t2_old) + call Add_restart_field ('fire_q2_old', this%fire_q2_old) + call Add_restart_field ('fire_psfc_old', this%fire_psfc_old) + call Add_restart_field ('fire_rain_old', this%fire_rain_old) + end if + end if + if (config_flags%fmoist_run) call Write_restart_fmc () + + call Restart_io_barrier (this) + + if (DEBUG_LOCAL) call Print_message ('Leaving Write_restart...') + + contains + + subroutine Add_restart_field (var_name, var) + + implicit none + + character (len = *), intent (in) :: var_name + real, dimension(this%ifms:this%ifme, this%jfms:this%jfme), intent (in) :: var + + + call Add_netcdf_var_mpi (file_restart, this%cfbm_comm, this%nx, this%ny, this%ifps, this%ifpe, this%jfps, this%jfpe, & + var_name, var(this%ifps:this%ifpe, this%jfps:this%jfpe)) + + end subroutine Add_restart_field + + subroutine Write_restart_fmc () + + implicit none + + character (len = 32), dimension(3) :: dim_names_fmc_gc + integer :: n_moisture_classes + + + if (.not. allocated (this%fmc_param)) call Stop_simulation ('FMC restart write requires allocated fmc_param') + + select type (fmc_param => this%fmc_param) + type is (fmc_wrffire_t) + if (.not. allocated (fmc_param%fmc_gc)) call Stop_simulation ('FMC restart write requires allocated fmc_gc') + + n_moisture_classes = size (fmc_param%fmc_gc, 2) + if (rank == 0) call Add_netcdf_dim (file_restart, NAME_DIM_MOISTURE_CLASS, n_moisture_classes) + call Restart_io_barrier (this) + + dim_names_fmc_gc(1) = NAME_DIM_X + dim_names_fmc_gc(2) = NAME_DIM_MOISTURE_CLASS + dim_names_fmc_gc(3) = NAME_DIM_Y + call Add_netcdf_var_mpi (file_restart, dim_names_fmc_gc, this%cfbm_comm, this%nx, this%ny, n_moisture_classes, & + this%ifps, this%ifpe, this%jfps, this%jfpe, NAME_VAR_FMC_GC, & + fmc_param%fmc_gc(this%ifps:this%ifpe, 1:n_moisture_classes, this%jfps:this%jfpe)) + + if (rank == 0) then + call Add_netcdf_att (file_restart, 'global', NAME_ATT_FMOIST_LASTTIME, & + real (fmc_param%fmoist_lasttime, kind = REAL32)) + call Add_netcdf_att (file_restart, 'global', NAME_ATT_FMOIST_NEXTTIME, & + real (fmc_param%fmoist_nexttime, kind = REAL32)) + end if + + class default + call Stop_simulation ('Restart write is not implemented for selected FMC component') + end select + + end subroutine Write_restart_fmc + + end procedure Write_restart + + end submodule state_restart_submod From b8d3934c16d8659830d71f8d84ccb0fd650656c7 Mon Sep 17 00:00:00 2001 From: masih Date: Fri, 25 Sep 2026 12:26:38 -0600 Subject: [PATCH 03/19] Move global 2D distribution helper to mpi_mod Move Distribute_global_var2d out of netcdf_mod and into mpi_mod so the domain-distribution helper lives with the other MPI utilities. Keep the restart-specific variable names at the restart read call sites, but use generic naming and messages in the MPI helper itself. --- io/netcdf_mod.F90 | 77 +---------------------------------- share/mpi_mod.F90 | 100 +++++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 99 insertions(+), 78 deletions(-) diff --git a/io/netcdf_mod.F90 b/io/netcdf_mod.F90 index 1bb38e39..3232fb05 100644 --- a/io/netcdf_mod.F90 +++ b/io/netcdf_mod.F90 @@ -1,7 +1,7 @@ module netcdf_mod use stderrout_mod, only : Stop_simulation, Print_message - use mpi_mod, only : Gather_var2d + use mpi_mod, only : Gather_var2d, Distribute_global_var2d implicit none @@ -283,81 +283,6 @@ subroutine Get_netcdf_var_real32_2d_mpi (file_name, cfbm_comm, nx, ny, ifps, ifp end subroutine Get_netcdf_var_real32_2d_mpi - subroutine Distribute_global_var2d (cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, var2d_global, var2d_local) - -#ifdef DM_PARALLEL - use mpi -#endif - implicit none - - integer, intent (in) :: cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe - real, dimension(:, :), allocatable, intent (in out) :: var2d_global - real, dimension(ifps:ifpe, jfps:jfpe), intent (out) :: var2d_local - - integer :: i, ierr, j, kbuf, nprocs, nx_local, ny_local, r, rank - integer, dimension(:), allocatable :: all_ifps, all_ifpe, all_jfps, all_jfpe, displs, sendcounts - real, dimension(:), allocatable :: sendbuf - - -#ifdef DM_PARALLEL - call Mpi_comm_rank (cfbm_comm, rank, ierr) - if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_comm_rank ') - call Mpi_comm_size (cfbm_comm, nprocs, ierr) - if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems getting the number of MPI tasks') - - nx_local = ifpe - ifps + 1 - ny_local = jfpe - jfps + 1 - - if (rank == 0) then - allocate (all_ifps(nprocs), all_ifpe(nprocs), all_jfps(nprocs), all_jfpe(nprocs)) - else - allocate (all_ifps(1), all_ifpe(1), all_jfps(1), all_jfpe(1)) - end if - - call MPI_Gather (ifps, 1, MPI_INTEGER, all_ifps, 1, MPI_INTEGER, 0, cfbm_comm, ierr) - if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with MPI_Gather for restart ifps') - call MPI_Gather (ifpe, 1, MPI_INTEGER, all_ifpe, 1, MPI_INTEGER, 0, cfbm_comm, ierr) - if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with MPI_Gather for restart ifpe') - call MPI_Gather (jfps, 1, MPI_INTEGER, all_jfps, 1, MPI_INTEGER, 0, cfbm_comm, ierr) - if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with MPI_Gather for restart jfps') - call MPI_Gather (jfpe, 1, MPI_INTEGER, all_jfpe, 1, MPI_INTEGER, 0, cfbm_comm, ierr) - if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with MPI_Gather for restart jfpe') - - if (rank == 0) then - allocate (sendcounts(nprocs), displs(nprocs)) - do r = 1, nprocs - sendcounts(r) = (all_ifpe(r) - all_ifps(r) + 1) * (all_jfpe(r) - all_jfps(r) + 1) - end do - - displs(1) = 0 - do r = 2, nprocs - displs(r) = displs(r - 1) + sendcounts(r - 1) - end do - - allocate (sendbuf(sum (sendcounts))) - kbuf = 0 - do r = 1, nprocs - do j = all_jfps(r), all_jfpe(r) - do i = all_ifps(r), all_ifpe(r) - kbuf = kbuf + 1 - sendbuf(kbuf) = var2d_global(i, j) - end do - end do - end do - else - allocate (sendcounts(1), displs(1), sendbuf(1)) - end if - - call MPI_Scatterv (sendbuf, sendcounts, displs, MPI_REAL, var2d_local, nx_local * ny_local, MPI_REAL, 0, cfbm_comm, ierr) - if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with MPI_Scatterv for restart variable') - - deallocate (all_ifps, all_ifpe, all_jfps, all_jfpe, sendcounts, displs, sendbuf) -#else - var2d_local(1:nx, 1:ny) = var2d_global(1:nx, 1:ny) -#endif - - end subroutine Distribute_global_var2d - subroutine Add_netcdf_var_real32_3d (file_name, name_dims, varname, var) use netcdf diff --git a/share/mpi_mod.F90 b/share/mpi_mod.F90 index 34291f89..76cb4ea8 100644 --- a/share/mpi_mod.F90 +++ b/share/mpi_mod.F90 @@ -1,6 +1,6 @@ module mpi_mod - use, intrinsic :: iso_fortran_env, only : OUTPUT_UNIT + use, intrinsic :: iso_fortran_env, only : ERROR_UNIT, OUTPUT_UNIT implicit none @@ -8,7 +8,7 @@ module mpi_mod public :: Calc_tasks_in_x_and_y, Calc_patch_dims, Gather_var2d, Do_halo_exchange, Do_halo_exchange_with_corners, & Max_across_mpi_tasks, Sum_across_mpi_tasks, Distribute_var2d, Min_across_mpi_tasks, Convert_mpi_comm_to_f08, & - Print_cart_info, topology_dim_order + Distribute_global_var2d, Print_cart_info, topology_dim_order ! --- Topology Ordering Flag --- ! 0 = Default (Fortran order: X is Dim 0, Y is Dim 1) @@ -204,6 +204,102 @@ subroutine Distribute_var2d (local_field, ips, ipe, jps, jpe, comm) end subroutine Distribute_var2d + subroutine Distribute_global_var2d (cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, var2d_global, var2d_local) + +#ifdef DM_PARALLEL + use mpi +#endif + implicit none + + integer, intent (in) :: cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe + real, dimension(:, :), allocatable, intent (in out) :: var2d_global + real, dimension(ifps:ifpe, jfps:jfpe), intent (out) :: var2d_local + + integer :: i, ierr, j, kbuf, nprocs, nx_local, ny_local, r, rank + integer, dimension(:), allocatable :: all_ifps, all_ifpe, all_jfps, all_jfpe, displs, sendcounts + real, dimension(:), allocatable :: sendbuf + + +#ifdef DM_PARALLEL + call Mpi_comm_rank (cfbm_comm, rank, ierr) + if (ierr /= MPI_SUCCESS) call Abort_mpi_operation ('Problems with Mpi_comm_rank ') + call Mpi_comm_size (cfbm_comm, nprocs, ierr) + if (ierr /= MPI_SUCCESS) call Abort_mpi_operation ('Problems getting the number of MPI tasks') + + nx_local = ifpe - ifps + 1 + ny_local = jfpe - jfps + 1 + + if (rank == 0) then + allocate (all_ifps(nprocs), all_ifpe(nprocs), all_jfps(nprocs), all_jfpe(nprocs)) + else + allocate (all_ifps(1), all_ifpe(1), all_jfps(1), all_jfpe(1)) + end if + + call MPI_Gather (ifps, 1, MPI_INTEGER, all_ifps, 1, MPI_INTEGER, 0, cfbm_comm, ierr) + if (ierr /= MPI_SUCCESS) call Abort_mpi_operation ('Problems with MPI_Gather for ifps') + call MPI_Gather (ifpe, 1, MPI_INTEGER, all_ifpe, 1, MPI_INTEGER, 0, cfbm_comm, ierr) + if (ierr /= MPI_SUCCESS) call Abort_mpi_operation ('Problems with MPI_Gather for ifpe') + call MPI_Gather (jfps, 1, MPI_INTEGER, all_jfps, 1, MPI_INTEGER, 0, cfbm_comm, ierr) + if (ierr /= MPI_SUCCESS) call Abort_mpi_operation ('Problems with MPI_Gather for jfps') + call MPI_Gather (jfpe, 1, MPI_INTEGER, all_jfpe, 1, MPI_INTEGER, 0, cfbm_comm, ierr) + if (ierr /= MPI_SUCCESS) call Abort_mpi_operation ('Problems with MPI_Gather for jfpe') + + if (rank == 0) then + allocate (sendcounts(nprocs), displs(nprocs)) + do r = 1, nprocs + sendcounts(r) = (all_ifpe(r) - all_ifps(r) + 1) * (all_jfpe(r) - all_jfps(r) + 1) + end do + + displs(1) = 0 + do r = 2, nprocs + displs(r) = displs(r - 1) + sendcounts(r - 1) + end do + + allocate (sendbuf(sum (sendcounts))) + kbuf = 0 + do r = 1, nprocs + do j = all_jfps(r), all_jfpe(r) + do i = all_ifps(r), all_ifpe(r) + kbuf = kbuf + 1 + sendbuf(kbuf) = var2d_global(i, j) + end do + end do + end do + else + allocate (sendcounts(1), displs(1), sendbuf(1)) + end if + + call MPI_Scatterv (sendbuf, sendcounts, displs, MPI_REAL, var2d_local, nx_local * ny_local, MPI_REAL, 0, cfbm_comm, ierr) + if (ierr /= MPI_SUCCESS) call Abort_mpi_operation ('Problems with MPI_Scatterv for distributed variable') + + deallocate (all_ifps, all_ifpe, all_jfps, all_jfpe, sendcounts, displs, sendbuf) +#else + var2d_local(1:nx, 1:ny) = var2d_global(1:nx, 1:ny) +#endif + + end subroutine Distribute_global_var2d + + subroutine Abort_mpi_operation (msg) + +#ifdef DM_PARALLEL + use mpi +#endif + implicit none + + character (len = *), intent (in) :: msg + integer :: ierr + + + write (ERROR_UNIT, *) 'STOP:' // trim (msg) +#ifdef DM_PARALLEL + call MPI_Abort (MPI_COMM_WORLD, 1, ierr) + call MPI_Finalize (ierr) +#else + stop +#endif + + end subroutine Abort_mpi_operation + subroutine Do_halo_exchange (patch, ims, ime, jms, jme, ips, ipe, jps, jpe, nghost, cart_comm) ! It does not update the corners in the halo From 6c023a9fcb706406927c7bffec1367d905aeecb4 Mon Sep 17 00:00:00 2001 From: masih Date: Fri, 25 Sep 2026 14:37:06 -0600 Subject: [PATCH 04/19] Refactor restart initialization and scheduling Move standalone restart metadata initialization into state initialization and remove the initialize_mod restart wrapper. Move restart output interval validation and write timing into state_fire_t%Handle_restart so the main driver loop stays focused on high-level model actions. --- driver/fire_behavior.F90 | 22 +------ driver/initialize_mod.F90 | 102 +++++++++++++-------------------- state/state_mod.F90 | 46 +++++++++++---- state/state_restart_submod.F90 | 36 ++++++++++++ 4 files changed, 111 insertions(+), 95 deletions(-) diff --git a/driver/fire_behavior.F90 b/driver/fire_behavior.F90 index ed1d2ab2..9367c7cb 100644 --- a/driver/fire_behavior.F90 +++ b/driver/fire_behavior.F90 @@ -15,15 +15,11 @@ program fire_behavior implicit none integer :: ierr, rank, mpi_comm_cfbm - integer :: restart_step_interval - real :: restart_interval_seconds, restart_interval_error type (state_fire_t) :: grid type (wrfdata_t) :: atm_state type (namelist_t) :: config_flags type (datetime_t) :: datetime_check - character (len = 256) :: msg logical, parameter :: DEBUG_LOCAL = .false. - real, parameter :: RESTART_INTERVAL_TOL = 1.0e-6 if (DEBUG_LOCAL) write (OUTPUT_UNIT, *) 'Running fire_behavior...' @@ -87,19 +83,7 @@ program fire_behavior call grid%Save_state () end if - restart_step_interval = -1 - if (config_flags%restart_interval > 0) then - restart_step_interval = nint (real (config_flags%restart_interval) / grid%dt) - restart_interval_seconds = restart_step_interval * grid%dt - restart_interval_error = abs (restart_interval_seconds - real (config_flags%restart_interval)) - - if (restart_step_interval <= 0 .or. & - restart_interval_error > max (RESTART_INTERVAL_TOL, abs (real (config_flags%restart_interval)) * RESTART_INTERVAL_TOL)) then - write (msg, '(a, i0, a, f12.6)') 'restart_interval must map to an integer number of time steps: restart_interval = ', & - config_flags%restart_interval, ', dt = ', grid%dt - call Stop_simulation (msg) - end if - end if + call grid%Handle_restart (config_flags, initialize = .true.) if (DEBUG_LOCAL) write (OUTPUT_UNIT, *) ' Starting temporal loop...' do while (grid%datetime_now < grid%datetime_end) @@ -112,9 +96,7 @@ program fire_behavior call grid%Handle_output (config_flags) if (config_flags%ideal_opt == 0) call grid%Handle_wrfdata_update (atm_state, config_flags) - if (restart_step_interval > 0) then - if (mod (grid%itimestep, restart_step_interval) == 0) call grid%Write_restart (config_flags) - end if + call grid%Handle_restart (config_flags) end do if (DEBUG_LOCAL) write (OUTPUT_UNIT, *) ' Completed temporal loop' diff --git a/driver/initialize_mod.F90 b/driver/initialize_mod.F90 index 23869f43..8f3a46f5 100644 --- a/driver/initialize_mod.F90 +++ b/driver/initialize_mod.F90 @@ -1,11 +1,10 @@ module initialize_mod - use state_mod, only : state_fire_t, Build_restart_file_name + use state_mod, only : state_fire_t use namelist_mod, only : namelist_t use geogrid_mod, only : geogrid_t use wrfdata_mod, only : wrfdata_t use fire_driver_mod, only : Init_fire_components - use datetime_mod, only : datetime_t use stderrout_mod, only: Print_message, Stop_simulation #ifdef DM_PARALLEL use mpi_mod, only : Convert_mpi_comm_to_f08 @@ -60,52 +59,54 @@ subroutine Init_fire_state (grid, config_flags, wrf) if (DEBUG_LOCAL) call Print_message (' Entering subroutine Init_state') if (DEBUG_LOCAL) call Print_message (' Initialization...') - if (config_flags%ideal_opt == 0 .and. config_flags%restart) then - ! Real world, restart run: the grid metadata comes from the restart file - call Init_fire_state_from_restart (grid, config_flags) - - else if (config_flags%ideal_opt == 0) then + if (config_flags%ideal_opt == 0) then ! Real world - if (DEBUG_LOCAL) call Print_message (' Reading geogrid file') + if (.not. config_flags%restart) then + if (DEBUG_LOCAL) call Print_message (' Reading geogrid file') #ifdef DM_PARALLEL - if (grid%is_cfbm_comm_set) then - call Convert_mpi_comm_to_f08 (grid%cfbm_comm, cfbm_comm_f08) - call Mpi_comm_rank (cfbm_comm_f08, rank, ierr) - else - call Stop_simulation ('The MPI communicator cfbm_comm has not been set') - end if + if (grid%is_cfbm_comm_set) then + call Convert_mpi_comm_to_f08 (grid%cfbm_comm, cfbm_comm_f08) + call Mpi_comm_rank (cfbm_comm_f08, rank, ierr) + else + call Stop_simulation ('The MPI communicator cfbm_comm has not been set') + end if #else - rank = 0 + rank = 0 #endif - if (rank == 0) geogrid = geogrid_t (file_name = 'geo_em.d01.nc') + if (rank == 0) geogrid = geogrid_t (file_name = 'geo_em.d01.nc') #ifdef DM_PARALLEL - call MPI_Bcast (geogrid%cen_lon, 1, MPI_REAL, 0, cfbm_comm_f08, ierr) - call MPI_Bcast (geogrid%cen_lat, 1, MPI_REAL, 0, cfbm_comm_f08, ierr) - call MPI_Bcast (geogrid%dx, 1, MPI_REAL, 0, cfbm_comm_f08, ierr) - call MPI_Bcast (geogrid%dy, 1, MPI_REAL, 0, cfbm_comm_f08, ierr) - call MPI_Bcast (geogrid%true_lat_1, 1, MPI_REAL, 0, cfbm_comm_f08, ierr) - call MPI_Bcast (geogrid%true_lat_2, 1, MPI_REAL, 0, cfbm_comm_f08, ierr) - call MPI_Bcast (geogrid%stand_lon, 1, MPI_REAL, 0, cfbm_comm_f08, ierr) - - call MPI_Bcast(geogrid%map_proj, 1, MPI_INTEGER, 0, cfbm_comm_f08, ierr) - call MPI_Bcast(geogrid%sr_x, 1, MPI_INTEGER, 0, cfbm_comm_f08, ierr) - call MPI_Bcast(geogrid%sr_y, 1, MPI_INTEGER, 0, cfbm_comm_f08, ierr) - - call MPI_Bcast(geogrid%ifds, 1, MPI_INTEGER, 0, cfbm_comm_f08, ierr) - call MPI_Bcast(geogrid%ifde, 1, MPI_INTEGER, 0, cfbm_comm_f08, ierr) - call MPI_Bcast(geogrid%jfds, 1, MPI_INTEGER, 0, cfbm_comm_f08, ierr) - call MPI_Bcast(geogrid%jfde, 1, MPI_INTEGER, 0, cfbm_comm_f08, ierr) - - call MPI_Bcast(geogrid%ids, 1, MPI_INTEGER, 0, cfbm_comm_f08, ierr) - call MPI_Bcast(geogrid%ide, 1, MPI_INTEGER, 0, cfbm_comm_f08, ierr) - call MPI_Bcast(geogrid%jds, 1, MPI_INTEGER, 0, cfbm_comm_f08, ierr) - call MPI_Bcast(geogrid%jde, 1, MPI_INTEGER, 0, cfbm_comm_f08, ierr) + call MPI_Bcast (geogrid%cen_lon, 1, MPI_REAL, 0, cfbm_comm_f08, ierr) + call MPI_Bcast (geogrid%cen_lat, 1, MPI_REAL, 0, cfbm_comm_f08, ierr) + call MPI_Bcast (geogrid%dx, 1, MPI_REAL, 0, cfbm_comm_f08, ierr) + call MPI_Bcast (geogrid%dy, 1, MPI_REAL, 0, cfbm_comm_f08, ierr) + call MPI_Bcast (geogrid%true_lat_1, 1, MPI_REAL, 0, cfbm_comm_f08, ierr) + call MPI_Bcast (geogrid%true_lat_2, 1, MPI_REAL, 0, cfbm_comm_f08, ierr) + call MPI_Bcast (geogrid%stand_lon, 1, MPI_REAL, 0, cfbm_comm_f08, ierr) + + call MPI_Bcast(geogrid%map_proj, 1, MPI_INTEGER, 0, cfbm_comm_f08, ierr) + call MPI_Bcast(geogrid%sr_x, 1, MPI_INTEGER, 0, cfbm_comm_f08, ierr) + call MPI_Bcast(geogrid%sr_y, 1, MPI_INTEGER, 0, cfbm_comm_f08, ierr) + + call MPI_Bcast(geogrid%ifds, 1, MPI_INTEGER, 0, cfbm_comm_f08, ierr) + call MPI_Bcast(geogrid%ifde, 1, MPI_INTEGER, 0, cfbm_comm_f08, ierr) + call MPI_Bcast(geogrid%jfds, 1, MPI_INTEGER, 0, cfbm_comm_f08, ierr) + call MPI_Bcast(geogrid%jfde, 1, MPI_INTEGER, 0, cfbm_comm_f08, ierr) + + call MPI_Bcast(geogrid%ids, 1, MPI_INTEGER, 0, cfbm_comm_f08, ierr) + call MPI_Bcast(geogrid%ide, 1, MPI_INTEGER, 0, cfbm_comm_f08, ierr) + call MPI_Bcast(geogrid%jds, 1, MPI_INTEGER, 0, cfbm_comm_f08, ierr) + call MPI_Bcast(geogrid%jde, 1, MPI_INTEGER, 0, cfbm_comm_f08, ierr) #endif + end if if (DEBUG_LOCAL) call Print_message (' Initializing fire state') - call grid%Initialization (config_flags, geogrid) + if (config_flags%restart) then + call grid%Initialization (config_flags) + else + call grid%Initialization (config_flags, geogrid) + end if else ! Ideal @@ -148,31 +149,6 @@ subroutine Init_fire_state (grid, config_flags, wrf) end subroutine Init_fire_state - subroutine Init_fire_state_from_restart (grid, config_flags) - - implicit none - - type (state_fire_t), intent (in out) :: grid - type (namelist_t), intent (in) :: config_flags - - type (datetime_t) :: datetime_restart - character (len = :), allocatable :: file_restart - logical, parameter :: DEBUG_LOCAL = .false. - - - if (DEBUG_LOCAL) call Print_message (' Entering subroutine Init_fire_state_from_restart') - - datetime_restart = datetime_t (config_flags%start_year, config_flags%start_month, config_flags%start_day, & - config_flags%start_hour, config_flags%start_minute, config_flags%start_second) - file_restart = Build_restart_file_name (datetime_restart%datetime) - - if (DEBUG_LOCAL) call Print_message (' Initializing fire state from restart metadata') - call grid%Initialization (config_flags, restart_file = file_restart) - - if (DEBUG_LOCAL) call Print_message (' Leaving subroutine Init_fire_state_from_restart') - - end subroutine Init_fire_state_from_restart - subroutine Init_fire_state_within_wrf (state, config_flags, & ifds, ifde, ifms, ifme, ifps, ifpe, & jfds, jfde, jfms, jfme, jfps, jfpe, & diff --git a/state/state_mod.F90 b/state/state_mod.F90 index fbc6efb9..e65895f6 100644 --- a/state/state_mod.F90 +++ b/state/state_mod.F90 @@ -109,6 +109,8 @@ module state_mod ! Output integer :: output_level + integer :: restart_step_interval = -1 + logical :: is_restart_output_initialized = .false. ! For MPI tasks integer :: cfbm_comm ! The MPI communicator before the domain decomposition @@ -121,6 +123,7 @@ module state_mod procedure, public :: Apply_wafs => Apply_wafs procedure, public :: Convert_sb_to_ander => Convert_scottburgan_to_anderson procedure, public :: Handle_output => Handle_output + procedure, public :: Handle_restart => Handle_restart procedure, public :: Handle_wrfdata_update => Handle_wrfdata_update procedure, public :: Init_fuel_vars => Init_fuel_vars procedure, public :: Initialization => Init_domain @@ -167,6 +170,12 @@ module subroutine Read_restart (this, config_flags) type (namelist_t), intent (in) :: config_flags end subroutine Read_restart + module subroutine Handle_restart (this, config_flags, initialize) + class (state_fire_t), intent (in out) :: this + type (namelist_t), intent (in) :: config_flags + logical, intent (in), optional :: initialize + end subroutine Handle_restart + module subroutine Write_restart (this, config_flags) class (state_fire_t), intent (in) :: this type (namelist_t), intent (in) :: config_flags @@ -357,28 +366,41 @@ subroutine Init_domain (this, config_flags, geogrid, & real, dimension(:, :), intent (in), optional :: nfuel_cat, zsf, dzdxf, dzdyf integer, parameter :: INIT_MODE_NONE = 0, INIT_MODE_GEOGRID = 1, INIT_MODE_WRF = 2, INIT_MODE_IDEAL = 3, INIT_MODE_RESTART = 4 + type (datetime_t) :: datetime_restart type (proj_lc_t) :: proj logical, parameter :: DEBUG_LOCAL = .false. + logical :: has_wrf_metadata integer :: ids0, ide0, jds0, jde0, i, j, init_mode, px, py, ntasks, ierr, cart_comm, rank, ips, ipe, jps, jpe, & is_lfn_init_allocated integer, dimension(2) :: coords character (len = 300) :: msg + character (len = :), allocatable :: file_restart if (DEBUG_LOCAL) call Print_message ('Entering Init_domain...') + has_wrf_metadata = present (ifds) .and. present (ifde) .and. present (ifms) .and. present (ifme) .and. & + present (ifps) .and. present (ifpe) .and. present (jfds) .and. present (jfde) .and. present (jfms) .and. & + present (jfme) .and. present (jfps) .and. present (jfpe) .and. present (kfds) .and. present (kfde) .and. & + present (kfms) .and. present (kfme) .and. present (kfps) .and. present (kfpe) .and. present (kfts) .and. & + present (kfte) .and. present (ide) .and. present (jde) .and. present (cen_lat) .and. present (cen_lon) .and. & + present (truelat1) .and. present (truelat2) .and. present (stand_lon) .and. present (dx) .and. present (dy) .and. & + present (sr_x) .and. present (sr_y) .and. present (nfuel_cat) .and. present (zsf) .and. present (dzdxf) .and. & + present (dzdyf) + + if (present (restart_file)) file_restart = restart_file + if (config_flags%ideal_opt == 0 .and. config_flags%restart .and. .not. present (geogrid) .and. & + .not. has_wrf_metadata .and. .not. allocated (file_restart)) then + datetime_restart = datetime_t (config_flags%start_year, config_flags%start_month, config_flags%start_day, & + config_flags%start_hour, config_flags%start_minute, config_flags%start_second) + file_restart = Build_restart_file_name (datetime_restart%datetime) + end if + init_mode = INIT_MODE_NONE if (config_flags%ideal_opt == 1) init_mode = INIT_MODE_IDEAL if (present (geogrid)) init_mode = INIT_MODE_GEOGRID - if (present (restart_file)) init_mode = INIT_MODE_RESTART - if (present (ifds) .and. present (ifde) .and. present (ifms) .and. present (ifme) .and. present (ifps) .and. present (ifpe) .and. & - present (jfds) .and. present (jfde) .and. present (jfms) .and. present (jfme) .and. present (jfps) .and. present (jfpe) .and. & - present (kfds) .and. present (kfde) .and. present (kfms) .and. present (kfme) .and. present (kfps) .and. present (kfpe) .and. & - present (kfts) .and. present (kfte) .and. present (ide) .and. present (jde) .and. & - present (cen_lat) .and. present (cen_lon) .and. present (truelat1) .and. present (truelat2) .and. present (stand_lon) .and. & - present (dx) .and. present (dy) .and. present (sr_x) .and. present (sr_y) .and. present (nfuel_cat) .and. present (zsf) .and. & - present (dzdxf) .and. present (dzdyf)) & - init_mode = INIT_MODE_WRF + if (allocated (file_restart)) init_mode = INIT_MODE_RESTART + if (has_wrf_metadata) init_mode = INIT_MODE_WRF if (init_mode == INIT_MODE_NONE) & call Stop_simulation ('Not enough information to initialize domain') @@ -483,7 +505,7 @@ subroutine Init_domain (this, config_flags, geogrid, & #endif else if (init_mode == INIT_MODE_RESTART) then - call Init_restart_dimensions (this, trim (restart_file), ids0, ide0, jds0, jde0, ips, ipe, jps, jpe) + call Init_restart_dimensions (this, trim (file_restart), ids0, ide0, jds0, jde0, ips, ipe, jps, jpe) end if this%ifds = ids0 @@ -608,7 +630,7 @@ subroutine Init_domain (this, config_flags, geogrid, & call this%Init_latlons (proj) case (INIT_MODE_RESTART) - call Init_restart_projection (this, trim (restart_file), proj) + call Init_restart_projection (this, trim (file_restart), proj) case default call Stop_simulation ('Not ready to complete fire state initialization 2') @@ -661,7 +683,7 @@ subroutine Init_domain (this, config_flags, geogrid, & call Stop_simulation ('Not ready to initialize from fire perimeter in idealized mode') case (INIT_MODE_RESTART) - call Read_restart_static_fields (this, trim (restart_file)) + call Read_restart_static_fields (this, trim (file_restart)) case default call Stop_simulation ('Not ready to complete fire state initialization 3') diff --git a/state/state_restart_submod.F90 b/state/state_restart_submod.F90 index 65a706f2..cbf360b3 100644 --- a/state/state_restart_submod.F90 +++ b/state/state_restart_submod.F90 @@ -13,6 +13,7 @@ character (len = *), parameter :: NAME_VAR_FMC_GC = 'fmc_gc' character (len = *), parameter :: NAME_ATT_FMOIST_LASTTIME = 'fmc_fmoist_lasttime' character (len = *), parameter :: NAME_ATT_FMOIST_NEXTTIME = 'fmc_fmoist_nexttime' + real, parameter :: RESTART_INTERVAL_TOL = 1.0e-6 contains @@ -22,6 +23,41 @@ end procedure Build_restart_file_name + module procedure Handle_restart + + real :: restart_interval_seconds, restart_interval_error + character (len = 256) :: msg + logical :: do_initialize + + + do_initialize = .false. + if (present (initialize)) do_initialize = initialize + + if (do_initialize .or. .not. this%is_restart_output_initialized) then + this%restart_step_interval = -1 + if (config_flags%restart_interval > 0) then + this%restart_step_interval = nint (real (config_flags%restart_interval) / this%dt) + restart_interval_seconds = this%restart_step_interval * this%dt + restart_interval_error = abs (restart_interval_seconds - real (config_flags%restart_interval)) + + if (this%restart_step_interval <= 0 .or. & + restart_interval_error > max (RESTART_INTERVAL_TOL, abs (real (config_flags%restart_interval)) * RESTART_INTERVAL_TOL)) then + write (msg, '(a, i0, a, f12.6)') 'restart_interval must map to an integer number of time steps: restart_interval = ', & + config_flags%restart_interval, ', dt = ', this%dt + call Stop_simulation (msg) + end if + end if + this%is_restart_output_initialized = .true. + end if + + if (do_initialize) return + + if (this%restart_step_interval > 0) then + if (mod (this%itimestep, this%restart_step_interval) == 0) call this%Write_restart (config_flags) + end if + + end procedure Handle_restart + module procedure Init_restart_dimensions integer :: restart_nx, restart_ny From cb37820940a66dcc1b73ac11f54054d83263de89 Mon Sep 17 00:00:00 2001 From: masih Date: Fri, 25 Sep 2026 15:49:16 -0600 Subject: [PATCH 05/19] Clean up standalone driver restart flow Move the standalone restart read/save decision into Init_fire_state so the main fire_behavior driver stays focused on top-level model control. Move the per-step model-clock consistency check into state_fire_t%Check_model_clock. Keep it separate from restart scheduling because the clock check is a general model consistency check, not restart-only logic. --- driver/fire_behavior.F90 | 16 +--------------- driver/initialize_mod.F90 | 8 ++++++++ state/state_mod.F90 | 16 ++++++++++++++++ 3 files changed, 25 insertions(+), 15 deletions(-) diff --git a/driver/fire_behavior.F90 b/driver/fire_behavior.F90 index 9367c7cb..60a2ef15 100644 --- a/driver/fire_behavior.F90 +++ b/driver/fire_behavior.F90 @@ -8,8 +8,6 @@ program fire_behavior use initialize_mod, only : Init_fire_state, Init_atm_state use advance_mod, only : Advance_state use wrfdata_mod, only : wrfdata_t - use datetime_mod, only : datetime_t - use stderrout_mod, only : Stop_simulation use, intrinsic :: iso_fortran_env, only : ERROR_UNIT, OUTPUT_UNIT implicit none @@ -18,7 +16,6 @@ program fire_behavior type (state_fire_t) :: grid type (wrfdata_t) :: atm_state type (namelist_t) :: config_flags - type (datetime_t) :: datetime_check logical, parameter :: DEBUG_LOCAL = .false. @@ -74,24 +71,13 @@ program fire_behavior end select - if (config_flags%restart) then - if (DEBUG_LOCAL) write (OUTPUT_UNIT, *) ' Reading restart state...' - call grid%Read_restart (config_flags) - - else - if (DEBUG_LOCAL) write (OUTPUT_UNIT, *) ' Saving fire state...' - call grid%Save_state () - end if - call grid%Handle_restart (config_flags, initialize = .true.) if (DEBUG_LOCAL) write (OUTPUT_UNIT, *) ' Starting temporal loop...' do while (grid%datetime_now < grid%datetime_end) call Advance_state (grid, config_flags) - datetime_check = grid%datetime_start - call datetime_check%Add_seconds (grid%itimestep * grid%dt) - if (datetime_check /= grid%datetime_now) call Stop_simulation ('Model clock is inconsistent with itimestep and dt') + call grid%Check_model_clock () call grid%Handle_output (config_flags) if (config_flags%ideal_opt == 0) call grid%Handle_wrfdata_update (atm_state, config_flags) diff --git a/driver/initialize_mod.F90 b/driver/initialize_mod.F90 index 8f3a46f5..14b8ed9e 100644 --- a/driver/initialize_mod.F90 +++ b/driver/initialize_mod.F90 @@ -121,6 +121,14 @@ subroutine Init_fire_state (grid, config_flags, wrf) call grid%Handle_wrfdata_update (wrf, config_flags) end if + if (config_flags%restart) then + if (DEBUG_LOCAL) call Print_message (' Reading restart state') + call grid%Read_restart (config_flags) + else + if (DEBUG_LOCAL) call Print_message (' Saving fire state') + call grid%Save_state () + end if + if (DEBUG_LOCAL) then ! print lat/lons open (newunit = unit_out, file = 'latlons_c.dat') diff --git a/state/state_mod.F90 b/state/state_mod.F90 index e65895f6..3eb7b4da 100644 --- a/state/state_mod.F90 +++ b/state/state_mod.F90 @@ -121,6 +121,7 @@ module state_mod contains procedure, public :: Allocate_vars => Allocate_vars procedure, public :: Apply_wafs => Apply_wafs + procedure, public :: Check_model_clock => Check_model_clock procedure, public :: Convert_sb_to_ander => Convert_scottburgan_to_anderson procedure, public :: Handle_output => Handle_output procedure, public :: Handle_restart => Handle_restart @@ -295,6 +296,21 @@ subroutine Convert_scottburgan_to_anderson (this) end subroutine Convert_scottburgan_to_anderson + subroutine Check_model_clock (this) + + implicit none + + class (state_fire_t), intent (in) :: this + + type (datetime_t) :: datetime_check + + + datetime_check = this%datetime_start + call datetime_check%Add_seconds (this%itimestep * this%dt) + if (datetime_check /= this%datetime_now) call Stop_simulation ('Model clock is inconsistent with itimestep and dt') + + end subroutine Check_model_clock + subroutine Handle_output (this, config_flags) implicit none From 5225310a3318658defa567962362592a761a5654 Mon Sep 17 00:00:00 2001 From: masih Date: Tue, 29 Sep 2026 13:45:08 -0600 Subject: [PATCH 06/19] Clean up restart helper dependencies --- driver/fire_behavior.F90 | 6 +++++ driver/initialize_mod.F90 | 3 --- io/netcdf_mod.F90 | 48 +++++++++++++++++----------------- share/mpi_mod.F90 | 30 +-------------------- state/state_mod.F90 | 6 ++--- state/state_restart_submod.F90 | 2 ++ 6 files changed, 35 insertions(+), 60 deletions(-) diff --git a/driver/fire_behavior.F90 b/driver/fire_behavior.F90 index 60a2ef15..99b3f828 100644 --- a/driver/fire_behavior.F90 +++ b/driver/fire_behavior.F90 @@ -8,6 +8,7 @@ program fire_behavior use initialize_mod, only : Init_fire_state, Init_atm_state use advance_mod, only : Advance_state use wrfdata_mod, only : wrfdata_t + use stderrout_mod, only : Print_message use, intrinsic :: iso_fortran_env, only : ERROR_UNIT, OUTPUT_UNIT implicit none @@ -71,6 +72,11 @@ program fire_behavior end select + if (.not. config_flags%restart) then + if (DEBUG_LOCAL) call Print_message (' Saving fire state') + call grid%Save_state () + end if + call grid%Handle_restart (config_flags, initialize = .true.) if (DEBUG_LOCAL) write (OUTPUT_UNIT, *) ' Starting temporal loop...' diff --git a/driver/initialize_mod.F90 b/driver/initialize_mod.F90 index 14b8ed9e..fedeeaff 100644 --- a/driver/initialize_mod.F90 +++ b/driver/initialize_mod.F90 @@ -124,9 +124,6 @@ subroutine Init_fire_state (grid, config_flags, wrf) if (config_flags%restart) then if (DEBUG_LOCAL) call Print_message (' Reading restart state') call grid%Read_restart (config_flags) - else - if (DEBUG_LOCAL) call Print_message (' Saving fire state') - call grid%Save_state () end if if (DEBUG_LOCAL) then diff --git a/io/netcdf_mod.F90 b/io/netcdf_mod.F90 index 3232fb05..e013d8e7 100644 --- a/io/netcdf_mod.F90 +++ b/io/netcdf_mod.F90 @@ -247,7 +247,7 @@ subroutine Get_netcdf_var_real32_2d_mpi (file_name, cfbm_comm, nx, ny, ifps, ifp integer, intent (in) :: cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe real, dimension(ifps:ifpe, jfps:jfpe), intent (out) :: var2d_local - real, dimension(:, :), allocatable :: var2d_restart + real, dimension(:, :), allocatable :: var2d_global character (len = :), allocatable :: msg integer :: rank, ierr logical, parameter :: DEBUG_LOCAL = .false. @@ -260,24 +260,24 @@ subroutine Get_netcdf_var_real32_2d_mpi (file_name, cfbm_comm, nx, ny, ifps, ifp if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_comm_rank ') if (rank == 0) then - call Get_netcdf_var (file_name, var_name, var2d_restart) - if (size (var2d_restart, 1) /= nx .or. size (var2d_restart, 2) /= ny) then - msg = 'Restart variable has unexpected dimensions: '//trim (var_name) + call Get_netcdf_var (file_name, var_name, var2d_global) + if (size (var2d_global, 1) /= nx .or. size (var2d_global, 2) /= ny) then + msg = 'NetCDF variable has unexpected dimensions: '//trim (var_name) call Stop_simulation (msg) end if end if - call Distribute_global_var2d (cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, var2d_restart, var2d_local) + call Distribute_global_var2d (cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, var2d_global, var2d_local) #else - call Get_netcdf_var (file_name, var_name, var2d_restart) - if (size (var2d_restart, 1) /= nx .or. size (var2d_restart, 2) /= ny) then - msg = 'Restart variable has unexpected dimensions: '//trim (var_name) + call Get_netcdf_var (file_name, var_name, var2d_global) + if (size (var2d_global, 1) /= nx .or. size (var2d_global, 2) /= ny) then + msg = 'NetCDF variable has unexpected dimensions: '//trim (var_name) call Stop_simulation (msg) end if - var2d_local(1:nx, 1:ny) = var2d_restart(1:nx, 1:ny) + var2d_local(1:nx, 1:ny) = var2d_global(1:nx, 1:ny) #endif - if (allocated (var2d_restart)) deallocate (var2d_restart) + if (allocated (var2d_global)) deallocate (var2d_global) if (DEBUG_LOCAL) call Print_message ('Leaving Get_netcdf_var_real32_2d_mpi...') @@ -380,8 +380,8 @@ subroutine Get_netcdf_var_real32_3d_mpi (file_name, cfbm_comm, nx, ny, n2, ifps, integer, intent (in) :: cfbm_comm, nx, ny, n2, ifps, ifpe, jfps, jfpe real, dimension(ifps:ifpe, n2, jfps:jfpe), intent (out) :: var3d_local - real, dimension(:, :), allocatable :: var2d_restart - real, dimension(:, :, :), allocatable :: var3d_restart + real, dimension(:, :), allocatable :: var2d_global + real, dimension(:, :, :), allocatable :: var3d_global character (len = :), allocatable :: msg integer :: rank, ierr, k logical, parameter :: DEBUG_LOCAL = .false. @@ -394,31 +394,31 @@ subroutine Get_netcdf_var_real32_3d_mpi (file_name, cfbm_comm, nx, ny, n2, ifps, if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_comm_rank ') if (rank == 0) then - call Get_netcdf_var (file_name, var_name, var3d_restart) - if (size (var3d_restart, 1) /= nx .or. size (var3d_restart, 2) /= n2 .or. size (var3d_restart, 3) /= ny) then - msg = 'Restart variable has unexpected dimensions: '//trim (var_name) + call Get_netcdf_var (file_name, var_name, var3d_global) + if (size (var3d_global, 1) /= nx .or. size (var3d_global, 2) /= n2 .or. size (var3d_global, 3) /= ny) then + msg = 'NetCDF variable has unexpected dimensions: '//trim (var_name) call Stop_simulation (msg) end if end if do k = 1, n2 if (rank == 0) then - allocate (var2d_restart(nx, ny)) - var2d_restart(1:nx, 1:ny) = var3d_restart(1:nx, k, 1:ny) + allocate (var2d_global(nx, ny)) + var2d_global(1:nx, 1:ny) = var3d_global(1:nx, k, 1:ny) end if - call Distribute_global_var2d (cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, var2d_restart, var3d_local(ifps:ifpe, k, jfps:jfpe)) - if (allocated (var2d_restart)) deallocate (var2d_restart) + call Distribute_global_var2d (cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, var2d_global, var3d_local(ifps:ifpe, k, jfps:jfpe)) + if (allocated (var2d_global)) deallocate (var2d_global) end do #else - call Get_netcdf_var (file_name, var_name, var3d_restart) - if (size (var3d_restart, 1) /= nx .or. size (var3d_restart, 2) /= n2 .or. size (var3d_restart, 3) /= ny) then - msg = 'Restart variable has unexpected dimensions: '//trim (var_name) + call Get_netcdf_var (file_name, var_name, var3d_global) + if (size (var3d_global, 1) /= nx .or. size (var3d_global, 2) /= n2 .or. size (var3d_global, 3) /= ny) then + msg = 'NetCDF variable has unexpected dimensions: '//trim (var_name) call Stop_simulation (msg) end if - var3d_local(1:nx, 1:n2, 1:ny) = var3d_restart(1:nx, 1:n2, 1:ny) + var3d_local(1:nx, 1:n2, 1:ny) = var3d_global(1:nx, 1:n2, 1:ny) #endif - if (allocated (var3d_restart)) deallocate (var3d_restart) + if (allocated (var3d_global)) deallocate (var3d_global) if (DEBUG_LOCAL) call Print_message ('Leaving Get_netcdf_var_real32_3d_mpi...') diff --git a/share/mpi_mod.F90 b/share/mpi_mod.F90 index 76cb4ea8..be0bf326 100644 --- a/share/mpi_mod.F90 +++ b/share/mpi_mod.F90 @@ -1,6 +1,6 @@ module mpi_mod - use, intrinsic :: iso_fortran_env, only : ERROR_UNIT, OUTPUT_UNIT + use, intrinsic :: iso_fortran_env, only : OUTPUT_UNIT implicit none @@ -222,9 +222,7 @@ subroutine Distribute_global_var2d (cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, v #ifdef DM_PARALLEL call Mpi_comm_rank (cfbm_comm, rank, ierr) - if (ierr /= MPI_SUCCESS) call Abort_mpi_operation ('Problems with Mpi_comm_rank ') call Mpi_comm_size (cfbm_comm, nprocs, ierr) - if (ierr /= MPI_SUCCESS) call Abort_mpi_operation ('Problems getting the number of MPI tasks') nx_local = ifpe - ifps + 1 ny_local = jfpe - jfps + 1 @@ -236,13 +234,9 @@ subroutine Distribute_global_var2d (cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, v end if call MPI_Gather (ifps, 1, MPI_INTEGER, all_ifps, 1, MPI_INTEGER, 0, cfbm_comm, ierr) - if (ierr /= MPI_SUCCESS) call Abort_mpi_operation ('Problems with MPI_Gather for ifps') call MPI_Gather (ifpe, 1, MPI_INTEGER, all_ifpe, 1, MPI_INTEGER, 0, cfbm_comm, ierr) - if (ierr /= MPI_SUCCESS) call Abort_mpi_operation ('Problems with MPI_Gather for ifpe') call MPI_Gather (jfps, 1, MPI_INTEGER, all_jfps, 1, MPI_INTEGER, 0, cfbm_comm, ierr) - if (ierr /= MPI_SUCCESS) call Abort_mpi_operation ('Problems with MPI_Gather for jfps') call MPI_Gather (jfpe, 1, MPI_INTEGER, all_jfpe, 1, MPI_INTEGER, 0, cfbm_comm, ierr) - if (ierr /= MPI_SUCCESS) call Abort_mpi_operation ('Problems with MPI_Gather for jfpe') if (rank == 0) then allocate (sendcounts(nprocs), displs(nprocs)) @@ -270,7 +264,6 @@ subroutine Distribute_global_var2d (cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, v end if call MPI_Scatterv (sendbuf, sendcounts, displs, MPI_REAL, var2d_local, nx_local * ny_local, MPI_REAL, 0, cfbm_comm, ierr) - if (ierr /= MPI_SUCCESS) call Abort_mpi_operation ('Problems with MPI_Scatterv for distributed variable') deallocate (all_ifps, all_ifpe, all_jfps, all_jfpe, sendcounts, displs, sendbuf) #else @@ -279,27 +272,6 @@ subroutine Distribute_global_var2d (cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, v end subroutine Distribute_global_var2d - subroutine Abort_mpi_operation (msg) - -#ifdef DM_PARALLEL - use mpi -#endif - implicit none - - character (len = *), intent (in) :: msg - integer :: ierr - - - write (ERROR_UNIT, *) 'STOP:' // trim (msg) -#ifdef DM_PARALLEL - call MPI_Abort (MPI_COMM_WORLD, 1, ierr) - call MPI_Finalize (ierr) -#else - stop -#endif - - end subroutine Abort_mpi_operation - subroutine Do_halo_exchange (patch, ims, ime, jms, jme, ips, ipe, jps, jpe, nghost, cart_comm) ! It does not update the corners in the halo diff --git a/state/state_mod.F90 b/state/state_mod.F90 index 3eb7b4da..83aab158 100644 --- a/state/state_mod.F90 +++ b/state/state_mod.F90 @@ -10,15 +10,13 @@ module state_mod use geogrid_mod, only : geogrid_t use ignition_line_mod, only : ignition_line_t use namelist_mod, only : namelist_t - use netcdf_mod, only : Create_netcdf_file, Add_netcdf_att, Add_netcdf_dim, Add_netcdf_var_mpi, Get_netcdf_att, & - Get_netcdf_var_mpi, Is_netcdf_file_present, NAME_DIM_X, NAME_DIM_Y + use netcdf_mod, only : Create_netcdf_file, Add_netcdf_dim, Add_netcdf_var_mpi, NAME_DIM_X, NAME_DIM_Y use proj_lc_mod, only : proj_lc_t use ros_mod, only : ros_t use stderrout_mod, only : Stop_simulation, Print_message use tiles_mod, only : Calc_tiles_dims use wrfdata_mod, only : wrfdata_t, G, RERADIUS - use mpi_mod, only : Calc_tasks_in_x_and_y, Calc_patch_dims, Distribute_var2d, Do_halo_exchange_with_corners, & - Print_cart_info, topology_dim_order + use mpi_mod, only : Calc_tasks_in_x_and_y, Calc_patch_dims, Distribute_var2d, Print_cart_info, topology_dim_order implicit none diff --git a/state/state_restart_submod.F90 b/state/state_restart_submod.F90 index cbf360b3..37d16757 100644 --- a/state/state_restart_submod.F90 +++ b/state/state_restart_submod.F90 @@ -4,6 +4,8 @@ use mpi #endif use fmc_wrffire_mod, only : fmc_wrffire_t + use mpi_mod, only : Do_halo_exchange_with_corners + use netcdf_mod, only : Add_netcdf_att, Get_netcdf_att, Get_netcdf_var_mpi, Is_netcdf_file_present use, intrinsic :: iso_fortran_env, only : INT32, REAL32 implicit none From 922aa41b406cce839779c77c81178ffe38fcde3c Mon Sep 17 00:00:00 2001 From: masih Date: Tue, 29 Sep 2026 14:12:21 -0600 Subject: [PATCH 07/19] Move FMC restart writes into FMC model --- physics/fmc_wrffire_mod.F90 | 53 +++++++++++++++++++++++++++++++++- share/fmc_mod.F90 | 15 ++++++++++ state/state_restart_submod.F90 | 50 ++++---------------------------- 3 files changed, 73 insertions(+), 45 deletions(-) diff --git a/physics/fmc_wrffire_mod.F90 b/physics/fmc_wrffire_mod.F90 index 252f0c50..a7dab511 100644 --- a/physics/fmc_wrffire_mod.F90 +++ b/physics/fmc_wrffire_mod.F90 @@ -13,7 +13,9 @@ module fmc_wrffire_mod ! 50% Entering dormancy, coloration starting, some leaves may have dropped from stem ! 30% Completely cured, treat as dead fuel + use, intrinsic :: iso_fortran_env, only : REAL32 use stderrout_mod, only : Stop_simulation + use netcdf_mod, only : Add_netcdf_att, Add_netcdf_dim, Add_netcdf_var_mpi, NAME_DIM_X, NAME_DIM_Y use ros_mod, only : ros_t use fmc_mod, only : fmc_t use fuel_mod, only : fuel_t @@ -22,10 +24,14 @@ module fmc_wrffire_mod private - public :: fmc_wrffire_t + public :: fmc_wrffire_t, NAME_VAR_FMC_GC, NAME_ATT_FMOIST_LASTTIME, NAME_ATT_FMOIST_NEXTTIME integer, parameter :: MOISTURE_CLASSES = 5, NUM_FMEP = 2 real, parameter :: FMEP_DECAY_TLAG = 999999 ! time constant of assimilated adjustments of equilibria decay + character (len = *), parameter :: NAME_DIM_MOISTURE_CLASS = 'moisture_class' + character (len = *), parameter :: NAME_VAR_FMC_GC = 'fmc_gc' + character (len = *), parameter :: NAME_ATT_FMOIST_LASTTIME = 'fmc_fmoist_lasttime' + character (len = *), parameter :: NAME_ATT_FMOIST_NEXTTIME = 'fmc_fmoist_nexttime' type, extends(fmc_t) :: fmc_wrffire_t real :: fmoist_lasttime, fmoist_nexttime, dt_moisture @@ -40,6 +46,7 @@ module fmc_wrffire_mod procedure, public :: Advance_fmc_model => Advance_fmc_model procedure, public :: Advance_moisture_classes => Advance_moisture_classes procedure, public :: Average_moisture_classes => Average_moisture_classes + procedure, public :: Add_restart_vars => Add_restart_vars procedure, public :: Init => Init_fmc_wrffire end type fmc_wrffire_t @@ -62,6 +69,50 @@ module fmc_wrffire_mod contains + subroutine Add_restart_vars (this, file_restart, cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, rank) + +#ifdef DM_PARALLEL + use mpi +#endif + implicit none + + class (fmc_wrffire_t), intent (in) :: this + character (len = *), intent (in) :: file_restart + integer, intent (in) :: cfbm_comm + integer, intent (in) :: nx + integer, intent (in) :: ny + integer, intent (in) :: ifps + integer, intent (in) :: ifpe + integer, intent (in) :: jfps + integer, intent (in) :: jfpe + integer, intent (in) :: rank + + character (len = 32), dimension(3) :: dim_names_fmc_gc + integer :: ierr, n_moisture_classes + + + if (.not. allocated (this%fmc_gc)) call Stop_simulation ('FMC restart write requires allocated fmc_gc') + + n_moisture_classes = size (this%fmc_gc, 2) + if (rank == 0) call Add_netcdf_dim (file_restart, NAME_DIM_MOISTURE_CLASS, n_moisture_classes) +#ifdef DM_PARALLEL + call MPI_Barrier (cfbm_comm, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with MPI_Barrier for FMC restart write') +#endif + + dim_names_fmc_gc(1) = NAME_DIM_X + dim_names_fmc_gc(2) = NAME_DIM_MOISTURE_CLASS + dim_names_fmc_gc(3) = NAME_DIM_Y + call Add_netcdf_var_mpi (file_restart, dim_names_fmc_gc, cfbm_comm, nx, ny, n_moisture_classes, & + ifps, ifpe, jfps, jfpe, NAME_VAR_FMC_GC, this%fmc_gc(ifps:ifpe, 1:n_moisture_classes, jfps:jfpe)) + + if (rank == 0) then + call Add_netcdf_att (file_restart, 'global', NAME_ATT_FMOIST_LASTTIME, real (this%fmoist_lasttime, kind = REAL32)) + call Add_netcdf_att (file_restart, 'global', NAME_ATT_FMOIST_NEXTTIME, real (this%fmoist_nexttime, kind = REAL32)) + end if + + end subroutine Add_restart_vars + subroutine Init_fmc_wrffire (this, fuels, fuelmc_g, fuelmc_g_live, ifms, ifme, jfms, jfme, itimestep, dt) implicit none diff --git a/share/fmc_mod.F90 b/share/fmc_mod.F90 index 55009685..165ee5f9 100644 --- a/share/fmc_mod.F90 +++ b/share/fmc_mod.F90 @@ -14,10 +14,25 @@ module fmc_mod type, abstract :: fmc_t contains procedure (Advance_fmc_model), deferred :: Advance_fmc_model + procedure (Add_restart_vars), deferred :: Add_restart_vars procedure (Init), deferred :: Init end type fmc_t abstract interface + subroutine Add_restart_vars (this, file_restart, cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, rank) + import :: fmc_t + class (fmc_t), intent (in) :: this + character (len = *), intent (in) :: file_restart + integer, intent (in) :: cfbm_comm + integer, intent (in) :: nx + integer, intent (in) :: ny + integer, intent (in) :: ifps + integer, intent (in) :: ifpe + integer, intent (in) :: jfps + integer, intent (in) :: jfpe + integer, intent (in) :: rank + end subroutine Add_restart_vars + subroutine Init (this, fuels, fuelmc_g, fuelmc_g_live, ifms, ifme, jfms, jfme, itimestep, dt) import :: fmc_t, fuel_t class (fmc_t), intent (in out) :: this diff --git a/state/state_restart_submod.F90 b/state/state_restart_submod.F90 index 37d16757..ab6789b1 100644 --- a/state/state_restart_submod.F90 +++ b/state/state_restart_submod.F90 @@ -3,7 +3,7 @@ #ifdef DM_PARALLEL use mpi #endif - use fmc_wrffire_mod, only : fmc_wrffire_t + use fmc_wrffire_mod, only : fmc_wrffire_t, NAME_VAR_FMC_GC, NAME_ATT_FMOIST_LASTTIME, NAME_ATT_FMOIST_NEXTTIME use mpi_mod, only : Do_halo_exchange_with_corners use netcdf_mod, only : Add_netcdf_att, Get_netcdf_att, Get_netcdf_var_mpi, Is_netcdf_file_present use, intrinsic :: iso_fortran_env, only : INT32, REAL32 @@ -11,10 +11,6 @@ implicit none integer, parameter :: RESTART_IO_ROOT = 0 - character (len = *), parameter :: NAME_DIM_MOISTURE_CLASS = 'moisture_class' - character (len = *), parameter :: NAME_VAR_FMC_GC = 'fmc_gc' - character (len = *), parameter :: NAME_ATT_FMOIST_LASTTIME = 'fmc_fmoist_lasttime' - character (len = *), parameter :: NAME_ATT_FMOIST_NEXTTIME = 'fmc_fmoist_nexttime' real, parameter :: RESTART_INTERVAL_TOL = 1.0e-6 contains @@ -822,7 +818,11 @@ end subroutine Validate_restart_real call Add_restart_field ('fire_rain_old', this%fire_rain_old) end if end if - if (config_flags%fmoist_run) call Write_restart_fmc () + if (config_flags%fmoist_run) then + if (.not. allocated (this%fmc_param)) call Stop_simulation ('FMC restart write requires allocated fmc_param') + call this%fmc_param%Add_restart_vars (file_restart, this%cfbm_comm, this%nx, this%ny, & + this%ifps, this%ifpe, this%jfps, this%jfpe, rank) + end if call Restart_io_barrier (this) @@ -843,44 +843,6 @@ subroutine Add_restart_field (var_name, var) end subroutine Add_restart_field - subroutine Write_restart_fmc () - - implicit none - - character (len = 32), dimension(3) :: dim_names_fmc_gc - integer :: n_moisture_classes - - - if (.not. allocated (this%fmc_param)) call Stop_simulation ('FMC restart write requires allocated fmc_param') - - select type (fmc_param => this%fmc_param) - type is (fmc_wrffire_t) - if (.not. allocated (fmc_param%fmc_gc)) call Stop_simulation ('FMC restart write requires allocated fmc_gc') - - n_moisture_classes = size (fmc_param%fmc_gc, 2) - if (rank == 0) call Add_netcdf_dim (file_restart, NAME_DIM_MOISTURE_CLASS, n_moisture_classes) - call Restart_io_barrier (this) - - dim_names_fmc_gc(1) = NAME_DIM_X - dim_names_fmc_gc(2) = NAME_DIM_MOISTURE_CLASS - dim_names_fmc_gc(3) = NAME_DIM_Y - call Add_netcdf_var_mpi (file_restart, dim_names_fmc_gc, this%cfbm_comm, this%nx, this%ny, n_moisture_classes, & - this%ifps, this%ifpe, this%jfps, this%jfpe, NAME_VAR_FMC_GC, & - fmc_param%fmc_gc(this%ifps:this%ifpe, 1:n_moisture_classes, this%jfps:this%jfpe)) - - if (rank == 0) then - call Add_netcdf_att (file_restart, 'global', NAME_ATT_FMOIST_LASTTIME, & - real (fmc_param%fmoist_lasttime, kind = REAL32)) - call Add_netcdf_att (file_restart, 'global', NAME_ATT_FMOIST_NEXTTIME, & - real (fmc_param%fmoist_nexttime, kind = REAL32)) - end if - - class default - call Stop_simulation ('Restart write is not implemented for selected FMC component') - end select - - end subroutine Write_restart_fmc - end procedure Write_restart end submodule state_restart_submod From 3d759917794c5df794f4a96ef598c34ad2da76f6 Mon Sep 17 00:00:00 2001 From: masih Date: Fri, 2 Oct 2026 09:48:35 -0600 Subject: [PATCH 08/19] Make restart submodule signatures explicit --- state/state_restart_submod.F90 | 66 ++++++++++++++++++++++++++-------- 1 file changed, 52 insertions(+), 14 deletions(-) diff --git a/state/state_restart_submod.F90 b/state/state_restart_submod.F90 index ab6789b1..c6e0acfc 100644 --- a/state/state_restart_submod.F90 +++ b/state/state_restart_submod.F90 @@ -15,13 +15,24 @@ contains - module procedure Build_restart_file_name + module function Build_restart_file_name (restart_datetime) result (file_restart) + + implicit none + + character (len = *), intent (in) :: restart_datetime + character (len = :), allocatable :: file_restart file_restart = 'fire_restart_'//trim (restart_datetime)//'.nc' - end procedure Build_restart_file_name + end function Build_restart_file_name + + module subroutine Handle_restart (this, config_flags, initialize) + + implicit none - module procedure Handle_restart + class (state_fire_t), intent (in out) :: this + type (namelist_t), intent (in) :: config_flags + logical, intent (in), optional :: initialize real :: restart_interval_seconds, restart_interval_error character (len = 256) :: msg @@ -54,9 +65,15 @@ if (mod (this%itimestep, this%restart_step_interval) == 0) call this%Write_restart (config_flags) end if - end procedure Handle_restart + end subroutine Handle_restart + + module subroutine Init_restart_dimensions (this, restart_file, ids0, ide0, jds0, jde0, ips, ipe, jps, jpe) + + implicit none - module procedure Init_restart_dimensions + class (state_fire_t), intent (in out) :: this + character (len = *), intent (in) :: restart_file + integer, intent (out) :: ids0, ide0, jds0, jde0, ips, ipe, jps, jpe integer :: restart_nx, restart_ny integer (kind = INT32) :: att_int32 @@ -112,9 +129,15 @@ jpe = jde0 #endif - end procedure Init_restart_dimensions + end subroutine Init_restart_dimensions + + module subroutine Init_restart_projection (this, restart_file, proj) - module procedure Init_restart_projection + implicit none + + class (state_fire_t), intent (in out) :: this + character (len = *), intent (in) :: restart_file + type (proj_lc_t), intent (out) :: proj integer :: restart_map_proj, restart_sr_x, restart_sr_y integer (kind = INT32) :: att_int32 @@ -188,9 +211,14 @@ call this%Init_latlons (proj, srx = restart_sr_x, sry = restart_sr_y) - end procedure Init_restart_projection + end subroutine Init_restart_projection + + module subroutine Read_restart_static_fields (this, restart_file) + + implicit none - module procedure Read_restart_static_fields + class (state_fire_t), intent (in out) :: this + character (len = *), intent (in) :: restart_file call Read_restart_field_2d (trim (restart_file), 'zsf', this%nx, this%ny, & this%ifms, this%ifme, this%jfms, this%jfme, this%ifps, this%ifpe, this%jfps, this%jfpe, this%cfbm_comm, this%zsf) @@ -203,7 +231,7 @@ call Read_restart_field_2d (trim (restart_file), 'fz0', this%nx, this%ny, & this%ifms, this%ifme, this%jfms, this%jfme, this%ifps, this%ifpe, this%jfps, this%jfpe, this%cfbm_comm, this%fz0) - end procedure Read_restart_static_fields + end subroutine Read_restart_static_fields subroutine Require_restart_mpi_comm (this) @@ -293,7 +321,12 @@ subroutine Broadcast_restart_real (this, value) end subroutine Broadcast_restart_real - module procedure Read_restart + module subroutine Read_restart (this, config_flags) + + implicit none + + class (state_fire_t), intent (in out) :: this + type (namelist_t), intent (in) :: config_flags type (datetime_t) :: datetime_restart, datetime_check character (len = :), allocatable :: file_restart @@ -640,7 +673,7 @@ subroutine Set_next_datetime_after_restart (datetime_next, interval_seconds) end subroutine Set_next_datetime_after_restart - end procedure Read_restart + end subroutine Read_restart subroutine Read_restart_field_2d (file_name, var_name, nx, ny, ifms, ifme, jfms, jfme, ifps, ifpe, jfps, jfpe, cfbm_comm, var) @@ -695,7 +728,12 @@ subroutine Validate_restart_real (file_name, att_name, expected_value) end subroutine Validate_restart_real - module procedure Write_restart + module subroutine Write_restart (this, config_flags) + + implicit none + + class (state_fire_t), intent (in) :: this + type (namelist_t), intent (in) :: config_flags character (len = :), allocatable :: file_restart integer :: start_year, start_month, start_day, start_hour, start_minute, start_second, & @@ -843,6 +881,6 @@ subroutine Add_restart_field (var_name, var) end subroutine Add_restart_field - end procedure Write_restart + end subroutine Write_restart end submodule state_restart_submod From e73f23c7af3eaf3e3784ad0ff574cd68a258d272 Mon Sep 17 00:00:00 2001 From: masih Date: Fri, 2 Oct 2026 13:13:56 -0600 Subject: [PATCH 09/19] Clean up restart initialization interface --- state/state_mod.F90 | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/state/state_mod.F90 b/state/state_mod.F90 index ab6b069e..1ee03e45 100644 --- a/state/state_mod.F90 +++ b/state/state_mod.F90 @@ -23,7 +23,7 @@ module state_mod private - public :: state_fire_t, N_POINTS_IN_HALO, Build_restart_file_name + public :: state_fire_t, N_POINTS_IN_HALO integer, parameter :: INIT_MODE_NONE = 0, INIT_MODE_GEOGRID = 1, INIT_MODE_WRF = 2, INIT_MODE_IDEAL = 3, INIT_MODE_RESTART = 4 @@ -368,7 +368,7 @@ subroutine Init_domain (this, config_flags, geogrid, & kfds, kfde, kfms, kfme, kfps, kfpe, & kfts, kfte, ide, jde, & cen_lat, cen_lon, truelat1, truelat2, stand_lon, & - dx, dy, sr_x, sr_y, nfuel_cat, zsf, dzdxf, dzdyf, restart_file) + dx, dy, sr_x, sr_y, nfuel_cat, zsf, dzdxf, dzdyf) implicit none @@ -376,7 +376,6 @@ subroutine Init_domain (this, config_flags, geogrid, & class (state_fire_t), intent(in out) :: this type (namelist_t), intent (in) :: config_flags type (geogrid_t), intent (in out), optional :: geogrid - character (len = *), intent (in), optional :: restart_file integer, intent (in), optional :: ifds, ifde, ifms, ifme, ifps, ifpe, & jfds, jfde, jfms, jfme, jfps, jfpe, & kfds, kfde, kfms, kfme, kfps, kfpe, & @@ -406,9 +405,8 @@ subroutine Init_domain (this, config_flags, geogrid, & present (sr_x) .and. present (sr_y) .and. present (nfuel_cat) .and. present (zsf) .and. present (dzdxf) .and. & present (dzdyf) - if (present (restart_file)) file_restart = restart_file if (config_flags%ideal_opt == 0 .and. config_flags%restart .and. .not. present (geogrid) .and. & - .not. has_wrf_metadata .and. .not. allocated (file_restart)) then + .not. has_wrf_metadata) then datetime_restart = datetime_t (config_flags%start_year, config_flags%start_month, config_flags%start_day, & config_flags%start_hour, config_flags%start_minute, config_flags%start_second) file_restart = Build_restart_file_name (datetime_restart%datetime) From 3291f13f258067a09f8d1715452ebe3bcec1313a Mon Sep 17 00:00:00 2001 From: masih Date: Tue, 20 Oct 2026 12:24:53 -0600 Subject: [PATCH 10/19] Preserve perimeter activation state across restart Write and read fire_perimeter_ignited as an integer restart attribute and broadcast it on every rank. --- state/state_restart_submod.F90 | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/state/state_restart_submod.F90 b/state/state_restart_submod.F90 index c6e0acfc..821b3931 100644 --- a/state/state_restart_submod.F90 +++ b/state/state_restart_submod.F90 @@ -332,6 +332,7 @@ module subroutine Read_restart (this, config_flags) character (len = :), allocatable :: file_restart integer (kind = INT32) :: att_int, restart_year, restart_month, restart_day, restart_hour, restart_minute, restart_second, & start_year, start_month, start_day, start_hour, start_minute, start_second + integer (kind = INT32) :: perimeter_ignited integer :: ij, expected_sr_x, expected_sr_y, ierr, rank logical, parameter :: DEBUG_LOCAL = .false. @@ -415,6 +416,10 @@ module subroutine Read_restart (this, config_flags) call Get_netcdf_att (file_restart, 'global', 'restart_minute', restart_minute) call Get_netcdf_att (file_restart, 'global', 'restart_second', restart_second) + call Get_netcdf_att (file_restart, 'global', 'fire_perimeter_ignited', perimeter_ignited) + if (perimeter_ignited /= 0 .and. perimeter_ignited /= 1) & + call Stop_simulation ('Restart fire_perimeter_ignited must be 0 or 1') + call Get_netcdf_att (file_restart, 'global', 'itimestep', att_int) end if @@ -432,6 +437,8 @@ module subroutine Read_restart (this, config_flags) call Broadcast_restart_integer (this, restart_second) call Broadcast_restart_integer (this, att_int) this%itimestep = att_int + call Broadcast_restart_integer (this, perimeter_ignited) + this%fire_perimeter_ignited = perimeter_ignited == 1 this%datetime_start = datetime_t (start_year, start_month, start_day, start_hour, start_minute, start_second) this%datetime_now = datetime_t (restart_year, restart_month, restart_day, restart_hour, restart_minute, restart_second) @@ -781,6 +788,8 @@ module subroutine Write_restart (this, config_flags) call Add_netcdf_att (file_restart, 'global', 'restart_minute', int (restart_minute, kind = INT32)) call Add_netcdf_att (file_restart, 'global', 'restart_second', int (restart_second, kind = INT32)) call Add_netcdf_att (file_restart, 'global', 'itimestep', int (this%itimestep, kind = INT32)) + call Add_netcdf_att (file_restart, 'global', 'fire_perimeter_ignited', & + int (merge (1, 0, this%fire_perimeter_ignited), kind = INT32)) call Add_netcdf_att (file_restart, 'global', 'nx', int (this%nx, kind = INT32)) call Add_netcdf_att (file_restart, 'global', 'ny', int (this%ny, kind = INT32)) call Add_netcdf_att (file_restart, 'global', 'dt', real (this%dt, kind = REAL32)) From 59b9081accba8a1f9414cb943a464c8f47382d69 Mon Sep 17 00:00:00 2001 From: masih Date: Tue, 20 Oct 2026 12:53:30 -0600 Subject: [PATCH 11/19] Preserve double precision in fuel restart fields Read and write fuel_frac and fuel_frac_burnt_dt as NetCDF doubles. Halo exchange for the variables as doubles. --- io/netcdf_mod.F90 | 209 +++++++++++++++++++++ share/mpi_mod.F90 | 320 ++++++++++++++++++++++++++++++++- state/state_restart_submod.F90 | 24 ++- 3 files changed, 540 insertions(+), 13 deletions(-) diff --git a/io/netcdf_mod.F90 b/io/netcdf_mod.F90 index bd57433a..eeea7d15 100644 --- a/io/netcdf_mod.F90 +++ b/io/netcdf_mod.F90 @@ -1,5 +1,6 @@ module netcdf_mod + use, intrinsic :: iso_fortran_env, only : REAL64 use netcdf, only : NF90_FILL_FLOAT use stderrout_mod, only : Stop_simulation, Print_message use mpi_mod, only : Gather_var2d, Distribute_global_var2d @@ -20,11 +21,13 @@ module netcdf_mod interface Add_netcdf_var module procedure Add_netcdf_var_real32_2d + module procedure Add_netcdf_var_real64_2d module procedure Add_netcdf_var_real32_3d end interface Add_netcdf_var interface Add_netcdf_var_mpi module procedure Add_netcdf_var_real32_2d_mpi + module procedure Add_netcdf_var_real64_2d_mpi module procedure Add_netcdf_var_real32_3d_mpi end interface Add_netcdf_var_mpi @@ -32,6 +35,7 @@ module netcdf_mod module procedure Get_netcdf_var_char_1d module procedure Get_netcdf_var_int32_3d module procedure Get_netcdf_var_real32_2d + module procedure Get_netcdf_var_real64_2d module procedure Get_netcdf_var_real32_3d module procedure Get_netcdf_var_real64_3d module procedure Get_netcdf_var_real32_4d @@ -39,6 +43,7 @@ module netcdf_mod interface Get_netcdf_var_mpi module procedure Get_netcdf_var_real32_2d_mpi + module procedure Get_netcdf_var_real64_2d_mpi module procedure Get_netcdf_var_real32_3d_mpi end interface Get_netcdf_var_mpi @@ -213,6 +218,63 @@ subroutine Add_netcdf_var_real32_2d (file_name, name_dims, varname, var, fill_va end subroutine Add_netcdf_var_real32_2d + subroutine Add_netcdf_var_real64_2d (file_name, name_dims, varname, var, fill_value, units, long_name) + + use netcdf + + implicit none + + character (len = *), intent (in) :: file_name, varname + character (len = *), dimension(:), intent (in) :: name_dims + real (kind = REAL64), dimension(:, :), intent (in) :: var + real (kind = REAL64), intent (in), optional :: fill_value + character (len = *), intent (in), optional :: units, long_name + + integer, parameter :: N_DIMS = 2 + integer :: ncidout, status, varid, n + integer :: dimids(N_DIMS) + + + status = nf90_open (file_name, NF90_WRITE, ncidout) + call Check_status (status) + + status = nf90_redef (ncidout) + call Check_status (status) + + do n = 1, N_DIMS + status = nf90_inq_dimid(ncidout, name_dims(n), dimids(n)) + call Check_status (status) + end do + + status = nf90_inq_varid (ncidout, varname, varid) + if (status == NF90_ENOTVAR) then + status = nf90_def_var (ncidout, varname, NF90_DOUBLE, dimids, varid) + call Check_status (status) + if (present (fill_value)) then + status = nf90_put_att (ncidout, varid, '_FillValue', fill_value) + call Check_status (status) + end if + if (present (units)) then + status = nf90_put_att (ncidout, varid, 'units', units) + call Check_status (status) + end if + if (present (long_name)) then + status = nf90_put_att (ncidout, varid, 'long_name', long_name) + call Check_status (status) + end if + end if + + status = nf90_enddef (ncidout) + call Check_status (status) + + status = nf90_put_var (ncidout, varid, var) + call Check_status (status) + + status = nf90_close (ncidout) + call Check_status (status) + + end subroutine Add_netcdf_var_real64_2d + subroutine Add_netcdf_var_real32_2d_mpi (file_name, cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, var_name, var2d_local, & fill_value, units, long_name) @@ -258,6 +320,51 @@ subroutine Add_netcdf_var_real32_2d_mpi (file_name, cfbm_comm, nx, ny, ifps, ifp end subroutine Add_netcdf_var_real32_2d_mpi + subroutine Add_netcdf_var_real64_2d_mpi (file_name, cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, var_name, var2d_local, & + fill_value, units, long_name) + +#ifdef DM_PARALLEL + use mpi +#endif + implicit none + + integer, intent (in) :: cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe + character (len = *), intent (in) :: file_name, var_name + real (kind = REAL64), dimension(ifps:ifpe, jfps:jfpe), intent (in) :: var2d_local + real (kind = REAL64), intent (in), optional :: fill_value + character (len = *), intent (in), optional :: units, long_name + + real (kind = REAL64), dimension(nx, ny) :: var2d + integer :: rank, ierr + logical, parameter :: DEBUG_LOCAL = .false. + + + if (DEBUG_LOCAL) call Print_message ('Entering Add_netcdf_var_real64_2d_mpi...') + + if (DEBUG_LOCAL) call Print_message ('file name = ' // trim (file_name)) + if (DEBUG_LOCAL) call Print_message ('var name = ' // trim (var_name)) + if (DEBUG_LOCAL) call Print_message ('dim X name = ' // trim (NAME_DIM_X)) + if (DEBUG_LOCAL) call Print_message ('dim Y name = ' // trim (NAME_DIM_Y)) + +#ifdef DM_PARALLEL + call Mpi_comm_rank (cfbm_comm, rank, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_comm_rank ') + + call Gather_var2d (cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, var2d_local(ifps:ifpe, jfps:jfpe), var2d) + + if (rank == 0) then + call Add_netcdf_var (file_name, [NAME_DIM_X, NAME_DIM_Y], var_name, var2d(1:nx, 1:ny), & + fill_value, units, long_name) + end if +#else + call Add_netcdf_var (file_name, [NAME_DIM_X, NAME_DIM_Y], var_name, var2d_local(1:nx, 1:ny), & + fill_value, units, long_name) +#endif + + if (DEBUG_LOCAL) call Print_message ('Leaving Add_netcdf_var_real64_2d_mpi...') + + end subroutine Add_netcdf_var_real64_2d_mpi + subroutine Get_netcdf_var_real32_2d_mpi (file_name, cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, var_name, var2d_local) #ifdef DM_PARALLEL @@ -305,6 +412,53 @@ subroutine Get_netcdf_var_real32_2d_mpi (file_name, cfbm_comm, nx, ny, ifps, ifp end subroutine Get_netcdf_var_real32_2d_mpi + subroutine Get_netcdf_var_real64_2d_mpi (file_name, cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, var_name, var2d_local) + +#ifdef DM_PARALLEL + use mpi +#endif + implicit none + + character (len = *), intent (in) :: file_name, var_name + integer, intent (in) :: cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe + real (kind = REAL64), dimension(ifps:ifpe, jfps:jfpe), intent (out) :: var2d_local + + real (kind = REAL64), dimension(:, :), allocatable :: var2d_global + character (len = :), allocatable :: msg + integer :: rank, ierr + logical, parameter :: DEBUG_LOCAL = .false. + + + if (DEBUG_LOCAL) call Print_message ('Entering Get_netcdf_var_real64_2d_mpi...') + +#ifdef DM_PARALLEL + call Mpi_comm_rank (cfbm_comm, rank, ierr) + if (ierr /= MPI_SUCCESS) call Stop_simulation ('Problems with Mpi_comm_rank ') + + if (rank == 0) then + call Get_netcdf_var (file_name, var_name, var2d_global) + if (size (var2d_global, 1) /= nx .or. size (var2d_global, 2) /= ny) then + msg = 'NetCDF variable has unexpected dimensions: '//trim (var_name) + call Stop_simulation (msg) + end if + end if + + call Distribute_global_var2d (cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, var2d_global, var2d_local) +#else + call Get_netcdf_var (file_name, var_name, var2d_global) + if (size (var2d_global, 1) /= nx .or. size (var2d_global, 2) /= ny) then + msg = 'NetCDF variable has unexpected dimensions: '//trim (var_name) + call Stop_simulation (msg) + end if + var2d_local(1:nx, 1:ny) = var2d_global(1:nx, 1:ny) +#endif + + if (allocated (var2d_global)) deallocate (var2d_global) + + if (DEBUG_LOCAL) call Print_message ('Leaving Get_netcdf_var_real64_2d_mpi...') + + end subroutine Get_netcdf_var_real64_2d_mpi + subroutine Add_netcdf_var_real32_3d (file_name, name_dims, varname, var) use netcdf @@ -745,6 +899,61 @@ subroutine Get_netcdf_var_real32_2d (file_name, var_name, output) end subroutine Get_netcdf_var_real32_2d + subroutine Get_netcdf_var_real64_2d (file_name, var_name, output) + + use netcdf + use, intrinsic :: iso_fortran_env, only : REAL64 + + implicit none + + character (len = *), intent(in) :: file_name, var_name + real (kind = REAL64), dimension (:, :), allocatable :: output + + integer :: status, ncid, ivar, nf_type, nvdims + integer :: i, n1, n2, io_stat + integer, dimension(NF90_MAX_VAR_DIMS) :: dimids, idims + character (len = :), allocatable :: msg + + + ! Open file + status = nf90_open (trim(file_name), NF90_NOWRITE, ncid) + call Check_status (status) + + ! Get var + status = nf90_inq_varid (ncid, trim (var_name), ivar) + call Check_status (status) + status = nf90_inquire_variable (ncid, ivar, xtype = nf_type, ndims = nvdims, dimids = dimids) + call Check_status (status) + + if (nf_type == NF90_FLOAT .or. nf_type == NF90_DOUBLE) then + if (nvdims == 2) then + do i = 1, nvdims + status = nf90_inquire_dimension (ncid, dimids(i), len = idims(i)) + call Check_status (status) + end do + + n1 = idims(1) + n2 = idims(2) + allocate (output(n1, n2), stat = io_stat) + if (io_stat /= 0) Call Stop_simulation ('Problems allocating output variable') + + status = nf90_get_var (ncid, ivar, output) + call Check_status (status) + else + msg = 'FATAL ERROR: ' // trim (var_name) // ' is not a 2D array' + call Stop_simulation (msg) + end if + else + msg = 'FATAL ERROR: ' // trim (var_name) // ' is not a floating-point variable' + call Stop_simulation (msg) + end if + + ! Closing file + status = nf90_close (ncid) + call Check_status (status) + + end subroutine Get_netcdf_var_real64_2d + subroutine Get_netcdf_var_real32_3d (file_name, var_name, output) use netcdf diff --git a/share/mpi_mod.F90 b/share/mpi_mod.F90 index be0bf326..283dea62 100644 --- a/share/mpi_mod.F90 +++ b/share/mpi_mod.F90 @@ -1,6 +1,6 @@ module mpi_mod - use, intrinsic :: iso_fortran_env, only : OUTPUT_UNIT + use, intrinsic :: iso_fortran_env, only : OUTPUT_UNIT, REAL64 implicit none @@ -16,6 +16,18 @@ module mpi_mod integer :: topology_dim_order = 0 ! ------------------------------ + interface Distribute_global_var2d + module procedure Distribute_global_var2d_real32, Distribute_global_var2d_real64 + end interface Distribute_global_var2d + + interface Gather_var2d + module procedure Gather_var2d_real32, Gather_var2d_real64 + end interface Gather_var2d + + interface Do_halo_exchange_with_corners + module procedure Do_halo_exchange_with_corners_real32, Do_halo_exchange_with_corners_real64 + end interface Do_halo_exchange_with_corners + contains subroutine Calc_patch_dims (nx, ny, px, py, coords, istart, iend, jstart, jend) @@ -204,7 +216,7 @@ subroutine Distribute_var2d (local_field, ips, ipe, jps, jpe, comm) end subroutine Distribute_var2d - subroutine Distribute_global_var2d (cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, var2d_global, var2d_local) + subroutine Distribute_global_var2d_real32 (cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, var2d_global, var2d_local) #ifdef DM_PARALLEL use mpi @@ -270,7 +282,79 @@ subroutine Distribute_global_var2d (cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, v var2d_local(1:nx, 1:ny) = var2d_global(1:nx, 1:ny) #endif - end subroutine Distribute_global_var2d + end subroutine Distribute_global_var2d_real32 + + subroutine Distribute_global_var2d_real64 (cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, var2d_global, var2d_local) + +#ifdef DM_PARALLEL + use mpi_f08 +#endif + implicit none + +#ifdef DM_PARALLEL + type (MPI_Comm) :: cfbm_comm_f08 +#endif + integer, intent (in) :: cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe + real (kind = REAL64), dimension(:, :), allocatable, intent (in out) :: var2d_global + real (kind = REAL64), dimension(ifps:ifpe, jfps:jfpe), intent (out) :: var2d_local + + integer :: i, ierr, j, kbuf, nprocs, nx_local, ny_local, r, rank + integer, dimension(:), allocatable :: all_ifps, all_ifpe, all_jfps, all_jfpe, displs, sendcounts + real (kind = REAL64), dimension(:), allocatable :: sendbuf + + +#ifdef DM_PARALLEL + call Convert_mpi_comm_to_f08 (cfbm_comm, cfbm_comm_f08) + call Mpi_comm_rank (cfbm_comm_f08, rank, ierr) + call Mpi_comm_size (cfbm_comm_f08, nprocs, ierr) + + nx_local = ifpe - ifps + 1 + ny_local = jfpe - jfps + 1 + + if (rank == 0) then + allocate (all_ifps(nprocs), all_ifpe(nprocs), all_jfps(nprocs), all_jfpe(nprocs)) + else + allocate (all_ifps(1), all_ifpe(1), all_jfps(1), all_jfpe(1)) + end if + + call MPI_Gather (ifps, 1, MPI_INTEGER, all_ifps, 1, MPI_INTEGER, 0, cfbm_comm_f08, ierr) + call MPI_Gather (ifpe, 1, MPI_INTEGER, all_ifpe, 1, MPI_INTEGER, 0, cfbm_comm_f08, ierr) + call MPI_Gather (jfps, 1, MPI_INTEGER, all_jfps, 1, MPI_INTEGER, 0, cfbm_comm_f08, ierr) + call MPI_Gather (jfpe, 1, MPI_INTEGER, all_jfpe, 1, MPI_INTEGER, 0, cfbm_comm_f08, ierr) + + if (rank == 0) then + allocate (sendcounts(nprocs), displs(nprocs)) + do r = 1, nprocs + sendcounts(r) = (all_ifpe(r) - all_ifps(r) + 1) * (all_jfpe(r) - all_jfps(r) + 1) + end do + + displs(1) = 0 + do r = 2, nprocs + displs(r) = displs(r - 1) + sendcounts(r - 1) + end do + + allocate (sendbuf(sum (sendcounts))) + kbuf = 0 + do r = 1, nprocs + do j = all_jfps(r), all_jfpe(r) + do i = all_ifps(r), all_ifpe(r) + kbuf = kbuf + 1 + sendbuf(kbuf) = var2d_global(i, j) + end do + end do + end do + else + allocate (sendcounts(1), displs(1), sendbuf(1)) + end if + + call MPI_Scatterv (sendbuf, sendcounts, displs, MPI_DOUBLE_PRECISION, var2d_local, nx_local * ny_local, MPI_DOUBLE_PRECISION, 0, cfbm_comm_f08, ierr) + + deallocate (all_ifps, all_ifpe, all_jfps, all_jfpe, sendcounts, displs, sendbuf) +#else + var2d_local(1:nx, 1:ny) = var2d_global(1:nx, 1:ny) +#endif + + end subroutine Distribute_global_var2d_real64 subroutine Do_halo_exchange (patch, ims, ime, jms, jme, ips, ipe, jps, jpe, nghost, cart_comm) @@ -432,7 +516,7 @@ subroutine Do_halo_exchange (patch, ims, ime, jms, jme, ips, ipe, jps, jpe, ngho end subroutine Do_halo_exchange - subroutine Do_halo_exchange_with_corners (patch, ims, ime, jms, jme, ips, ipe, jps, jpe, nghost, cart_comm) + subroutine Do_halo_exchange_with_corners_real32 (patch, ims, ime, jms, jme, ips, ipe, jps, jpe, nghost, cart_comm) #ifdef DM_PARALLEL use mpi @@ -588,9 +672,173 @@ subroutine Do_halo_exchange_with_corners (patch, ims, ime, jms, jme, ips, ipe, j deallocate (sendbuf_up, recvbuf_down, sendbuf_down, recvbuf_up) #endif - end subroutine Do_halo_exchange_with_corners + end subroutine Do_halo_exchange_with_corners_real32 + + subroutine Do_halo_exchange_with_corners_real64 (patch, ims, ime, jms, jme, ips, ipe, jps, jpe, nghost, cart_comm) + +#ifdef DM_PARALLEL + use mpi_f08 +#endif + + implicit none + +#ifdef DM_PARALLEL + type (MPI_Comm) :: cart_comm_f08 +#endif + integer, intent (in) :: cart_comm, ims, ime, jms, jme, ips, ipe, jps, jpe, nghost + real (kind = REAL64), dimension(ims:ime, jms:jme), intent (inout) :: patch + + integer :: ierr, nbr_left, nbr_right, nbr_up, nbr_down, tag_base + integer :: nx, ny, rank, i, j, k +#ifdef DM_PARALLEL + type (MPI_Request), dimension(8) :: reqs +#endif + integer, dimension(2) :: coords + real (kind = REAL64), dimension(:), allocatable :: sendbuf_right, recvbuf_left, sendbuf_left, recvbuf_right + real (kind = REAL64), dimension(:), allocatable :: sendbuf_up, recvbuf_down, sendbuf_down, recvbuf_up + + ! New variables to handle the topology mapping + integer :: dim_x, dim_y + +#ifdef DM_PARALLEL + call Convert_mpi_comm_to_f08 (cart_comm, cart_comm_f08) + call MPI_Comm_rank(cart_comm_f08, rank, ierr) + call MPI_Cart_coords(cart_comm_f08, rank, 2, coords, ierr) + + nx = ipe - ips + 1 + ny = jpe - jps + 1 + tag_base = 1000 + + ! --------------------------------------------------------- + ! ADAPT DIMENSIONS BASED ON MODULE FLAG + ! --------------------------------------------------------- + if (topology_dim_order == 0) then + ! Default internal behavior [X, Y] + dim_x = 0 + dim_y = 1 + else + ! External framework behavior [Y, X] + dim_x = 1 + dim_y = 0 + end if + + ! Neighbor ranks using mapped dimensions + call MPI_Cart_shift (cart_comm_f08, dim_x, 1, nbr_left, nbr_right, ierr) + call MPI_Cart_shift (cart_comm_f08, dim_y, 1, nbr_down, nbr_up, ierr) + ! --------------------------------------------------------- + + ! Halo exchange in X direction + allocate (sendbuf_right(ny * nghost), recvbuf_left(ny * nghost)) + allocate (sendbuf_left(ny * nghost), recvbuf_right(ny * nghost)) + + ! Pack right send buffer + k = 0 + do j = jps, jpe + do i = ipe - nghost + 1, ipe + k = k + 1 + sendbuf_right(k) = patch(i, j) + end do + end do + + ! Pack left send buffer + k = 0 + do j = jps, jpe + do i = ips, ips + nghost - 1 + k = k + 1 + sendbuf_left(k) = patch(i, j) + end do + end do + + ! Exchange left/right + call MPI_Irecv (recvbuf_left, ny * nghost, MPI_DOUBLE_PRECISION, nbr_left, tag_base + 0, cart_comm_f08, reqs(1), ierr) + call MPI_Isend (sendbuf_right, ny * nghost, MPI_DOUBLE_PRECISION, nbr_right, tag_base + 0, cart_comm_f08, reqs(2), ierr) + call MPI_Irecv (recvbuf_right, ny * nghost, MPI_DOUBLE_PRECISION, nbr_right, tag_base + 1, cart_comm_f08, reqs(3), ierr) + call MPI_Isend (sendbuf_left, ny * nghost, MPI_DOUBLE_PRECISION, nbr_left, tag_base + 1, cart_comm_f08, reqs(4), ierr) + + call MPI_Waitall (4, reqs, MPI_STATUSES_IGNORE, ierr) + + ! Unpack left halo + if (nbr_left /= MPI_PROC_NULL) then + k = 0 + do j = jps, jpe + do i = ips - nghost, ips - 1 + k = k + 1 + patch(i, j) = recvbuf_left(k) + end do + end do + end if + + ! Unpack right halo + if (nbr_right /= MPI_PROC_NULL) then + k = 0 + do j = jps, jpe + do i = ipe + 1, ipe + nghost + k = k + 1 + patch(i, j) = recvbuf_right(k) + end do + end do + end if + + deallocate (sendbuf_right, recvbuf_left, sendbuf_left, recvbuf_right) + + ! Exchange in Y direction (including halos in X direction) + allocate (sendbuf_up((nx + 2 * nghost) * nghost), recvbuf_down((nx + 2 * nghost) * nghost)) + allocate (sendbuf_down((nx + 2 * nghost) * nghost), recvbuf_up((nx + 2 * nghost) * nghost)) + + ! Pack up send buffer + k = 0 + do j = jpe - nghost + 1, jpe + do i = ips - nghost, ipe + nghost + k = k + 1 + sendbuf_up(k) = patch(i, j) + end do + end do + + ! Pack down send buffer + k = 0 + do j = jps, jps + nghost - 1 + do i = ips - nghost, ipe + nghost + k = k + 1 + sendbuf_down(k) = patch(i, j) + end do + end do + + ! Exchange up/down + call MPI_Irecv (recvbuf_down, (nx + 2 * nghost) * nghost, MPI_DOUBLE_PRECISION, nbr_down, tag_base + 2, cart_comm_f08, reqs(1), ierr) + call MPI_Isend (sendbuf_up, (nx + 2 * nghost) * nghost, MPI_DOUBLE_PRECISION, nbr_up, tag_base + 2, cart_comm_f08, reqs(2), ierr) + call MPI_Irecv (recvbuf_up, (nx + 2 * nghost) * nghost, MPI_DOUBLE_PRECISION, nbr_up, tag_base + 3, cart_comm_f08, reqs(3), ierr) + call MPI_Isend (sendbuf_down, (nx + 2 * nghost) * nghost, MPI_DOUBLE_PRECISION, nbr_down, tag_base + 3, cart_comm_f08, reqs(4), ierr) + + call MPI_Waitall(4, reqs, MPI_STATUSES_IGNORE, ierr) + + ! Unpack down halo + if (nbr_down /= MPI_PROC_NULL) then + k = 0 + do j = jps - nghost, jps - 1 + do i = ips - nghost, ipe + nghost + k = k + 1 + patch(i, j) = recvbuf_down(k) + end do + end do + end if + + ! Unpack up halo + if (nbr_up /= MPI_PROC_NULL) then + k = 0 + do j = jpe + 1, jpe + nghost + do i = ips - nghost, ipe + nghost + k = k + 1 + patch(i, j) = recvbuf_up(k) + end do + end do + end if + + deallocate (sendbuf_up, recvbuf_down, sendbuf_down, recvbuf_up) +#endif + + end subroutine Do_halo_exchange_with_corners_real64 - subroutine Gather_var2d (cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, var2d_local, var2d_global) + subroutine Gather_var2d_real32 (cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, var2d_local, var2d_global) #ifdef DM_PARALLEL use mpi_f08 @@ -646,7 +894,65 @@ subroutine Gather_var2d (cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, var2d_local, end if #endif - end subroutine Gather_var2d + end subroutine Gather_var2d_real32 + + subroutine Gather_var2d_real64 (cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe, var2d_local, var2d_global) + +#ifdef DM_PARALLEL + use mpi_f08 +#endif + + implicit none + + integer, intent (in) :: cfbm_comm, nx, ny, ifps, ifpe, jfps, jfpe + real (kind = REAL64), dimension(ifps:ifpe, jfps:jfpe), intent (in) :: var2d_local + real (kind = REAL64), dimension(nx, ny), intent (out) :: var2d_global + + real (kind = REAL64), dimension(:, :), allocatable :: buf + integer :: rank, ierr, ntasks, src, i_start, j_start, nx_local, ny_local, nxl, nyl + + integer, dimension(4) :: metadata +#ifdef DM_PARALLEL + type(MPI_Comm) :: cfbm_comm_f08 +#endif + + +#ifdef DM_PARALLEL + call Convert_mpi_comm_to_f08 (cfbm_comm, cfbm_comm_f08) + + call Mpi_comm_size (cfbm_comm_f08, ntasks, ierr) + call Mpi_comm_rank (cfbm_comm_f08, rank, ierr) + + nx_local = ifpe - ifps + 1 + ny_local = jfpe - jfps + 1 + if (rank /= 0) then + metadata = [ifps, jfps, nx_local, ny_local] + call Mpi_Send (metadata, 4, MPI_INTEGER, 0, 200, cfbm_comm_f08, ierr) + call Mpi_Send (var2d_local, nx_local * ny_local, MPI_DOUBLE_PRECISION, 0, 100, cfbm_comm_f08, ierr) + else + ! Rank 0 places its own data + var2d_global(ifps:ifpe, jfps:jfpe) = var2d_local + + ! Receive from others + do src = 1, ntasks - 1 + ! Receive metadata + call Mpi_Recv (metadata, 4, MPI_INTEGER, src, 200, cfbm_comm_f08, MPI_STATUS_IGNORE, ierr) + i_start = metadata(1) + j_start = metadata(2) + nxl = metadata(3) + nyl = metadata(4) + allocate (buf(nxl, nyl)) + call Mpi_Recv (buf, nxl * nyl, MPI_DOUBLE_PRECISION, src, 100, cfbm_comm_f08, MPI_STATUS_IGNORE, ierr) + + ! Place in global array + var2d_global(i_start:i_start + nxl - 1, j_start:j_start + nyl - 1) = buf + + deallocate (buf) + end do + end if +#endif + + end subroutine Gather_var2d_real64 subroutine Max_across_mpi_tasks (local_max, cart_comm, global_max) diff --git a/state/state_restart_submod.F90 b/state/state_restart_submod.F90 index 821b3931..48d41fc5 100644 --- a/state/state_restart_submod.F90 +++ b/state/state_restart_submod.F90 @@ -461,9 +461,13 @@ module subroutine Read_restart (this, config_flags) call Read_restart_field (file_restart, 'lfn_s3', this%lfn_s3) call Read_restart_field (file_restart, 'lfn_out', this%lfn_out) call Read_restart_field (file_restart, 'tign_g', this%tign_g) - call Read_restart_field (file_restart, 'fuel_frac', this%fuel_frac) + call Get_netcdf_var_mpi (file_restart, this%cfbm_comm, this%nx, this%ny, & + this%ifps, this%ifpe, this%jfps, this%jfpe, 'fuel_frac', & + this%fuel_frac(this%ifps:this%ifpe, this%jfps:this%jfpe)) call Read_restart_field (file_restart, 'fire_area', this%fire_area) - call Read_restart_field (file_restart, 'fuel_frac_burnt_dt', this%fuel_frac_burnt_dt) + call Get_netcdf_var_mpi (file_restart, this%cfbm_comm, this%nx, this%ny, & + this%ifps, this%ifpe, this%jfps, this%jfpe, 'fuel_frac_burnt_dt', & + this%fuel_frac_burnt_dt(this%ifps:this%ifpe, this%jfps:this%jfpe)) call Read_restart_field (file_restart, 'fgrnhfx', this%fgrnhfx) call Read_restart_field (file_restart, 'fgrnqfx', this%fgrnqfx) call Read_restart_field (file_restart, 'fcanhfx', this%fcanhfx) @@ -527,9 +531,13 @@ subroutine Exchange_restart_halos () call Exchange_restart_field (this%lfn_s3) call Exchange_restart_field (this%lfn_out) call Exchange_restart_field (this%tign_g) - call Exchange_restart_field (this%fuel_frac) + call Do_halo_exchange_with_corners (this%fuel_frac, & + this%ifms, this%ifme, this%jfms, this%jfme, & + this%ifps, this%ifpe, this%jfps, this%jfpe, N_POINTS_IN_HALO, this%cart_comm) call Exchange_restart_field (this%fire_area) - call Exchange_restart_field (this%fuel_frac_burnt_dt) + call Do_halo_exchange_with_corners (this%fuel_frac_burnt_dt, & + this%ifms, this%ifme, this%jfms, this%jfme, & + this%ifps, this%ifpe, this%jfps, this%jfpe, N_POINTS_IN_HALO, this%cart_comm) call Exchange_restart_field (this%fgrnhfx) call Exchange_restart_field (this%fgrnqfx) call Exchange_restart_field (this%fcanhfx) @@ -831,9 +839,13 @@ module subroutine Write_restart (this, config_flags) call Add_restart_field ('lfn_s3', this%lfn_s3) call Add_restart_field ('lfn_out', this%lfn_out) call Add_restart_field ('tign_g', this%tign_g) - call Add_restart_field ('fuel_frac', this%fuel_frac) + call Add_netcdf_var_mpi (file_restart, this%cfbm_comm, this%nx, this%ny, & + this%ifps, this%ifpe, this%jfps, this%jfpe, 'fuel_frac', & + this%fuel_frac(this%ifps:this%ifpe, this%jfps:this%jfpe)) call Add_restart_field ('fire_area', this%fire_area) - call Add_restart_field ('fuel_frac_burnt_dt', this%fuel_frac_burnt_dt) + call Add_netcdf_var_mpi (file_restart, this%cfbm_comm, this%nx, this%ny, & + this%ifps, this%ifpe, this%jfps, this%jfpe, 'fuel_frac_burnt_dt', & + this%fuel_frac_burnt_dt(this%ifps:this%ifpe, this%jfps:this%jfpe)) call Add_restart_field ('fgrnhfx', this%fgrnhfx) call Add_restart_field ('fgrnqfx', this%fgrnqfx) call Add_restart_field ('fcanhfx', this%fcanhfx) From 1f611014e421f2e50de3c2dc8b35258a21a958e7 Mon Sep 17 00:00:00 2001 From: Maria Frediani Date: Tue, 20 Oct 2026 20:45:48 -0600 Subject: [PATCH 12/19] Add standalone restart-chain regression coverage --- tests/regression/CMakeLists.txt | 2 +- tests/regression/README.md | 1 + tests/regression/cases.yaml | 40 +++ tests/regression/compare_outputs.py | 11 +- tests/regression/config.py | 24 +- tests/regression/doc/comparison.md | 4 + tests/regression/doc/configuration.md | 4 + tests/regression/doc/deferred.md | 2 +- tests/regression/doc/restart.md | 79 ++++++ tests/regression/doc/running.md | 3 +- tests/regression/regression.py | 3 +- tests/regression/restart.py | 265 ++++++++++++++++++ tests/regression/run_case.py | 81 ++++-- tests/regression/run_suite.py | 17 +- tests/regression/tests/python_harness_test.py | 164 +++++++++++ 15 files changed, 662 insertions(+), 38 deletions(-) create mode 100644 tests/regression/doc/restart.md create mode 100644 tests/regression/restart.py diff --git a/tests/regression/CMakeLists.txt b/tests/regression/CMakeLists.txt index 97c58276..b97b317b 100644 --- a/tests/regression/CMakeLists.txt +++ b/tests/regression/CMakeLists.txt @@ -109,5 +109,5 @@ endif() add_test(NAME regression_python COMMAND "${Python3_EXECUTABLE}" -B -m unittest discover -s "${CMAKE_CURRENT_SOURCE_DIR}/tests" -p "*_test.py" -v) set_tests_properties(regression_python PROPERTIES - LABELS "unit;quick;pr;full" + LABELS "unit;quick;pr;full;restart" ENVIRONMENT "CFBM_TEST_TMP=${PROJECT_BINARY_DIR}/python-test-artifacts") diff --git a/tests/regression/README.md b/tests/regression/README.md index 96dc80ca..a1963624 100644 --- a/tests/regression/README.md +++ b/tests/regression/README.md @@ -42,6 +42,7 @@ difference between one `compile.sh` call and the complete four-build matrix. | Understand terrain, fuel, ignition, winds, and timing | [Case configurations and figures](doc/cases.md) | | Follow the build/run flow and connections between files | [Execution overview and file relationships](doc/structure.md) | | Find every compared field and its acceptance rule | [Variable comparison table](doc/comparison.md) | +| Check continuous versus restarted integrations | [Restart continuity](doc/restart.md) | | Understand improvements and remaining legacy coverage | [New and legacy tests](doc/advantages.md) | | Find outputs or prepare an unapproved reference | [Results and references](doc/references.md) | | Add or extend a case, namelist option, or output | [Contributor guide](doc/contributing.md) | diff --git a/tests/regression/cases.yaml b/tests/regression/cases.yaml index 709b8010..a830dfce 100644 --- a/tests/regression/cases.yaml +++ b/tests/regression/cases.yaml @@ -115,6 +115,12 @@ cases: base_elevation_m: 0.0 configurations: base: {} + restart_prescribed: + restart_splits: [28] + namelist: + time: + restart_interval: 4 + interval_output: 12 godunov: namelist: fire: @@ -202,6 +208,31 @@ cases: namelist: fire: wind_vinterp_opt: 1 + restart_time: + # Updates at 4, 20, 36, 52 s; splits bracket perimeter activation + # and include times between moisture updates and forcing records. + restart_splits: [4, 12, 28] + namelist: + time: + restart_interval: 4 + interval_output: 12 + atm: + interval_atm: 12 + fire: + fmoist_freq: 0 + fmoist_dt: 16.0 + restart_step: + # Delayed first update exercises the initialization defect owned by P02. + restart_splits: [4, 12, 28] + namelist: + time: + restart_interval: 4 + interval_output: 12 + atm: + interval_atm: 12 + fire: + fmoist_freq: 3 + fmoist_dt: 12.0 u3d: namelist: fire: @@ -269,6 +300,15 @@ executions: timeout_seconds: 1800 build: mpi suites: + restart: + scale: small + cases: + circle: + configurations: [restart_prescribed] + executions: [serial, omp4, mpi1, mpi4, hybrid4] + terrain: + configurations: [restart_time, restart_step] + executions: [serial, omp4, mpi1, mpi4, hybrid4] quick: scale: small cases: diff --git a/tests/regression/compare_outputs.py b/tests/regression/compare_outputs.py index ab469cfa..39ed7d67 100644 --- a/tests/regression/compare_outputs.py +++ b/tests/regression/compare_outputs.py @@ -242,6 +242,7 @@ def compare_file( test_path: Path, static_fields: set[str], volatile_global_attributes: set[str] | None = None, + required_fields: set[str] | None = None, ) -> dict[str, Any]: """Compare two NetCDF files and every field under the fixed policy.""" result: dict[str, Any] = { @@ -280,7 +281,15 @@ def compare_file( if _attributes(reference, volatile_global_attributes) != _attributes( test, volatile_global_attributes): result["reasons"].append("global metadata differs") - for name in sorted(set(reference.variables) & set(test.variables)): + shared = set(reference.variables) & set(test.variables) + if required_fields is not None: + result["uncompared_fields"] = sorted(shared - required_fields) + if required_fields - shared: + result["reasons"].append( + f"required fields missing: {sorted(required_fields - shared)}" + ) + shared &= required_fields + for name in sorted(shared): result["fields"].append( compare_variable(reference.variables[name], test.variables[name], name in static_fields)) diff --git a/tests/regression/config.py b/tests/regression/config.py index 9351faba..8eea4222 100644 --- a/tests/regression/config.py +++ b/tests/regression/config.py @@ -19,6 +19,7 @@ "time": { "dt": float, "interval_output": float, + "restart_interval": int, "num_tiles": int, "tile_strategy": int }, @@ -89,7 +90,9 @@ "v_profile_m_s", }, } -SCIENCE_KEYS = {"start", "duration_seconds", "namelist", "inputs"} +SCIENCE_KEYS = { + "start", "duration_seconds", "namelist", "inputs", "restart_splits" +} DRIVERS = {"standalone", "nuopc", "esmx"} BUILDS = {"serial", "omp", "mpi", "hybrid"} ALLOWED_METHOD_PAIRS = {(9, 4), (2, 4)} @@ -284,9 +287,12 @@ def registrations(document: dict[str, Any], variant: str, if layout["build"] != variant: continue # Validate scientific combinations while configuring CMake. - resolve_spec(document, case, scale, configuration, - execution) + spec = resolve_spec(document, case, scale, configuration, + execution) for driver in drivers: + if spec.get( + "restart_splits") and driver != "standalone": + continue if driver not in document["cases"][case]["drivers"]: continue name = f"{driver}_{case}_{configuration}_{scale}_{execution}" @@ -328,6 +334,18 @@ def validate_spec(spec: dict[str, Any]) -> None: if abs(duration / interval - round(duration / interval)) > 1.0e-10: raise ValueError( "Integration end must align with output and forcing records") + if "restart_splits" in spec: + splits = spec["restart_splits"] + interval = _require_number(time, "restart_interval", step, "time") + if interval / step != round(interval / step): + raise ValueError( + "restart_interval must align with the fire timestep") + if (not isinstance(splits, list) or not splits or any( + type(value) is not int or not 0 < value < duration or value % + interval for value in splits) or + splits != sorted(set(splits)) or duration % interval): + raise ValueError("restart_splits must increase within the run and " + "align with restart_interval, including the end") start = dt.datetime.strptime(spec["start"], "%Y-%m-%d_%H:%M:%S") spec["end"] = (start + dt.timedelta(seconds=duration)).strftime("%Y-%m-%d_%H:%M:%S") diff --git a/tests/regression/doc/comparison.md b/tests/regression/doc/comparison.md index 973260ee..bc5c26e8 100644 --- a/tests/regression/doc/comparison.md +++ b/tests/regression/doc/comparison.md @@ -2,6 +2,10 @@ [Documentation index](../README.md) +The separate [restart suite](restart.md) also compares physical checkpoint +state bitwise, validates REAL64 fuel storage, and checks clock and activation +metadata. The regular-output acceptance rules below remain unchanged. + The standard inventory contains 19 variables. Each expected output file must contain exactly the fields listed in `expected_output_fields` in [cases.yaml](../cases.yaml). The comparator checks all variables, not a selected diff --git a/tests/regression/doc/configuration.md b/tests/regression/doc/configuration.md index 33a952f4..bf4a1296 100644 --- a/tests/regression/doc/configuration.md +++ b/tests/regression/doc/configuration.md @@ -2,6 +2,10 @@ [Documentation index](../README.md) +Restart configurations additionally specify `restart_splits` and the exact +`namelist.time.restart_interval`; the runner derives `time.restart` and segment +dates. See [restart continuity](restart.md) for schedule and driver constraints. + Use one [cases.yaml](../cases.yaml) with exact Fortran block and option names under `namelist`. For example, `namelist.fire.wind_vinterp_opt: 0` writes `wind_vinterp_opt=0` in `&fire`. Input-generator settings live under `inputs`; diff --git a/tests/regression/doc/deferred.md b/tests/regression/doc/deferred.md index 90c99c10..d5f1fdc3 100644 --- a/tests/regression/doc/deferred.md +++ b/tests/regression/doc/deferred.md @@ -12,7 +12,7 @@ coverage limits. | Intermediate-time 3D standalone/coupled agreement | Earlier diagnostics exceeded tolerance between saved PR endpoints. The bilinear-precision experiment passed an every-4-second GitHub comparison through 60 s. These are diagnostic results, not an extension of the PR acceptance schedule. See [validation history](validation.md). | | Longer standalone/coupled agreement | The 600 s terrain diagnostics still exceed the short-run comparison tolerance, including at 6.096 m with double-precision bilinear arithmetic. A longer-duration acceptance policy would need separate scientific review; it is not required for this PR. See [validation history](validation.md). | | Humidity interpretation | Scientific review associated with PR #46 is independent of reproducibility; current moisture remains enabled. | -| Restart | Restart continuity, restart inputs, and segmented-run comparisons. | +| Restart | Standalone same-revision chains are covered by the [restart suite](restart.md). Coupled/WRF-host continuity and cross-revision compatibility remain untested. Known production failures remain failures. | | PR #39 changes and method `(4,5)` | Separate scientific review and suitable comparison cases before matrix inclusion. | | ESMX_Data feedback | The generated coupled cases use WRF-data forcing. Legacy `testx` still exercises a distinct feedback fixture. | | Coupled OpenMP-only | MPI and hybrid NUOPC/ESMX are included in the PR matrix. Coupled runs using an OpenMP-only build remain outside this selection. | diff --git a/tests/regression/doc/restart.md b/tests/regression/doc/restart.md new file mode 100644 index 00000000..9193ef1a --- /dev/null +++ b/tests/regression/doc/restart.md @@ -0,0 +1,79 @@ +# Restart continuity + +[Documentation index](../README.md) + +Run `regression.py suite --suite restart --build-dir=BUILD --run-root=RESULTS` +inside a PBS allocation on NCAR systems. Repeat `--build-dir` for serial, +OpenMP, MPI, and hybrid builds. `compile.sh` still owns compilation and CTest +owns invocation. No production physics is changed by these tests. + +The suite uses the existing small circle and terrain geometries, fuel maps, +forcing endpoints, and propagation methods. Their existing configuration +figures remain applicable. This table specifies the different event schedules; +all times are seconds from the original 2020-01-01 00:00:00 origin. + +| Configuration | Moisture | Atmospheric records | Restart splits | Perimeter | +| --- | --- | --- | --- | --- | +| circle/restart_prescribed | Prescribed 0.08 kg kg-1 | Idealized, no input file | 28 | Line/point ignition | +| terrain/restart_time | `fmoist_freq=0`, `fmoist_dt=16` | Every 12 s | 4, 12, 28 | Observed perimeter at 8 s | +| terrain/restart_step | `fmoist_freq=3`, `fmoist_dt=12` | Every 12 s | 4, 12, 28 | Observed perimeter at 8 s | + +All integrations use `dt=4`, run through 60 s, write regular output every 12 s, +and write checkpoints every 4 s. Time-based moisture updates occur at 4, 20, +36, and 52 s on the reviewed base. The split at 4 s precedes perimeter +activation; 12 and 28 s follow it. The 28 s split is between moisture updates +and atmospheric records. Step-based updates first occur at 12 s, exposing the +known initialization defect assigned to P02. A failure remains a failing CTest; +the suite has no expected-failure conversion. + +Each case runs one uninterrupted integration and a separate cold first segment, +then resumes only from the preceding segment's checkpoint. Generated input files +are copied unchanged to every segment. In particular, shortening a segment does +not regenerate the forcing or change its temperature/humidity trajectory. Input +and incoming-checkpoint hashes, namelists, original/resolved settings, command, +process status, and model logs are retained. Schedule inventories derive from +the original origin, including output after a split between output events. + +| Compared content | Acceptance | +| --- | --- | +| Regular output at every scheduled time | Existing static-field bitwise policy and dynamic `rtol=1e-4`, `atol=0`; unchanged schema/metadata checks | +| Physical checkpoint fields in `restart.STATE_FIELDS`, current forcing, previous forcing and five moisture classes where applicable | Bitwise equality for the same executable and execution layout | +| Checkpoint dimensions, full variable inventory, and global metadata | Exact equality, including clock origin, step count, perimeter activation, and moisture times | +| Fuel fraction and fuel fraction burned in the current timestep | REAL64 storage and a non-REAL32 signal at the final checkpoint; catches quantization even when stored as float64 | +| Independent checkpoint validity | Required fields, finite/unmasked owned cells, expected clock and activation state, five moisture-class dimensions, valid moisture times and moisture in (0,1) for these dry-forcing cases | + +The PR #39 writer also saves method work arrays `lfn_0/1/2`, `lfn_s0/1/2/3` +and unused crown fluxes `fcanhfx/fcanqfx`. Their values are not all initialized +for these methods. They remain in the schema comparison but are explicitly +listed as uncompared fields. Physical fields are not excluded to obtain a pass. +Regular-output tolerances do not weaken the exact checkpoint-state comparison. + +The complete selection contains 15 cases across `serial`, `omp4`, `mpi1`, +`mpi4`, and `hybrid4`: 65 model invocations, including the 15 continuous runs, +and 315 file comparisons (90 output and 225 checkpoint comparisons). Counts +of attempted/completed segments and failed comparisons are recorded separately +in `summary.json`; complete field metrics and checkpoint checks are in each +case's `restart.json`. Failed cases are excluded from cross-layout comparisons +and remain failures in the summary. Cross-layout comparisons use continuous +outputs and the existing comparison policy. + +The new configurations use same-revision comparisons and do not select the +historical `github-pr` reference, which does not contain their keys. The +existing `pr` suite remains unchanged and checks its approved reference. +No reference candidate is created or approved by this suite. + +Restart experiments currently require the standalone driver. NUOPC/ESMX and +WRF-host restart continuity are not established here. Existing coupled PR +tests exercise ordinary integrations only. Cross-layout checkpoint transfers, +legacy-file compatibility across revisions, fractional timesteps, and coupled +flux accumulation/reset are outside P00 and require their assigned sessions. + +## Extend a restart experiment + +Add `restart_splits` (an increasing list of whole seconds) and the exact +`namelist.time.restart_interval` to a named configuration in `cases.yaml`. +Splits and the final time must align with checkpoint writes; checkpoint writes +must align with the fire timestep. The segment runner derives `time.restart` +and the start/end dates. Add the configuration and executions explicitly to +the `restart` suite. Preserve original forcing and compare every expected +checkpoint and regular output, including the first resumed event. diff --git a/tests/regression/doc/running.md b/tests/regression/doc/running.md index 05a80565..51b960cb 100644 --- a/tests/regression/doc/running.md +++ b/tests/regression/doc/running.md @@ -130,7 +130,7 @@ python -B tests/regression/regression.py suite --suite pr \ selection accepts regex syntax. Internally, Python reads CTest's JSON listing and supplies a numeric list of selected tests to CTest. -`--suite` accepts **five values**. `--test` accepts **any exact CTest name +`--suite` accepts **six values**. `--test` accepts **any exact CTest name registered in the selected build**, so its available names depend on the build configuration. @@ -142,6 +142,7 @@ The suite options are defined in [regression.py](../regression.py): | `pr` | Unit tests plus all four default case/configuration combinations across `serial`, `omp1`, `omp4`, `mpi1`, `mpi4`, and `hybrid4`, where supported. | | `full` | Unit tests plus the larger experiments: 320 × 320 grids, 3600 s integrations, both method pairs `(9,4)` and `(2,4)`, and the additional `mpi8` execution. Requires `CFBM_FULL_TESTS=ON`. | | `unit` | Unit Fortran tests and the Python harness tests. No generated fire-case integrations. | +| `restart` | Python harness checks and standalone continuous/restarted cases in serial, OpenMP, MPI, and hybrid layouts. See [restart continuity](restart.md) for schedules, counts, and coverage limits. | | `legacy` | Original tests registered by the legacy build route. | These selections follow [cases.yaml](../cases.yaml). Each build contributes only diff --git a/tests/regression/regression.py b/tests/regression/regression.py index 356f0aca..db49c746 100644 --- a/tests/regression/regression.py +++ b/tests/regression/regression.py @@ -42,7 +42,8 @@ def build_parser() -> argparse.ArgumentParser: help="Select and run registered CTests") suite.add_argument("--build-dir", type=Path, action="append", required=True) suite.add_argument("--suite", - choices=("quick", "pr", "full", "unit", "legacy"), + choices=("quick", "pr", "full", "restart", "unit", + "legacy"), default="quick") suite.add_argument("--case") suite.add_argument("--driver", choices=("standalone", "nuopc", "esmx")) diff --git a/tests/regression/restart.py b/tests/regression/restart.py new file mode 100644 index 00000000..dbc8e041 --- /dev/null +++ b/tests/regression/restart.py @@ -0,0 +1,265 @@ +#!/usr/bin/env python3 +# Copyright 2026 Research Applications Laboratory (RAL), +# National Center for Atmospheric Research (NCAR), +# University Corporation for Atmospheric Research (UCAR) +#-------------------------------------------------------------------------------- +# Created by Maria Frediani (frediani@ucar.edu) on 2026-10-20 +# run python -B tests/regression/regression.py suite --suite restart --help +"""Check same-executable restart chains against an uninterrupted integration.""" +from __future__ import annotations + +#-------------------------------------------------------------------------------- +# Python modules and checkpoint contract +#-------------------------------------------------------------------------------- +import copy +import datetime as dt +from pathlib import Path +import shutil +from typing import Any + +import netCDF4 +import numpy as np + +from compare_outputs import compare_file +from generate_inputs import DATE_FORMAT, sha256_file +from render_namelist import render_namelist +from reports import write_result + +# Compare persistent physical state. Method work arrays lfn_0/1/2 and lfn_s0/1/2/3, +# and unused crown fluxes fcanhfx/fcanqfx, are written by PR #39 but are not all +# initialized for the selected methods. Their names remain in the schema check. +STATE_FIELDS = { + "lfn", "lfn_hist", "lfn_out", "tign_g", "fuel_frac", "fuel_frac_burnt_dt", + "fire_area", "fgrnhfx", "fgrnqfx", "flame_length", "ros", "ros_front", + "emis_smoke", "fmc_g", "fuel_load_g", "fuel_time", "zsf", "dzdxf", "dzdyf", + "nfuel_cat", "uf", "vf", "fz0" +} +FORCING_FIELDS = {"fire_t2", "fire_q2", "fire_psfc", "fire_rain"} +REAL64_FIELDS = {"fuel_frac", "fuel_frac_burnt_dt"} +MOISTURE_TIMES = ("fmc_fmoist_lasttime", "fmc_fmoist_nexttime") + +#-------------------------------------------------------------------------------- +# Original-origin schedules and checkpoint validation +#-------------------------------------------------------------------------------- + + +def timestamp(spec: dict[str, Any], seconds: int) -> str: + """Translate elapsed seconds from the original origin into a file timestamp.""" + return (dt.datetime.strptime(spec["start"], DATE_FORMAT) + + dt.timedelta(seconds=seconds)).strftime(DATE_FORMAT) + + +def segment_names(spec: dict[str, Any], + begin: int, + end: int, + checkpoint: bool = False) -> list[str]: + """Enumerate writes independently of a resumed segment's local start time.""" + key = "restart_interval" if checkpoint else "interval_output" + interval = int(spec["namelist"]["time"][key]) + seconds = list(range((begin // interval + 1) * interval, end + 1, interval)) + if begin == 0 and not checkpoint: + seconds.insert(0, 0) + prefix = "fire_restart" if checkpoint else "fire_output" + return [f"{prefix}_{timestamp(spec, second)}.nc" for second in seconds] + + +def checkpoint_fields(spec: dict[str, Any]) -> set[str]: + """Select physical fields whose owned cells must survive a restart exactly.""" + fields = set(STATE_FIELDS) + fire = spec["namelist"]["fire"] + if fire["ideal_opt"] == 0: + fields |= FORCING_FIELDS + if fire["fmoist_run"]: + fields |= {"fmc_gc"} | {f"{name}_old" for name in FORCING_FIELDS} + return fields + + +def check_checkpoint(path: Path, spec: dict[str, Any], + seconds: int) -> dict[str, Any]: + """Require valid clock, perimeter, class state, and double-precision fuel data.""" + result = {"path": str(path), "pass": False, "reasons": []} + if not path.is_file(): + result["reasons"].append("Checkpoint is missing") + return result + reasons = result["reasons"] + fire = spec["namelist"]["fire"] + with netCDF4.Dataset(path) as dataset: + required = checkpoint_fields(spec) + missing = required - set(dataset.variables) + if missing: + reasons.append(f"Checkpoint fields missing: {sorted(missing)}") + for name in required & set(dataset.variables): + variable = dataset[name] + values = np.ma.asarray(variable[:]) + if (not values.size or np.ma.count_masked(values) or + not np.isfinite(values).all()): + reasons.append( + f"Unavailable or nonfinite checkpoint state: {name}") + if name in REAL64_FIELDS and variable.dtype != np.dtype("float64"): + reasons.append(f"{name} must retain REAL64 storage") + if (name in REAL64_FIELDS and + seconds == spec["duration_seconds"] and not np.any( + values != values.astype("float32").astype("float64"))): + reasons.append(f"{name} lacks a non-REAL32 precision signal") + # Check dates and step count independently of the comparison file. + for prefix, date in (("start", spec["start"]), + ("restart", timestamp(spec, seconds))): + parsed = dt.datetime.strptime(date, DATE_FORMAT) + for part in ("year", "month", "day", "hour", "minute", "second"): + name = f"{prefix}_{part}" + if getattr(dataset, name, None) != getattr(parsed, part): + reasons.append(f"Incorrect checkpoint clock: {name}") + if getattr(dataset, "itimestep", + None) != seconds / spec["namelist"]["time"]["dt"]: + reasons.append("Incorrect checkpoint itimestep") + ignited = int(fire["fire_is_real_perim"] and + seconds >= fire["fire_ignition_start_time1"]) + if getattr(dataset, "fire_perimeter_ignited", None) != ignited: + reasons.append("Incorrect observed-perimeter activation state") + if fire["fmoist_run"]: + for name in MOISTURE_TIMES: + value = getattr(dataset, name, None) + if value is None or not np.isfinite(value) or value < 0: + reasons.append(f"Missing or invalid moisture time: {name}") + if "fmc_gc" in dataset.variables: + variable = dataset["fmc_gc"] + if (variable.dimensions != ("ny", "moisture_class", "nx") or + variable.shape[1] != 5): + reasons.append("Incorrect five-class moisture dimensions") + # Positive, finite moisture is required for these nonzero initial + # values and dry forcing. Zero-filled uninitialized storage fails. + if np.any(variable[:] <= 0) or np.any(variable[:] >= 1): + reasons.append( + "Class moisture is outside this case's physical range (0,1)" + ) + result["pass"] = not reasons + return result + + +def compare_checkpoint(reference: Path, test: Path, + spec: dict[str, Any]) -> dict[str, Any]: + """Require bitwise physical state and identical metadata for the same layout.""" + fields = checkpoint_fields(spec) + result = compare_file(reference, test, fields, required_fields=fields) + result["comparison_kind"] = "restart_checkpoint" + return result + + +#-------------------------------------------------------------------------------- +# Segmented integrations using the existing case launcher +#-------------------------------------------------------------------------------- + + +def run_restarts(spec: dict[str, Any], directory: Path, command: list[str], + environment: dict[str, str], + static_fields: set[str]) -> dict[str, Any]: + """Run a cold first segment and chained restarts against unchanged forcing.""" + from run_case import run_model + + result = { + "pass": False, + "reasons": [], + "segments": [], + "comparisons": [], + "checkpoint_checks": [] + } + duration = int(spec["duration_seconds"]) + splits = [0, *spec["restart_splits"], duration] + inputs = [ + path for name in ("wrf.nc", "geo_em.d01.nc") + if (path := directory / name).is_file() + ] + result["input_sha256"] = {path.name: sha256_file(path) for path in inputs} + expected = set(segment_names(spec, 0, duration, checkpoint=True)) + if {path.name for path in directory.glob("fire_restart_*.nc")} != expected: + result["reasons"].append("Continuous checkpoint inventory differs") + # Validate every continuous checkpoint, including times before first ignition. + for seconds in range(spec["namelist"]["time"]["restart_interval"], + duration + 1, + spec["namelist"]["time"]["restart_interval"]): + result["checkpoint_checks"].append( + check_checkpoint( + directory / f"fire_restart_{timestamp(spec, seconds)}.nc", spec, + seconds)) + previous = None + for index, (begin, end) in enumerate(zip(splits, splits[1:])): + segment = directory / f"segment-{index:02d}" + segment.mkdir() + segment_spec = copy.deepcopy(spec) + segment_spec.update(start=timestamp(spec, begin), + end=timestamp(spec, end), + duration_seconds=end - begin) + segment_spec["namelist"]["time"]["restart"] = begin > 0 + for path in inputs: + shutil.copy2(path, segment / path.name) + incoming = None + if begin: + incoming = f"fire_restart_{timestamp(spec, begin)}.nc" + source = previous / incoming + if not source.is_file(): + result["reasons"].append(f"Cannot resume: missing {source}") + break + shutil.copy2(source, segment / incoming) + (segment / "namelist.fire").write_text(render_namelist(segment_spec)) + write_result( + segment / "settings.json", { + "spec": + segment_spec, + "original_spec": + spec, + "input_sha256": + result["input_sha256"], + "restart_source": + str(previous / incoming) if incoming else None, + "restart_sha256": + sha256_file(segment / incoming) if incoming else None + }) + try: + record = run_model(command, segment, environment, + spec["execution"]["timeout_seconds"], + "standalone") + except (OSError, RuntimeError) as error: + record = {"reasons": [f"{type(error).__name__}: {error}"]} + record.update(directory=str(segment), + begin_seconds=begin, + end_seconds=end) + for checkpoint in (False, True): + expected = segment_names(spec, begin, end, checkpoint) + prefix = "fire_restart" if checkpoint else "fire_output" + actual = {path.name for path in segment.glob(f"{prefix}_*.nc")} + inventory = set(expected) | ({incoming} + if checkpoint and incoming else set()) + if actual != inventory: + record["reasons"].append( + f"{prefix} inventory differs: expected={sorted(inventory)}, actual={sorted(actual)}" + ) + for name in expected: + if checkpoint: + comparison = compare_checkpoint(directory / name, + segment / name, spec) + seconds = int( + (dt.datetime.strptime(name[13:-3], DATE_FORMAT) - + dt.datetime.strptime(spec["start"], + DATE_FORMAT)).total_seconds()) + result["checkpoint_checks"].append( + check_checkpoint(segment / name, spec, seconds)) + else: + comparison = compare_file(directory / name, segment / name, + static_fields) + comparison["comparison_kind"] = "restart_output" + result["comparisons"].append(comparison) + record["pass"] = not record["reasons"] + write_result(segment / "execution.json", record) + result["segments"].append(record) + previous = segment + if len(result["segments"]) != len(splits) - 1 or any( + not record["pass"] for record in result["segments"]): + result["reasons"].append( + "One or more restart segments did not complete") + if any(not item["pass"] for item in result["checkpoint_checks"]): + result["reasons"].append("Checkpoint contract failed") + if any(not item["pass"] for item in result["comparisons"]): + result["reasons"].append("Continuous/restarted comparison failed") + result["pass"] = not result["reasons"] + write_result(directory / "restart.json", result) + return result diff --git a/tests/regression/run_case.py b/tests/regression/run_case.py index 8c03287c..0abce560 100644 --- a/tests/regression/run_case.py +++ b/tests/regression/run_case.py @@ -79,6 +79,44 @@ def write_esmx_config(path: Path, spec: dict[str, Any]) -> None: path.write_text(yaml.safe_dump(config, sort_keys=False), encoding="utf-8") +def run_model(command: list[str], directory: Path, environment: dict[str, str], + timeout_seconds: int, driver: str) -> dict[str, Any]: + """Launch one integration and retain process and fatal-log evidence.""" + result = { + "reasons": [], + "command": command, + "started_utc": dt.datetime.now(dt.timezone.utc).isoformat() + } + with (directory / "model.log").open("w") as log: + process = subprocess.run(command, + cwd=directory, + env=environment, + stdout=log, + stderr=subprocess.STDOUT, + timeout=timeout_seconds, + check=False) + result["exit_status"] = process.returncode + if process.returncode: + result["reasons"].append( + f"Model exited with status {process.returncode}") + logs = [directory / "model.log", *directory.glob("PET*.ESMF_LogFile")] + for log in logs: + lines = log.read_text(errors="replace").splitlines() + result["reasons"].extend( + line.strip() + for line in lines + if line.strip().startswith(FATAL_PREFIXES) or " ERROR " in line) + if driver != "standalone": + marker = ("esmApp FINISHED" if driver == "nuopc" else + "ESMX (Earth System Model eXecutable) FINISHED") + completed = any( + marker in log.read_text(errors="replace") for log in logs) + result["coupled_finalize_observed"] = completed + if not completed: + result["reasons"].append("Coupled finalization was not observed") + return result + + def run_case(args: Any) -> dict[str, Any]: """Stage a case, launch the model, and reject incomplete integrations.""" if Path("/glade").exists() and not os.environ.get("PBS_JOBID"): @@ -89,6 +127,8 @@ def run_case(args: Any) -> dict[str, Any]: args.execution) if args.driver not in document["cases"][args.case]["drivers"]: raise ValueError(f"No coupled configuration for {args.case}") + if spec.get("restart_splits") and args.driver != "standalone": + raise ValueError("Restart experiments currently require standalone") executable = args.executable.resolve(strict=True) root = Path(os.environ.get("CFBM_RUN_ROOT", str(args.run_root))).resolve() if root.is_relative_to(SOURCE_ROOT) and not root.is_relative_to( @@ -135,37 +175,20 @@ def run_case(args: Any) -> dict[str, Any]: result.update(command=command, started_utc=dt.datetime.now(dt.timezone.utc).isoformat(), threads=execution["threads"]) - with (directory / "model.log").open("w") as log: - process = subprocess.run(command, - cwd=directory, - env=environment, - stdout=log, - stderr=subprocess.STDOUT, - timeout=execution["timeout_seconds"], - check=False) - result["exit_status"] = process.returncode - if process.returncode: - result["reasons"].append( - f"Model exited with status {process.returncode}") - logs = [directory / "model.log", *directory.glob("PET*.ESMF_LogFile")] - for log in logs: - lines = log.read_text(errors="replace").splitlines() - result["reasons"].extend( - line.strip() - for line in lines - if line.strip().startswith(FATAL_PREFIXES) or " ERROR " in line) - if args.driver != "standalone": - marker = ("esmApp FINISHED" if args.driver == "nuopc" else - "ESMX (Earth System Model eXecutable) FINISHED") - completed = any( - marker in log.read_text(errors="replace") for log in logs) - result["coupled_finalize_observed"] = completed - if not completed: - result["reasons"].append( - "Coupled finalization was not observed") + result.update( + run_model(command, directory, environment, + execution["timeout_seconds"], args.driver)) result["validation"] = validate_outputs(directory, result) result["reasons"].extend(result["validation"]["reasons"]) - if args.reference: + if spec.get("restart_splits"): + from restart import run_restarts + result["restart"] = run_restarts(spec, directory, command, + environment, + set(document["static_fields"])) + result["reference_policy"] = "same-revision continuous/restarted" + if not result["restart"]["pass"]: + result["reasons"].append("Restart continuity checks failed") + if args.reference and not spec.get("restart_splits"): from reference import compare_reference result["reference"] = compare_reference(args.reference, result, document) diff --git a/tests/regression/run_suite.py b/tests/regression/run_suite.py index 37b30f85..d831344b 100644 --- a/tests/regression/run_suite.py +++ b/tests/regression/run_suite.py @@ -155,7 +155,8 @@ def run_suite(args: Any) -> dict[str, Any]: json.loads(path.read_text()) for path in sorted(root.glob("*/result.json")) ] - if args.suite in ("quick", "pr", "full") and not args.test and not results: + if args.suite in ("quick", "pr", "full", + "restart") and not args.test and not results: summary["reasons"].append( "No generated cases ran; full requires CFBM_FULL_TESTS=ON") if any(not result["pass"] for result in results): @@ -165,6 +166,20 @@ def run_suite(args: Any) -> dict[str, Any]: "pass": result["pass"], "directory": result["directory"] } for result in results] + summary["restart_cases"] = [{ + "name": + result["name"], + "pass": + result["restart"]["pass"], + "segments": + len(result["restart"]["segments"]), + "comparisons": + len(result["restart"]["comparisons"]), + "failed_comparisons": + sum(not item["pass"] for item in result["restart"]["comparisons"]), + "report": + str(Path(result["directory"]) / "restart.json") + } for result in results if "restart" in result] comparisons = compare_executions( results, set(load_yaml(args.config)["static_fields"])) summary["comparisons"] = comparisons diff --git a/tests/regression/tests/python_harness_test.py b/tests/regression/tests/python_harness_test.py index 15e42135..d0e565e6 100644 --- a/tests/regression/tests/python_harness_test.py +++ b/tests/regression/tests/python_harness_test.py @@ -14,8 +14,10 @@ from __future__ import annotations import json +import datetime as dt import os from pathlib import Path +import shutil import sys import tempfile from types import SimpleNamespace @@ -35,6 +37,8 @@ from render_namelist import render_namelist from run_case import run_case, write_esmx_config from run_suite import select_tests +from restart import (check_checkpoint, checkpoint_fields, compare_checkpoint, + run_restarts, segment_names) # CTest supplies a directory under the build tree. Direct Python invocations # must supply CFBM_TEST_TMP, so artifacts never go to a developer's home/scratch. @@ -382,6 +386,166 @@ def test_selection_uses_exact_names(self) -> None: args.test = "terrain.*" self.assertEqual(select_tests(tests, args), []) + #------------------------------------------------------------------------ + # Restart configuration, precision, and segmentation + #------------------------------------------------------------------------ + + def test_restart_registration_and_phase(self) -> None: + """Select standalone restart cases without changing existing PR matrices.""" + records = registrations(self.document, "mpi", + ["standalone", "nuopc", "esmx"]) + selected = [r for r in records if "restart" in r["labels"]] + self.assertEqual(len(selected), 6) + self.assertEqual({r["driver"] for r in selected}, {"standalone"}) + self.assertTrue(all("pr" not in r["labels"] for r in selected)) + spec = resolve_spec(self.document, + "terrain", + configuration="restart_time") + self.assertEqual(segment_names(spec, 28, 60), [ + "fire_output_2020-01-01_00:00:36.nc", + "fire_output_2020-01-01_00:00:48.nc", + "fire_output_2020-01-01_00:01:00.nc" + ]) + self.assertEqual(segment_names(spec, 0, 4), + ["fire_output_2020-01-01_00:00:00.nc"]) + + def test_invalid_restart_splits_fail(self) -> None: + """Reject absent checkpoints, unordered splits, and invalid endpoints.""" + for values in ([], [0], [60], [5], [12, 4], [4, 4], [True], [4.0]): + with self.subTest(values=values): + self.document["cases"]["terrain"]["configurations"][ + "restart_time"]["restart_splits"] = values + with self.assertRaises(ValueError): + resolve_spec(self.document, + "terrain", + configuration="restart_time") + + def _checkpoint(self, + path: Path, + spec: dict, + seconds: int, + fuel_dtype: str = "f8") -> None: + """Write independent small physical-state fixtures with REAL64 signals.""" + with netCDF4.Dataset(path, "w") as dataset: + for name, size in (("nx", 2), ("ny", 2), ("moisture_class", 5)): + dataset.createDimension(name, size) + for name in checkpoint_fields(spec): + dimensions = ("ny", "moisture_class", + "nx") if name == "fmc_gc" else ("ny", "nx") + kind = fuel_dtype if name in { + "fuel_frac", "fuel_frac_burnt_dt" + } else "f4" + dataset.createVariable(name, kind, + dimensions)[:] = 0.123456789012345 + origin = dt.datetime(2020, 1, 1) + for prefix, date in (("start", origin), + ("restart", + origin + dt.timedelta(seconds=seconds))): + for part in ("year", "month", "day", "hour", "minute", + "second"): + setattr(dataset, f"{prefix}_{part}", getattr(date, part)) + dataset.itimestep = seconds // 4 + dataset.fire_perimeter_ignited = int( + spec["namelist"]["fire"]["fire_is_real_perim"] and seconds >= 8) + dataset.fmc_fmoist_lasttime = 4.0 + dataset.fmc_fmoist_nexttime = 20.0 + + def test_checkpoint_detects_real64_loss(self) -> None: + """Detect both float32 storage and quantization hidden in float64 storage.""" + spec = resolve_spec(self.document, + "circle", + configuration="restart_prescribed") + for kind in ("f4", "f8"): + path = self.root / f"{kind}.nc" + self._checkpoint(path, spec, 60, kind) + check = check_checkpoint(path, spec, 60) + self.assertEqual(check["pass"], kind == "f8", check) + path = self.root / "f8.nc" + with netCDF4.Dataset(path, "a") as dataset: + dataset["fuel_frac"][:] = dataset["fuel_frac"][:].astype("f4") + self.assertFalse(check_checkpoint(path, spec, 60)["pass"]) + + def test_checkpoint_rejects_state_and_metadata_damage(self) -> None: + """Require class state, clock origin, activation flag, and exact fuel bits.""" + spec = resolve_spec(self.document, + "terrain", + configuration="restart_time") + reference = self.root / "reference.nc" + self._checkpoint(reference, spec, 12) + self.assertTrue(check_checkpoint(reference, spec, 12)["pass"]) + for defect in ("fuel_bit", "perimeter", "clock", "class_zero", + "class_nan", "moisture_time"): + test = self.root / f"{defect}.nc" + shutil.copy2(reference, test) + with netCDF4.Dataset(test, "a") as dataset: + if defect == "fuel_bit": + value = dataset["fuel_frac"][0, 0] + dataset["fuel_frac"][0, 0] = np.nextafter(value, np.inf) + elif defect == "perimeter": + dataset.fire_perimeter_ignited = 0 + elif defect == "clock": + dataset.start_second = 4 + elif defect == "class_zero": + dataset["fmc_gc"][:] = 0 + elif defect == "class_nan": + dataset["fmc_gc"][:] = np.nan + else: + dataset.fmc_fmoist_nexttime = 999.0 + self.assertFalse( + compare_checkpoint(reference, test, spec)["pass"], defect) + if defect not in ("fuel_bit", "moisture_time"): + self.assertFalse( + check_checkpoint(test, spec, 12)["pass"], defect) + self.assertFalse( + compare_checkpoint(reference, self.root / "absent.nc", + spec)["pass"]) + + def test_restart_chain_preserves_forcing_and_uses_previous_segment( + self) -> None: + """Exercise staging with a synthetic launcher and reject an extra output.""" + spec = resolve_spec(self.document, + "terrain", + configuration="restart_time") + for extra in (False, True): + root = self.root / str(extra) + root.mkdir() + (root / "wrf.nc").write_bytes(b"original full-duration forcing") + for seconds in range(4, 61, 4): + name = f"fire_restart_2020-01-01_00:{seconds // 60:02d}:{seconds % 60:02d}.nc" + self._checkpoint(root / name, spec, seconds) + for name in segment_names(spec, 0, 60): + with netCDF4.Dataset(root / name, "w") as dataset: + dataset.createDimension("nx", 1) + dataset.createVariable("field", "f4", ("nx",))[:] = 1.0 + calls = [] + + def launch(command: list, directory: Path, environment: dict, + timeout: int, driver: str) -> dict: + """Emulate only file writes; no model process is launched.""" + begin, end = [(0, 4), (4, 12), (12, 28), (28, 60)][len(calls)] + settings = json.loads((directory / "settings.json").read_text()) + self.assertEqual((directory / "wrf.nc").read_bytes(), + (root / "wrf.nc").read_bytes()) + if calls: + self.assertEqual( + Path(settings["restart_source"]).parent, calls[-1]) + self.assertIn("restart=.true.", + (directory / "namelist.fire").read_text()) + for checkpoint in (False, True): + for name in segment_names(spec, begin, end, checkpoint): + shutil.copy2(root / name, directory / name) + if extra: + (directory / + "fire_output_unexpected.nc").write_bytes(b"unexpected") + calls.append(directory) + return {"reasons": [], "exit_status": 0} + + with patch("run_case.run_model", side_effect=launch): + result = run_restarts(spec, root, ["fixture"], {}, set()) + self.assertEqual(result["pass"], not extra, result["reasons"]) + self.assertEqual(len(calls), 4) + self.assertEqual(len(result["comparisons"]), 21) + #------------------------------------------------------------------------ # Stored reference integrity and approval #------------------------------------------------------------------------ From a863938ad1a00b232eec0db1f15983957813c097 Mon Sep 17 00:00:00 2001 From: Maria Frediani Date: Tue, 20 Oct 2026 20:51:09 -0600 Subject: [PATCH 13/19] Respect checkpoint field applicability and preserve run provenance --- tests/regression/compare_outputs.py | 2 +- tests/regression/doc/restart.md | 3 +++ tests/regression/restart.py | 17 +++++++++++------ tests/regression/run_case.py | 12 ++++++++++-- tests/regression/run_suite.py | 4 ++++ 5 files changed, 29 insertions(+), 9 deletions(-) diff --git a/tests/regression/compare_outputs.py b/tests/regression/compare_outputs.py index 39ed7d67..8bdc6026 100644 --- a/tests/regression/compare_outputs.py +++ b/tests/regression/compare_outputs.py @@ -244,7 +244,7 @@ def compare_file( volatile_global_attributes: set[str] | None = None, required_fields: set[str] | None = None, ) -> dict[str, Any]: - """Compare two NetCDF files and every field under the fixed policy.""" + """Compare full schemas and all fields, or an explicit required-field subset.""" result: dict[str, Any] = { "reference": str(reference_path), diff --git a/tests/regression/doc/restart.md b/tests/regression/doc/restart.md index 9193ef1a..d6f26228 100644 --- a/tests/regression/doc/restart.md +++ b/tests/regression/doc/restart.md @@ -46,6 +46,9 @@ The PR #39 writer also saves method work arrays `lfn_0/1/2`, `lfn_s0/1/2/3` and unused crown fluxes `fcanhfx/fcanqfx`. Their values are not all initialized for these methods. They remain in the schema comparison but are explicitly listed as uncompared fields. Physical fields are not excluded to obtain a pass. +`lfn_hist` is checked only for observed perimeters, whose initialization sets +it; `fz0` and atmospheric forcing are checked only for real cases. Idealized +runs do not use those atmospheric fields. Regular-output tolerances do not weaken the exact checkpoint-state comparison. The complete selection contains 15 cases across `serial`, `omp4`, `mpi1`, diff --git a/tests/regression/restart.py b/tests/regression/restart.py index dbc8e041..45abf45d 100644 --- a/tests/regression/restart.py +++ b/tests/regression/restart.py @@ -15,6 +15,7 @@ import datetime as dt from pathlib import Path import shutil +import subprocess from typing import Any import netCDF4 @@ -29,10 +30,10 @@ # and unused crown fluxes fcanhfx/fcanqfx, are written by PR #39 but are not all # initialized for the selected methods. Their names remain in the schema check. STATE_FIELDS = { - "lfn", "lfn_hist", "lfn_out", "tign_g", "fuel_frac", "fuel_frac_burnt_dt", - "fire_area", "fgrnhfx", "fgrnqfx", "flame_length", "ros", "ros_front", - "emis_smoke", "fmc_g", "fuel_load_g", "fuel_time", "zsf", "dzdxf", "dzdyf", - "nfuel_cat", "uf", "vf", "fz0" + "lfn", "lfn_out", "tign_g", "fuel_frac", "fuel_frac_burnt_dt", "fire_area", + "fgrnhfx", "fgrnqfx", "flame_length", "ros", "ros_front", "emis_smoke", + "fmc_g", "fuel_load_g", "fuel_time", "zsf", "dzdxf", "dzdyf", "nfuel_cat", + "uf", "vf" } FORCING_FIELDS = {"fire_t2", "fire_q2", "fire_psfc", "fire_rain"} REAL64_FIELDS = {"fuel_frac", "fuel_frac_burnt_dt"} @@ -67,8 +68,12 @@ def checkpoint_fields(spec: dict[str, Any]) -> set[str]: """Select physical fields whose owned cells must survive a restart exactly.""" fields = set(STATE_FIELDS) fire = spec["namelist"]["fire"] + # Init_fire_perimeter alone initializes lfn_hist; ideal runs leave fz0 + # unavailable. Neither is consumed by the corresponding excluded path. + if fire["fire_is_real_perim"]: + fields.add("lfn_hist") if fire["ideal_opt"] == 0: - fields |= FORCING_FIELDS + fields |= FORCING_FIELDS | {"fz0"} if fire["fmoist_run"]: fields |= {"fmc_gc"} | {f"{name}_old" for name in FORCING_FIELDS} return fields @@ -218,7 +223,7 @@ def run_restarts(spec: dict[str, Any], directory: Path, command: list[str], record = run_model(command, segment, environment, spec["execution"]["timeout_seconds"], "standalone") - except (OSError, RuntimeError) as error: + except (OSError, RuntimeError, subprocess.SubprocessError) as error: record = {"reasons": [f"{type(error).__name__}: {error}"]} record.update(directory=str(segment), begin_seconds=begin, diff --git a/tests/regression/run_case.py b/tests/regression/run_case.py index 0abce560..fbb526ae 100644 --- a/tests/regression/run_case.py +++ b/tests/regression/run_case.py @@ -13,6 +13,7 @@ from pathlib import Path import shutil import subprocess +import sys import tempfile import time from typing import Any @@ -21,7 +22,7 @@ from check_outputs import expected_output_names, validate_outputs from config import load_yaml, resolve_spec -from generate_inputs import generate_inputs +from generate_inputs import generate_inputs, sha256_file from render_namelist import render_namelist from reports import print_result, write_result @@ -145,7 +146,14 @@ def run_case(args: Any) -> dict[str, Any]: "driver": args.driver, "pass": False, "reasons": [], - "source": source_identity() + "source": source_identity(), + "executable": str(executable), + "executable_sha256": sha256_file(executable), + "environment": { + "python": sys.executable, + "PBS_JOBID": os.environ.get("PBS_JOBID"), + "LOADEDMODULES": os.environ.get("LOADEDMODULES") + } } started = time.monotonic() try: diff --git a/tests/regression/run_suite.py b/tests/regression/run_suite.py index d831344b..f76450ba 100644 --- a/tests/regression/run_suite.py +++ b/tests/regression/run_suite.py @@ -173,6 +173,10 @@ def run_suite(args: Any) -> dict[str, Any]: result["restart"]["pass"], "segments": len(result["restart"]["segments"]), + "completed_segments": + sum(item["pass"] for item in result["restart"]["segments"]), + "model_invocations": + 1 + len(result["restart"]["segments"]), "comparisons": len(result["restart"]["comparisons"]), "failed_comparisons": From ed18f91b1abc163cba22dc8d16b349ba1d2c2661 Mon Sep 17 00:00:00 2001 From: Maria Frediani Date: Tue, 20 Oct 2026 21:04:56 -0600 Subject: [PATCH 14/19] Document restart validation and the P02 initialization failure --- compile.sh | 2 +- tests/regression/doc/cases.md | 6 +++ tests/regression/doc/structure.md | 8 ++++ tests/regression/doc/validation.md | 61 ++++++++++++++++++++++++++++++ 4 files changed, 76 insertions(+), 1 deletion(-) diff --git a/compile.sh b/compile.sh index aebb8662..e66bf733 100755 --- a/compile.sh +++ b/compile.sh @@ -31,7 +31,7 @@ usage () { printf " build with verbose output\n" printf " --test[=TEST_NAME], -t[=TEST_NAME]\n" printf " run one exact test name, or the quick suite\n" - printf " --test-suite=quick|pr|full|unit\n" + printf " --test-suite=quick|pr|full|restart|unit\n" printf " --reference=DIRECTORY\n" printf " override the reference; strict defaults to the bundled PR baseline\n" printf " --no-reference\n" diff --git a/tests/regression/doc/cases.md b/tests/regression/doc/cases.md index abdc7c5c..d113e05e 100644 --- a/tests/regression/doc/cases.md +++ b/tests/regression/doc/cases.md @@ -13,6 +13,12 @@ scientific options within each case; see [configuration.md](configuration.md). | `fuels` | Line ignition, twelve fuel categories, nearest interpolation | | `terrain` / `u10m` | Terrain, twelve fuel strips, delayed observed perimeter, varying moisture, terrain-dependent 10 m winds | | `terrain` / `u3d` | Same terrain experiment with a terrain-dependent vertical wind profile | +| `circle` / `restart_prescribed` | Same circle geometry with prescribed moisture and a restart at 28 s | +| `terrain` / `restart_time`, `restart_step` | Same terrain geometry and 10 m winds with restart chains bracketing perimeter activation and moisture updates | + +The restart configurations use different output, atmospheric, and moisture +cadences. Their [schedule table](restart.md) supplements the geometry figures +below. Existing `quick`, `pr`, and `full` selections retain their settings. The shared configuration uses a 72 x 72 fire grid, 100 m spacing, 4 s timesteps, and 60 s integrations. NUOPC and ESMX run both terrain wind configurations with generated diff --git a/tests/regression/doc/structure.md b/tests/regression/doc/structure.md index 6f0b070c..10c2a7cf 100644 --- a/tests/regression/doc/structure.md +++ b/tests/regression/doc/structure.md @@ -43,6 +43,8 @@ flowchart TD CTEST -->|runs| UNIT CTEST -->|runs each model case through| CASE CASE -->|stages and launches| MODEL + CASE -->|for restart configurations| RESTART["restart.py
Cold first segment and chained restarts"] + RESTART -->|checks each segment and checkpoint| RESULT MODEL -->|produces fields and logs for| CHECK BASELINE -->|supplies accepted fields to| CHECK CHECK -->|records findings in| RESULT @@ -62,6 +64,12 @@ Strict selects the shared baseline automatically. `--reference` overrides it; cross-execution checks remain active. See [running tests](running.md) for commands and [comparison rules](comparison.md) for acceptance criteria. +The separately selected [restart suite](restart.md) uses `run_case.run_model` +for the continuous run and every segment. `restart.py` copies the original +forcing, derives segment namelists, and calls the shared comparison routines. +Its results enter the same case and suite reports; it does not build or launch +tests through a second framework. + ## File relationships inside the harness `regression.py` is the command interface. CMake uses its `registrations` command diff --git a/tests/regression/doc/validation.md b/tests/regression/doc/validation.md index e40a7f0b..f628fa78 100644 --- a/tests/regression/doc/validation.md +++ b/tests/regression/doc/validation.md @@ -5,6 +5,67 @@ These records describe specific revisions. They do not automatically validate later source, documentation examples, renamed registrations, or new cases. +## P00 restart foundation (2026-10-20) + +The review base is local merge `bc205bc`, combining upstream `26d57b6` with +restart PR #39 at `59b9081`. Remote heads were checked on 2026-10-20 and still +matched that snapshot. P00 changes only the harness and its documentation. +Its feature branch must retain this explicit prerequisite; targeting upstream +develop directly would also include the unmerged restart work. + +Derecho PBS `7889837.desched1` built this base in serial, OpenMP, MPI, and +hybrid GNU Strict configurations. All 190 PR CTests passed: 154 unit CTests +and 36 model runs. The 40 cross-execution comparisons and all 36 approved +`github-pr` reference comparisons passed. The largest cross-execution absolute +difference was 0.125 W m-2 in `fgrnhfx`; the applicable relative tolerance was +unchanged. This establishes the base's existing regression behavior, not +restart continuity. + +The environment was GNU 12.2.0, Cray MPICH 8.1.25, NetCDF 4.9.2, ESMF 8.5.0, +and the shared `npl-2026a` Python interpreter. P00's 45 Python tests also passed +in a serial-NetCDF development environment; YAPF 0.43.0 and Python compilation +checks passed. Both existing Python test modules remain in discovery. + +The first harness attempt, `1f61101` in PBS `7889874.desched1`, passed all five +time-based moisture restart cases and the unchanged PR matrix. Its prescribed +cases exposed a checker applicability error: idealized `fz0` is deliberately +unavailable. `a863938` corrects that check and restricts `lfn_hist` checks to +observed perimeters, whose initialization sets it. It also records executable +hashes and execution environment. The original failed evidence is retained. + +The step-based configuration exposes the pre-existing initialization defect +assigned to P02. In `physics/fmc_wrffire_mod.F90`, `Advance_fmc_model` gates the +call on update cadence, but `Advance_moisture_classes` initializes only when +`itimestep == 1`. With `fmoist_freq=3`, that first call is missed. Results include +invalid class moisture, nonfinite moisture/heat fluxes, differing restart state, +and MPI aborts reporting equilibrium-moisture NaNs. The tests remain failures; +P00 does not change the production implementation or claim scientific acceptance +of this configuration. + +The corrected harness at `a863938`, PBS `7889890.desched1`, passes all ten +prescribed/time-based cases across serial, OpenMP, one-rank MPI, four-rank MPI, +and hybrid layouts. Their 210 output/checkpoint comparisons have zero absolute +error, including REAL64 fuel state. All eight cross-layout comparisons pass. +All five step-based cases fail on the P02 defect described above. There are +61 attempted model invocations rather than the scheduled 65: the four-rank +MPI and hybrid chains cannot continue after their first resumed segment aborts. +The report contains 283 file-comparison attempts, including four missing-file +failures; 279 comparisons read both files. Agreement in an invalid step-based +case does not establish restart acceptance. The overall restart suite exits +nonzero, as intended for an unresolved production defect. + +The same job also passed all 190 existing PR CTests, all 40 cross-execution +comparisons, and all 36 approved-reference comparisons. Maximum cross-driver +absolute error remained 0.125 W m-2 in `fgrnhfx`. The PBS job exits nonzero +because of the restart suite's five P02 cases, not because the existing PR +suite failed. Subsequent P00 edits only document these results and list the +new suite in `compile.sh --help`; runtime validation applies to `a863938`. + +Evidence is preserved under `cfbm-P00-20261021T024500/` in the validation +submitter's Derecho scratch `tmp/`, with separate `base`, `attempt-01`, and +`attempt-02` source/build/run trees. See [restart continuity](restart.md) for +the comparison policy, scheduled counts, and intentionally untested paths. + ## Coupled hybrid baseline extension (2026-10-14) At Maria Frediani's request, revision `4b6535d` enables NUOPC and ESMX in From c8aeeaa70fdbd17e0629294c6354194098717e2d Mon Sep 17 00:00:00 2001 From: Maria Frediani Date: Thu, 22 Oct 2026 23:16:28 -0600 Subject: [PATCH 15/19] Record P00 validation on merged P02 develop --- tests/regression/doc/restart.md | 19 ++++++------ tests/regression/doc/validation.md | 49 +++++++++++++++++++++++++++--- 2 files changed, 54 insertions(+), 14 deletions(-) diff --git a/tests/regression/doc/restart.md b/tests/regression/doc/restart.md index 590cae8c..9b89527b 100644 --- a/tests/regression/doc/restart.md +++ b/tests/regression/doc/restart.md @@ -66,15 +66,16 @@ historical `github-pr` reference, which does not contain their keys. The existing `pr` suite remains unchanged and checks its approved reference. No reference candidate is created or approved by this suite. -P02 additionally runs 30 restart-loading probes across the five layouts. Each -time-based case resumes a copy with PR #39 metadata (five new attributes removed) -and requires an exact checkpoint match. Four damaged copies must reject an unset -initialization flag, changed cadence, future last-update time, or invalid previous -pressure before writing output. Each step-based case must reject the ambiguous -legacy schema. These probes add 30 model invocations and five checkpoint -comparisons to the continuity counts above. Reports keep expected rejection -diagnostics separate from failed continuity comparisons. Synthetic legacy files -exercise schema compatibility; they do not establish cross-revision equivalence. +The suite additionally runs 30 restart-loading probes for P02 across the five +layouts. Each time-based case resumes a copy with PR #39 metadata (five new +attributes removed) and requires an exact checkpoint match. Four damaged copies +must reject an unset initialization flag, changed cadence, future last-update +time, or invalid previous pressure before writing output. Each step-based case +must reject the ambiguous legacy schema. These probes add 30 model invocations +and five checkpoint comparisons to the continuity counts above. Reports keep +expected rejection diagnostics separate from failed continuity comparisons. +Synthetic legacy files exercise schema compatibility; they do not establish +cross-revision equivalence. Restart experiments currently require the standalone driver. NUOPC/ESMX and WRF-host restart continuity are not established here. Existing coupled PR diff --git a/tests/regression/doc/validation.md b/tests/regression/doc/validation.md index f628fa78..21b688a6 100644 --- a/tests/regression/doc/validation.md +++ b/tests/regression/doc/validation.md @@ -5,13 +5,52 @@ These records describe specific revisions. They do not automatically validate later source, documentation examples, renamed registrations, or new cases. +## P00 on merged P02/develop (2026-10-22) + +After PR #39 and P02/PR #62 were squash-merged, the review base became +`develop` at `622e191`. P00 merges that revision into its published history, +retaining the merged moisture component and restart reader. The 19-file review +diff contains only tests and documentation; production code and P02's focused +unit tests match `622e191` exactly. The harness now includes the initialization +metadata checks and 30 restart-loading probes from the earlier P00/P02 test +integration. + +Derecho PBS `7915414.desched1` tests candidate `9a64864`. The published-history +merge `ce551ee` has the identical complete file tree; subsequent edits only +update documentation. All four GNU Strict builds pass: serial, OpenMP, MPI, +and hybrid, with NUOPC/ESMX enabled in the latter two. The environment is GNU +12.2.0, Cray MPICH 8.1.25, NetCDF 4.9.2, ESMF 8.5.0, and shared NPL Python. + +All 67 restart CTests pass, comprising 15 restart cases, 48 P02 lifecycle +checks, and four Python harness registrations. All five formerly failing +step-based moisture cases now pass. The suite completes 65 continuity model +invocations and 30 loading probes. Its 315 continuity file comparisons and +five accepted-legacy checkpoint comparisons are bitwise equal, with zero +maximum absolute error. All 450 checkpoint validity checks, 30 loading probes, +and 12 cross-layout comparisons pass; cross-layout error is also zero. + +The same job passes all 238 existing PR CTests: 202 unit registrations and +36 model runs. All 40 execution comparisons and 36 approved-reference +comparisons pass. The reference error is zero; the maximum cross-driver error +is 0.125 W m-2 in `fgrnhfx`, within the unchanged tolerance. The PBS job exits +zero. The 45 Python tests also pass in the serial-NetCDF development environment; +YAPF 0.43.0, shell syntax, and whitespace checks pass. + +Evidence is preserved under `cfbm-P00-develop-20261022T230324-0600/` in the +validation submitter's Derecho scratch `tmp/`. The historical P00 failures +below remain preserved. This validation establishes same-revision standalone +restart continuity, including P02's delayed initialization correction. It does +not establish NUOPC/ESMX or WRF-host restart continuity, cross-layout checkpoint +transfer, or combined P01/P04 behavior. + ## P00 restart foundation (2026-10-20) -The review base is local merge `bc205bc`, combining upstream `26d57b6` with -restart PR #39 at `59b9081`. Remote heads were checked on 2026-10-20 and still -matched that snapshot. P00 changes only the harness and its documentation. -Its feature branch must retain this explicit prerequisite; targeting upstream -develop directly would also include the unmerged restart work. +The original review base was local merge `bc205bc`, combining upstream +`26d57b6` with restart PR #39 at `59b9081`. Remote heads were checked on +2026-10-20 and still matched that snapshot. P00 changes only the harness and +its documentation. +At that time, its feature branch required an explicit stacked review base; +targeting upstream develop would also have included the unmerged restart work. Derecho PBS `7889837.desched1` built this base in serial, OpenMP, MPI, and hybrid GNU Strict configurations. All 190 PR CTests passed: 154 unit CTests From 5de15ed3f1ff4d13c6ed097ab7efe8254eed25e7 Mon Sep 17 00:00:00 2001 From: Maria Frediani Date: Fri, 23 Oct 2026 08:28:16 -0600 Subject: [PATCH 16/19] Run restart regressions in CI and document suite selection --- .github/workflows/ci_test.yml | 11 +++++--- tests/regression/README.md | 5 ++++ tests/regression/doc/configuration.md | 38 +++++++++++++++++++++++++-- tests/regression/doc/references.md | 17 +++++++----- tests/regression/doc/running.md | 16 ++++++++--- 5 files changed, 72 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci_test.yml b/.github/workflows/ci_test.yml index dabcd7f6..dc2d61c3 100644 --- a/.github/workflows/ci_test.yml +++ b/.github/workflows/ci_test.yml @@ -171,14 +171,19 @@ jobs: --build-dir="$GITHUB_WORKSPACE/build/$variant" \ --prefix="$GITHUB_WORKSPACE/install/$variant" done - - name: Run the selected CTest suite and compare executions + - name: Run PR and restart CTest suites and compare executions run: | OPTIONS=() for variant in serial omp mpi hybrid; do OPTIONS+=(--build-dir "$GITHUB_WORKSPACE/build/$variant") done - python -B tests/regression/regression.py suite "${OPTIONS[@]}" \ - --suite pr --run-root "$GITHUB_WORKSPACE/build/results" + # Collect both suites even if one fails; either failure fails CI. + STATUS=0 + for suite in pr restart; do + python -B tests/regression/regression.py suite "${OPTIONS[@]}" \ + --suite "$suite" --run-root "$GITHUB_WORKSPACE/build/results" || STATUS=1 + done + exit "$STATUS" - name: Save an unapproved reference from the passing suite if: ${{ inputs.create_reference }} run: | diff --git a/tests/regression/README.md b/tests/regression/README.md index a1963624..e0c90c73 100644 --- a/tests/regression/README.md +++ b/tests/regression/README.md @@ -62,6 +62,11 @@ configuration names, and the `large` scale. GNU `Strict` builds select the shared `github-pr` baseline by default. Use `--reference` to override it or `--no-reference` to disable historical comparison deliberately. Configuration warns when no reference is selected. + +GitHub CI runs both `pr` and `restart` using the same four builds; either suite +failing makes the regression check fail. The restart suite compares uninterrupted +and segmented runs from the same revision and needs no stored baseline. Both +suites' reports and model files are retained in the test-evidence artifact. The [comparison table](doc/comparison.md) separates bitwise checks from numerical tolerance. Passing without an approved reference establishes the configured behavior checks and agreement among selected executions, not historical diff --git a/tests/regression/doc/configuration.md b/tests/regression/doc/configuration.md index bf4a1296..38d4f2fe 100644 --- a/tests/regression/doc/configuration.md +++ b/tests/regression/doc/configuration.md @@ -16,17 +16,22 @@ they describe synthetic fields and are not additional model namelist options. | Selection | Meaning | Examples | | --- | --- | --- | | Case | Shared physical setup and its purpose | `circle`, `fuels`, `terrain` | -| Configuration | Explicit set of scientific option/input overrides within a case | `base`, `u10m`, `u3d`, `godunov3d` | +| Configuration | Explicit set of scientific option/input overrides within a case | `base`, `u10m`, `u3d`, `godunov3d`, `restart_time` | | Scale | Grid dimensions, spacing, integration duration, and intervals | `small`, `large` | | Driver | Executable entry point | `standalone`, `nuopc`, `esmx` | | Execution | Compiled build, MPI ranks, OpenMP threads, timeout | `serial`, `omp4`, `mpi4`, `hybrid4` | -| Suite | Explicit selections of cases, configurations, scale, and executions | `quick`, `pr`, `full` | +| Suite | Explicit selections of cases, configurations, scale, and executions | `quick`, `pr`, `full`, `restart` | `unit` and `legacy` are additional CTest selections, not generated experiments. The suite `full` uses scale `large`. Scale is more than domain size: it also changes the integration schedule. The name `small` replaces `standard`, and `large` replaces the old scale name `full`. +Suite membership is explicit: selecting `pr` or `full` does not automatically +include the `restart` configurations. GitHub CI invokes both `pr` and `restart` +using the same four builds. The `restart` suite selects standalone runs even +when the parent case also supports NUOPC and ESMX. + Resolution is **defaults → case → configuration → scale**. Nested mappings merge; scalar values and lists replace earlier values. Execution settings are kept separately and cannot override science. If `case --configuration` is @@ -104,6 +109,12 @@ cases: godunov3d: namelist: fire: {wind_vinterp_opt: 0, fire_upwinding: 2} + restart_time: + restart_splits: [4, 12, 28] + namelist: + time: {restart_interval: 4, interval_output: 12} + atm: {interval_atm: 12} + fire: {fmoist_freq: 0, fmoist_dt: 16.0} scales: small: {} @@ -120,6 +131,12 @@ executions: mpi4: {build: mpi, ranks: 4, threads: 1, timeout_seconds: 900} suites: + restart: + scale: small + cases: + terrain: + configurations: [restart_time] + executions: [serial, mpi4] pr: scale: small cases: @@ -128,6 +145,16 @@ suites: executions: [serial, mpi4] ``` +The `restart_time` example inherits `dt=4` and a 60 s duration. It writes +checkpoints every 4 s and regular output every 12 s. For each selected execution, +the harness runs one uninterrupted integration from 0 to 60 s and a chain of +segments spanning 0-4, 4-12, 12-28, and 28-60 s. The 28 s split falls between +regular outputs at 24 and 36 s; the first output after resuming is still at +36 s. `restart_splits` controls the harness, while `restart_interval` is a +Fortran namelist option. The complete production selection also includes +prescribed and step-based moisture cases across five execution layouts; see +[restart continuity](restart.md). + ## Add options without adding a category system A configuration can set any supported option under its real namelist block. @@ -161,3 +188,10 @@ example `nuopc_terrain_u3d_small_mpi4`. Only runs sharing the same case, configuration, and scale are compared. Wind choices and numerical schemes are separate experiments, not numerical references for one another. Resolved settings are saved in each `result.json`. + +Restart configurations compare each resumed segment with the uninterrupted run +from the same revision and execution layout. Physical checkpoint state must +match exactly; regular output follows the existing comparison policy. These +cases do not use the stored `github-pr` baseline. Running `restart` alongside +`pr` in CI therefore requires no baseline update: `pr` keeps its approved +reference, and reference generation continues to use only the `pr` results. diff --git a/tests/regression/doc/references.md b/tests/regression/doc/references.md index 57f336ca..dd7ee90b 100644 --- a/tests/regression/doc/references.md +++ b/tests/regression/doc/references.md @@ -47,17 +47,22 @@ history](validation.md). ## GitHub review and activation -CI uses the fixed `pr` suite with all four GNU Strict builds: serial, OpenMP, -MPI and hybrid, both including NUOPC/ESMX. Its 36 generated runs produce 40 -cross-execution comparisons. Unit checks run in each build. PRs targeting -prerequisite branches are included, so a merge into `develop` is not needed. +CI runs `pr` and `restart` using all four GNU Strict builds: serial, OpenMP, +MPI and hybrid, with NUOPC/ESMX in the latter two. The `pr` suite's 36 generated +runs produce 40 cross-execution comparisons. The `restart` suite compares +continuous and segmented standalone integrations from the same revision and +does not use a stored baseline. Both suites must pass. Unit checks run in each +build. PRs targeting prerequisite branches are included, so a merge into +`develop` is not needed. Normal Strict checks default to the [approved GitHub reference](../baselines/github-pr/README.md) committed for the complete PR suite. A missing or unapproved reference fails the check. To generate a future candidate, select **Run workflow** for **Fire Behavior CI Test**, choose the PR branch, and enable `create_reference`. -That run checks the same complete PR selection without a reference, then -uploads an unapproved baseline only if all checks and comparisons pass. +That run checks both suites without a stored reference, then creates a candidate +from only the `pr` results. It uploads the unapproved baseline only if all +checks and comparisons pass. Adding restart coverage does not change the +approved baseline or include restart checkpoints in a reference candidate. Download and extract both `.tar.gz` artifacts (candidate and test evidence). Archiving preserves the colons in NetCDF timestamp filenames, which GitHub's diff --git a/tests/regression/doc/running.md b/tests/regression/doc/running.md index 51b960cb..58fcdd0b 100644 --- a/tests/regression/doc/running.md +++ b/tests/regression/doc/running.md @@ -142,13 +142,20 @@ The suite options are defined in [regression.py](../regression.py): | `pr` | Unit tests plus all four default case/configuration combinations across `serial`, `omp1`, `omp4`, `mpi1`, `mpi4`, and `hybrid4`, where supported. | | `full` | Unit tests plus the larger experiments: 320 × 320 grids, 3600 s integrations, both method pairs `(9,4)` and `(2,4)`, and the additional `mpi8` execution. Requires `CFBM_FULL_TESTS=ON`. | | `unit` | Unit Fortran tests and the Python harness tests. No generated fire-case integrations. | -| `restart` | Python harness checks and standalone continuous/restarted cases in serial, OpenMP, MPI, and hybrid layouts. See [restart continuity](restart.md) for schedules, counts, and coverage limits. | +| `restart` | Moisture initialization unit checks, Python harness checks, and standalone continuous/restarted cases in serial, OpenMP, MPI, and hybrid layouts. See [restart continuity](restart.md) for schedules, counts, and coverage limits. | | `legacy` | Original tests registered by the legacy build route. | These selections follow [cases.yaml](../cases.yaml). Each build contributes only the executions it supports, such as serial or MPI. **`full` selects only the larger experiments.** +GitHub CI runs `pr` and `restart` consecutively using the same serial, OpenMP, +MPI and hybrid builds. Both suites run even if one fails, and either failure +fails the regression check. Their separate `summary.json` files, JUnit reports, +model logs and outputs are included in the test-evidence artifact. Restart +comparisons use the uninterrupted run from the same revision; the existing +`github-pr` baseline remains specific to `pr`. + Enable their registration in an appropriately sized allocation: ```bash @@ -211,7 +218,7 @@ namelist_broadcast_mpi Here `*_unit` describes a naming pattern, not a literal selector. Copy the complete name from the CTest listing when using `--test`. `regression_python` tests **the Python regression harness**: input generation, configuration, output -checks, comparisons, and reporting. It currently contains **40 Python tests** +checks, comparisons, and reporting. It currently contains **45 Python tests** across two files and does not launch the Fortran model. | Area | What it checks | @@ -224,9 +231,10 @@ across two files and does not launch the Fortran model. | Numerical comparisons | Exercises relative tolerance, zero references, tolerance boundaries, bitwise static-field checks, NaNs, infinities, missing-data masks, and empty valid-data sets. | | Comparison policy | Checks cross-driver roughness tolerance, exact fuel categories, comparisons across execution layouts, and rejection of incompatible scientific settings. | | Failure detection and reporting | Checks rejection of corrupted or unapproved references, extra output files, and a successful process exit with no outputs; also exercises JSON reporting. | +| Restart chains | Checks checkpoint precision, state and metadata validity, segment schedules, and preservation of forcing across segments. | The implementation is in -[python_harness_test.py](../tests/python_harness_test.py) (23 tests) and +[python_harness_test.py](../tests/python_harness_test.py) (28 tests) and [generator_comparator_test.py](../tests/generator_comparator_test.py) (17 tests). @@ -347,7 +355,7 @@ The build loop calls `compile.sh` once per configuration; each call builds only that configuration. After all four builds finish, one suite invocation selects their tests and compares the outputs. Settings are grouped near the top of the template. -`SUITES=(quick pr)` can explicitly run both when separate reports are needed. +Set `SUITES=(pr restart)` to run both CI suites and retain separate reports. The template runs only the generated regression harness. Legacy investigations use `compile_legacy.sh` separately with their own build/install directories. From 2c6b5b98f94b79f845b73cef605f9ace41b73efa Mon Sep 17 00:00:00 2001 From: Maria Frediani Date: Fri, 23 Oct 2026 09:00:31 -0600 Subject: [PATCH 17/19] Explain regression test and checkpoint counts --- tests/regression/README.md | 74 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/tests/regression/README.md b/tests/regression/README.md index e0c90c73..3bdab1de 100644 --- a/tests/regression/README.md +++ b/tests/regression/README.md @@ -43,6 +43,7 @@ difference between one `compile.sh` call and the complete four-build matrix. | Follow the build/run flow and connections between files | [Execution overview and file relationships](doc/structure.md) | | Find every compared field and its acceptance rule | [Variable comparison table](doc/comparison.md) | | Check continuous versus restarted integrations | [Restart continuity](doc/restart.md) | +| Distinguish CTest invocations, model runs, and checkpoint checks | [How test counts are reported](#how-test-counts-are-reported) | | Understand improvements and remaining legacy coverage | [New and legacy tests](doc/advantages.md) | | Find outputs or prepare an unapproved reference | [Results and references](doc/references.md) | | Add or extend a case, namelist option, or output | [Contributor guide](doc/contributing.md) | @@ -86,6 +87,79 @@ investigations, use `compile_legacy.sh` with separate build/install directories. Coverage limits and postponed investigations are recorded in [deferred work](doc/deferred.md). +## How test counts are reported + +The counts below describe the complete four-build `pr` and `restart` selections +validated at `5de15ed` in [CI run 39970138677](https://github.com/NCAR/fire_behavior/actions/runs/39970138677). +They count different levels of work: CTest invocations, model integrations, +individual checkpoint-file checks, and comparisons between files. Selecting +fewer builds or filtering cases changes these totals. When registrations or +schedules change, update this breakdown from the suite summaries, per-build +CTest/JUnit reports, and each case's `restart.json`. + +### 450 checkpoint-validity checks + +These are checks of **450 checkpoint files** across 15 restart cases. Each case +writes checkpoints every 4 s, from 4 through 60 s: + +- 15 files from the uninterrupted run. +- 15 files produced across the segmented run. +- Total: **15 cases × 30 files = 450 checkpoint-file checks**. + +Each file is checked for required fields, finite/unmasked values, REAL64 fuel +storage, correct dates and timestep count, and the expected perimeter-activation +flag. The final checkpoint also checks for precision that would be lost if +fuel state were reduced to REAL32. Evolving-moisture cases additionally check +initialization status, update cadence and times, five-class dimensions, and +moisture bounds appropriate to these dry-forcing cases. The class-moisture +range `(0,1)` is specific to these cases, not a general bound on fuel moisture. +The checks are implemented in [check_checkpoint](restart.py). + +These checks test each file against the expected checkpoint contract. The +separately reported **320 file comparisons** compare continuous and restarted +results: 90 regular-output comparisons, 225 checkpoint comparisons, and five +accepted-legacy checkpoint comparisons. Each checkpoint comparison requires +exact physical-state agreement. The 450 validity checks run inside the 15 +restart cases; they are not 450 additional CTest invocations. See +[restart continuity](doc/restart.md) for field applicability and comparison rules. + +### 238 PR CTest invocations + +The `pr` suite executes the following registered CTests across four builds: + +| Build | Fortran unit checks | Python harness invocation | Model runs | Total | +| --- | ---: | ---: | ---: | ---: | +| Serial | 49 | 1 | 4 | 54 | +| OpenMP | 49 | 1 | 8 | 58 | +| MPI | 50 | 1 | 16 | 67 | +| Hybrid | 50 | 1 | 8 | 59 | +| **Total** | **198** | **4** | **36** | **238** | + +The [Fortran unit checks](../unit/CMakeLists.txt) cover namelist validation, +timing, tiles, perimeter activation, numerical precision, wind interpolation, +and moisture initialization. MPI and hybrid add the namelist-broadcast check. + +Each `regression_python` invocation runs **45 Python tests**, but CTest counts +that invocation as one entry. Many unit checks are repeated across builds to +exercise the corresponding compiled configuration. Thus 238 is a count of +CTest invocations, not 238 distinct scientific scenarios. The 40 cross-execution +comparisons and 36 approved-reference comparisons are reported separately from +these invocation counts. + +### 67 restart CTest invocations + +The `restart` suite is separate from the 238 PR invocations. It contains 15 +restart cases, 48 moisture-initialization unit invocations (12 in each build), +and four Python harness invocations: **15 + 48 + 4 = 67**. The unit and Python +checks overlap with `pr` and run again when both suites are selected. + +Each restart case invokes the model several times. Together they run 15 +uninterrupted integrations, 50 chain segments, and 30 restart-loading probes: +**95 model invocations** inside the 15 case-level CTests. The loading probes +include five accepted legacy files and 25 expected rejections of ambiguous or +damaged checkpoints. An expected rejection passes only when its diagnostic +and absence of regular output match the test requirements. + ## Python style Use Google-based YAPF formatting, four-space indentation, an 80-column target, From b2da9360ba330a118761090922c0059ce9aaea93 Mon Sep 17 00:00:00 2001 From: Maria Frediani Date: Fri, 23 Oct 2026 09:18:53 -0600 Subject: [PATCH 18/19] Describe regression behavior without development labels --- tests/AGENTS.md | 15 +++++++++++++++ tests/regression/doc/cases.md | 7 +++---- tests/regression/doc/deferred.md | 4 ++-- tests/regression/doc/restart.md | 29 +++++++++++++++++------------ 4 files changed, 37 insertions(+), 18 deletions(-) create mode 100644 tests/AGENTS.md diff --git a/tests/AGENTS.md b/tests/AGENTS.md new file mode 100644 index 00000000..3127cea1 --- /dev/null +++ b/tests/AGENTS.md @@ -0,0 +1,15 @@ +# Instructions for test documentation + +Write user-facing test documentation so readers can understand it without +knowing the development history. Describe the scientific behavior, configuration, +or compatibility condition being tested. Do not use internal planning labels +such as P00/P02 or PR numbers as substitutes for those descriptions. + +For example, use "restart-loading probes for moisture initialization" instead +of "probes for P02", and "legacy restart metadata" instead of "PR #39 metadata". +Explain which metadata is absent and why it matters when discussing legacy +compatibility; "restart metadata" alone does not identify that distinction. + +Keep PR references, planning labels, revisions, and historical validation results +in validation records such as regression/doc/validation.md or session notes. +Preserve that provenance when revising user-facing guides. diff --git a/tests/regression/doc/cases.md b/tests/regression/doc/cases.md index d113e05e..393ba3c1 100644 --- a/tests/regression/doc/cases.md +++ b/tests/regression/doc/cases.md @@ -49,10 +49,9 @@ between mass levels, where unscaled (U, V) = (12, 8) m/s. For 3D winds, both drivers first remap the wind and geopotential profiles horizontally to the fire grid, then sample vertically using mapped roughness. -PR #57 applies this order to standalone; previously it sampled vertically on -the atmospheric grid first. The logarithmic surface calculation is nonlinear, -so reversing these operations can change the result. Matching the order does -not make the two horizontal interpolation implementations bitwise identical. +The logarithmic surface calculation is nonlinear, so reversing these operations +can change the result. Matching the order does not make the two horizontal +interpolation implementations bitwise identical. Both wind paths retain their final values in the fire-state `uf`/`vf` arrays. The 10 m path maps `u10`/`v10` directly before applying WAF; it no longer diff --git a/tests/regression/doc/deferred.md b/tests/regression/doc/deferred.md index d5f1fdc3..03d0266a 100644 --- a/tests/regression/doc/deferred.md +++ b/tests/regression/doc/deferred.md @@ -11,9 +11,9 @@ coverage limits. | --- | --- | | Intermediate-time 3D standalone/coupled agreement | Earlier diagnostics exceeded tolerance between saved PR endpoints. The bilinear-precision experiment passed an every-4-second GitHub comparison through 60 s. These are diagnostic results, not an extension of the PR acceptance schedule. See [validation history](validation.md). | | Longer standalone/coupled agreement | The 600 s terrain diagnostics still exceed the short-run comparison tolerance, including at 6.096 m with double-precision bilinear arithmetic. A longer-duration acceptance policy would need separate scientific review; it is not required for this PR. See [validation history](validation.md). | -| Humidity interpretation | Scientific review associated with PR #46 is independent of reproducibility; current moisture remains enabled. | +| Humidity interpretation | Scientific review of the humidity convention is independent of reproducibility; current moisture remains enabled. | | Restart | Standalone same-revision chains are covered by the [restart suite](restart.md). Coupled/WRF-host continuity and cross-revision compatibility remain untested. Known production failures remain failures. | -| PR #39 changes and method `(4,5)` | Separate scientific review and suitable comparison cases before matrix inclusion. | +| Method `(4,5)` | Separate scientific review and suitable comparison cases before matrix inclusion. | | ESMX_Data feedback | The generated coupled cases use WRF-data forcing. Legacy `testx` still exercises a distinct feedback fixture. | | Coupled OpenMP-only | MPI and hybrid NUOPC/ESMX are included in the PR matrix. Coupled runs using an OpenMP-only build remain outside this selection. | | Atmospheric hosts | WRF/UFS host integrations, including UFS mass-centred winds and any future staggered interpolation option. | diff --git a/tests/regression/doc/restart.md b/tests/regression/doc/restart.md index 9b89527b..71964e54 100644 --- a/tests/regression/doc/restart.md +++ b/tests/regression/doc/restart.md @@ -20,11 +20,12 @@ all times are seconds from the original 2020-01-01 00:00:00 origin. All integrations use `dt=4`, run through 60 s, write regular output every 12 s, and write checkpoints every 4 s. Time-based moisture updates occur at 4, 20, -36, and 52 s on the reviewed base. The split at 4 s precedes perimeter -activation; 12 and 28 s follow it. The 28 s split is between moisture updates +36, and 52 s. The split at 4 s precedes perimeter activation; 12 and 28 s +follow it. The 28 s split is between moisture updates and atmospheric records. Step-based moisture initializes at 4 s and first advances at 12 s using an 8 s interval, then advances every 12 s. These cases -exposed the delayed-initialization defect on P00 and test its P02 correction. +verify initialization before the first scheduled step-based moisture update +and restoration of that state on restart. A failure remains a failing CTest; the suite has no expected-failure conversion. Each case runs one uninterrupted integration and a separate cold first segment, @@ -43,7 +44,7 @@ the original origin, including output after a split between output events. | Fuel fraction and fuel fraction burned in the current timestep | REAL64 storage and a non-REAL32 signal at the final checkpoint; catches quantization even when stored as float64 | | Independent checkpoint validity | Required fields, finite/unmasked owned cells, expected clock and activation state, five moisture-class dimensions, initialized moisture at 4 s, expected cadence and last/next times, and moisture in (0,1) for these dry-forcing cases | -The PR #39 writer also saves method work arrays `lfn_0/1/2`, `lfn_s0/1/2/3` +The restart writer also saves method work arrays `lfn_0/1/2`, `lfn_s0/1/2/3` and unused crown fluxes `fcanhfx/fcanqfx`. Their values are not all initialized for these methods. They remain in the schema comparison but are explicitly listed as uncompared fields. Physical fields are not excluded to obtain a pass. @@ -66,13 +67,16 @@ historical `github-pr` reference, which does not contain their keys. The existing `pr` suite remains unchanged and checks its approved reference. No reference candidate is created or approved by this suite. -The suite additionally runs 30 restart-loading probes for P02 across the five -layouts. Each time-based case resumes a copy with PR #39 metadata (five new -attributes removed) and requires an exact checkpoint match. Four damaged copies -must reject an unset initialization flag, changed cadence, future last-update -time, or invalid previous pressure before writing output. Each step-based case -must reject the ambiguous legacy schema. These probes add 30 model invocations -and five checkpoint comparisons to the continuity counts above. Reports keep +The suite additionally runs 30 restart-loading probes for moisture initialization +across the five layouts. Each time-based case resumes a checkpoint copy with +legacy restart metadata and requires an exact checkpoint match. The legacy copy +omits five attributes recording the initialization flag and time, whether +moisture evolution is enabled, and the step-based and time-based update cadences. +Four additional probes must reject current checkpoints with an unset +initialization flag, changed cadence, future last-update time, or invalid previous +pressure before writing output. Each step-based case must reject the ambiguous +legacy schema. These probes add 30 model invocations and five checkpoint +comparisons to the continuity counts above. Reports keep expected rejection diagnostics separate from failed continuity comparisons. Synthetic legacy files exercise schema compatibility; they do not establish cross-revision equivalence. @@ -81,7 +85,8 @@ Restart experiments currently require the standalone driver. NUOPC/ESMX and WRF-host restart continuity are not established here. Existing coupled PR tests exercise ordinary integrations only. Cross-layout checkpoint transfers, legacy-file compatibility across revisions, fractional timesteps, and coupled -flux accumulation/reset are outside P00 and require their assigned sessions. +flux accumulation/reset are outside this suite's coverage and require separate +validation. ## Extend a restart experiment From 56b832037a8e73d198d395e1c194afbdb46ed205 Mon Sep 17 00:00:00 2001 From: Maria Frediani Date: Fri, 23 Oct 2026 09:31:44 -0600 Subject: [PATCH 19/19] Consolidate test guidance and require unit-test explanations --- tests/AGENTS.md | 54 +++++++++++++++++++++++++++++++++++++- tests/regression/AGENTS.md | 35 +++--------------------- 2 files changed, 56 insertions(+), 33 deletions(-) diff --git a/tests/AGENTS.md b/tests/AGENTS.md index 3127cea1..f863c510 100644 --- a/tests/AGENTS.md +++ b/tests/AGENTS.md @@ -1,4 +1,19 @@ -# Instructions for test documentation +# Instructions for tests + +These instructions apply to all tests below this directory, including Fortran +unit tests in `unit/` and the Python regression harness in `regression/`. + +## Unit-test comments + +Each new or modified unit test must include an informative comment explaining +the behavior it checks, why the setup exercises that behavior, and the expected +result or rejection. For files covering several scenarios, summarize the shared +purpose near the top and explain distinct cases where they are defined. +Include relevant scientific assumptions, units, timing, or boundary conditions. +Explain what a failure would mean; avoid comments that merely repeat assertions +or depend on internal planning labels or PR numbers. + +## User-facing documentation Write user-facing test documentation so readers can understand it without knowing the development history. Describe the scientific behavior, configuration, @@ -13,3 +28,40 @@ compatibility; "restart metadata" alone does not identify that distinction. Keep PR references, planning labels, revisions, and historical validation results in validation records such as regression/doc/validation.md or session notes. Preserve that provenance when revising user-facing guides. + +## Regression harness + +Read [regression/README.md](regression/README.md) before editing the harness. +Use the Google-based Python formatting in `regression/.style.yapf`, with +four-space indentation and an 80-column target. Run the pinned YAPF check +documented in that README. Keep the formatter out of runtime dependencies. + +Use descriptive snake_case names, pathlib paths, and public-function type hints. +Separate logical operations with blank lines. Explain scientific choices and +synthetic test substitutions in comments. A formatter pass does not replace +review for readability. + +Do not add personal filesystem paths or require a developer's Conda installation. +Use the module-provided environment documented in `regression/README.md` on NCAR +systems. `CFBM_TEST_TMP` is required for direct Python tests; CTest supplies a +build-tree artifact directory. Keep generated files in the repository's ignored +`build/` directory or outside the checkout, and retain failure evidence. Use PBS +for model/MPI runs and for the shared MPI-enabled Python environment. + +Python test filenames end in `_test.py`. Use unittest discovery with +`-p '*_test.py'` and confirm that both Python test files are collected after any +rename. Keep the repository's `compile.sh` responsible for build options, CTest +responsible for test invocation, and `regression/cases.yaml` responsible for the +scientific cases and executions. Do not change scientific tolerances or legacy +references to conceal failures. + +Documentation lives in `regression/doc/`. Keep `regression/README.md` as the entry +point. Update the case figures and comparison table when scientific settings or +outputs change. Read [contributing.md](regression/doc/contributing.md) and +[pr-checklist.md](regression/doc/pr-checklist.md) before extending the cases. +Keep historical fixtures in `legacy/` and preserve their reference data. + +Use exact namelist names in `regression/cases.yaml`. Cases own explicit +configurations; do not add generic option-category expansion. Compare only +matching case, configuration, and scale. No creation or assistance header is +required. diff --git a/tests/regression/AGENTS.md b/tests/regression/AGENTS.md index 3d55c2a4..913dc9d3 100644 --- a/tests/regression/AGENTS.md +++ b/tests/regression/AGENTS.md @@ -1,33 +1,4 @@ -# Instructions for regression harness changes +# Test instructions -Read README.md here before editing. Use the Google-based Python formatting in -.style.yapf, with four-space indentation and an 80-column target. Run the pinned -YAPF check documented in README.md. Keep the formatter out of runtime dependencies. - -Use descriptive snake_case names, pathlib paths, and public-function type hints. -Separate logical operations with blank lines. Explain scientific choices and -synthetic test substitutions in comments. A formatter pass does not replace -review for readability. - -Do not add personal filesystem paths or require a developer's Conda installation. -Use the module-provided environment documented in README.md on NCAR systems. -CFBM_TEST_TMP is required for direct Python tests; CTest supplies a build-tree -artifact directory. Keep generated files in the ignored build/ directory or -outside the checkout, and retain -failure evidence. Use PBS for model/MPI runs and for the shared MPI-enabled -Python environment. - -Python test filenames end in _test.py. Use unittest discovery with -p '*_test.py' -and confirm that both Python test files are collected after any rename. -Keep compile.sh responsible for build options, CTest responsible for test -invocation, and cases.yaml responsible for the scientific cases and executions. -Do not change scientific tolerances or legacy references to conceal failures. - -Documentation lives in doc/. Keep README.md as the entry point. Update the -case figures and comparison table when scientific settings or outputs change. -Read doc/contributing.md and doc/pr-checklist.md before extending the cases. -Keep historical fixtures in ../legacy/ and preserve their reference data. - -Use exact namelist names in cases.yaml. Cases own explicit configurations; do -not add generic option-category expansion. Compare only matching case, -configuration, and scale. No creation or assistance header is required. +The shared test instructions, including regression-harness guidance, are in +[tests/AGENTS.md](../AGENTS.md).