diff --git a/.github/workflows/ci_env.yml b/.github/workflows/ci_env.yml
index 46964190..b48f8072 100644
--- a/.github/workflows/ci_env.yml
+++ b/.github/workflows/ci_env.yml
@@ -42,7 +42,7 @@ jobs:
- name: Install lisflood-module
shell: bash -el {0}
run: |
- pip install .
+ pip install .[test]
- name: Check installation
shell: bash -el {0}
run: |
@@ -52,5 +52,4 @@ jobs:
- name: Test with pytest
shell: bash -el {0}
run: |
- pip install pytest pytest-cov
pytest -m "slow or not slow"
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
index ef5f354a..77e96443 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,6 +1,9 @@
*.gem
*.sublime-project
*.sublime-workspace
+.kiro
+.glm53flash
+.agents
.bundle
.DS_Store
.jekyll-metadata
@@ -27,3 +30,15 @@ lisflood_model.egg-info
.vscode/
*.ipynb
.ipynb_checkpoints/
+tws_integration_master_diff.md
+implementation_guide_scale_offset_packing.md
+lisflood_optimization_report.md
+lisflood_optimization_report_v2.md
+# version file auto-generated by setuptools_scm (do not track)
+src/lisflood/_version.py
+# output folders generated by the test suite / example runs
+tests/data/**/out/
+# generated profiling artifacts
+profiling/
+# debug variants of source modules (not part of the package)
+*_debug.py
diff --git a/.zenodo.json b/.zenodo.json
index 3e35c5f6..ad9938dd 100644
--- a/.zenodo.json
+++ b/.zenodo.json
@@ -81,9 +81,9 @@
"orcid": "0000-0002-2430-3634"
},
{
- "name": "Jensen, Laura",
- "affiliation": "GFZ Helmholtz Centre for Geosciences",
- "orcid": "0000-0002-8318-791X"
+ "name": "Jensen, Laura",
+ "affiliation": "GFZ Helmholtz Centre for Geosciences",
+ "orcid": "0000-0002-8318-791X"
},
{
"name": "Morrison, Oisin",
diff --git a/Dockerfile b/Dockerfile
index eac44765..a1f3c0c4 100755
--- a/Dockerfile
+++ b/Dockerfile
@@ -25,7 +25,6 @@ COPY src/lisflood/. /lisflood/
COPY src/lisf1.py /
COPY src/lisfloodSettings_reference.xml /
COPY LICENSE /
-COPY VERSION /
# RUN Tests
COPY tests/. /tests/
diff --git a/MANIFEST.in b/MANIFEST.in
index bb1d0bd5..167470f0 100755
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -1,3 +1,2 @@
include requirements.txt
include *.xml
-include VERSION
diff --git a/VERSION b/VERSION
deleted file mode 100644
index 0062ac97..00000000
--- a/VERSION
+++ /dev/null
@@ -1 +0,0 @@
-5.0.0
diff --git a/docs/3_step2_preparing-setting-file/index.md b/docs/3_step2_preparing-setting-file/index.md
index 3b991c83..2e495f4a 100644
--- a/docs/3_step2_preparing-setting-file/index.md
+++ b/docs/3_step2_preparing-setting-file/index.md
@@ -751,12 +751,41 @@ Within the 'lfoptions' element of the settings file, each option is defined usin
+ **NetCDFTimeChunks**: chunking size in the time dimension. This option was implemented to optimize the loading of the meteo maps. Recommended value is “auto" but chunking size can be specified manually (e.g. "10") or set to “-1" to load the whole time series into memory (very fast but expensive in terms of memory).
+ **MapsCaching** (True or False): option designed for lisflood calibration. If set to True, all the static maps and forcings will be stored in a cache so that they don't have to be loaded by each lisflood instance. This option sets the value of NetCDFTimeChunks to "-1", meaning that the whole time series in the NetCDF inputs is loaded into memory.
++ **NetCDFTimeChunksPrefetch** (1 or 0, default 0): when set to 1, the next forcing time chunk is loaded in a background thread while the model is still computing with the current one, so the chunk boundary no longer stalls the run. This only helps when forcing reads are slow compared to the computation (e.g. forcings on a network file system); on fast local disks the benefit is negligible. It requires `readNetcdfStack=1` and has no effect when `NetCDFTimeChunks` results in a single chunk (e.g. `-1` or `MapsCaching=1`), because there is no next chunk to load. Model outputs are identical with the option on or off.
### Options to manage output files
+ **OutputMapsChunks**: this option is used to write output maps every X steps (default 1).
+ **OutputMapsDataType**: this option sets the output data type and may take the following values: "float64" or "float32" (default float64). This option applies to all output maps.
+### Option to select the reporting engine
++ **useEarthkitHydro** (1 or 0, default 1): selects the engine used for the river-network reductions of the reporting step, i.e. upstream accumulation (`accuflux`/`catchmenttotal`-type series) and the zonal means at gauge locations written to time series. When 1, these are computed with [earthkit-hydro](https://github.com/ecmwf/earthkit-hydro) directly on the model's compressed 1D arrays, which avoids the conversions to and from 2D PCRaster maps and the single-threaded PCRaster engine; it reduces run time compared to PCRaster. When 0, the original PCRaster operations are used. If the `earthkit-hydro` package is not installed, LISFLOOD falls back to the PCRaster path automatically. Install it with `pip install earthkit-hydro` (listed in `requirements.txt`). To force the PCRaster path (for example to reproduce results of earlier versions), add to the settings file:
+```xml
+
+```
+The thread pool used by earthkit-hydro can be limited with `numCPUs_earthkit` (see below).
+
+
+LISFLOOD uses several independent parallelization mechanisms. They all affect run time only, never results: model outputs are identical regardless of the values below. Every `numCPUs_*` option shares the same value legend: **absent or `"auto"`** selects the automatic policy (the default; each option's policy is described below), **`"all"` or `"0"`** forces all available CPU cores, and a **positive integer** sets a fixed thread/process count. Left at `auto`, LISFLOOD sizes each pool automatically from the host core count and the domain size, so most users do not need to set these at all.
+
+These are `lfuser`/`lfbinding` variables (add them like any other `textvar`), for example:
+
+```xml
+
+
+
+
+
+```
+
++ **numCPUs_parallelNumba**: threads for the **land-phase** Numba kernels (the soil/vegetation loop, water abstraction, surface routing). `auto` is domain-size-aware: serial on small catchments (where thread fork/join overhead makes multi-threaded execution slower) and all cores on large/global domains. `0`/`all` forces all cores; a positive integer sets a fixed count.
++ **numCPUs_routingNumba**: threads for the **channel-routing substep** Numba kernels (kinematic-wave / Muskingum-Cunge), capped *independently* of the land-phase pool above. These kernels run many times per model step with a small per-call payload, so beyond a modest thread count the OpenMP fork/join cost dominates and more threads make routing *slower*. `auto` applies a host-relative cap (about half the cores, clamped to the 8–16 band, never above the land-phase count); `all`/`0` removes the routing-specific cap; a positive integer sets a fixed count. This is scheduling only — routing results are identical for any thread count.
++ **numCPUs_BLAS**: threads for the BLAS/OpenMP backend of NumPy/SciPy. In LISFLOOD, BLAS is used only by the cold-start soil-hydraulics least-squares solve (a scalar per-pixel fit, no matrix algebra) and is idle during the dynamic loop, so extra BLAS threads do not speed the model up and only risk oversubscribing against the soil-init process pool. `auto` is therefore a single thread (1); `0`/`all` forces all cores; a positive integer sets a fixed count.
++ **numCPUs_soilInit**: number of worker **processes** used to solve the soil layer-2 saturation during a cold start. This is a per-pixel SciPy solve that can dominate initialization time on large/global domains (turning it from serial minutes/hours into a few minutes with many cores). This is process-based parallelism, independent of the thread options above. Default `0` (use all available cores); the solve is bit-identical regardless of the worker count.
++ **numCPUs_earthkit**: thread count of the earthkit-hydro (rayon) pool used when `useEarthkitHydro` is 1. Default (absent or `auto`) is 1, which avoids oversubscribing the node against the routing kernel threads; a positive integer caps the pool to that many threads; `0` or `all` leaves it uncapped. Thread count affects scheduling only, not results.
+ - **Set `numCPUs_soilInit` to `1` when an outer tool already runs many LISFLOOD instances in parallel** — most importantly [lisflood-calibration](https://github.com/ec-jrc/lisflood-calibration), which runs one LISFLOOD per DEAP individual and expects each instance to use a single core. As a safeguard, this parallelization automatically disables itself (falls back to serial) when LISFLOOD detects it is running inside a forked/daemon worker, so an accidental high value cannot cause oversubscription under calibration.
++ **modelPrecision**: floating-point precision of the LISFLOOD model state and the kinematic-wave routing kernel. This is a performance/precision lever, separate from the thread options above. Default `float64` reproduces previous LISFLOOD output bit-for-bit (use it for calibration and any bitwise-reproducibility need). Setting it to `float32` makes the routing kernel faster by halving its memory traffic; results are **not** bit-identical but stay within the LISFLOOD reference tolerance (`atol=1e-4, rtol=1e-3`) on tested catchments, with discharge and channel storage differing only at the last reported digit. The Muskingum-Cunge-Todini (MCT) routing and all mass-balance reductions always stay `float64` regardless of this setting. An unknown value falls back to `float64` (precision is never silently degraded).
+
### Reference settings file
In order to facilitate the preparation of the settings file, a complete example is provided [here](https://github.com/ec-jrc/lisflood-code/tree/master/src/lisfloodSettings_reference.xml). The user is encouraged to update the paths, the names of the maps and of the tables in the provided template.
diff --git a/docs/3_step6_model-output/index.md b/docs/3_step6_model-output/index.md
index 08e6982d..27847d63 100644
--- a/docs/3_step6_model-output/index.md
+++ b/docs/3_step6_model-output/index.md
@@ -44,3 +44,140 @@ Time series file have *.tss* extension and they can be opened with any text edit
It is important to be aware of the spatial domain for which each time series is computed. For instance, all *rate variables* are reported as pixel-average values. Soil moisture and groundwater storage are reported for the permeable fraction of each pixel only. The reported snow cover is the average of the snow depths in snow zones A, B and C.
This [**Annex**](../5_annex_output-files/index.md) summarises most of the options to report additional output maps.
+
+## Output data compression (int16 packing)
+
+By default, output maps are written as floating-point values (`float32` or `float64`, controlled by the `OutputMapsDataType` setting). For large simulations (long time series, global domains, many variables) this can produce very large files.
+
+LISFLOOD offers an optional lossy compression that stores output maps as signed 16-bit integers (`int16`) using the CF-convention `scale_factor`/`add_offset` packing scheme. This roughly halves the file size compared to `float32` (2 bytes per value instead of 4) and reduces it fourfold compared to `float64`.
+
+### How to enable it
+
+Add the following to the settings file:
+
+```xml
+
+```
+
+- `"False"` (default): output maps are written as raw floating-point values.
+- `"True"`: output maps are packed into `int16` using per-variable `scale_factor` and `add_offset`.
+
+No further user action is required. The `scale_factor` and `add_offset` for each variable are defined internally (see `default_options.py`). Standard netCDF readers (xarray, CDO, NCO, ncview, Panoply) automatically unpack the values on read, so from a user perspective the data appears as normal floating-point.
+
+### Packing and unpacking formulas
+
+When writing, each floating-point value is encoded into an integer:
+
+```
+packed_int = round( (value - add_offset) / scale_factor )
+```
+
+When reading, the CF-compliant reader restores the physical value:
+
+```
+value = packed_int * scale_factor + add_offset
+```
+
+The packed integer is stored in the range `[-32766, +32767]`. The value `-32767` is reserved as the fill value (`_FillValue`) for masked / outside-domain pixels, so it is never used for real data.
+
+### Precision and value range
+
+The `scale_factor` directly determines both the resolution and the representable range of a packed variable:
+
+- **Resolution (quantization step):** equal to `scale_factor`. A value read back can differ from the original by at most `scale_factor / 2`.
+- **Representable range:**
+ - minimum = `add_offset + scale_factor * (-32766)`
+ - maximum = `add_offset + scale_factor * (+32767)`
+
+For a target physical range `[min, max]`, the parameters are chosen as:
+
+```
+scale_factor = (max - min) / 65533
+add_offset = (min + max) / 2
+```
+
+(65533 is the number of usable integer levels, i.e. `32767 - (-32766)`.)
+
+**Example** — actual soil evaporation (`ESActMaps`) uses `scale_factor = 3.1e-4`, `add_offset = 10.0`:
+- minimum = `10.0 + 3.1e-4 * (-32766)` ≈ `0.0 mm/day`
+- maximum = `10.0 + 3.1e-4 * (+32767)` ≈ `+20.2 mm/day`
+- resolution ≈ `3.1e-4 mm/day` (values accurate to about `±1.5e-4 mm/day`)
+
+The `add_offset` is placed at the centre of the target range (here 10.0, the midpoint of roughly 0–20 mm/day), so the available integer levels are distributed symmetrically around it.
+
+Values falling outside the representable range are clipped to the nearest bound, and a warning is issued once per variable during the run. Because packing is lossy, a tiny quantization error is expected: for instance an input of exactly `0.0` may be read back as a value on the order of `1e-4`. This is far below any hydrologically meaningful threshold and is normal for packed datasets (the same behaviour applies to ERA5, CMIP6, and other CF-packed products).
+
+### Variables that are never packed
+
+- **State and "end" maps** (used to initialise warm-start runs) are always written at full floating-point precision, regardless of the `OutputPacking` setting. This preserves the exact model state needed for a bit-reproducible restart.
+- Variables whose value range is too wide to pack meaningfully with a *linear* scale/offset (for example **discharge**, which spans several orders of magnitude globally) are intentionally excluded from `OutputPacking`. Discharge has its own dedicated logarithmic packing option, described next.
+
+### Logarithmic packing of discharge (`OutputPackingDischarge`)
+
+Discharge cannot be packed with the linear `scale_factor`/`add_offset` scheme without unacceptable precision loss, because on a global domain it spans several orders of magnitude (from near-zero headwater flows to hundreds of thousands of m³/s at major river mouths). A single linear step size is either far too coarse for small flows or wastes the entire integer range on the largest ones.
+
+`OutputPackingDischarge` solves this by packing discharge in **log10 space**, which gives near-constant *relative* precision (about 0.027% per step) across the whole range.
+
+This option is **completely independent** of `OutputPacking`. You can enable either, both, or neither:
+
+```xml
+
+```
+
+- `"False"` (default): discharge is written as raw floating-point values.
+- `"True"`: the `DischargeMaps` output is packed into `int16` using the logarithmic encoding below.
+
+**Encoding.** The resolved range is `0.01` to `500000` m³/s. The transform is:
+
+```
+stored = round( (log10(q) - packing_log_min) / packing_log_scale ) + packing_int_offset
+q = 10 ** ( packing_log_min + packing_log_scale * (stored - packing_int_offset) )
+```
+
+**This is a non-CF encoding**, so standard netCDF readers will **not** auto-unpack it via `scale_factor`/`add_offset`. Instead, the writer tags the variable with explicit custom attributes describing the transform so a post-processing step can invert it:
+
+| Attribute | Meaning |
+|-----------|---------|
+| `packing` | `"log10"` — identifies the scheme |
+| `packing_log_min` | `log10(0.01) = -2.0` |
+| `packing_log_scale` | log-space step size per integer level |
+| `packing_int_offset` | integer offset (`-32766`) mapped to `packing_log_min` |
+| `packing_valid_min` / `packing_valid_max` | resolved physical range (`0.01` / `500000` m³/s) |
+
+**Value handling.**
+- Discharge **below 0.01 m³/s** — including zero, negative, and missing values — is stored as `_FillValue` (read back as masked/NaN). Such values carry no useful information at this precision. **Please note that same `_FillValue` is also used in masked region, thus you will need mask map to check whether the value is outside the mask or is just below 0.01**
+- Discharge **above 500000 m³/s** is clipped to the maximum.
+- As with any lossy packing, a small relative quantization error (~0.027% per step, well under 0.02% round-trip) is expected.
+
+**Interaction with temporal aggregation.** Monthly/yearly **mean** discharge keeps log packing (it stays within the resolved range). **Sum**-aggregated discharge disables log packing automatically (a period sum can exceed 500000 m³/s), and is written as floating-point — the same policy used for linear packing.
+
+## Temporal aggregation of output maps
+
+Instead of writing a value at every model time step, LISFLOOD can aggregate selected output variables to daily, monthly or yearly means or sums. This dramatically reduces output volume for long simulations while retaining the climatologically relevant signal.
+
+Six settings control temporal aggregation, each taking a semicolon-separated list of output variable names:
+
+```xml
+
+
+
+
+
+
+```
+
+- **Mean** aggregation writes the average of all time steps within each period.
+- **Sum** aggregation writes the accumulated total over each period.
+- A value is written only at the end of a completed period (day-end, month-end or year-end). Partial periods at the end of a simulation are not written.
+- When a variable is listed for aggregation, its normal per-time-step output is suppressed to avoid duplication.
+
+The aggregation boundaries are detected directly from the calendar date and therefore work for daily as well as sub-daily time steps. Daily aggregation is mainly useful with sub-daily steps (e.g. 6-hourly), where it writes one value per calendar day; with a daily time step it reproduces the per-step values.
+
+**Requirement:** temporal aggregation is only available with netCDF output (`writeNetcdf` or `writeNetcdfStack`). Listing a variable in an aggregation setting while writing PCRaster maps stops the run at start-up with an error.
+
+**Threaded output (`outputThreads`):** when enabled, map output is written by background threads. The output is identical to synchronous output, including aggregated values and file or time-step selection: the model clock state needed for a write (step, date, end-of-run flag, reporting checkpoint) is captured when the write is staged.
+
+**Interaction with packing:**
+
+- **Mean** aggregates stay within the same value range as the daily values, so `int16` packing (if `OutputPacking = True`) is applied as usual.
+- **Sum** aggregates can greatly exceed the daily value range (a monthly sum can be up to ~30× a daily value). To prevent overflow, packing is automatically disabled for sum-aggregated outputs, which are always written as floating-point.
diff --git a/docs/4_Static-Maps_reservoirs-lakes/index.md b/docs/4_Static-Maps_reservoirs-lakes/index.md
index bdde1f94..ce38feeb 100644
--- a/docs/4_Static-Maps_reservoirs-lakes/index.md
+++ b/docs/4_Static-Maps_reservoirs-lakes/index.md
@@ -4,6 +4,8 @@ Lakes and reservoirs can be defined as a significant volume of water, which occu
The modelling of lakes and reservoirs requires three maps and a set of txt files.
+For the generation of total water storage output (option `repTWSMaps`), two additional maps are needed to distribute the storage mass over the extent of the lakes/reservoirs (see [Lake and reservoir extent maps](#lake-and-reservoir-extent-maps) below).
+
## Lake mask map
@@ -20,7 +22,7 @@ The lake mask map represents the area covered by lakes and reservoirs, it is use
| :---| :--- | :--- | :--- |
|Global Lakes and Wetlands Database (GLWD):
Large Lake Polygons (Level 1) |[GLWD leve1](https://www.worldwildlife.org/publications/global-lakes-and-wetlands-database-large-lake-polygons-level-1)|2004|Global, 1:1 to 1:3 million resolution|
|Global Lakes and Wetlands Database (GLWD):
Small Lake Polygons (Level 2) |[GLWD leve1](https://www.worldwildlife.org/publications/global-lakes-and-wetlands-database-large-lake-polygons-level-2)|2004|Global, 1:1 to 1:3 million resolution|
-|Fraction of inland water| It can be prepared by using
the methodology explained [here](../4_Static-Maps_land-use#land-use)|NA|Global|
+|Fraction of inland water| It can be prepared by using
the methodology explained [here](../4_Static-Maps_land-use#land-use/index.md)|NA|Global|
### Methodology
@@ -54,7 +56,7 @@ The reservoirs map shows the outflow location of each reservoir: each outflow po
|Reservoir normal outflow| res_normal_outflow.txt;
2 columms: ID VALUE; 1 row for each reservoir|Units: m3/s|Normal outflow|
|Reseervoir minimum outflow| res_min_outflow.txt;
2 columms: ID VALUE; 1 row for each reservoir|Units: m3/s|Minimum outflow|
-The well-known Global Reservoir and Dam Database[GDW](https://www.globaldamwatch.org/grand) now superseeded by the Global Dam Watch [GDW](https://www.globaldamwatch.org/database) is a relevant example of source of data for lakes map and tables.
+The well-known Global Reservoir and Dam Database [GDW](https://www.globaldamwatch.org/grand) now superseeded by the Global Dam Watch [GDW](https://www.globaldamwatch.org/database) is a relevant example of source of data for lakes map and tables.
### Methodology
As a first step, it is recommended to create a file including all reservoir information required by OS LISFLOOD and some relevant metadata that can help with model analysis and results description.
@@ -137,7 +139,7 @@ The following paragraphs provide guidelines for the generation of the lake map a
Lake unique identifier (1) and coordinates of the outlet mapped on the OS LISFLOOD local drainage direction map ([ldd](../4_Static-Maps_topography/index.md)) (3) are required to generate the lake map. Geographic coordinates of the lake outlet (2) and OS LISFLOOD local drainage direction map ([ldd](../4_Static-Maps_topography/index.md)) are essential to generate (3). Adequate model representation requires the agreement between lake catchment area (7) and OS LISFLOOD [upstream area map](../4_Static-Maps_topography/index.md).
-Lake surface area can be retrieved from local datasets or global datasets such as HydroLAKES](https://www.hydrosheds.org/products/hydrolakes), [GLWD](https://www.hydrosheds.org/products/glwd), [GRAND](https://www.globaldamwatch.org/grand).
+Lake surface area can be retrieved from local datasets or global datasets such as [HydroLAKES](https://www.hydrosheds.org/products/hydrolakes), [GLWD](https://www.hydrosheds.org/products/glwd), [GRAND](https://www.globaldamwatch.org/grand).
Where lake outlet width cannot be retrieved from external dataset, it can be measured with GIS tools.
@@ -146,4 +148,32 @@ Finally, lake average inflow can be retrieved from observed time series (where a
Lake maps and tables of the European 1arcmin domain and global 3arcmin domain are based on information from [HydroLAKES](https://www.hydrosheds.org/products/hydrolakes): waterbodies classified in HydroLakes as natural lake were considered for inclusion into the lakes dataset.
Lakes included in the European 1arcmin domain had a minimum volume of 10 hm3, a minimum lake surface area of 5 km2, a minimum upstream catchment area of 50 km2.
Lakes included in the global 3arcmin domain had a minimum volume of 100 hm3, a minimum lake surface area of 50 km2, a minimum upstream catchment area of 250 km2.
-Lake outlet width was generally measured with GIS tools; lake average inflow was computed using OS LISFLOOD CEMS EFAS and CEMS GloFAS discharge reanalysis (GloFASv4 reanalysis for GloFASv5 tables; EFASv5 naturalized flow for EFASv6 tables).
\ No newline at end of file
+Lake outlet width was generally measured with GIS tools; lake average inflow was computed using OS LISFLOOD CEMS EFAS and CEMS GloFAS discharge reanalysis (GloFASv4 reanalysis for GloFASv5 tables; EFASv5 naturalized flow for EFASv6 tables).
+
+
+## Lake and reservoir extent maps
+
+The lake and reservoir modules output lake/reservoir levels at the outlet locations (output files `lakeh.nc` for locations in `lakes.nc` and `res.nc`). For the generation of total water storage maps (option `repTWSMaps`), these levels have to be converted to mass changes and spatially distributed over the extent of the lakes/reservoirs. For this, two additional maps are needed that contain the lake/reservoir ID for each pixel belonging to the area covered by the respective lake/reservoir.
+
+### General map information
+
+| Map/table name | File name; type | Units; range | Description |
+| :--- | :--- | :--- | :--- |
+| Lake extent | lake_extent.nc;
Type: Float32 | Units: -;
Range: integer ID number to identify each lake | Lake ID (ID of outflow location) for each pixel belonging to the surface extent of the lake |
+| Reservoir extent | res_extent.nc;
Type: Float32 | Units: -;
Range: integer ID number to identify each reservoir | Reservoir ID (ID of outflow location) for each pixel belonging to the surface extent of the reservoir |
+
+### Methodology
+
+The creation of the lake extent map requires:
+
+1. Lake outflow location with unique lake identifier (e.g., `lakes.nc`)
+2. External local or global dataset containing the vectorized polygons of the lake outlines (e.g., in shapefile format), such as [HydroLAKES](https://www.hydrosheds.org/products/hydrolakes), as well as the surface area of the lake
+3. Table relating the unique lake identifier of OS LISFLOOD to the lake identifier in the external data set
+
+The creation of a raster file fulfilling the criteria for a consistent lake extent map (e.g., lakes with an area smaller than the grid cell size must not be neglected; lakes assigned to only one grid cell must not be overwritten by other lakes) is not straight forward, thus a standard rasterize command is not appropriate. A dedicated script should loop over each lake defined in the outlet location file and perform the following actions:
+
+- If the total lake area is too small in comparison to the grid cell area (e.g., smaller than 5% of the cell area), the outlet location pixel is set as the lake extent.
+- Otherwise, the corresponding polygon from the shapefile is intersected with the grid cells and each grid cell containing a fraction of the lake extent is checked. If the lake covers a sufficiently large fraction (e.g., more than 7%) of the grid cell, it is assigned the lake ID, else it is neglected.
+- If no intersection fraction is large enough (e.g., if the lake is very thin but long), the outlet location pixel is set as the lake extent.
+
+The creation of the reservoir extent map can be done analogously to the lake extent map using the same methodology.
diff --git a/docs/5_annex_output-files/index.md b/docs/5_annex_output-files/index.md
index fd7cfbf0..cbe65dfb 100644
--- a/docs/5_annex_output-files/index.md
+++ b/docs/5_annex_output-files/index.md
@@ -61,7 +61,7 @@ Output time series can be classified in the following categories:
| infiltration | $\frac{mm}{timestep}$ | InfiltrationAvUpsTS | infiltrationUps.tss |
| preferential (bypass) flow | $\frac{mm}{timestep}$ | PrefFlowAvUpsTS | prefFlowUps.tss |
| percolation upper to lower soil layer | $\frac{mm}{timestep}$ | PercolationAvUpsTS | dTopToSubUps.tss |
-| percolation lower soil layer to subsoil | $\frac{mm}{timestep}$ | SeepSubToGWAvUpsTS | dSubToUzUps.tss |
+| percolation lower soil layer to Upper Zone | $\frac{mm}{timestep}$ | SeepSubToGWAvUpsTS | dSubToUzUps.tss |
| surface runoff | $\frac{mm}{timestep}$ | SurfaceRunoffAvUpsTS | surfaceRunoffUps.tss |
| outflow from upper zone | $\frac{mm}{timestep}$ | UZOutflowAvUpsTS | qUzUps.tss |
| outflow from lower zone | $\frac{mm}{timestep}$ | LZOutflowAvUpsTS | qLzUps.tss |
@@ -148,7 +148,9 @@ To speed up the pre-run and to prevent that results are taken from the pre-run,
| storage in lower groundwater zone | repLZMaps | $mm$ | LZMaps | lz |
| number of days since last rain | repDSLRMaps | $days$ | DSLRMaps
DSLRForestMaps | dslr
dslF |
| frost index | repFrostIndexMaps | $\frac{°C}{days}$ | FrostIndexMaps | frost |
-| Total Water Storage | repTotalWaterStorageMaps | $mm$ | TotalWaterStorageMaps | tws |
+| Total Water Storage | repTWSMaps | $m$ | TWSMaps | TotalWS |
+| Individual Water Storage Compartments | repStorageMaps | $m$ | LakeSMaps
RiverSMaps
SoilSMaps
GWSMaps
SnowSMaps
CumSMaps | Lakestor
Riverstor
Soilstor
GWstor
Snowstor
Cumstor |
+| Flow Momentum | repFlowMomMaps | $\frac{kgm}{s}$ | FlowMomMaps | FlowMomentum |
| **RATE VARIABLES** | | | | |
| rain (excluding snow) | repRainMaps | $\frac{mm}{timestep}$ | RainMaps | rain |
| snow | repSnowMaps | $\frac{mm}{timestep}$ | SnowMaps | snow |
@@ -157,7 +159,7 @@ To speed up the pre-run and to prevent that results are taken from the pre-run,
| actual transpiration | repTaMaps | $\frac{mm}{timestep}$ | TaMaps | tact |
| rainfall interception | repInterceptionMaps | $\frac{mm}{timestep}$ | InterceptionMaps
InterceptionForestMaps | int
intF |
| evaporation of intercepted water | repEWIntMaps | $\frac{mm}{timestep}$ | EWIntMaps | ewint |
-| leaf drainage | repLeafDrainageMaps | $\frac{mm}{timestep}$ | LeafDrainageMaps
LeafDrainageForestMaps | ldra
draF |
+| leaf drainage | repLeafDrainageMaps | $\frac{mm}{timestep}$ | LeafDrainageMaps
LeafDrainageForestMaps no | ldra
draF |
| infiltration | repInfiltrationMaps | $\frac{mm}{timestep}$ | InfiltrationMaps
InfiltrationForestMaps | inf
infF |
| preferential (bypass) flow | repPrefFlowMaps | $\frac{mm}{timestep}$ | PrefFlowMaps
PrefFlowtherMaps
PrefFlowForestMaps
PrefFlowIrrigationMaps | pflowpixel
pflow
pflowF
pflowi |
| percolation upper to lower soil layer | repPercolationMaps | $\frac{mm}{timestep}$ | Percolation1ato1bOtherMaps
Percolation1to1bForestMaps
Percolation1ato1bIrrigationMaps
Percolation1bto2OtherMaps
Percolation1bto2ForestMaps
Percolation1bto2IrrigationMaps | Percolation1ato1bOther
Percolation1ato1bForest
Percolation1to2Irrigation
Percolation1bto2Other
Percolation1bto2Forest
Percolation1bto2Irrigation |
@@ -181,8 +183,9 @@ The users should be aware that some state maps are generated only if the relevan
**Note**
-Some cumulative storages and volumes are computed internally by LISFLOOD. Some relevant example is described below:
-- Total Water Storage is the total water volume stored in channels, lakes, reservoirs, snow cover, sealed surfaces depressions, surface runoff, canopy interception, upper and lower groundwater zones.
+Some cumulative storages and volumes are computed internally by LISFLOOD. Some relevant examples are described below:
+- Total Water Storage is the total water mass (expressed in equivalent water height) stored in channels, lakes, reservoirs, all soil layers, snow cover, sealed surfaces depressions, surface runoff, canopy interception, uppper and lower groundwater zones. Using the option "repStorageMaps" outputs the individual storage compartments (Rivers, Lakes/Reservoirs, Soil, Groundwater, Snow, Interception storage) which add up to TWS ("repTWSMaps"). For the computation of mass stored in lakes and reservoirs (i.e. "simulateLakes" and/or "simulateReservoirs" is set to 1), the initial LISFLOOD output of lake levels centered at the outlet location are distributed over the actual extents of the lakes/reservoirs, which requires two additional input maps when using the options "repTWSMaps" and "repStorageMaps" (see [Reservoirs and lakes](../4_Static-Maps_reservoirs-lakes/)).
+- Flow Momentum is the product of flow velocity and river mass, which is needed for special applications, e.g. the computation of Earth rotation excitation functions.
- Surface runoff is the sum of direct runoff (from sealed and water fractions) and runoff generated by the pervious land cover fractions (forest, irrigation, other).
- Total runoff is the sum of surface runoff and sub-surface runoff. Sub-surface runoff is the outflow from upper and lower groundwater zones.
diff --git a/docs/5_annex_settings_and_options/index.md b/docs/5_annex_settings_and_options/index.md
index 18effa70..e495191f 100644
--- a/docs/5_annex_settings_and_options/index.md
+++ b/docs/5_annex_settings_and_options/index.md
@@ -39,6 +39,8 @@ The table below presents the list of available switches to activate optional mod
| IO | readNetcdfStack | switch (1 or 0) | nan | Read meteorological data in NetCDF format (Precip, Tavg, ET0, E0,ES0) |
| IO | writeNetcdfStack | switch (1 or 0) | nan | Write NetCDF stacks for output files (the pr.nc is read to get the metadata like projection) |
| IO | writeNetcdf | switch (1 or 0) | nan | Write NetCDF files for END files (single netcdf) |
+| IO | NetCDFTimeChunksPrefetch | switch (1 or 0, default = 0) | nan | Prefetch the next NetCDF forcing time chunk in a background thread while the current one is consumed, overlapping chunk I/O with model computation. Requires `readNetcdfStack=1` and has no effect unless `NetCDFTimeChunks` defines more than one chunk (e.g. `-1` loads everything upfront, so there is no next chunk). Intended for runs where forcing reads are slow relative to computation (e.g. network file systems); outputs are identical to the synchronous loader. Default 0 = historical synchronous loading. |
+| IO | useEarthkitHydro | switch (1 or 0, default = 1) | nan | Compute reporting reductions (upstream accumulation, gauge zonal means) with earthkit-hydro instead of PCRaster. Falls back to PCRaster if earthkit-hydro is not installed. Set to 0 to force the PCRaster path. |
| DISCHARGE | repDischargeTs | switch (1 or 0, default = 1) rep tss | output | Report discharge time series at gauges |
| LOG | repMBTs | switch (1 or 0) rep tss | output | Report timeseries of absolute cumulative mass balance error |
| STATE | repStateSites | switch (1 or 0) rep tss | output | Report state variables at sites |
diff --git a/environment.yml b/environment.yml
index 46f8a41d..908ae794 100644
--- a/environment.yml
+++ b/environment.yml
@@ -291,7 +291,6 @@ dependencies:
- networkx==3.6.1
- nine==1.2.0
- numba==0.65.1
- - numexpr==2.14.1
- numpngw==0.1.4
- numpy==2.2.6
- odclib==1.6.3.19
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 00000000..da695298
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,92 @@
+[build-system]
+requires = ["setuptools>=65", "setuptools_scm[toml]>=8"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "lisflood-model"
+description = "LISFLOOD model python module"
+authors = [
+ { name = "Ad de Roo" },
+ { name = "Emiliano Gelati" },
+ { name = "Peter Burek" },
+ { name = "Johan van der Knijff" },
+]
+maintainers = [
+ { name = "Stefania Grimaldi" },
+ { name = "Timo Schaffhauser" },
+ { name = "Carlo Russo" },
+ { name = "Cinzia Mazzetti" },
+ { name = "Corentin Carton De Wiart" },
+]
+license = "EUPL-1.2"
+keywords = ["lisflood", "lisvap", "efas", "glofas", "copernicus", "ecmwf"]
+requires-python = ">=3.10"
+classifiers = [
+ "Development Status :: 5 - Production/Stable",
+ "Intended Audience :: Developers",
+ "Intended Audience :: Education",
+ "Intended Audience :: Financial and Insurance Industry",
+ "Intended Audience :: Other Audience",
+ "Intended Audience :: Science/Research",
+ "Operating System :: Unix",
+ "Operating System :: POSIX",
+ "Operating System :: Microsoft :: Windows",
+ "Operating System :: MacOS :: MacOS X",
+ "Programming Language :: Python",
+ "Programming Language :: Python :: 3 :: Only",
+ "Programming Language :: Python :: 3.10",
+ "Programming Language :: Python :: 3.11",
+ "Programming Language :: Python :: 3.12",
+ "Topic :: Scientific/Engineering :: Physics",
+]
+# version is derived from git by setuptools_scm; readme and (runtime)
+# dependencies are resolved from README.md and requirements.txt
+# (see [tool.setuptools.dynamic]).
+dynamic = ["version", "readme", "dependencies"]
+
+[project.optional-dependencies]
+test = [
+ "pytest>=6.2.5",
+ "pytest-cov>=3.0.0",
+ "pytest-mock>=3.6.1",
+ "pytest-xdist>=3.0.0",
+ "coverage>=6.0",
+]
+
+[project.urls]
+homepage = "https://github.com/ec-jrc/lisflood-code"
+repository = "https://github.com/ec-jrc/lisflood-code"
+documentation = "https://ec-jrc.github.io/lisflood-code/"
+issues = "https://github.com/ec-jrc/lisflood-code/issues"
+
+# ---------------------------------------------------------------------------
+# Versioning
+# ----------
+# The package version is managed by setuptools_scm and derived from git tags.
+# - On branches/dev: version is auto-generated (e.g. 5.0.1.dev61), so every
+# commit produces a distinct version number.
+# - For releases: the publish/testpypi commands in setup.py accept a --version
+# argument to create the git tag and build the package with that exact version.
+# ---------------------------------------------------------------------------
+[tool.setuptools_scm]
+version_file = "src/lisflood/_version.py"
+version_file_template = '''# Do not change! Do not track in version control!
+__version__ = "{version}"
+'''
+# get version from GitHub-like tarballs (e.g. lisflood-code-5.0.0.tar.gz)
+parentdir_prefix_version = 'lisflood-code-'
+# used when building outside a git checkout with no version file present
+fallback_version = '5.0.0'
+# omit the local +g segment so builds are uploadable to PyPI
+local_scheme = "no-local-version"
+
+[tool.setuptools]
+package-dir = { "" = "src" }
+include-package-data = true
+
+[tool.setuptools.packages.find]
+where = ["src"]
+
+[tool.setuptools.dynamic]
+readme = { file = ["README.md"], content-type = "text/markdown" }
+dependencies = { file = ["requirements.txt"] }
diff --git a/pytest.ini b/pytest.ini
index 4736beac..3fc566d6 100755
--- a/pytest.ini
+++ b/pytest.ini
@@ -14,4 +14,15 @@ env =
D:NUMEXPR_MAX_THREADS=12
; D:PCRASTER_NR_WORKER_THREADS=4 gives errors with pcraster.multicore - need changes in many hydro modules
; run pytest tests/ -m "slow" before release !!!
-addopts = -ra -x -l --cov=lisflood --cov-config=.coveragerc -m "slow or not slow" -vv
+; Parallel test options (default): run the suite across CPU cores with pytest-xdist
+; while still collecting coverage (pytest-cov combines per-worker coverage into one
+; report). "-n auto" uses all available cores; "--dist loadscope" keeps every test
+; in a class on the same worker, which is required for safe parallelism here because
+; tests reuse per-class output dirs and different files share 'out/' leaf names
+; (cross-worker collisions are avoided by the per-worker output isolation in
+; tests/test_utils.py). "-x" is omitted because stop-on-first-failure is racy under
+; xdist. This runs both locally and in CI at full speed with coverage.
+addopts = -ra -l --cov=lisflood --cov-config=.coveragerc -m "slow or not slow" -vv -n auto --dist loadscope
+; Serial fallback (uncomment this line and comment the parallel one above) - useful
+; for debugging a single test or when pytest-xdist is unavailable; keeps -x.
+; addopts = -ra -x -l --cov=lisflood --cov-config=.coveragerc -m "slow or not slow" -vv
diff --git a/requirements.txt b/requirements.txt
index 36976755..05400906 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,12 +1,11 @@
attrs>=19.3.0
beautifulsoup4>=4.8.1
cftime>=1.0.4.2
-coverage>=6.0
-dask>=2021.10.0
+dask>=2021.10.0
+earthkit-hydro>=1.1.0
future>=0.18.0
GDAL>=3.12.4
importlib-metadata>=5.0.0
-iniconfig>=1.1.1
lisflood-utilities>=1.0.1
llvmlite>=0.37.0
lxml>=4.6.5
@@ -14,19 +13,13 @@ more-itertools>=7.2.0
netCDF4>=1.7.4
nine>=1.0.0
numba>=0.54.0
-numexpr>=2.14.1
numpy>=1.21.0,<2.3
packaging>=19.2
pandas>=2.3.3
pathlib2>=2.3.5
pkginfo>=1.5.0.1
-pluggy>=0.13.0
-py>=1.10.0
pyparsing>=2.4.2
pyproj>=2.4.0
-pytest>=6.2.5
-pytest-cov>=3.0.0
-pytest-mock>=3.6.1
python-dateutil>=2.8.1
pytz>=2019.3
PyYAML>=5.4
diff --git a/setup.cfg b/setup.cfg
index e69de29b..57d1cded 100755
--- a/setup.cfg
+++ b/setup.cfg
@@ -0,0 +1,3 @@
+[flake8]
+ignore = E501
+exclude = .git,__pycache__,.eggs,.pytest_cache,.tox,build,dist
diff --git a/setup.py b/setup.py
index ba2c8c77..83db40fe 100755
--- a/setup.py
+++ b/setup.py
@@ -13,173 +13,199 @@
See the Licence for the specific language governing permissions and limitations under the Licence.
---------------------------------------------------------------------------------------------------------------------------------------
-Use python setup.y publish to publish versioned tags and pypi package
+Project metadata (name, description, authors, classifiers, dependencies, ...)
+lives in pyproject.toml. This file only carries the pieces that cannot be
+expressed declaratively there: the file-based `scripts`/`data_files` install
+and the custom publish/testpypi commands.
+
+Versioning
+----------
+The package version is managed by setuptools_scm and derived from git tags
+(see pyproject.toml [tool.setuptools_scm]).
+- On branches/dev: version is auto-generated (e.g. 5.0.1.dev61), so every
+ commit produces a distinct version number.
+- For releases: the publish/testpypi commands accept a --version argument
+ to create the git tag and build the package with that exact version.
-IMPORTANT Note:
To test pip installation:
-python setup.py testpypi
-pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple lisflood-model==5.0.0
+ python setup.py testpypi --version 5.0.1rc1
+ pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple lisflood-model==5.0.1rc1
-To publish on PyPi:
+ Note: testpypi creates a temporary local tag for the build, then deletes it.
+ No tag is pushed to the remote.
-python setup.py publish
+To publish on PyPI:
+ python setup.py publish --version 5.0.1
-It's equivalent to:
+This will:
+ 1. git tag v{version} (creates the tag so setuptools_scm picks it up)
+ 2. python -m build (builds sdist + wheel with that version)
+ 3. twine upload dist/* (uploads to PyPI)
+ 4. git push --tags (pushes the tag to the remote)
-1. python setup.py sdist
-2. twine upload dist/*
-3. git tag {version}
-4. git push --tags
-
-Install
-
-pip install lisflood-model
+Install:
+ pip install lisflood-model
"""
import os
import subprocess
import sys
-from glob import glob
from shutil import rmtree
-from setuptools import find_packages, Extension, setup, Command, __path__ as setuppath
-
-pip_package = os.path.normpath(os.path.join(setuppath[0], '../lisflood'))
-if os.path.exists(pip_package):
- print('[-] Removing current package installed {}'.format(pip_package))
- rmtree(pip_package)
+from setuptools import setup, Command
current_dir = os.path.dirname(os.path.abspath(__file__))
-sys.path.append(os.path.join(current_dir, './src/'))
-from lisflood import __authors__
-version_file = os.path.join(current_dir, 'VERSION')
-with open(version_file, 'r') as f:
- version = f.read().strip()
+def _check_publish_dependencies():
+ """Check that 'build' and 'twine' are installed before publishing."""
+ missing = []
+ if os.system('{} -m build --version > /dev/null 2>&1'.format(sys.executable)) != 0:
+ missing.append('build')
+ if os.system('{} -m twine --version > /dev/null 2>&1'.format(sys.executable)) != 0:
+ missing.append('twine')
+ if missing:
+ raise SystemExit(
+ 'Error: missing required packages: {}.\n'
+ 'Install them with: pip install {}'.format(
+ ', '.join(missing), ' '.join(missing)
+ )
+ )
-print(">>>>>>>>>>>>>>>> Building LISFLOOD version {} <<<<<<<<<<<<<<<<<<".format(version))
-readme_file = os.path.join(current_dir, 'README.md')
-with open(readme_file, 'r') as f:
- long_description = f.read()
+def _run_or_fail(cmd, error_msg):
+ """Run a shell command and abort if it fails."""
+ if os.system(cmd) != 0:
+ raise SystemExit('Error: {}'.format(error_msg))
class UploadCommand(Command):
- """Support setup.py upload."""
+ """Build, tag, and publish lisflood-model package to PyPI."""
- description = 'Publish lisflood-model package.'
- user_options = []
+ description = 'Build and publish lisflood-model package to PyPI.'
+ user_options = [
+ ('version=', 'V', 'Version to publish (e.g. 5.0.1). Required.'),
+ ]
@staticmethod
def print_console(s):
print('\033[1m{0}\033[0m'.format(s))
def initialize_options(self):
- pass
+ self.version = None
def finalize_options(self):
- pass
+ if self.version is None:
+ raise SystemExit(
+ 'Error: --version is required.\n'
+ 'Usage: python setup.py publish --version 5.0.1'
+ )
def run(self):
+ _check_publish_dependencies()
+
try:
self.print_console('Removing previous builds...')
rmtree(os.path.join(current_dir, 'dist'))
except OSError:
pass
- self.print_console('Building Source and Wheel (universal) distribution...')
- os.system('{} setup.py sdist'.format(sys.executable))
+ self.print_console('Tagging version v{}...'.format(self.version))
+ if os.system('git tag v{}'.format(self.version)) != 0:
+ raise SystemExit('Error: git tag failed. Does v{} already exist?'.format(self.version))
+
+ self.print_console('Building Source and Wheel distribution...')
+ _run_or_fail(
+ '{} -m build'.format(sys.executable),
+ 'Build failed. Fix the errors above before publishing.'
+ )
self.print_console('Uploading the package to PyPI via Twine...')
- os.system('twine upload dist/*')
+ _run_or_fail(
+ 'twine upload dist/*',
+ 'Twine upload to PyPI failed.'
+ )
self.print_console('Pushing git tags...')
- os.system('git tag v{}'.format(version))
os.system('git push --tags')
sys.exit()
-class UploadCommandTest(UploadCommand):
+class UploadCommandTest(Command):
+ """Build, tag, and publish lisflood-model package to Test PyPI."""
+
+ description = 'Build and publish lisflood-model package to Test PyPI.'
+ user_options = [
+ ('version=', 'V', 'Version to publish (e.g. 5.0.1rc1). Required.'),
+ ]
+
+ @staticmethod
+ def print_console(s):
+ print('\033[1m{0}\033[0m'.format(s))
+
+ def initialize_options(self):
+ self.version = None
+
+ def finalize_options(self):
+ if self.version is None:
+ raise SystemExit(
+ 'Error: --version is required.\n'
+ 'Usage: python setup.py testpypi --version 5.0.1rc1'
+ )
def run(self):
+ _check_publish_dependencies()
+
try:
self.print_console('Removing previous builds...')
rmtree(os.path.join(current_dir, 'dist'))
except OSError:
pass
- self.print_console('Building Source and Wheel (universal) distribution...')
- os.system('{} setup.py sdist'.format(sys.executable))
+ self.print_console('Creating temporary local tag v{}...'.format(self.version))
+ if os.system('git tag v{}'.format(self.version)) != 0:
+ raise SystemExit('Error: git tag failed. Does v{} already exist?'.format(self.version))
- self.print_console('Uploading the package to test PyPI via Twine...')
- os.system('twine upload --repository testpypi dist/*')
+ try:
+ self.print_console('Building Source and Wheel distribution...')
+ _run_or_fail(
+ '{} -m build'.format(sys.executable),
+ 'Build failed. Fix the errors above before publishing.'
+ )
+
+ self.print_console('Uploading the package to Test PyPI via Twine...')
+ _run_or_fail(
+ 'twine upload --repository testpypi dist/*',
+ 'Twine upload to Test PyPI failed.'
+ )
+ finally:
+ self.print_console('Removing temporary tag v{}...'.format(self.version))
+ os.system('git tag -d v{}'.format(self.version))
sys.exit()
-def _get_gdal_version():
+def _check_gdal_installed():
+ """Fail early with a clear message if the GDAL binaries are missing."""
try:
- p = subprocess.Popen(['gdal-config', '--version'], stdout=subprocess.PIPE)
+ subprocess.Popen(['gdal-config', '--version'], stdout=subprocess.PIPE).communicate()
except FileNotFoundError:
- raise SystemError('gdal-config not found.'
+ raise SystemError('gdal-config not found. '
'GDAL seems not installed. '
'Please, install GDAL binaries and libraries for your system '
'and then install the relative pip package.')
- else:
- return p.communicate()[0].splitlines()[0].decode()
-gdal_version = _get_gdal_version()
-req_file = 'requirements.txt'
-requirements = [l for l in open(req_file).readlines() if l and not l.startswith('#')]
-# GDAL requirement is commented to avoid fixing GDAL version in pypi package
-# requirements += ['GDAL=={}'.format(gdal_version)]
+_check_gdal_installed()
setup(
- name='lisflood-model',
- version=version,
- package_dir={'': 'src/'},
- py_modules=[os.path.splitext(os.path.basename(path))[0] for path in glob('src/*.py*')],
- include_package_data=True,
- data_files=[('', ['VERSION']), ('settings', ['src/lisfloodSettings_reference.xml'])],
- packages=find_packages('src'),
- long_description=long_description,
- long_description_content_type='text/markdown',
- description='LISFLOOD model python module',
- author=__authors__,
- author_email='carlo.russo@ext.ec.europa.eu',
- keywords=['lisflood', 'lisvap', 'efas', 'glofas', 'copernicus', 'ecmwf'],
- license='EUPL 1.2',
- url='https://github.com/ec-jrc/lisflood-code',
- download_url='https://github.com/ec-jrc/lisflood-code/archive/{}.tar.gz'.format(version),
- setup_requires=[
- 'nine',
- 'setuptools>=41.0',
- 'numpy>=1.15',
- ],
- install_requires=requirements,
+ # All static metadata (name, description, authors, classifiers,
+ # dependencies, urls, readme) is declared in pyproject.toml.
+ # These entries cover the file-based installs that pyproject cannot express.
+ data_files=[('settings', ['src/lisfloodSettings_reference.xml'])],
scripts=['bin/lisflood'],
zip_safe=True,
- classifiers=[
- # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers
- 'Development Status :: 5 - Production/Stable',
- 'Intended Audience :: Developers',
- 'Intended Audience :: Education',
- 'Intended Audience :: Financial and Insurance Industry',
- 'Intended Audience :: Other Audience',
- 'Intended Audience :: Science/Research',
- 'License :: OSI Approved :: European Union Public Licence 1.2 (EUPL 1.2)',
- 'Operating System :: Unix',
- 'Operating System :: POSIX',
- 'Operating System :: Microsoft :: Windows',
- 'Operating System :: MacOS :: MacOS X',
- 'Programming Language :: Python',
- 'Programming Language :: Python :: 3.7',
- 'Topic :: Scientific/Engineering :: Physics',
- ],
- # setup.py publish to pypi.
cmdclass={
'upload': UploadCommand,
'publish': UploadCommand,
diff --git a/src/lisflood/Lisflood_dynamic.py b/src/lisflood/Lisflood_dynamic.py
index abf8d1cc..ada327b2 100644
--- a/src/lisflood/Lisflood_dynamic.py
+++ b/src/lisflood/Lisflood_dynamic.py
@@ -32,7 +32,6 @@
from .global_modules.settings import CDFFlags, LisSettings, MaskInfo
-from pcraster import upstream
from lisflood.global_modules.add1 import decompress, compressArray
@@ -184,9 +183,29 @@ def splitlanduse(array1, array2=None, array3=None):
self.delta_avg_total = maskinfo.in_zero()
self.delta_outlet_total = maskinfo.in_zero()
# sums up discharge of the sub steps
- for NoRoutingExecuted in range(self.NoRoutSteps):
- self.routing_module.dynamic(NoRoutingExecuted)
- # routing sub steps
+ # The routing substep kernels have a small per-substep payload (one
+ # topological traversal of the river network each), so beyond a modest
+ # OpenMP worker count the fork/join cost dominates. The governor caps
+ # the numba pool to a routing-specific count once around this loop and
+ # restores the land-phase count afterwards (try/finally => restored on
+ # every exit path, including an exception inside a routing substep). The
+ # land-phase kernels (soil loop, water abstraction, surface routing) all
+ # run before this loop and are therefore outside the capped window.
+ # Scheduling-only: the routing result is identical across thread counts.
+ from lisflood.global_modules.parallelization import (
+ get_routing_numba_threads, get_land_numba_threads, set_numba_threads,
+ )
+ _routing_k = get_routing_numba_threads()
+ _land_k = get_land_numba_threads()
+ if _routing_k is not None:
+ set_numba_threads(_routing_k)
+ try:
+ for NoRoutingExecuted in range(self.NoRoutSteps):
+ self.routing_module.dynamic(NoRoutingExecuted)
+ # routing sub steps
+ finally:
+ if _routing_k is not None:
+ set_numba_threads(_land_k)
# ----------------------------------------------------------------------
@@ -248,6 +267,10 @@ def splitlanduse(array1, array2=None, array3=None):
# Calculate water level
self.waterlevel_module.dynamic()
+ # %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+ # Calculate water storage
+ self.waterstorage_module.dynamic()
+
# %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
# ************************************************************
diff --git a/src/lisflood/Lisflood_initial.py b/src/lisflood/Lisflood_initial.py
index 7a6a864e..cae18c48 100644
--- a/src/lisflood/Lisflood_initial.py
+++ b/src/lisflood/Lisflood_initial.py
@@ -21,20 +21,14 @@
from collections import OrderedDict
import numpy as np
-try:
- from numba import njit, set_num_threads, get_num_threads, config as numba_config
- numba_full = True
-except ImportError:
- import numba
- print('WARNING! Numba version is probably too old. Only limited functionalities will be available for Numba')
- print('Numba version: {}'.format(numba.__version__))
- from numba import njit, config as numba_config
- numba_full = False
+from numba import njit, prange
+# Thread-pool governance (numba/BLAS) lives in parallelization.py.
from .global_modules.settings import CutMap, LisSettings, NetCDFMetadata, EPICSettings, MaskInfo
from .global_modules.zusatz import DynamicModel
from .global_modules.add1 import loadsetclone, mapattrNetCDF
+from .global_modules.parallelization import configure_parallelism
from .hydrological_modules.miscInitial import miscInitial
from .hydrological_modules.readmeteo import readmeteo
@@ -59,6 +53,7 @@
from .hydrological_modules.opensealed import opensealed
from .hydrological_modules.waterbalance import waterbalance
from .hydrological_modules.waterlevel import waterlevel
+from .hydrological_modules.waterstorage import waterstorage
from .hydrological_modules.structures import structures
from .global_modules.output import outputTssMap
@@ -67,10 +62,62 @@
from .global_modules.add1 import NumpyModified
@njit(fastmath=False, cache=True)
-def _vegSum(ax_veg, variable, soil_fracs):
+def _vegSum_serial(ax_veg, variable, soil_fracs):
return (soil_fracs * variable).sum(ax_veg)
+# numpy switches its reduction from a straight sequential sum to pairwise
+# summation once an axis has >= 8 elements, so a plain sequential accumulation
+# only reproduces ``(a).sum(0)`` bit-for-bit for small vegetation counts. The
+# production SoilFraction has 3 vegetation rows (Rainfed/Forest/Irrigated), so
+# the parallel path is taken; anything with >= 8 vegetation rows falls back to
+# the serial numpy reduction to stay bitwise-identical (see _vegSum).
+_VEGSUM_PARALLEL_MAX_VEG = 7
+
+
+@njit(parallel=True, fastmath=False, cache=True)
+def _vegSum_2d_axis0(variable, soil_fracs):
+ """Parallel weighted sum over the vegetation axis of a (vegetation, pixel)
+ array, equivalent to ``(soil_fracs * variable).sum(axis=0)``.
+
+ Only used when the vegetation axis has <= ``_VEGSUM_PARALLEL_MAX_VEG``
+ elements, where a straight sequential reduction matches numpy's own
+ sequential summation bit-for-bit. Parallelism is over the retained *pixel*
+ axis; the per-pixel reduction over the (small) vegetation axis runs in a
+ fixed ascending order, so the result is bit-identical to
+ ``(soil_fracs * variable).sum(0)`` (``fastmath=False`` preserves FP order).
+ ``prange`` only engages when numba is allowed more than one thread; the
+ central governance (parallelization.py) caps numba to 1 thread on small
+ domains, so this runs serially there.
+ """
+ nveg = variable.shape[0]
+ npix = variable.shape[1]
+ out = np.empty(npix, dtype=variable.dtype)
+ for p in prange(npix):
+ acc = 0.0
+ for v in range(nveg):
+ acc += soil_fracs[v, p] * variable[v, p]
+ out[p] = acc
+ return out
+
+
+def _vegSum(ax_veg, variable, soil_fracs):
+ """Weighted sum over the soil fractions of each pixel.
+
+ Fast path: the production case is a 2-D ``(vegetation, pixel)`` array
+ reduced over the vegetation axis (``ax_veg == 0``) with matching
+ ``soil_fracs`` shape; that is handled by the parallel kernel
+ ``_vegSum_2d_axis0``. Any other shape/axis (defensive — not produced by the
+ current callers) falls back to the original serial reduction so behaviour is
+ unchanged.
+ """
+ if (ax_veg == 0 and variable.ndim == 2 and soil_fracs.ndim == 2
+ and soil_fracs.shape == variable.shape
+ and variable.shape[0] <= _VEGSUM_PARALLEL_MAX_VEG):
+ return _vegSum_2d_axis0(variable, soil_fracs)
+ return _vegSum_serial(ax_veg, variable, soil_fracs)
+
+
# --------------------------------------------
class LisfloodModel_ini(DynamicModel):
@@ -97,11 +144,16 @@ def __init__(self):
flags = self.settings.flags
report_steps = self.settings.report_steps
- # set the maximum number of threads that numba should use (now used in soilloop only)
- num_threads = int(binding["numCPUs_parallelNumba"])
- if (num_threads>0):
- if num_threads<=numba_config.NUMBA_NUM_THREADS:
- set_num_threads(num_threads)
+ # Centrally govern the numba and BLAS/OpenMP thread pools to
+ # avoid oversubscription. Nested thread pools each sized to the host
+ # core count cause heavy context-switching and can make runs slower
+ # than serial, especially on small/narrow domains. Thread counts affect
+ # performance only, not results.
+ num_pixels = int(self.maskinfo.info.mapC[0]) if self.maskinfo.info.mapC else None
+ # verbose=False: the effective thread counts are reported once, in the
+ # LisfloodRunInfo startup banner (see LisfloodRunInfo in settings.py),
+ # to keep the startup output in a single coherent block.
+ configure_parallelism(binding, num_pixels=num_pixels)
# Mapping of vegetation types to land use fractions (and the other way around)
##global VEGETATION_LANDUSE, LANDUSE_VEGETATION, PRESCRIBED_VEGETATION, PRESCRIBED_LAI
@@ -150,6 +202,7 @@ def __init__(self):
self.opensealed_module = opensealed(self)
self.waterbalance_module = waterbalance(self)
self.waterlevel_module = waterlevel(self)
+ self.waterstorage_module = waterstorage(self)
self.structures_module = structures(self)
self.prescribed_vegetation = self.epic_settings.prescribed_vegetation
@@ -208,6 +261,7 @@ def __init__(self):
self.reservoir_module.initial()
self.lakes_module.initial()
self.polder_module.initial()
+ self.waterstorage_module.initial()
self.transmission_module.initial()
@@ -345,15 +399,22 @@ def get_indexes_from_landuse_and_veg_list_GLOBAL(self, landuse, veg_list):
iveg_list.append(self.vegetation.index(veg))
return iveg_list,iveg_list_pres,ilanduse
- def allocateVariableAllVegetation(self, dtype=float):
- """Allocate xarray.DataArray filled by 0 with dimensions 'vegetation' and 'pixel'. It covers all vegetation types (including EPIC crops, if simulated)."""
+ def allocateVariableAllVegetation(self, dtype=None):
+ """Allocate xarray.DataArray filled by 0 with dimensions 'vegetation' and 'pixel'. It covers all vegetation types (including EPIC crops, if simulated).
+ dtype=None defers to the resolved model dtype (float64, or float32 when modelPrecision=float32)."""
return self.allocateDataArray(self.coord_vegetation, dtype)
- def allocateDataArray(self, dimensions, dtype=float):
+ def allocateDataArray(self, dimensions, dtype=None):
"""Allocate xarray.DataArray filled by 0 with input dimensions.
Argument 'dimensions' is a list of tuples of the type ('dimension name', coordinate list/array)."""
coords = OrderedDict(dimensions)
option = self.settings.options
+ if dtype is None:
+ # Default to the resolved model dtype (float64 unless modelPrecision=
+ # float32), so vegetation/soil state stacks (UZ, W1, ...) are born in
+ # the model precision. An explicit dtype argument is honoured as-is.
+ from .global_modules.parallelization import get_model_dtype
+ dtype = get_model_dtype()
if option.get('cropsEPIC'):
return xr.DataArray(np.zeros([len(v) for v in coords.values()], dtype), coords=coords, dims=coords.keys())
else:
diff --git a/src/lisflood/__init__.py b/src/lisflood/__init__.py
index 37c449d8..067c90da 100644
--- a/src/lisflood/__init__.py
+++ b/src/lisflood/__init__.py
@@ -1,14 +1,18 @@
import os
-current_dir = os.path.dirname(os.path.abspath(__file__))
-version_file = os.path.join(current_dir, '../../VERSION')
-if not os.path.exists(version_file):
- version_file = os.path.join(current_dir, '../../../../VERSION')
+# Version is managed by setuptools_scm and written to _version.py at build time
+# (see pyproject.toml [tool.setuptools_scm]). The file is not tracked in git.
+try:
+ from ._version import __version__
+except ImportError:
+ # Not built yet (e.g. running from a fresh git checkout). Derive the
+ # version live from git; fall back to a sentinel if that also fails.
+ try:
+ from setuptools_scm import get_version
+ __version__ = get_version(root=os.path.join(os.path.dirname(__file__), '../..'))
+ except Exception:
+ __version__ = '0.0.0'
-with open(version_file, 'r') as f:
- version = f.read().strip()
-
-__version__ = version
__authors__ = "Ad de Roo, Emiliano Gelati, Peter Burek, Johan van der Knijff"
__date__ = "30/06/2026"
__copyright__ = "Copyright 2019-2026, European Commission - Joint Research Centre"
diff --git a/src/lisflood/global_modules/add1.py b/src/lisflood/global_modules/add1.py
index 3b92e33a..869909f0 100755
--- a/src/lisflood/global_modules/add1.py
+++ b/src/lisflood/global_modules/add1.py
@@ -286,30 +286,54 @@ def compressArray(map, pcr=True, name=None, force_load_with_nans=False):
msg = name + " has less valid pixels than area or ldd \n"
raise LisfloodError(msg)
# test if map has less valid pixel than area.map (or ldd)
- return mapC.astype(float)
+ # Cast to the resolved MODEL dtype (float64 by default -> bit-identical; the
+ # copy=False keeps the earlier no-op fast path when mapC is already that
+ # dtype). When modelPrecision=float32, compressed STATE arrays are born
+ # float32 here (the array-creation funnel), which is what avoids the
+ # mixed-dtype upcast trap downstream. Accuracy-critical consumers
+ # (mass-balance np.bincount weights, cumulative accumulators, MCT/reservoir
+ # coupling) cast back to float64 explicitly at their call sites. Integer
+ # topology callers (Catchments/downstruct) apply their own .astype(int32)
+ # afterwards, so the float dtype here is irrelevant to them.
+ from .parallelization import get_model_dtype
+ return mapC.astype(get_model_dtype(), copy=False)
def decompress(map):
maskinfo = MaskInfo.instance()
- dmap = maskinfo.info.maskall.copy()
- dmap[~maskinfo.info.maskflat] = map[:]
- dmap = dmap.reshape(maskinfo.info.shape)
# check if integer map (like outlets, lakes etc)
try:
checkint = str(map.dtype)
except:
checkint = None
- if checkint in ("int16", "int32", "int64"):
- dmap[dmap.mask] = -9999
- map = numpy2pcr(Nominal, dmap, -9999)
- elif checkint == "int8":
+ if checkint == "int8":
+ # Rare, special-cased branch: the original relied on np.ma semantics
+ # (dmap[dmap < 0] excludes masked cells, leaving their uninitialized
+ # masked_all .data untouched). Keep the exact original implementation
+ # here to preserve behaviour bit-for-bit; it is not on any hot path
+ # (compressArray returns float, so decompress almost always hits the
+ # Scalar branch below).
+ dmap = maskinfo.info.maskall.copy()
+ dmap[~maskinfo.info.maskflat] = map[:]
+ dmap = dmap.reshape(maskinfo.info.shape)
dmap[dmap < 0] = -9999
- map = numpy2pcr(Nominal, dmap, -9999)
- else:
- dmap[dmap.mask] = -9999
- map = numpy2pcr(Scalar, dmap, -9999)
- return map
+ return numpy2pcr(Nominal, dmap, -9999)
+
+ # Fast path (float / int16 / int32 / int64): build the full-domain array
+ # with a plain numpy buffer instead of copying the masked `maskall` and
+ # operating on it through the np.ma machinery. `maskall` is float64 and the
+ # result handed to numpy2pcr is simply: valid cells = map values,
+ # outside-domain cells = -9999 (the missing-value marker). This avoids the
+ # full-domain masked-array copy and the np.ma wrapped-method overhead that
+ # dominate decompress on large/global domains. Verified bit-identical to the
+ # original masked-array path for all these dtype branches.
+ dmap = np.full(maskinfo.info.shapeflat, -9999.0, dtype=np.float64)
+ dmap[~maskinfo.info.maskflat] = map[:]
+ dmap = dmap.reshape(maskinfo.info.shape)
+ if checkint in ("int16", "int32", "int64"):
+ return numpy2pcr(Nominal, dmap, -9999)
+ return numpy2pcr(Scalar, dmap, -9999)
def makenumpy(map):
@@ -322,6 +346,61 @@ def makenumpy(map):
return map
+# Cache of parsed 2-column lookup tables: {abspath: {id(int): value(float)}}.
+_LOOKUP_TABLE_CACHE = {}
+
+
+def _read_lookup_table(table_path):
+ """Parse a PCRaster-style 2-column 'key value' lookup table into an
+ {int(key): float(value)} dict. Only the simple exact-match form used by the
+ reservoir/lake parameter tables is supported (no range/interval rows).
+ Cached by absolute path."""
+ key = os.path.abspath(str(table_path))
+ cached = _LOOKUP_TABLE_CACHE.get(key)
+ if cached is not None:
+ return cached
+ mapping = {}
+ with open(str(table_path)) as f:
+ for line in f:
+ parts = line.split()
+ if len(parts) < 2:
+ continue
+ # keys are integer point ids (reservoir/lake ids); skip non-numeric
+ # (e.g. accidental header lines) rather than crash.
+ try:
+ k = int(float(parts[0]))
+ v = float(parts[1])
+ except ValueError:
+ continue
+ mapping[k] = v
+ _LOOKUP_TABLE_CACHE[key] = mapping
+ return mapping
+
+
+def lookup_table_values(table_path, point_ids):
+ """Vectorized equivalent of
+ np.compress(sites>0, compressArray(pcraster.lookupscalar(table, siteMapPcr)))
+ for a simple exact-match 2-column lookup table.
+
+ For each id in `point_ids` (the compressed point ids, e.g.
+ ReservoirSitesCC / LakeSitesCC) return the table value for that id, or NaN if
+ the id is absent from the table -- exactly as pcraster.lookupscalar returns a
+ missing value (which compressArray turns into NaN). This replaces a full-grid
+ PCRaster table lookup (~30 s/call on the global domain) with an O(n_points)
+ dict lookup.
+ """
+ mapping = _read_lookup_table(table_path)
+ ids = np.asarray(point_ids)
+ out = np.empty(ids.shape[0], dtype=float)
+ for i in range(ids.shape[0]):
+ out[i] = mapping.get(int(ids[i]), np.nan)
+ # Cast to float32 then back, matching pcraster.lookupscalar's Scalar (float32)
+ # precision. Without this, the full float64 table values introduce ~1e-7
+ # per-value differences (visible as ~1e-4 in discharge after amplification
+ # through reservoir/lake routing), breaking bit-identical output.
+ return out.astype(np.float32).astype(np.float64)
+
+
def loadmap(*args, **kwargs):
settings = LisSettings.instance()
binding = settings.binding
@@ -558,7 +637,8 @@ def loadmap_base(name, pcr=False, lddflag=False, timestampflag='exact', averagey
nanCheckMap(map, filename, name)
return map
elif isinstance(mapC, np.ndarray):
- return mapC.astype(float)
+ from .parallelization import get_model_dtype
+ return mapC.astype(get_model_dtype(), copy=False)
else:
if flags['nancheck'] and name != 'Ldd':
nanCheckMap(mapC, filename, name)
diff --git a/src/lisflood/global_modules/checkers.py b/src/lisflood/global_modules/checkers.py
index 33bc1c24..d33947d0 100755
--- a/src/lisflood/global_modules/checkers.py
+++ b/src/lisflood/global_modules/checkers.py
@@ -21,7 +21,11 @@
import inspect
import warnings
+from pcraster.operations import boolean
+import numpy as np
+
from .errors import LisfloodError, LisfloodWarning
+from .add1 import loadmap, loadsetclone, compressArray
from ..hydrological_modules import HydroModule
from ..hydrological_modules import (surface_routing, evapowater, snow, routing, leafarea, inflow, waterlevel,
waterbalance, wateruse, waterabstraction, lakes, riceirrigation, indicatorcalc,
@@ -29,6 +33,26 @@
reservoir, transmission)
+def lakes_present(lake_type):
+ """Check whether there are any lakes/reservoirs."""
+ present = True
+ sites_dict = {'lakes': 'LakeSites', 'reservoir': 'ReservoirSites'}
+ MaskMap = loadsetclone('MaskMap') # need to define mask map to use loadmap
+ LakeSitesC = loadmap(sites_dict[lake_type])
+ LakeSitesC[LakeSitesC < 1] = 0
+
+ # Get rid of any lakes/reservoirs that are not part of the channel network
+ IsChannelPcr = boolean(loadmap('Channels', pcr=True))
+ IsChannel = np.bool(compressArray(IsChannelPcr))
+ LakeSitesC[IsChannel == 0] = 0
+
+ LakeSitesCC = np.compress(LakeSitesC > 0, LakeSitesC)
+ if LakeSitesCC.size == 0:
+ present = False
+
+ return present
+
+
class ModulesInputs:
root_package = 'lisflood.hydrological_modules'
# dict representing modules activated per option
@@ -82,6 +106,12 @@ def check(cls, settings):
and obj is not HydroModule and str(obj.__module__) == hydro_module.__name__]
total_checks += len(clzzs)
for clz in clzzs:
+ if clz.__name__ in ['lakes', 'reservoir']:
+ if not lakes_present(clz.__name__):
+ # No lakes/reservoirs so skip the check for these input files.
+ successful_checks += 1
+ continue
+
errors = clz.check_input_files(option)
successful_checks += int(bool(not errors))
all_errors += errors
diff --git a/src/lisflood/global_modules/default_options.py b/src/lisflood/global_modules/default_options.py
index 32de212c..fe6cabaf 100644
--- a/src/lisflood/global_modules/default_options.py
+++ b/src/lisflood/global_modules/default_options.py
@@ -1,7 +1,8 @@
from collections import namedtuple
TimeSeries = namedtuple('TimeSeries', 'name, output_var, where, repoption, restrictoption, operation')
-ReportedMap = namedtuple('ReportedMap', 'name, output_var, unit, end, steps, all, restrictoption, monthly, yearly')
+ReportedMap = namedtuple('ReportedMap', 'name, output_var, unit, end, steps, all, restrictoption, monthly, yearly, scale_factor, add_offset')
+ReportedMap.__new__.__defaults__ = (None, None) # scale_factor, add_offset
default_options = {'EnKF': False,
'InitLisflood': False,
@@ -53,7 +54,6 @@
'repLZOutflowMaps': False,
'repLeafDrainageMaps': False,
'repMBTs': False,
- 'repTotalWaterStorageMaps': False,
'repMeteoUpsGauges': False,
'repOverlandFlowMaps': False,
'repPFMaps': False,
@@ -90,6 +90,9 @@
'repWaterLevelMaps': False,
'repWaterLevelTs': False,
'repEtactBudykoTs': False,
+ 'repTWSMaps': False,
+ 'repStorageMaps': False,
+ 'repFlowMomMaps': False,
'reportedmaps': {'AreaTotalAbstractionFromSurfaceWaterM3': ReportedMap(
name='AreaTotalAbstractionFromSurfaceWaterM3',
output_var='areatotal_withdrawal_SW_actual_M3', unit='m3', end=[],
@@ -316,7 +319,11 @@
unit='m3',
end=['InitLisflood'],
steps=[], all=[], restrictoption=[],
- monthly=False, yearly=False),
+ monthly=False, yearly=False),
+ 'CumSMaps': ReportedMap(name='CumSMaps', output_var='cumstor',
+ unit='m', end=[], steps=['repStorageMaps'],
+ all=[], restrictoption=['nonInit'],
+ monthly=False, yearly=False),
'DSLREnd': ReportedMap(name='DSLREnd', output_var='DSLR[0]', unit='-',
end=['repEndMaps'], steps=[], all=[],
restrictoption=[], monthly=False, yearly=False),
@@ -351,7 +358,7 @@
output_var='DirectRunoff', unit='mm', end=[],
steps=['repSurfaceRunoffMaps'], all=[''],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.0062, add_offset=200.0),
'DischargeEnd': ReportedMap(name='DischargeEnd', output_var='ChanQAvg',
unit='m3/s', end=[], steps=[],
all=[], restrictoption=[], monthly=False,
@@ -368,38 +375,46 @@
'ESActForestMaps': ReportedMap(name='ESActForestMaps', output_var='ESAct[1]',
unit='mm', end=[], steps=[''],
all=['repESActMaps'], restrictoption=['nonInit'],
- monthly=False, yearly=False),
+ monthly=False, yearly=False,scale_factor=0.00031, add_offset=10.),
'ESActMaps': ReportedMap(name='ESActMaps', output_var='ESActPixel', unit='mm',
end=[], steps=[], all=['repESActMaps','repE2O1'],
- restrictoption=['nonInit'], monthly=False, yearly=False),
+ restrictoption=['nonInit'], monthly=False, yearly=False,
+ scale_factor=0.00031, add_offset=10.0),
'ESRefMapsOut': ReportedMap(name='ESRefMapsOut', output_var='ESRef', unit='mm',
end=[], steps=[], all=['repESRefMaps'],
- restrictoption=[], monthly=False, yearly=False),
+ restrictoption=[], monthly=False, yearly=False,
+ scale_factor=0.00031, add_offset=10.0),
'ETActMaps': ReportedMap(name='ETActMaps',
output_var='ESActPixel+self.var.TaPixel+self.var.TaInterceptionAll+self.var.EvaAddM3*self.var.M3toMM',
unit='mm', end=[], steps=[], all=['repETActMaps','repE2O1'],
- restrictoption=['nonInit'], monthly=False, yearly=False),
+ restrictoption=['nonInit'], monthly=False, yearly=False,
+ scale_factor=0.016, add_offset=495.0),
'ETActBudykoMaps': ReportedMap(name='ETActBudykoMaps',
output_var='ESActNoIrrPixel+self.var.TaNoIrrPixel+self.var.TaInterceptionNoIrrPixel',
unit='mm', end=[], steps=[], all=['repETActMaps','repE2O1'],
- restrictoption=['nonInit'], monthly=False, yearly=False),
+ restrictoption=['nonInit'], monthly=False, yearly=False,
+ scale_factor=0.016, add_offset=495.0),
'ETRefMapsOut': ReportedMap(name='ETRefMapsOut', output_var='ETRef', unit='mm',
end=[], steps=[], all=['repETRefMaps'],
- restrictoption=[], monthly=False, yearly=False),
+ restrictoption=[], monthly=False, yearly=False,
+ scale_factor=0.00031, add_offset=10.0),
'EWIntForestMaps': ReportedMap(name='EWIntForestMaps',
output_var='TaInterception[1]', unit='mm',
end=[], steps=[], all=['repEWIntMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.00031, add_offset=10.0),
'EWIntMaps': ReportedMap(name='EWIntMaps', output_var='TaInterceptionAll',
unit='mm', end=[], steps=[], all=['repEWIntMaps','repE2O1'],
- restrictoption=['nonInit'], monthly=False, yearly=False),
+ restrictoption=['nonInit'], monthly=False, yearly=False,
+ scale_factor=0.00031, add_offset=10.0),
'EWRefMapsOut': ReportedMap(name='EWRefMapsOut', output_var='EWRef', unit='mm',
end=[], steps=[], all=['repEWRefMaps'],
- restrictoption=[], monthly=False, yearly=False),
+ restrictoption=[], monthly=False, yearly=False,
+ scale_factor=0.00031, add_offset=10.0),
'EWater': ReportedMap(name='EWater', output_var='EvaAddM3*self.var.M3toMM',
unit='mm', end=[], steps=[], all=['repE2O1'],
- restrictoption=['nonInit'], monthly=False, yearly=False),
+ restrictoption=['nonInit'], monthly=False, yearly=False,
+ scale_factor=0.00031, add_offset=10.0),
'FalkenmarkM3Capita1': ReportedMap(name='FalkenmarkM3Capita1',
output_var='FalkenmarkM3Capita1',
unit='M3Capita', end=[],
@@ -426,7 +441,7 @@
output_var='SurfaceRunoff+self.var.UZOutflowPixel',
unit='mm', end=[], steps=[], all=['repFastRunoffMaps','repE2O1'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.0077, add_offset=250),
'FossilGroundwaterUsedM3': ReportedMap(name='FossilGroundwaterUsedM3',
output_var='FossilGroundwaterUsedM3',
unit='m3', end=[],
@@ -451,6 +466,10 @@
unit='-', end=[], steps=['repStateMaps'],
all=[], restrictoption=['nonInit'],
monthly=False, yearly=False),
+ 'FlowMomMaps': ReportedMap(name='FlowMomMaps', output_var='FlowMomentum',
+ unit='kgm/s', end=[], steps=['repFlowMomMaps'],
+ all=[], restrictoption=['nonInit'],
+ monthly=False, yearly=False),
'GwLossMaps': ReportedMap(name='GwLossMaps', output_var='GwLossPixel',
unit='mm', end=[], steps=[], all=['repGwLossMaps','repE2O2'],
restrictoption=['nonInit'], monthly=False,
@@ -460,49 +479,52 @@
end=[], steps=['repGwPercUZLZMaps'],
all=['repGwPercUZLZMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
- 'GwPercUZLZIrrigationMaps': ReportedMap(name='GwPercUZLZForestMaps',
+ yearly=False, scale_factor=4.6e-05, add_offset=1.5),
+ 'GwPercUZLZIrrigationMaps': ReportedMap(name='GwPercUZLZIrrigationMaps',
output_var='GwPercUZLZ[2]', unit='mm',
end=[], steps=['repGwPercUZLZMaps'],
all=['repGwPercUZLZMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
- 'GwPercUZLZOtherMaps': ReportedMap(name='GwPercUZLZForestMaps',
+ yearly=False, scale_factor=4.6e-05, add_offset=1.5),
+ 'GwPercUZLZOtherMaps': ReportedMap(name='GwPercUZLZOtherMaps',
output_var='GwPercUZLZ[0]', unit='mm',
end=[], steps=[],
all=['repGwPercUZLZMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
-
+ yearly=False, scale_factor=4.6e-05, add_offset=1.5),
'GwPercUZLZMaps': ReportedMap(name='GwPercUZLZMaps',
output_var='GwPercUZLZPixel', unit='mm', end=[],
steps=[], all=['repGwPercUZLZMaps','repE2O2'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=4.6e-05, add_offset=1.5),
+ 'GWSMaps': ReportedMap(name='GWSMaps', output_var='gwstor',
+ unit='m', end=[], steps=['repStorageMaps'],
+ all=[], restrictoption=['nonInit'],
+ monthly=False, yearly=False),
'InfiltrationForestMaps': ReportedMap(name='InfiltrationForestMaps',
output_var='Infiltration[1]', unit='mm',
end=[], steps=[],
all=['repInfiltrationMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.0077, add_offset=250.),
'InfiltrationMaps': ReportedMap(name='InfiltrationMaps',
output_var='Infiltration[0]', unit='mm',
end=[], steps=[],
all=['repInfiltrationMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.0077, add_offset=250.),
'InterceptionForestMaps': ReportedMap(name='InterceptionForestMaps',
output_var='Interception[1]', unit='mm',
end=[], steps=[],
all=['repInterceptionMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.00023, add_offset=7.5),
'InterceptionMaps': ReportedMap(name='InterceptionMaps',
output_var='Interception[0]', unit='mm',
end=[], steps=[],
all=['repInterceptionMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.00023, add_offset=7.5),
'LocalFreshwaterM3': ReportedMap(name='LocalFreshwaterM3',
output_var='LocalFreshwaterM3',
unit='m3', end=[],
@@ -530,15 +552,16 @@
restrictoption=[], monthly=False, yearly=False),
'LZMaps': ReportedMap(name='LZMaps', output_var='LZ', unit='mm', end=[],
steps=[], all=['repLZMaps', 'repE2O2'], restrictoption=['nonInit'],
- monthly=False, yearly=False),
+ monthly=False, yearly=False,scale_factor=0.23, add_offset=7500.0),
'LZOutflowMaps': ReportedMap(name='LZOutflowMaps',
output_var='LZOutflowToChannelPixel', unit='mm',
end=[], steps=[], all=['repLZOutflowMaps','repE2O1'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=4.6e-05, add_offset=1.5),
'LZState': ReportedMap(name='LZState', output_var='LZ', unit='mm', end=[],
steps=['repStateMaps'], all=[],
- restrictoption=['nonInit'], monthly=False, yearly=False),
+ restrictoption=['nonInit'], monthly=False, yearly=False,
+ scale_factor=0.092, add_offset=-1000.0),
'LakeAbstractionM3': ReportedMap(name='LakeAbstractionM3',
output_var='LakeAbstractionM3', unit='m3',
end=[], steps=['repTotalAbs'], all=[],
@@ -573,6 +596,10 @@
all=[],
restrictoption=['nonInit', 'simulateLakes'],
monthly=False, yearly=False),
+ 'LakeSMaps': ReportedMap(name='LakeSMaps', output_var='lakestor',
+ unit='m', end=[], steps=['repStorageMaps'],
+ all=[], restrictoption=['nonInit'],
+ monthly=False, yearly=False),
'LakeStorageM3': ReportedMap(name='LakeStorageM3', output_var='LakeStorageM3',
unit='m3', end=[], steps=['repTotalAbs'],
all=[], restrictoption=['nonInit', 'wateruse',
@@ -697,34 +724,34 @@
output_var='SeepTopToSubA[0]', unit='mm',
end=[], steps=[], all=['repPercolationMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=4.6e-05, add_offset=1.5),
'Percolation1ato1bForestMaps': ReportedMap(name='Percolation1ato1bForestMaps',
output_var='SeepTopToSubA[1]', unit='mm',
end=[], steps=[],
all=['repPercolationMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=4.6e-05, add_offset=1.5),
'Percolation1ato1bIrrigationMaps': ReportedMap(name='Percolation1ato1bIrrigationMaps',
output_var='SeepTopToSubA[2]', unit='mm',
end=[], steps=[], all=['repPercolationMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=4.6e-05, add_offset=1.5),
'Percolation1bto2OtherMaps': ReportedMap(name='Percolation1bto2OtherMaps',
output_var='SeepTopToSubB[0]', unit='mm',
end=[], steps=[], all=['repPercolationMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=4.6e-05, add_offset=1.5),
'Percolation1bto2ForestMaps': ReportedMap(name='Percolation1bto2ForestMaps',
output_var='SeepTopToSubB[1]', unit='mm',
end=[], steps=[],
all=['repPercolationMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=4.6e-05, add_offset=1.5),
'Percolation1bto2IrrigationMaps': ReportedMap(name='Percolation1bto2IrrigationMaps',
output_var='SeepTopToSubB[2]', unit='mm',
end=[], steps=[], all=['repPercolationMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=4.6e-05, add_offset=1.5),
'PrevCmMCTEnd': ReportedMap(name='PrevCmMCTEnd', output_var='PrevCm0', unit='-',
end=['repEndMaps'], steps=[], all=[],
restrictoption=['nonInit','MCTRouting'], monthly=False, yearly=False),
@@ -759,7 +786,7 @@
unit='mm/timestep', end=[], steps=[],
all=['repPrecipitationMaps'],
restrictoption=[], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.016, add_offset=500.),
'PrefFlowOtherMaps': ReportedMap(name='PrefFlowOtherMaps',
output_var='PrefFlow[0]', unit='mm', end=[],
steps=[], all=['repPrefFlowMaps'],
@@ -781,7 +808,8 @@
yearly=False),
'RainMaps': ReportedMap(name='RainMaps', output_var='Rain', unit='mm',
end=[], steps=[], all=['repRainMaps','repE2O1'],
- restrictoption=['nonInit'], monthly=False, yearly=False),
+ restrictoption=['nonInit'], monthly=False, yearly=False,
+ scale_factor=0.016, add_offset=500.),
'RegionMonthExternalInflowM3': ReportedMap(name='RegionMonthExternalInflowM3',
output_var='RegionMonthExternalInflowM3',
unit='M3', end=[],
@@ -874,6 +902,11 @@
restrictoption=['nonInit', 'wateruse',
'simulateReservoirs'],
monthly=False, yearly=False),
+ 'RiverSMaps': ReportedMap(name='RiverSMaps', output_var='riverstor',
+ unit='m', end=[], steps=['repStorageMaps'],
+ all=[], restrictoption=['nonInit'],
+ monthly=False, yearly=False),
+
'SMStressForestMap': ReportedMap(name='SMStressForestMap',
output_var='SoilMoistureStressDays[1]',
unit='days', end=[], steps=[],
@@ -890,39 +923,39 @@
end=[], steps=[],
all=['repSeepSubToGWMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.00092, add_offset=30.),
'SeepSubToGWforestMaps': ReportedMap(name='SeepSubToGWForestMaps',
output_var='SeepSubToGW[1]', unit='mm',
end=[], steps=[],
all=['repSeepSubToGWMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.00092, add_offset=30.),
'SeepSubToGWirrigationMaps': ReportedMap(name='SeepSubToGWIrrigationMaps',
output_var='SeepSubToGW[2]', unit='mm',
end=[], steps=[],
all=['repSeepSubToGWMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.00092, add_offset=30.),
'SeepSubToGWMaps': ReportedMap(name='SeepSubToGWMaps',
output_var='SeepSubToGWPixel', unit='mm',
end=[], steps=[], all=['repSeepSubToGWMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.00092, add_offset=30.),
'SeepTopToSubBAverageOtherMap': ReportedMap(name='SeepTopToSubBAverageOtherMap',
output_var='SeepTopToSubBAv[0]', unit='mm/timestep',
end=['InitLisflood'], steps=[], all=[],
restrictoption=[], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.00092, add_offset=30.),
'SeepTopToSubBAverageForestMap': ReportedMap(name='SeepTopToSubBAverageForestMap',
output_var='SeepTopToSubBAv[1]', unit='mm/timestep',
end=['InitLisflood'], steps=[], all=[],
restrictoption=[], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.00092, add_offset=30.),
'SeepTopToSubBAverageIrrigationMap': ReportedMap(name='SeepTopToSubBAverageIrrigationMap',
output_var='SeepTopToSubBAv[2]', unit='mm/timestep',
end=['InitLisflood'], steps=[], all=[],
restrictoption=[], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.00092, add_offset=30.),
'SnowCoverAEnd': ReportedMap(name='SnowCoverAEnd', output_var='SnowCoverS[0]',
unit='mm', end=['repEndMaps'], steps=[],
all=[], restrictoption=[], monthly=False,
@@ -931,7 +964,7 @@
output_var='SnowCoverS[0]', unit='mm', end=[],
steps=['repStateMaps'], all=[],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.16, add_offset=5000.),
'SnowCoverBEnd': ReportedMap(name='SnowCoverBEnd', output_var='SnowCoverS[1]',
unit='mm', end=['repEndMaps'], steps=[],
all=[], restrictoption=[], monthly=False,
@@ -940,7 +973,7 @@
output_var='SnowCoverS[1]', unit='mm', end=[],
steps=['repStateMaps'], all=[],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.16, add_offset=5000.),
'SnowCoverCEnd': ReportedMap(name='SnowCoverCEnd', output_var='SnowCoverS[2]',
unit='mm', end=['repEndMaps'], steps=[],
all=[], restrictoption=[], monthly=False,
@@ -949,39 +982,48 @@
output_var='SnowCoverS[2]', unit='mm', end=[],
steps=['repStateMaps'], all=[],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.16, add_offset=5000.),
'SnowCoverMaps': ReportedMap(name='SnowCoverMaps', output_var='SnowCover',
unit='mm', end=[], steps=[], all=['repSnowCoverMaps','repE2O1'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.16, add_offset=5000.),
'SnowMaps': ReportedMap(name='SnowMaps', output_var='Snow', unit='mm',
end=[], steps=[], all=['repSnowMaps'],
- restrictoption=['nonInit'], monthly=False, yearly=False),
+ restrictoption=['nonInit'], monthly=False, yearly=False,
+ scale_factor=0.0031, add_offset=100.),
'SnowMeltMaps': ReportedMap(name='SnowMeltMaps', output_var='SnowMelt',
unit='mm', end=[], steps=[], all=['repSnowMeltMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.0039, add_offset=125.),
+ 'SnowSMaps': ReportedMap(name='SnowSMaps', output_var='snowstor',
+ unit='m', end=[], steps=['repStorageMaps'],
+ all=[], restrictoption=['nonInit'],
+ monthly=False, yearly=False),
+ 'SoilSMaps': ReportedMap(name='SoilSMaps', output_var='soilstor',
+ unit='m', end=[], steps=['repStorageMaps'],
+ all=[], restrictoption=['nonInit'],
+ monthly=False, yearly=False),
'SurfaceRunoffMaps': ReportedMap(name='SurfaceRunoffMaps',
output_var='SurfaceRunoff', unit='mm',
end=[], steps=[],
all=['repSurfaceRunoffMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.0077, add_offset=250.),
'TaOtherMaps': ReportedMap(name='TaOtherMaps', output_var='Ta[0]', unit='mm',
end=[], steps=[], all=['repTaMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=4.6e-4, add_offset=5.0),
'TaForestMaps': ReportedMap(name='TaForestMaps', output_var='Ta[1]', unit='mm',
end=[], steps=[], all=['repTaMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=4.6e-4, add_offset=5.0),
'TaIrrigationMaps': ReportedMap(name='TaIrrigationMaps', output_var='Ta[2]', unit='mm',
end=[], steps=[], all=['repTaMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=4.6e-4, add_offset=5.0),
'TaMaps': ReportedMap(name='TaMaps', output_var='TaPixel', unit='mm', end=[],
steps=[], all=['repE2O1','repTaMaps'], restrictoption=['nonInit'],
- monthly=False, yearly=False),
+ monthly=False, yearly=False, scale_factor=4.6e-4, add_offset=5.0),
'TavgMapsOut': ReportedMap(name='TavgMapsOut', output_var='Tavg', unit='degree',
end=[], steps=[], all=['repTavgMaps'],
restrictoption=[], monthly=False, yearly=False),
@@ -996,12 +1038,12 @@
unit='', end=[], steps=[],
all=['repThetaForestMaps','repThetaMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=1.6e-05, add_offset=0.5),
'Theta1ForestState': ReportedMap(name='Theta1ForestState',
output_var='Theta1a[1]', unit='', end=[],
steps=['repStateMaps'], all=[],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=1.6e-05, add_offset=0.5),
'Theta1IrrigationEnd': ReportedMap(name='Theta1IrrigationEnd',
output_var='Theta1a[2]', unit='',
end=['repEndMaps','InitLisflood'], steps=[], all=[],
@@ -1012,20 +1054,20 @@
end=[], steps=[],
all=['repThetaIrrigationMaps','repThetaMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=1.6e-05, add_offset=0.5),
'Theta1IrrigationState': ReportedMap(name='Theta1IrrigationState',
output_var='Theta1a[2]', unit='',
end=[], steps=['repStateMaps'], all=[],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=1.6e-05, add_offset=0.5),
'Theta1Maps': ReportedMap(name='Theta1Maps', output_var='Theta1a[0]', unit='',
end=[], steps=[], all=['repThetaMaps','repE2O2'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=1.6e-05, add_offset=0.5),
'Theta1State': ReportedMap(name='Theta1State', output_var='Theta1a[0]',
unit='', end=[], steps=['repStateMaps'],
all=[], restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=1.6e-05, add_offset=0.5),
'Theta2End': ReportedMap(name='Theta2End', output_var='Theta1b[0]', unit='',
end=['repEndMaps','InitLisflood'], steps=[], all=[],
restrictoption=[], monthly=False, yearly=False),
@@ -1037,12 +1079,12 @@
unit='', end=[], steps=[],
all=['repThetaForestMaps','repThetaMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=1.6e-05, add_offset=0.5),
'Theta2ForestState': ReportedMap(name='Theta2ForestState',
output_var='Theta1b[1]', unit='', end=[],
steps=['repStateMaps'], all=[],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=1.6e-05, add_offset=0.5),
'Theta2IrrigationEnd': ReportedMap(name='Theta2IrrigationEnd',
output_var='Theta1b[2]', unit='',
end=['repEndMaps','InitLisflood'], steps=[], all=[],
@@ -1053,20 +1095,20 @@
end=[], steps=[],
all=['repThetaIrrigationMaps','repThetaMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=1.6e-05, add_offset=0.5),
'Theta2IrrigationState': ReportedMap(name='Theta2IrrigationState',
output_var='Theta1b[2]', unit='',
end=[], steps=['repStateMaps'], all=[],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=1.6e-05, add_offset=0.5),
'Theta2Maps': ReportedMap(name='Theta2Maps', output_var='Theta1b[0]', unit='',
end=[], steps=[], all=['repThetaMaps','repE2O2'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=1.6e-05, add_offset=0.5),
'Theta2State': ReportedMap(name='Theta2State', output_var='Theta1b[0]',
unit='', end=[], steps=['repStateMaps'],
all=[], restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=1.6e-05, add_offset=0.5),
'Theta3End': ReportedMap(name='Theta3End', output_var='Theta2[0]', unit='',
end=['repEndMaps','InitLisflood'], steps=[], all=[],
restrictoption=[], monthly=False, yearly=False),
@@ -1078,12 +1120,12 @@
unit='', end=[], steps=[],
all=['repThetaForestMaps','repThetaMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=1.6e-05, add_offset=0.5),
'Theta3ForestState': ReportedMap(name='Theta3ForestState',
output_var='Theta2[1]', unit='', end=[],
steps=['repStateMaps'], all=[],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=1.6e-05, add_offset=0.5),
'Theta3IrrigationEnd': ReportedMap(name='Theta3IrrigationEnd',
output_var='Theta2[2]', unit='',
end=['repEndMaps','InitLisflood'], steps=[], all=[],
@@ -1093,20 +1135,20 @@
output_var='Theta2[2]', unit='', end=[],
steps=[], all=['repThetaIrrigationMaps','repThetaMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=1.6e-05, add_offset=0.5),
'Theta3IrrigationState': ReportedMap(name='Theta3IrrigationState',
output_var='Theta2[2]', unit='',
end=[], steps=['repStateMaps'], all=[],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=1.6e-05, add_offset=0.5),
'Theta3Maps': ReportedMap(name='Theta3Maps', output_var='Theta2[0]', unit='',
end=[], steps=[], all=['repThetaMaps','repE2O2'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=1.6e-05, add_offset=0.5),
'Theta3State': ReportedMap(name='Theta3State', output_var='Theta2[0]',
unit='', end=[], steps=['repStateMaps'],
all=[], restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=1.6e-05, add_offset=0.5),
'TotalAbsGroundwater': ReportedMap(name='TotalAbsGroundwater',
output_var='abstraction_GW_actual_M3*self.var.M3toMM',
unit='mm', end=[], steps=['repTotalAbs'],
@@ -1151,17 +1193,21 @@
output_var='TotalRunoff',
unit='mm', end=[], steps=[],
all=['repTotalRunoffMaps'], restrictoption=['nonInit'],
- monthly=False, yearly=False),
+ monthly=False, yearly=False, scale_factor=0.0077, add_offset=250.),
'TotalToChanMaps': ReportedMap(name='TotalToChanMaps',
output_var='ToChanM3Runoff*self.var.M3toMM',
unit='mm', end=[], steps=[],
all=['repTotalRunoffMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.0054, add_offset=175.),
+ 'TWSMaps': ReportedMap(name='TWSMaps', output_var='twsstor',
+ unit='m', end=[], steps=['repTWSMaps'],
+ all=[], restrictoption=['nonInit'],
+ monthly=False, yearly=False),
'TransLossMaps': ReportedMap(name='TransLossMaps', output_var='TransLossM3*self.var.M3toMM',
unit='mm', end=[], steps=[], all=['repGwLossMaps','repE2O2'],
restrictoption=['nonInit','TransLoss'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.0092, add_offset=300.),
'UpstreamInflowUsedM3': ReportedMap(name='UpstreamInflowUsedM3',
output_var='UpstreamInflowUsedM3',
unit='M3', end=[], steps=['repWaterUse'],
@@ -1182,52 +1228,53 @@
'UZForestMaps': ReportedMap(name='UZForestMaps', output_var='UZ[1]', unit='mm',
end=[], steps=[], all=['repUZMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.054, add_offset=1750.),
'UZForestState': ReportedMap(name='UZForestState', output_var='UZ[1]',
unit='mm', end=[], steps=['repStateMaps'],
all=[], restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.054, add_offset=1750.),
'UZIrrigationEnd': ReportedMap(name='UZIrrigationEnd', output_var='UZ[2]',
unit='mm', end=['repEndMaps','InitLisflood'], steps=[],
all=[], restrictoption=[],
- monthly=False, yearly=False),
+ monthly=False, yearly=False,),
'UZIrrigationMaps': ReportedMap(name='UZIrrigationMaps', output_var='UZ[2]',
unit='mm', end=[], steps=[],
all=['repUZMaps'], restrictoption=['nonInit'],
- monthly=False, yearly=False),
+ monthly=False, yearly=False, scale_factor=0.054, add_offset=1750.),
'UZIrrigationState': ReportedMap(name='UZIrrigationState', output_var='UZ[2]',
unit='mm', end=[], steps=['repStateMaps'],
all=[], restrictoption=['nonInit'],
- monthly=False, yearly=False),
+ monthly=False, yearly=False, scale_factor=0.054, add_offset=1750.),
'UZMaps': ReportedMap(name='UZMaps', output_var='UZ[0]', unit='mm', end=[],
steps=[], all=['repUZMaps'], restrictoption=['nonInit'],
- monthly=False, yearly=False),
+ monthly=False, yearly=False, scale_factor=0.054, add_offset=1750.),
'UZOutflowForestMaps': ReportedMap(name='UZOutflowForestMaps',
output_var='UZOutflow[1]', unit='mm',
end=[], steps=[],
all=['repUZOutflowMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.0046, add_offset=150.),
'UZOutflowIrrigationMaps': ReportedMap(name='UZOutflowIrrigationMaps',
output_var='UZOutflow[2]', unit='mm',
end=[], steps=[],
all=['repUZOutflowMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.0046, add_offset=150.),
'UZOutflowOtherMaps': ReportedMap(name='UZOutflowOtherMaps',
output_var='UZOutflow[0]', unit='mm',
end=[], steps=[],
all=['repUZOutflowMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.0046, add_offset=150.),
'UZOutflowMaps': ReportedMap(name='UZOutflowMaps', output_var='UZOutflowPixel',
unit='mm', end=[], steps=[],
all=['repUZOutflowMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.0046, add_offset=150.),
'UZState': ReportedMap(name='UZState', output_var='UZ[0]', unit='mm', end=[],
steps=['repStateMaps'], all=[],
- restrictoption=['nonInit'], monthly=False, yearly=False),
+ restrictoption=['nonInit'], monthly=False, yearly=False,
+ scale_factor=0.054, add_offset=1750.),
'WEI_Abs': ReportedMap(name='WEI_Abs', output_var='WEI_Abs', unit='-',
end=[], steps=['repWIndex'], all=[],
restrictoption=['nonInit', 'wateruse', 'indicator'],
@@ -1258,17 +1305,11 @@
unit='m', end=[], steps=[],
all=['repWaterDepthMaps'],
restrictoption=['nonInit'], monthly=False,
- yearly=False),
+ yearly=False, scale_factor=0.0039, add_offset=125.),
'WaterDepthState': ReportedMap(name='WaterDepthState', output_var='WaterDepth',
unit='m', end=[], steps=[],
all=[], restrictoption=['nonInit'],
- monthly=False, yearly=False),
- 'TotalWaterStorageMaps': ReportedMap(name='TotalWaterStorageMaps',
- output_var='TotalWaterStorageMM',
- unit='mm', end=[], steps=[],
- all=['repTotalWaterStorageMaps'],
- restrictoption=['nonInit'],
- monthly=False, yearly=False),
+ monthly=False, yearly=False, scale_factor=0.0039, add_offset=125.),
'WaterSecurityIndex': ReportedMap(name='WaterSecurityIndex',
output_var='WaterSecurityIndex', unit='-',
end=[], steps=['repWIndex'], all=[],
@@ -1601,10 +1642,17 @@
'TransLossQAvgTS' : TimeSeries(name='TransLossQAvgTS', output_var='TransLossQAvg',
where='Gauges', repoption=['repRateUpsGauges'],
restrictoption=['nonInit', 'TransLoss'],
- operation=[''])},
+ operation=['']),
+ 'TWSAvUpsTS': TimeSeries(name='TWSAvUpsTS', output_var='twsstor',
+ where='Gauges', repoption=['repStateUpsGauges'],
+ restrictoption=['nonInit', 'repTWS'],
+ operation=['total'])},
'useWaterDemandAveYear': False,
'varfractionwater': False,
'wateruse': False,
'wateruseRegion': False,
'writeNetcdf': False,
- 'writeNetcdfStack': False}
+ 'writeNetcdfStack': False,
+ 'outputThreads': False,
+ 'NetCDFTimeChunksPrefetch': False,
+ 'useEarthkitHydro': True}
\ No newline at end of file
diff --git a/src/lisflood/global_modules/earthkit_network.py b/src/lisflood/global_modules/earthkit_network.py
new file mode 100644
index 00000000..034ca943
--- /dev/null
+++ b/src/lisflood/global_modules/earthkit_network.py
@@ -0,0 +1,295 @@
+"""
+
+Copyright 2019 European Union
+
+Licensed under the EUPL, Version 1.2 or as soon they will be approved by the European Commission subsequent versions of the EUPL (the "Licence");
+
+You may not use this work except in compliance with the Licence.
+You may obtain a copy of the Licence at:
+
+https://joinup.ec.europa.eu/sites/default/files/inline-files/EUPL%20v1_2%20EN(1).txt
+
+Unless required by applicable law or agreed to in writing, software distributed
+under the Licence is distributed on an "AS IS" basis,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the Licence for the specific language governing permissions and limitations under the Licence.
+
+"""
+import numpy as np
+from decimal import Decimal
+import numpy as np
+
+from .errors import LisfloodWarning
+
+# ------------------------------------------------------------------------
+# earthkit-hydro bridge (opt-in via the `useEarthkitHydro` setoption)
+# ------------------------------------------------------------------------
+# Replaces the slow single-threaded PCRaster engine reductions on the
+# reporting path (catchmenttotal/accuflux/areaaverage) with earthkit-hydro's
+# parallel, numpy-native topological accumulation on a RiverNetwork built
+# once from the model Ldd. Verified against PCRaster accuflux:
+# - earthkit node order == LISFLOOD compressed 1D (maskflat) order,
+# - accumulation results BIT-IDENTICAL on finite (in-domain) cells
+# (see tests/test_earthkit_network.py).
+# The PCRaster path is kept verbatim behind the flag.
+
+try:
+ import earthkit.hydro as _ekh
+ from earthkit.hydro._readers import from_d8 as _ek_from_d8
+ from earthkit.hydro.data_structures._network import RiverNetwork as _EKRiverNetwork
+
+ _EKH_IMPORT_ERROR = None
+except Exception as _e: # pragma: no cover - optional dependency
+ # ImportError (package missing) or AttributeError (the private layout moved
+ # in a newer earthkit-hydro release): the bridge is pinned to the 1.x
+ # private layout, so degrade to "unavailable" and let callers fall back to
+ # the PCRaster path instead of crashing the import chain.
+ _ekh = None
+ _ek_from_d8 = None
+ _EKRiverNetwork = None
+ _EKH_IMPORT_ERROR = '%s: %s' % (type(_e).__name__, _e)
+
+
+# Return type used for all earthkit operations. 'masked' returns a 1D array
+# over the network nodes (== LISFLOOD compressed order) with NaN outside the
+# network domain. Since LISFLOOD's Ldd is exactly the network domain, all
+# in-domain cells are finite.
+_EK_RETURN_TYPE = 'masked'
+
+# Storage-level key of the currently built network, to detect Ldd changes
+# across instances (defensive; within one process the Ldd never changes).
+_EK_CACHE_KEY = None
+_EK_CACHE_NETWORK = None
+
+
+def earthkit_available():
+ """ True if earthkit-hydro is importable in this environment. """
+ return _ekh is not None
+
+
+def _storage_mask(storage):
+ """ Boolean flat mask of the network domain (True = in-domain).
+
+ earthkit's storage.mask is an int64 INDEX array over the flat grid
+ (used as v.flat[mask] in the xarray decorator), not a boolean mask, so
+ materialise the boolean explicitly.
+ """
+ idx = np.asarray(storage.mask).ravel()
+ if idx.dtype == bool:
+ return idx
+ boolean = np.zeros(int(storage.shape[0]) * int(storage.shape[1]), dtype=bool)
+ boolean[idx] = True
+ return boolean
+
+
+def _network_from_ldd(ldd_np):
+ """ Build an earthkit RiverNetwork from a numpy Ldd (PCRaster d8 codes,
+ 0/-9999 outside the model mask).
+
+ The network node order equals the row-major (C-order) in-domain cell
+ order, which is exactly LISFLOOD's compressed 1D array order
+ (MaskInfo.info.maskflat, True = outside). The two masks are asserted
+ equal at build time so any mismatch fails loudly rather than silently
+ scrambling the reduction.
+ """
+ global _EK_CACHE_KEY, _EK_CACHE_NETWORK
+
+ ldd_np = np.asarray(ldd_np)
+ shape = ldd_np.shape
+ key = (shape, ldd_np.tobytes())
+
+ if _EK_CACHE_NETWORK is not None and _EK_CACHE_KEY == key:
+ return _EK_CACHE_NETWORK
+
+ storage = _ek_from_d8(ldd_np, 'pcr_d8')
+
+ # node order must match the LISFLOOD compressed order: earthkit keeps
+ # cells whose d8 value is a valid pcr code (1..9); the model mask is
+ # derived from the same Ldd, so the two must coincide exactly.
+ ek_valid = _storage_mask(storage)
+ lis_valid = (ldd_np > 0) & (ldd_np <= 9) # same rule used by lddmask/boolean(Ldd)
+ if not np.array_equal(ek_valid, lis_valid.ravel()):
+ raise LisfloodWarning(
+ 'earthkit-hydro network mask does not match the model Ldd mask; '
+ 'falling back to PCRaster reductions.'
+ )
+
+ net = _EKRiverNetwork(storage)
+ net.set_default_return_type(_EK_RETURN_TYPE)
+ # coords are only used by the xarray decorator for metadata; supply a
+ # minimal dict so the numpy path does not fail on None.items().
+ net.coords = {'y': np.arange(shape[0]), 'x': np.arange(shape[1])}
+
+ _EK_CACHE_KEY = key
+ _EK_CACHE_NETWORK = net
+ return net
+
+
+def get_earthkit_network(ldd_np):
+ """ Public entry: build (or fetch the cached) earthkit RiverNetwork for
+ the given numpy Ldd. Raises ImportError if earthkit-hydro is missing so
+ callers can fall back to the PCRaster path with a clear message.
+ """
+ if _ekh is None:
+ raise ImportError(
+ 'useEarthkitHydro=True requires the earthkit-hydro package '
+ '(pip install earthkit-hydro): %s' % _EKH_IMPORT_ERROR
+ )
+ return _network_from_ldd(ldd_np)
+
+
+def ek_upstream_sum(net, field_1d, node_weights=None):
+ """ earthkit equivalent of PCRaster accuflux / catchmenttotal over the
+ full Ldd: for every node, the sum of the field over its whole upstream
+ subnetwork (node itself included).
+
+ field_1d : 1D compressed array (LISFLOOD order == earthkit node order)
+ node_weights : optional per-node weights (e.g. PixelArea for m3 sums)
+ returns : 1D array over the same compressed order (values at
+ outside-network nodes are NaN by construction of the
+ masked return type; with LISFLOOD's Ldd there are none)
+ """
+ out = _ekh.upstream.sum(net, field_1d, node_weights=node_weights,
+ return_type=_EK_RETURN_TYPE)
+ return np.asarray(out)
+
+
+def ek_upstream_sum_batch(net, fields_2d, node_weights=None):
+ """ Batched form of ek_upstream_sum: one topological traversal for a
+ stack of fields that share the same network and node_weights.
+
+ fields_2d : 2D numpy array with the NODE AXIS LAST, i.e. shape
+ (n_series, n_nodes) in LISFLOOD compressed order. The
+ leading axis is the batch axis; each row is one field.
+ node_weights : optional per-node weights (shape (n_nodes,)), broadcast
+ across the batch (e.g. PixelArea for m3 sums).
+ returns : 2D array of shape (n_series, n_nodes), one upstream-sum
+ row per input field, in the same compressed order.
+
+ The batch axis is a pure outer vectorization: each node accumulates its
+ upstream contributions in the same topological order regardless of batch
+ width, so every row is bit-identical to the scalar ek_upstream_sum of
+ that field. The node axis MUST be last because node_weights broadcasts
+ against the trailing axis; a transposed array raises a broadcast error.
+ """
+ arr = np.asarray(fields_2d, dtype=np.float64)
+ if arr.ndim != 2:
+ raise ValueError(
+ 'ek_upstream_sum_batch expects a 2D array with layout '
+ '(n_series, n_nodes) (node axis last); got ndim=%d' % arr.ndim)
+ out = _ekh.upstream.sum(net, arr, node_weights=node_weights,
+ return_type=_EK_RETURN_TYPE)
+ return np.asarray(out)
+
+
+def model_precision():
+ """ np dtype of the configured model precision
+ (parallelization.get_model_dtype).
+
+ Indirected here so that (a) bridge consumers do not each import the
+ parallelization module and (b) tests can monkeypatch a deterministic
+ dtype. Only callable when the import shim exposed it, i.e. earthkit-hydro
+ is installed; guard call sites with earthkit_available().
+ """
+ from .parallelization import get_model_dtype
+ return get_model_dtype()
+
+
+def ek_upstream_mean(net, field_1d, node_weights, uparea_1d):
+ """ earthkit equivalent of the PCRaster upstream-average pattern
+ compressArray(catchmenttotal(x * PixelAreaPcr, Ldd) * InvUpArea).
+ Returns the upstream-weighted mean directly, without ever building a
+ 2D PCRaster map (no decompress / numpy2pcr / pcrMul round-trips).
+ Callers should cast the result to model_precision() where the historical
+ compressArray dtype (the model dtype) matters downstream.
+ """
+ tot = ek_upstream_sum(net, field_1d, node_weights=node_weights)
+ return tot * uparea_1d
+
+
+def ek_zone_mean_at_gauges(net, field_1d, zone_codes_flat, n_codes):
+ """ Numpy equivalent of PCRaster areaaverage(expr, spatialId): for each
+ gauge-zone code, the mean of the field over the cells bearing that code
+ (the zone cells themselves, NOT the upstream catchment).
+
+ zone_codes_flat : 1D int array over the compressed cells; 0/negative =
+ not a gauge cell (mirrors the pcr2numpy gauge map with
+ nan replaced by -9999)
+ n_codes : number of gauge codes (max code)
+
+ Returns a 1D array of length n_codes+1 where entry k is the zone-k mean
+ (entry 0 unused). Cast to float32 by the caller to match PCRaster's
+ engine output precision (same convention as sample_compressed).
+ """
+ valid = zone_codes_flat > 0
+ codes = zone_codes_flat[valid].astype(np.int64)
+ # codes may be non-contiguous (e.g. 5, 17, 42): size the arrays by the
+ # largest actual code so every caller-side code index is addressable
+ n_bins = max(int(codes.max()) + 1 if codes.size else 1, n_codes + 1)
+ counts = np.bincount(codes, minlength=n_bins)
+ sums = np.bincount(codes, weights=np.asarray(field_1d)[valid],
+ minlength=n_bins)
+ with np.errstate(invalid='ignore', divide='ignore'):
+ means = sums / counts
+ return means
+
+
+def compute_gauge_zone_codes(outletsmapnp, maskflat, n_codes):
+ """ Zone codes over the compressed cells for multi-cell gauge zones.
+
+ Equivalent of the PCRaster spatialId map flattened to LISFLOOD's compressed
+ 1D order: cells whose gauge code is finite and positive keep their code,
+ every other in-domain cell becomes 0 (= not a gauge cell).
+
+ outletsmapnp : 2D numpy gauge map (pcr2numpy of the spatialId, nan outside)
+ maskflat : MaskInfo.info.maskflat (True = outside the model domain)
+ n_codes : number of distinct gauge codes (self._ncodesId)
+
+ Returns (zone_codes, n_codes) for ek_sample_zone_means /
+ ek_zone_mean_at_gauges. Kept next to the sampling helper so all earthkit
+ logic lives in this module (zusatz only calls it).
+ """
+ codes_flat_full = np.asarray(outletsmapnp).ravel(order='C')
+ zone_codes = np.asarray(codes_flat_full[~np.asarray(maskflat, dtype=bool)])
+ zone_codes = np.where(np.isfinite(zone_codes) & (zone_codes > 0),
+ zone_codes, 0).astype(np.int64)
+ return zone_codes, int(n_codes)
+
+
+def ek_sample_zone_means(tss_obj, array_1d):
+ """ Standalone numpy replacement for TimeoutputTimeseries.sample() on
+ MULTI-CELL gauge zones (the pcraster.areaaverage(expr, spatialId) path),
+ operating on the 1D compressed array without any 2D map round-trip.
+
+ tss_obj must carry the zone attributes prepared in
+ TimeoutputTimeseries.__init__ via compute_gauge_zone_codes
+ (_ek_zone_codes, _ek_n_codes). Semantics match the inherited .sample():
+ per-zone mean for Scalar series, values stored as float32 (PCRaster engine
+ output precision, verified against the engine in
+ tests/test_earthkit_network.py), invalid/NaN stored as Decimal('NaN')
+ (written as 1e31), and the tss file written on the final time step.
+ """
+ arrayRowPos = tss_obj._userModel.currentTimeStep() - tss_obj._userModel.firstTimeStep()
+ # tss header datatype: the .sample() path records the expression's
+ # PCRaster dataType; for the Scalar series routed here it is 'VALUESCALE.Scalar'.
+ if tss_obj._spatialDatatype is None:
+ tss_obj._spatialDatatype = 'VALUESCALE.Scalar'
+ vals = np.asarray(array_1d, dtype=np.float64)
+ # per-zone mean over the zone cells (entry 0 = not-a-zone, discarded)
+ with np.errstate(invalid='ignore'):
+ zone_means = ek_zone_mean_at_gauges(None, vals, tss_obj._ek_zone_codes,
+ tss_obj._ek_n_codes)
+ row = tss_obj._sampleValues[arrayRowPos]
+ nan = Decimal("NaN")
+ # map each gauge code to its column: codesId[k] is the k-th code
+ # (sorted ascending, as produced by np.unique); column order in the
+ # tss row follows codesId order exactly like the inherited .sample().
+ for col in range(tss_obj._ncodesId):
+ code = tss_obj._codesId[col]
+ m = zone_means[code] if 0 < code < zone_means.size else np.nan
+ # PCRaster areaaverage returns float32; cast for identical %14g
+ m32 = np.float32(m) if np.isfinite(m) else m
+ row[col] = float(m32) if np.isfinite(m32) else nan
+ # Write the tss on the final step, mirroring the inherited .sample().
+ if tss_obj._userModel.currentTimeStep() == tss_obj._userModel.nrTimeSteps():
+ tss_obj._writeTssFile()
\ No newline at end of file
diff --git a/src/lisflood/global_modules/netcdf.py b/src/lisflood/global_modules/netcdf.py
index 5d08ed19..9ff0057f 100644
--- a/src/lisflood/global_modules/netcdf.py
+++ b/src/lisflood/global_modules/netcdf.py
@@ -1,7 +1,9 @@
import os
import glob
import warnings
+import threading
import xarray as xr
+import dask
import numpy as np
import datetime
import pcraster
@@ -19,8 +21,56 @@
from .add1 import nanCheckMap, decompress
from netCDF4 import default_fillvals
+# Global HDF5 lock shared by ALL netCDF4/HDF5 Dataset access in the process
+# (reads and writes). The conda-forge netCDF4/HDF5 libraries are not built
+# --enable-threadsafe, so *any* concurrent call into the HDF5 library -- even
+# on DISTINCT files -- corrupts HDF5 global state and aborts the process
+# (SIGSEGV or Fatal Python error: Aborted).
+# The lock is deliberately NARROW: it is held only around actual HDF5
+# library operations (file open, variable/attribute read, array write, close),
+# never around pure-numpy/dask work such as CF decoding, packing/unpacking or
+# array arithmetic. Readers pass it to xarray via `lock=hdf5_lock` on the
+# open_dataset/open_mfdataset calls, so xarray acquires it around every
+# C-level netCDF4 access (including the GC-triggered lazy close of cached
+# file handles). Writers (output.py) acquire it per operation around their
+# open/header, slice writes and close. Holding several file handles open at
+# once across threads is fine: only non-overlapping *calls* matter, and this
+# per-operation serialisation is the same model the reader side has always
+# used. An RLock is used so a thread that legitimately re-enters (e.g. a
+# writer calling a helper that also takes the lock) is not deadlocked.
+# NOTE: building netCDF4/HDF5 with --enable-threadsafe would make this lock
+# unnecessary.
+hdf5_lock = threading.RLock()
+
from .. import __authors__, __version__, __date__, __status__, __institution__
+_OUTPUT_DTYPE_WARNED = False
+
+
+def _coherent_output_dtype(binding, dtype):
+ """Return the output map dtype, downgrading float64 -> float32 (with a
+ one-time warning) when the model itself runs in float32 (modelPrecision).
+ Writing float64 maps for a float32 model stores non-existent precision and
+ doubles file size. Any non-float64 OutputMapsDataType (e.g. an explicit
+ float32) is left untouched."""
+ global _OUTPUT_DTYPE_WARNED
+ try:
+ from .parallelization import resolve_model_dtype
+ model_dt = resolve_model_dtype(binding)
+ except Exception:
+ return dtype
+ if model_dt == np.float32 and str(dtype).strip().lower() in ("float64", "f8", "double"):
+ if not _OUTPUT_DTYPE_WARNED:
+ warnings.warn(LisfloodWarning(
+ "OutputMapsDataType=float64 but modelPrecision=float32: the model "
+ "state is float32, so float64 output would store no extra precision. "
+ "Writing output as float32 instead. Set OutputMapsDataType=float32 "
+ "to silence this, or modelPrecision=float64 for full precision."))
+ _OUTPUT_DTYPE_WARNED = True
+ return "float32"
+ return dtype
+
+
def mask_array_np(data, mask, crop, name, valid_min, valid_max, func_x, func_y):
data_cut = func_x(func_y(data))[:, crop[2]:crop[3], crop[0]:crop[1]]
if (valid_min is not None):
@@ -92,9 +142,14 @@ def check_dataset_calendar_type(ds, path):
# check calendar type
# if using multiple files, the encoding will be droppped (bug in xarray), let's check the calendar fom the first file
if '*' in path:
- ds = xr.open_dataset(glob.glob(path)[0], chunks={'time': -1})
-
- calendar = ds.time.encoding['calendar']
+ # lock=hdf5_lock: every C-level netCDF4 access on this ad-hoc handle
+ # (open, header read, and its close when the with-block exits) must be
+ # serialised against the writer threads; closing it deterministically
+ # here also removes a GC-timed unlocked close from the dynamic phase.
+ with xr.open_dataset(glob.glob(path)[0], chunks={'time': -1}, lock=hdf5_lock) as ds:
+ calendar = ds.time.encoding['calendar']
+ else:
+ calendar = ds.time.encoding['calendar']
if calendar != binding['calendar_type']:
print('WARNING! Wrong calendar type in dataset {}, is \"{}\" and should be \"{}\"\n Please double check your forcing datasets and update them to use the correct calendar type'.format(path, ds.time.encoding['calendar'], binding['calendar_type']))
@@ -171,18 +226,46 @@ class XarrayChunked():
""" Class that handles the reading of netcdf files with temporal chunks
"""
- def __init__(self, data_path, time_chunk, dates, indexer=None, climatology=False):
+ def __init__(self, data_path, time_chunk, dates, indexer=None, climatology=False, prefetch=False):
# load dataset using xarray
if time_chunk != 'auto' and time_chunk is not None:
time_chunk = int(time_chunk)
data_path = data_path + ".nc" if not data_path.endswith('.nc') else data_path
+ # Forcing decode precision. Default float64 keeps the historical
+ # mask_and_scale=True decode, bit-identical. When modelPrecision=float32
+ # (opt-in), decode the forcing NATIVELY in float32: open raw (no decode),
+ # retype the CF scale_factor/add_offset to float32, then xr.decode_cf ->
+ # xarray runs the WHOLE decode (scale/offset + fill mask) in float32,
+ # halving the per-element decode cost (_scale_offset_decoding/_apply_mask/
+ # astype) AND letting the forcing meet float32 model state with no upcast.
+ # decode_cf restores scale_factor/add_offset into .encoding, so the
+ # valid_min/max scaling below is unchanged; forcing is read-only input and
+ # the int16xscale packing carries far fewer significant digits than float32,
+ # so no real precision is lost (verified within reference tolerance).
+ from .parallelization import get_model_dtype
+ _f32_forcing = (get_model_dtype() == np.float32)
try:
- ds = xr.open_mfdataset(
- data_path, engine='netcdf4',
- chunks={'time': time_chunk}, combine='by_coords',
- mask_and_scale=True
- )
+ if _f32_forcing:
+ ds = xr.open_mfdataset(
+ data_path, engine='netcdf4',
+ chunks={'time': time_chunk}, combine='by_coords',
+ mask_and_scale=False,
+ lock=hdf5_lock # serialise C-level HDF5 access with the writer threads
+ )
+ for _v in ds.data_vars:
+ _a = ds[_v].attrs
+ for _k in ('scale_factor', 'add_offset'):
+ if _k in _a and np.issubdtype(np.asarray(_a[_k]).dtype, np.floating):
+ _a[_k] = np.float32(_a[_k])
+ ds = xr.decode_cf(ds, mask_and_scale=True)
+ else:
+ ds = xr.open_mfdataset(
+ data_path, engine='netcdf4',
+ chunks={'time': time_chunk}, combine='by_coords',
+ mask_and_scale=True,
+ lock=hdf5_lock # serialise C-level HDF5 access with the writer threads
+ )
except OSError:
raise OSError(f"Could not open file {data_path}")
@@ -237,11 +320,51 @@ def __init__(self, data_path, time_chunk, dates, indexer=None, climatology=False
if not flags['skipvalreplace']:
valid_min_scaled = (da.attrs['valid_min']*scale_factor+add_offset) if 'valid_min' in da.attrs else None
valid_max_scaled = (da.attrs['valid_max']*scale_factor+add_offset) if 'valid_max' in da.attrs else None
+
self.dataset = compress_xarray(mask, crop, da, var_name, valid_min_scaled, valid_max_scaled, func_x, func_y) # final dataset to store
# initialise class variables and load first chunk
+ # (self.prefetch must exist before init_chunks: load_next_chunk reads it;
+ # it is switched on below, after the first synchronous chunk load)
+ self.prefetch = False
self.init_chunks(self.dataset, time_chunk)
+ # Opt-in background prefetch of the next time chunk. The loaded chunk
+ # is held in _pf_ready and promoted to dataset_chunk by the model
+ # thread at the boundary, so only one next chunk is ever in flight.
+ if prefetch and len(self.chunk_indexes) > 2:
+ # no benefit with a single chunk (no next chunk to load)
+ self.prefetch = True
+ self._pf_cond = threading.Condition()
+ self._pf_state = 'idle' # idle -> running -> ready | error -> (promoted) idle
+ self._pf_error = None
+ self._pf_ready = None # prefetched chunk awaiting promotion
+ self._pf_scheduled_ichunk = None # next-chunk index already in flight
+
+ # ------------------------------------------------------------------
+ # background chunk prefetch machinery
+ # ------------------------------------------------------------------
+ def _prefetch_worker(self, ichunk):
+ """Load chunk `ichunk` into the holding slot _pf_ready. Exceptions are
+ captured and re-raised in the model thread by load_next_chunk."""
+ try:
+ begin = self.chunk_indexes[ichunk]
+ end = self.chunk_indexes[ichunk + 1]
+ chunk = self.dataset.isel(time=range(begin, end))
+ # Same lock discipline as the synchronous path (see load_next_chunk):
+ # the chunk is never written to dataset_chunk here; the model thread
+ # promotes it at the boundary.
+ self._pf_ready = chunk.load()
+ except BaseException as e: # re-raised in the model thread
+ self._pf_error = e
+ finally:
+ with self._pf_cond:
+ if self._pf_error is not None:
+ self._pf_state = 'error'
+ else:
+ self._pf_state = 'ready'
+ self._pf_cond.notify_all()
+
def init_chunks(self, dataset, time_chunk):
# compute chunks indexes in dataset
chunks = self.dataset.chunks[0] # list of chunks indexes in dataset
@@ -256,12 +379,62 @@ def init_chunks(self, dataset, time_chunk):
self.ichunk = -1
self.load_next_chunk()
+ def _schedule_prefetch(self, next_ichunk):
+ """Start loading chunk `next_ichunk` in the background if it is not
+ already in flight."""
+ with self._pf_cond:
+ if self._pf_state == 'running' or self._pf_scheduled_ichunk == next_ichunk:
+ return # already loading this (or a later) chunk
+ self._pf_state = 'running'
+ self._pf_scheduled_ichunk = next_ichunk
+ t = threading.Thread(target=self._prefetch_worker, args=(next_ichunk,),
+ name=f'lisflood-prefetch-{next_ichunk}', daemon=True)
+ t.start()
+
+ def _wait_prefetch_done(self):
+ """Block until any in-flight prefetch finishes; promote the prefetched
+ chunk, re-raising any worker error in the calling thread."""
+ with self._pf_cond:
+ while self._pf_state == 'running':
+ self._pf_cond.wait()
+ if self._pf_state == 'error':
+ self._pf_state = 'idle'
+ err = self._pf_error
+ self._pf_error = None
+ self._pf_scheduled_ichunk = None
+ raise err
+ # success: promote the prefetched chunk
+ self.ichunk += 1
+ self.dataset_chunk = self._pf_ready
+ self._pf_ready = None
+ self._pf_state = 'idle'
+ self._pf_scheduled_ichunk = None
+
def load_next_chunk(self):
+ if self.prefetch:
+ # Boundary reached while the prefetch of this chunk is running or
+ # ready: promote it. Otherwise fall through to the synchronous load.
+ with self._pf_cond:
+ pf_active = self._pf_state in ('running', 'ready')
+ if pf_active:
+ self._wait_prefetch_done()
+ return
+
self.ichunk += 1
begin = self.chunk_indexes[self.ichunk]
end = self.chunk_indexes[self.ichunk+1]
-
+
chunk = self.dataset.isel(time=range(begin, end))
+ # The conda-forge netCDF4/HDF5 libraries are not thread-safe, so every
+ # C-level netCDF4 access must hold the process-wide hdf5_lock: the
+ # datasets are opened with `lock=hdf5_lock` (see __init__) and xarray
+ # acquires it around each netCDF4 operation. The lock is NOT held for
+ # the whole chunk.load(): decoding/masking run outside it, so chunk
+ # loads only serialise on the actual HDF5 reads (also with the
+ # prefetch thread and the threaded map-output writer).
+ # TODO: check the synchronous issue, as actually this slows down the loading of chunks
+ # so it is commented right now
+ #with dask.config.set(scheduler='synchronous'):
self.dataset_chunk = chunk.load() # triggers xarray computation
def __getitem__(self, step):
@@ -272,6 +445,14 @@ def __getitem__(self, step):
if dataset_index == self.chunk_indexes[self.ichunk+1]:
self.load_next_chunk()
+ # Prefetch (opt-in): near the end of the current chunk, start loading
+ # the next one in the background.
+ if self.prefetch:
+ next_boundary = self.chunk_indexes[self.ichunk+1]
+ # schedule only if a next chunk exists (ichunk+1 <= last chunk index)
+ if dataset_index >= next_boundary - 2 and self.ichunk + 2 <= len(self.chunk_indexes) - 1:
+ self._schedule_prefetch(self.ichunk + 1)
+
local_index = dataset_index - self.chunk_indexes[self.ichunk]
if local_index < 0:
@@ -330,13 +511,17 @@ def xarray_reader(dataname, indexer=None, climatology=False):
# extract chunk from bindings
time_chunk = binding['NetCDFTimeChunks'] # -1 to load everything, 'auto' to let xarray decide
+ # opt-in background prefetch of the next time chunk (no effect with a
+ # single chunk: time_chunk=-1, MapsCaching, climatology)
+ prefetch = bool(settings.options.get('NetCDFTimeChunksPrefetch', False))
+
if binding['MapsCaching'] == "True":
data = XarrayCached(data_path, dates, indexer, climatology)
else:
if climatology: # for climatology, load the entire dataset
data = XarrayChunked(data_path, None, dates, indexer, climatology)
else:
- data = XarrayChunked(data_path, time_chunk, dates, indexer, climatology)
+ data = XarrayChunked(data_path, time_chunk, dates, indexer, climatology, prefetch=prefetch)
return data
@@ -439,7 +624,8 @@ def write_netcdf_header(settings,
start_date,
rep_steps,
frequency,
- ):
+ map_value=None,
+ log_packed=None):
""" Writes a netcdf header without the data inside
@@ -465,7 +651,16 @@ def write_netcdf_header(settings,
list of reporting steps
frequency:
output frequency (all, monthly or yearly)
-
+ map_value: ReportedMap namedtuple or None, optional
+ Output variable metadata from default_options.py. When provided and OutputPacking is
+ enabled, its scale_factor and add_offset attributes are used for int16 packing.
+ If None (default), no packing is applied.
+ log_packed: bool or None, optional
+ Explicit override for the logarithmic discharge packing decision. When None
+ (default), the decision is derived from OutputPackingDischarge and the map
+ identity. The writer passes its own decision here so the on-disk dtype and
+ the per-step packing stay in sync (e.g. SUM aggregates disable log packing).
+
Returns
-------
object
@@ -476,6 +671,11 @@ def write_netcdf_header(settings,
binding = settings.binding
dtype = binding['OutputMapsDataType']
+ # Coherence: writing float64 output while the model runs in float32 wastes
+ # space and implies a precision the model does not have. If modelPrecision is
+ # float32 but OutputMapsDataType is float64, switch the output to float32 and
+ # warn once (the state carries no more than float32 precision anyway).
+ dtype = _coherent_output_dtype(binding, dtype)
# general Attributes
nf1.settingsfile = os.path.realpath(settings.settings_path)
@@ -537,6 +737,14 @@ def write_netcdf_header(settings,
if frequency == "all":
steps = all_steps
time_stamps = all_dates
+ elif frequency == 'daily':
+ # check next date (step+1) and see if we are still in the same day;
+ # selects one stamp per calendar day (the last sub-daily step of the
+ # day), matching MapOutputAggregated's daily period boundary.
+ next_date_times = np.array([j + datetime.timedelta(seconds=int(binding["DtSec"])) for j in all_dates])
+ days_end = np.array([all_dates[j].day != next_date_times[j].day for j in range(n_steps)])
+ steps = all_steps[days_end]
+ time_stamps = all_dates[days_end]
elif frequency == 'monthly':
# check next date (step+1) and see if we are still in the same month
next_date_times = np.array([j + datetime.timedelta(seconds=int(binding["DtSec"])) for j in all_dates])
@@ -550,7 +758,7 @@ def write_netcdf_header(settings,
steps = all_steps[years_end]
time_stamps= all_dates[years_end]
else:
- raise ValueError(f'ERROR! Frequency {frequency} not supported! Value accepted: [all, monthly, yearly]')
+ raise ValueError(f'ERROR! Frequency {frequency} not supported! Value accepted: [all, daily, monthly, yearly]')
nf1.createDimension('time', steps.size)
time = nf1.createVariable('time', float, ('time'))
@@ -569,7 +777,46 @@ def write_netcdf_header(settings,
time.units = 'minutes since %s' % start_date.strftime("%Y-%m-%d %H:%M:%S.0")
nf1.variables["time"][:] = date2num(time_stamps, time.units, time.calendar)
- value = nf1.createVariable(var_name, dtype, ('time', dim_lat_y, dim_lon_x), zlib=True, fill_value=-9999, chunksizes=(1, nrow, ncol))
+ # value = nf1.createVariable(var_name, dtype, ('time', dim_lat_y, dim_lon_x), zlib=True, fill_value=-9999, chunksizes=(1, nrow, ncol))
+ # Packing: use int16 with CF scale/offset if enabled and variable has packing metadata
+ packing_enabled = binding.get('OutputPacking', 'False') == 'True'
+ has_packing = (map_value is not None
+ and getattr(map_value, 'scale_factor', None) is not None
+ and getattr(map_value, 'add_offset', None) is not None)
+ # Logarithmic discharge packing: a COMPLETELY independent opt-in
+ # (OutputPackingDischarge), separate from OutputPacking above. It only
+ # applies to the DischargeMaps output and writes a custom log10 int16
+ # encoding (non-CF). This predicate MUST stay in sync with
+ # NetcdfStepsWriter.write's `is_log_packed`.
+ if log_packed is not None:
+ is_log_packed = bool(log_packed)
+ else:
+ is_log_packed = (binding.get('OutputPackingDischarge', 'False') == 'True'
+ and map_value is not None
+ and getattr(map_value, 'name', None) == 'DischargeMaps')
+ if is_log_packed:
+ var_dtype = 'i2'
+ var_fill = default_fillvals['i2']
+ elif packing_enabled and has_packing:
+ var_dtype = 'i2'
+ var_fill = default_fillvals['i2']
+ else:
+ var_dtype = dtype
+ var_fill = -9999
+
+ value = nf1.createVariable(var_name, var_dtype, ('time', dim_lat_y, dim_lon_x),
+ zlib=True, fill_value=var_fill, chunksizes=(1, nrow, ncol))
+ if is_log_packed:
+ # Custom (non-CF) log10 encoding. Do NOT set scale_factor/add_offset
+ # here: that would make CF readers auto-decode the int16 as a linear
+ # value. Instead tag the explicit transform so tooling can invert it.
+ from .output import LOG_PACK_ATTRS
+ for _attr, _val in LOG_PACK_ATTRS.items():
+ value.setncattr(_attr, _val)
+ elif packing_enabled and has_packing:
+ value.scale_factor = np.float64(map_value.scale_factor)
+ value.add_offset = np.float64(map_value.add_offset)
+
else:
value = nf1.createVariable(var_name, dtype, (dim_lat_y, dim_lon_x), zlib=True, fill_value=-9999)
diff --git a/src/lisflood/global_modules/output.py b/src/lisflood/global_modules/output.py
index e238f21e..2b9ab5e3 100644
--- a/src/lisflood/global_modules/output.py
+++ b/src/lisflood/global_modules/output.py
@@ -15,17 +15,148 @@
"""
import os
+import datetime
import numpy as np
-from pcraster import ifthen, catchmenttotal, mapmaximum
+from pcraster import ifthen, catchmenttotal, mapmaximum, pcr2numpy
+from pcraster import report as pcraster_report
+from pcraster.framework.frameworkBase import generateNameT
import sys
+import warnings
from .zusatz import TimeoutputTimeseries
from .add1 import decompress, valuecell, loadmap, compressArray
from .netcdf import write_netcdf_header, iterOpenNetcdf, nanCheckMap, uncompress_array
-from .errors import LisfloodFileError, LisfloodWarning
-from .settings import inttodate, CDFFlags, LisSettings
+from .netcdf import hdf5_lock as _hdf5_lock
+from .errors import LisfloodError, LisfloodFileError, LisfloodWarning
+from .settings import inttodate, CDFFlags, LisSettings, MaskInfo
+from netCDF4 import default_fillvals
+
+# ------------------------------------------------------------------------
+# Optional earthkit-hydro acceleration of the PCRaster reductions
+# (`useEarthkitHydro` setoption, default True; set it to 0 to use the
+# historical PCRaster path). See earthkit_network.py.
+# ------------------------------------------------------------------------
+try:
+ from .earthkit_network import (
+ earthkit_available,
+ get_earthkit_network,
+ ek_upstream_sum,
+ ek_upstream_sum_batch,
+ ek_upstream_mean,
+ model_precision,
+ )
+except ImportError: # pragma: no cover - earthkit-hydro not installed
+ earthkit_available = staticmethod(lambda: False)
+ get_earthkit_network = None
+ ek_upstream_sum = None
+ ek_upstream_sum_batch = None
+ ek_upstream_mean = None
+
+ def model_precision():
+ from .parallelization import get_model_dtype
+ return get_model_dtype()
+# ------------------------------------------------------------------------
+# Packing constants for int16 CF scale/offset encoding
+# ------------------------------------------------------------------------
+PACK_FILL = np.int16(default_fillvals['i2']) # -32767
+# Keep PACK_MIN / PACK_MAX as plain Python ints: arithmetic between np.int16
+# scalars silently overflows under numpy 2.x NEP 50 semantics (e.g.
+# PACK_MAX - PACK_MIN wraps to -3), which would corrupt derived quantities such
+# as LOG_PACK_SCALE below. Python ints are unbounded, so the span is exact, and
+# they still broadcast/clip correctly against int16 arrays.
+PACK_MIN = int(PACK_FILL) + 1 # -32766
+PACK_MAX = int(np.iinfo(np.int16).max) # 32767
+
+# ------------------------------------------------------------------------
+# Logarithmic int16 packing for discharge (OutputPackingDischarge)
+# ------------------------------------------------------------------------
+# Discharge spans several orders of magnitude on a global domain, so the
+# linear CF scale/offset packing used for the other variables would lose too
+# much precision. Instead we pack log10(discharge) linearly into int16, which
+# gives (near) constant RELATIVE precision across the whole range.
+#
+# This is a NON-CF encoding: readers cannot auto-decode it via the standard
+# scale_factor/add_offset attributes. Instead, the writer tags the variable
+# with explicit custom attributes (see LOG_PACK_ATTRS below) describing the
+# transform so a decoder can invert it:
+#
+# stored = round((log10(q) - LOG_PACK_MIN) / LOG_PACK_SCALE) + PACK_MIN
+# q = 10 ** (LOG_PACK_MIN + LOG_PACK_SCALE * (stored - PACK_MIN))
+#
+# Discharge values below LOG_PACK_QMIN (0.01 m3/s) -- including zero and
+# negatives -- carry no useful information at this precision and are stored as
+# _FillValue (read back as masked). Values above LOG_PACK_QMAX are clipped.
+LOG_PACK_QMIN = 0.01 # m3/s, smallest resolved discharge
+LOG_PACK_QMAX = 500000.0 # m3/s, largest resolved discharge
+LOG_PACK_MIN = np.log10(LOG_PACK_QMIN) # -2.0
+LOG_PACK_MAX = np.log10(LOG_PACK_QMAX) # ~5.69897
+# Map [LOG_PACK_MIN, LOG_PACK_MAX] onto the usable int16 range
+# [PACK_MIN, PACK_MAX] (PACK_FILL is reserved as _FillValue).
+LOG_PACK_SCALE = (LOG_PACK_MAX - LOG_PACK_MIN) / float(PACK_MAX - PACK_MIN)
+
+# Custom attributes written on a log-packed discharge variable so downstream
+# tooling can invert the transform. These are NOT CF scale_factor/add_offset.
+LOG_PACK_ATTRS = {
+ 'packing': 'log10',
+ 'packing_log_min': np.float64(LOG_PACK_MIN),
+ 'packing_log_scale': np.float64(LOG_PACK_SCALE),
+ 'packing_int_offset': np.float64(PACK_MIN),
+ 'packing_valid_min': np.float64(LOG_PACK_QMIN),
+ 'packing_valid_max': np.float64(LOG_PACK_QMAX),
+}
+
+
+def pack_discharge_log(values):
+ """Pack a discharge array (float, physical m3/s) into int16 log space.
+
+ Parameters
+ ----------
+ values : numpy.ndarray
+ Discharge values in m3/s. May contain NaN / fill (-9999) / non-positive
+ entries; all of those are stored as PACK_FILL.
+
+ Returns
+ -------
+ numpy.ndarray (int16)
+ Log-packed discharge, same shape as `values`.
+ """
+ q = np.asarray(values, dtype=np.float64)
+ # Resolvable cells: finite and >= the minimum resolved discharge. Zero,
+ # negative, NaN, and the -9999 sentinel all fall out here and become fill.
+ resolved = np.isfinite(q) & (q >= LOG_PACK_QMIN)
+
+ packed = np.full(q.shape, PACK_FILL, dtype=np.int16)
+ if resolved.any():
+ qr = np.clip(q[resolved], LOG_PACK_QMIN, LOG_PACK_QMAX)
+ idx = np.round((np.log10(qr) - LOG_PACK_MIN) / LOG_PACK_SCALE) + PACK_MIN
+ idx = np.clip(idx, PACK_MIN, PACK_MAX)
+ packed[resolved] = idx.astype(np.int16)
+ return packed
+
+
+def unpack_discharge_log(packed):
+ """Invert :func:`pack_discharge_log`.
+
+ Parameters
+ ----------
+ packed : numpy.ndarray (int16)
+ Log-packed discharge as produced by `pack_discharge_log`.
+
+ Returns
+ -------
+ numpy.ndarray (float64)
+ Discharge in m3/s. Fill cells are returned as NaN.
+ """
+ stored = np.asarray(packed)
+ out = np.full(stored.shape, np.nan, dtype=np.float64)
+ valid = stored != PACK_FILL
+ if valid.any():
+ exponent = LOG_PACK_MIN + LOG_PACK_SCALE * (stored[valid].astype(np.float64) - PACK_MIN)
+ out[valid] = np.power(10.0, exponent)
+ return out
+
# ------------------------------------------------------------------------
# Writer classes
# ------------------------------------------------------------------------
@@ -93,15 +224,20 @@ def stage(self):
def write(self, start_date, rep_steps):
if self.data is not None:
- nf1 = write_netcdf_header(self.settings, self.map_name, self.map_path, self.var.DtDay,
- self.map_key, self.map_value.output_var, self.map_value.unit,
- start_date, rep_steps, self.frequency)
+ # HDF5 lock only around the actual netCDF4 calls (open/header,
+ # slice write, close); everything else is pure numpy and runs
+ # outside the lock so writer threads can overlap on it.
+ with _hdf5_lock:
+ nf1 = write_netcdf_header(self.settings, self.map_name, self.map_path, self.var.DtDay,
+ self.map_key, self.map_value.output_var, self.map_value.unit,
+ start_date, rep_steps, self.frequency, map_value=self.map_value)
map_np = uncompress_array(self.data)
- nf1.variables[self.map_name][:, :] = map_np
+ with _hdf5_lock:
+ nf1.variables[self.map_name][:, :] = map_np
- nf1.close()
+ nf1.close()
else:
raise Exception('You need to stage the variable data before writing!')
@@ -120,6 +256,12 @@ def __init__(self, var, map_key, map_value, map_path, frequency, flag, end_step)
self.end_step = end_step
self.step_range = []
self.data_steps = []
+ # End-of-run flag snapshotted in stage() (main thread); see _checkpoint.
+ self._stage_end_run = False
+ # Set True by MapOutputAggregated for monthly/yearly SUM aggregates to
+ # suppress the logarithmic discharge packing (those sums can exceed the
+ # calibrated range). Daily sums and mean aggregates leave it False.
+ self.log_pack_disabled = False
super().__init__(var, map_key, map_value, map_path, frequency)
@@ -127,7 +269,9 @@ def _checkpoint(self):
""" Private, makes sure this is the right time to write
"""
write = False
- end_run = self.var.currentTimeStep() == self.end_step
+ # Use the snapshot taken in stage(): write() may run on a worker thread
+ # while the main thread has already advanced the model clock.
+ end_run = self._stage_end_run
if len(self.data_steps) == self.chunks or end_run:
write = True
return write
@@ -144,23 +288,124 @@ def stage(self):
self.step_range.append(step)
self.data_steps.append(map_np)
+ self._stage_end_run = self.var.currentTimeStep() == self.end_step
def write(self, start_date, rep_steps):
if self._checkpoint():
if self.data_steps:
- if self.step_range[0] == 0:
- nf1 = write_netcdf_header(self.settings, self.map_name, self.map_path, self.var.DtDay,
- self.map_key, self.map_value.output_var, self.map_value.unit,
- start_date, rep_steps, self.frequency)
- else:
- nf1 = iterOpenNetcdf(self.map_path, "", 'a', format='NETCDF4')
-
+ # Pre-compute all packed arrays OUTSIDE the HDF5 lock: the int16
+ # packing arithmetic (round/divide/clip/fill) is pure numpy and
+ # is the expensive part of compressed output. Only the actual
+ # netCDF4 calls (open/header, slice writes, close) are serialised.
+ # `prepared_steps` is thread-local to this writer instance, and
+ # the flush handshake guarantees one flush generation at a time
+ # per file, so no cross-thread data race is possible here.
+ # The packing parameters come from the staged step metadata and
+ # are identical for every step of one flush, so they can be read
+ # once before the loop. This mirrors exactly how
+ # write_netcdf_header decides the variable dtype: packing is
+ # int16 with CF scale/offset only when OutputPacking is enabled
+ # AND the map_value carries scale_factor/add_offset (as float64,
+ # matching the values stored in the header).
+ packing_enabled = self.settings.binding.get('OutputPacking', 'False') == 'True'
+ # Decide packing from the RAW scale_factor/add_offset, testing
+ # for None BEFORE any coercion: np.float64(None) is nan, which
+ # `is not None`, so coercing first would enable packing for maps
+ # that carry no packing metadata. This predicate must stay
+ # identical to write_netcdf_header's `has_packing` (netcdf.py) so
+ # the on-disk variable dtype and the per-step packing decision
+ # agree for every map.
+ raw_scale = getattr(self.map_value, 'scale_factor', None) if self.map_value is not None else None
+ raw_offset = getattr(self.map_value, 'add_offset', None) if self.map_value is not None else None
+ is_packed = packing_enabled and raw_scale is not None and raw_offset is not None
+ packed_scale = np.float64(raw_scale) if is_packed else None
+ packed_offset = np.float64(raw_offset) if is_packed else None
+ # Logarithmic discharge packing is a COMPLETELY separate opt-in
+ # (OutputPackingDischarge), independent of OutputPacking above.
+ # It only applies to the DischargeMaps output and uses the
+ # custom log10 int16 encoding (pack_discharge_log), never the
+ # linear CF scale/offset path. This predicate MUST stay in sync
+ # with write_netcdf_header's `is_log_packed`.
+ is_log_packed = (self.settings.binding.get('OutputPackingDischarge', 'False') == 'True'
+ and self.map_key == 'DischargeMaps'
+ and not self.log_pack_disabled)
+ maskinfo = MaskInfo.instance()
+
+ prepared_steps = []
for step, data in zip(self.step_range, self.data_steps):
- nf1.variables[self.map_name][step, :, :] = uncompress_array(data)
-
- nf1.close()
+ if is_log_packed:
+ # Pack BEFORE uncompressing (same rationale as the
+ # linear path): operate on the staged compressed
+ # in-domain vector so masking can never hide NaN.
+ map_in = np.ma.getdata(data).astype(np.float64)
+ packed_in = pack_discharge_log(map_in)
+ packed = np.full(maskinfo.info.shape, PACK_FILL, dtype=np.int16)
+ packed.reshape(-1)[~maskinfo.info.maskflat] = packed_in
+ elif is_packed:
+ # Pack BEFORE uncompressing: operate on the staged
+ # compressed in-domain vector (np.ma.getdata = the
+ # positional raw buffer, as a plain ndarray) so np.ma
+ # auto-masking can never hide NaN from the guard below
+ # (isnan ignores masked entries, while astype(int16)
+ # still casts the raw NaN buffer -> RuntimeWarning +
+ # silent 0). The int16 result is then scattered into a
+ # full-domain array pre-filled with PACK_FILL.
+ map_in = np.ma.getdata(data).astype(np.float64)
+ packed_in = np.round((map_in - packed_offset) / packed_scale)
+ clipped = ((packed_in < PACK_MIN) | (packed_in > PACK_MAX)) & (map_in != -9999)
+ if clipped.any():
+ vmin = packed_offset + packed_scale * PACK_MIN
+ vmax = packed_offset + packed_scale * PACK_MAX
+ warnings.warn(LisfloodWarning(
+ f"OutputPacking: {clipped.sum()} values in '{self.map_name}' outside "
+ f"packing range [{vmin:.4g}, {vmax:.4g}] and will be clipped."
+ ))
+ packed_in = np.clip(packed_in, PACK_MIN, PACK_MAX)
+ # NaN cannot be represented in int16: after the clip it
+ # would silently cast to 0 (i.e. read back as add_offset,
+ # a plausible but wrong value). Store the fill value
+ # instead, so those cells read back as masked, and warn
+ # so the run reports which maps contain non-finite data.
+ nonfinite = np.isnan(packed_in)
+ if nonfinite.any():
+ warnings.warn(LisfloodWarning(
+ f"OutputPacking: {nonfinite.sum()} non-finite (NaN) values in '{self.map_name}' "
+ f"cannot be packed and are stored as _FillValue."
+ ))
+ packed_in[nonfinite] = PACK_FILL
+ packed_in = packed_in.astype(np.int16)
+ # Scatter into the full domain, pre-filled with the
+ # packing fill value (outside-mask cells were written
+ # as PACK_FILL before, and their uninitialised buffer
+ # content never enters the arithmetic now).
+ packed = np.full(maskinfo.info.shape, PACK_FILL, dtype=np.int16)
+ packed.reshape(-1)[~maskinfo.info.maskflat] = packed_in
+ else:
+ map_np = uncompress_array(data)
+ # For non-packed: convert masked array to plain array with fill value
+ if hasattr(map_np, 'filled'):
+ map_np = map_np.filled(-9999)
+ packed = map_np
+ prepared_steps.append((step, packed))
+
+ # HDF5 lock around the actual netCDF4 calls only
+ with _hdf5_lock:
+ if self.step_range[0] == 0:
+ nf1 = write_netcdf_header(self.settings, self.map_name, self.map_path, self.var.DtDay,
+ self.map_key, self.map_value.output_var, self.map_value.unit,
+ start_date, rep_steps, self.frequency, map_value=self.map_value,
+ log_packed=is_log_packed)
+ else:
+ nf1 = iterOpenNetcdf(self.map_path, "", 'a', format='NETCDF4')
+
+ nc_var = nf1.variables[self.map_name]
+ nc_var.set_auto_maskandscale(False)
+
+ for step, packed in prepared_steps:
+ nc_var[step, :, :] = packed
+
+ nf1.close()
- # clear lists for next chunk
self.step_range.clear()
self.data_steps.clear()
else:
@@ -171,6 +416,14 @@ class PCRasterWriter(Writer):
""" Main PCRaster writer class
Doesn't require another class to handle multiple steps as PCRaster doesn't support temporal metadata
Writing outputs in temporal chunks not supported for PCRaster
+
+ Thread-safety: with 'outputThreads' write() runs on worker threads.
+ All model state is snapshotted in stage() on the main thread, and each
+ writer reports to its own file name, so no two workers write the same
+ file. The pcraster.report calls themselves are NOT serialised by a
+ lock (unlike netCDF/HDF5 calls, which go through hdf5_lock); the
+ assumption that concurrent reports to distinct files are safe is
+ verified empirically by tests, not guaranteed by the PCRaster library.
"""
def __init__(self, var, map_value, map_path):
@@ -181,10 +434,23 @@ def __init__(self, var, map_value, map_path):
def stage(self):
self.data = self._extract_map()
+ # Snapshot the model clock (and the Monte Carlo sample directory) on
+ # the main thread: the PCRaster framework names dynamic reports from
+ # the live currentTimeStep(), which the main thread advances while
+ # write() may still be running on a worker thread.
+ self._stage_step = self.var.currentTimeStep()
+ self._stage_prefix = ''
+ if hasattr(self.var, '_inStochastic') and self.var._inStochastic():
+ self._stage_prefix = str(self.var.currentSampleNumber())
def write(self, start_date, rep_steps):
if self.data is not None:
- self.var.report(decompress(self.data), str(self.map_path))
+ # Build the file name from the staged snapshot and call
+ # pcraster.report directly, mirroring DynamicFramework._reportNew
+ # for the dynamic section without reading any live model state.
+ # generateNameT rejects extensions and over-long names, like report().
+ path = os.path.join(self._stage_prefix, generateNameT(str(self.map_path), self._stage_step))
+ pcraster_report(decompress(self.data), path)
else:
raise Exception('You need to stage data before writing!')
@@ -289,13 +555,17 @@ def stage(self):
""" Registers map for output at current step
"""
self.step = self.var.currentTimeStep()
- if self._output_checkpoint():
+ # Evaluate the checkpoint here (main thread) and reuse it in write():
+ # it may read live model state (monthend/yearend flags) that the main
+ # thread overwrites while write() runs on a worker thread.
+ self._staged_checkpoint = self._output_checkpoint()
+ if self._staged_checkpoint:
self.writer.stage()
def write(self):
""" Writes staged maps
"""
- if self._output_checkpoint():
+ if self._staged_checkpoint:
return self.writer.write(self._start_date, self._rep_steps)
@@ -381,6 +651,114 @@ def _start_date(self):
def _rep_steps(self):
return self._rep_steps_val
+
+class MapOutputAggregated(MapOutput):
+ """Handles temporal aggregation (monthly/yearly mean/sum) for a variable."""
+
+ def __init__(self, var, map_key, map_value, frequency, operation):
+ out_type = 'all' # accumulates every timestep
+ settings = LisSettings.instance()
+ binding = settings.binding
+ self._start_date_val = var.CalendarDayStart
+ self._rep_steps_val = range(binding['StepStartInt'], binding['StepEndInt'] + 1)
+
+ self._operation = operation # 'mean' or 'sum'
+ self._accum_buffer = None
+ self._accum_count = 0
+ self._write_step = 0 # own step counter for NetCDF time dimension
+
+ # Packing of SUM aggregates depends on the aggregation period:
+ # - MONTHLY / YEARLY sums accumulate many timesteps and overflow the
+ # int16 range that scale_factor/add_offset were calibrated for, so
+ # they are written UNPACKED (float), as originally.
+ # - DAILY sums stay close to the per-step value range (at most a few
+ # timesteps per day), so they keep int16 packing.
+ # MEAN aggregates always keep packing (they stay in the per-step range).
+ pack_disabled = (operation == 'sum' and frequency in ('monthly', 'yearly'))
+ if pack_disabled:
+ map_value_no_pack = map_value._replace(scale_factor=None, add_offset=None)
+ else:
+ map_value_no_pack = map_value
+
+ super().__init__(var, out_type, frequency, map_key, map_value_no_pack)
+
+ # Same reasoning for the independent logarithmic DISCHARGE packing: a
+ # monthly/yearly SUM of discharge can exceed the calibrated log range
+ # (max 500000 m3/s), so disable log packing for those. Daily sums and
+ # mean aggregates keep it. The linear _replace() above cannot express
+ # this (log packing does not use scale_factor/add_offset), so flag it
+ # on the writer directly.
+ if pack_disabled and hasattr(self, 'writer'):
+ self.writer.log_pack_disabled = True
+
+ # Force immediate write for aggregated outputs (one slice per period)
+ if hasattr(self, 'writer') and hasattr(self.writer, 'chunks'):
+ self.writer.chunks = 1
+
+ def _output_checkpoint(self):
+ """Always True — we accumulate every timestep."""
+ return True
+
+ @property
+ def _start_date(self):
+ return self._start_date_val
+
+ @property
+ def _rep_steps(self):
+ return self._rep_steps_val
+
+ def stage(self):
+ """Accumulate instead of storing instantaneous values."""
+ self.step = self.var.currentTimeStep()
+ # Snapshot the model clock here (main thread). write() may run on a
+ # worker thread while the main thread advances CalendarDate and the
+ # step counter, so write() must only use these snapshots.
+ self._stage_date = self.var.CalendarDate
+ self._stage_dt_day = self.var.DtDay
+ self.writer._stage_end_run = self.step == self.writer.end_step
+ map_np = self.writer._extract_map()
+
+ if self._accum_buffer is None:
+ self._accum_buffer = np.zeros_like(map_np)
+
+ self._accum_buffer += map_np
+ self._accum_count += 1
+
+ def write(self):
+ """Write only at the end of a completed period (day, month or year).
+
+ A value is flushed when the next step crosses into a new day/month/year.
+ A partial period left at the end of the run is not written. Only the
+ snapshots taken in stage() are used (never live model state).
+ """
+ current_date = self._stage_date
+ next_date = current_date + datetime.timedelta(days=self._stage_dt_day)
+ if self.frequency == 'daily':
+ is_boundary = current_date.day != next_date.day
+ elif self.frequency == 'monthly':
+ is_boundary = current_date.month != next_date.month
+ elif self.frequency == 'yearly':
+ is_boundary = current_date.year != next_date.year
+ else:
+ is_boundary = True
+
+ if is_boundary and self._accum_buffer is not None:
+ # Finalize
+ if self._operation == 'mean':
+ result = self._accum_buffer / self._accum_count
+ else: # sum
+ result = self._accum_buffer
+
+ # Stage the aggregated result into the writer
+ self.writer.data_steps.append(result)
+ self.writer.step_range.append(self._write_step)
+ self.writer.write(self._start_date, self._rep_steps)
+
+ # Increment own step counter and reset accumulator
+ self._write_step += 1
+ self._accum_buffer = None
+ self._accum_count = 0
+
# ------------------------------------------------------------------------
# Output factory
# ------------------------------------------------------------------------
@@ -428,11 +806,63 @@ def __init__(self, var):
if out.is_valid():
outputs.append(out)
- check_duplicates = []
+ # --- Temporal aggregation outputs ---
+ binding = settings.binding
+ aggregation_configs = {
+ 'OutputDailyMean': ('daily', 'mean'),
+ 'OutputDailySum': ('daily', 'sum'),
+ 'OutputMonthlyMean': ('monthly', 'mean'),
+ 'OutputMonthlySum': ('monthly', 'sum'),
+ 'OutputYearlyMean': ('yearly', 'mean'),
+ 'OutputYearlySum': ('yearly', 'sum'),
+ }
+ aggregated_vars = set() # track which vars are aggregated
+ # Aggregated outputs are written by the netCDF steps writer only.
+ netcdf_output = settings.options['writeNetcdf'] or settings.options['writeNetcdfStack']
+
+ # Match aggregation-list entries against ReportedMap.name, which is the
+ # identifier used everywhere else in the output system: the per-step/all
+ # outputs are keyed and path-resolved by rm.name (see
+ # settings._reported_maps), and the output settings textvars are named
+ # after rm.name too. The reportedmaps DICT KEY is only an internal dict
+ # handle and for a few entries differs from rm.name (e.g. key
+ # 'SeepSubToGWforestMaps' vs name 'SeepSubToGWForestMaps'). Matching the
+ # user's list against the dict key (as before) meant that for those
+ # entries no spelling worked: the dict-key spelling passed the lookup but
+ # then failed path resolution (aggregate dropped, per-step file kept at
+ # full resolution), while the rm.name spelling failed the lookup outright.
+ # Keying by rm.name makes the aggregate resolve to the right file AND
+ # match the surviving per-step output in the cleanup below.
+ reportedmaps = settings.options['reportedmaps']
+ maps_by_name = {rm.name: rm for rm in reportedmaps.values()}
+ for setting_key, (frequency, operation) in aggregation_configs.items():
+ var_list = binding.get(setting_key, '').split(';')
+ for var_name in var_list:
+ var_name = var_name.strip()
+ if var_name and var_name in maps_by_name:
+ if not netcdf_output:
+ raise LisfloodError(
+ 'Temporal aggregation (%s) requires netCDF output, but '
+ 'writeNetcdf/writeNetcdfStack is not set; "%s" cannot be '
+ 'aggregated with PCRaster map output. Enable netCDF output '
+ 'or remove it from the OutputDaily*/OutputMonthly*/'
+ 'OutputYearly* settings.' % (setting_key, var_name))
+ map_value = maps_by_name[var_name]
+ out = MapOutputAggregated(var, var_name, map_value, frequency, operation)
+ if out.is_valid():
+ outputs.append(out)
+ aggregated_vars.add(var_name)
+
+ # Remove normal outputs for variables that are now aggregated
outputs_clean = []
+ check_duplicates = []
for out in outputs:
+ # Skip normal Maps/All output if variable is aggregated
+ if hasattr(out, 'map_key') and out.map_key in aggregated_vars:
+ if not isinstance(out, (MapOutputEnd, MapOutputAggregated)):
+ continue
if out.map_path in check_duplicates:
- print(f'Warning! Output map {out.map_path} is duplicated, check list of outputs')
+ print(f'Warning! Output map {out.map_path} is duplicated')
else:
check_duplicates.append(out.map_path)
outputs_clean.append(out)
@@ -449,36 +879,83 @@ def write(self):
class OutputMapsFactoryThreads(OutputMapsFactory):
""" Extension of the OutputMapsFactory class
- Allows asynchronous writing on threads
- NOT FULLY TESTED, USE AS A PROOF OF CONTEXT
+ Allows asynchronous writing on threads, overlapping NetCDF disk I/O
+ with the model computation of the next time step. Opt-in via the
+ 'outputThreads' option (default off = synchronous writing).
+
+ Thread-safety contract (HDF5 is not thread-safe, single writer per
+ file):
+ - stage() runs on the MAIN thread: map extraction, decompression and
+ nancheck never race with writers.
+ - Every actual netCDF4/HDF5 library call (file open, variable slice
+ write, close) is serialised through netcdf.py's process-wide
+ hdf5_lock (imported below as _hdf5_lock), not a writer-local lock:
+ the dynamic loop's forcing readers (netcdf.py load_next_chunk under
+ dask's threaded scheduler) also touch HDF5, and concurrent access on
+ a non-threadsafe HDF5 build corrupts global state and aborts the
+ process. The lock is held ONLY around the HDF5 calls themselves --
+ metadata handling, uncompress_array and int16 packing arithmetic are
+ pure numpy and run OUTSIDE the lock, so compression/encoding cost is
+ not serialised. Holding several file handles open at once across
+ threads is fine: only non-overlapping *calls* matter.
+ - At most one flush generation is in flight: write() waits for the
+ previous map_async batch before staging the next step, so each
+ NetCDF file is touched by one worker thread at a time. The write
+ path per file is open -> write -> close within one worker call,
+ and iterOpenNetcdf/write_netcdf_header are stateless per call.
"""
+ # Upper bound on writer threads; the pool is anyway sized to the number
+ # of map outputs, so this only matters for very map-heavy configurations.
+ max_threads = 16
+
+ # Serialises the actual netCDF4/HDF5 library calls among worker threads and
+ # with the reader side (netcdf.py load_next_chunk). The HDF5 library used
+ # in this build is not compiled --enable-threadsafe, so concurrent Dataset
+ # calls segfault even when they target *distinct* files. The lock is held
+ # only around the HDF5 operations themselves (see NetcdfWriter.write /
+ # NetcdfStepsWriter.write), not around the whole per-map write, so numpy
+ # packing/encoding work overlaps across threads. This keeps the feature's
+ # actual goal: background writing overlapping with main-thread computation
+ # of the next time step (the handshake already ensures only one flush
+ # generation is in flight).
+ hdf5_write_lock = _hdf5_lock
+
def __init__(self, var):
super().__init__(var)
from multiprocessing import pool
- self.thread_pool = pool.ThreadPool(16)
+ # One thread per output map writer at most, capped at max_threads;
+ # avoids idle threads on small configurations (the prototype
+ # hardcoded 16 regardless of the actual map count).
+ self.thread_pool = pool.ThreadPool(min(len(self.output_maps), self.max_threads) or 1)
self.thread_out = None
def write(self):
# make sure that previous step finished writing, then stage data
if self.thread_out is not None:
self.thread_out.wait()
-
+
for out in self.output_maps:
out.stage()
- # quickly wraps OutputMaps write function to allow the use of map_async
+ # quickly wraps OutputMaps write function to allow the use of map_async.
+ # NOTE: the HDF5 lock is NOT taken here -- the writers acquire it
+ # themselves around the individual netCDF4 operations (open, slice
+ # write, close), leaving packing/encoding work outside the lock.
def write_output(output_map):
output_map.write()
-
+
# write outputs using thread pool
if self.output_maps:
self.thread_out = self.thread_pool.map_async(write_output, self.output_maps)
- # for last step, wait until everything is finished
+ # for last step, re-raise any exception the workers hit (get() propagates
+ # worker exceptions, wait() would silently swallow them and truncate the
+ # output files without failing the run)
if self.var.currentTimeStep() == self.var.nrTimeSteps() and self.thread_out is not None:
- self.thread_out.wait()
+ self.thread_out.get()
+
# ------------------------------------------------------------------------
# Output Module
@@ -493,6 +970,20 @@ class outputTssMap(object):
def __init__(self, out_variable):
self.var = out_variable
+ # earthkit-hydro RiverNetwork (lazily built in initial(), see below)
+ self._ek_net = None
+
+ def _ek_network(self):
+ """ Build (once) the earthkit RiverNetwork from the model Ldd for the
+ opt-in useEarthkitHydro reporting path. The network node order equals
+ LISFLOOD's compressed 1D order, so compressed arrays can be passed
+ directly. Raises if earthkit-hydro is unavailable (caller only calls
+ this when option['useEarthkitHydro'] and earthkit_available()).
+ """
+ if self._ek_net is None:
+ ldd_np = pcr2numpy(self.var.Ldd, 0)
+ self._ek_net = get_earthkit_network(ldd_np)
+ return self._ek_net
def initial(self):
""" initial part of the output module
@@ -502,27 +993,50 @@ def initial(self):
flags = settings.flags
binding = settings.binding
report_time_serie_act = settings.report_timeseries
+ maskinfo = MaskInfo.instance()
binding['Catchments'] = self.var.Catchments
# output for single column eg mapmaximum
self.var.Tss = {}
+ # --- opt-in earthkit-hydro acceleration of the reporting path --------
+ # Precompute the 1D arrays the earthkit path needs (upstream area and
+ # pixel area in compressed order), and the per-series zone-code arrays
+ # for multi-cell gauge zones. Nothing here touches the PCRaster path.
+ if option['useEarthkitHydro'] and earthkit_available():
+ net = self._ek_network()
+ pixelarea_1d = np.asarray(self.var.PixelArea, dtype=np.float64)
+ self.var.EkPixelArea = pixelarea_1d
+ # UpArea recomputed on the network (fast, parallel) instead of
+ # reusing the PCRaster map: same accumulation, compressed order.
+ uparea_1d = ek_upstream_sum(net, pixelarea_1d, node_weights=None)
+ self.var.EkInvUpArea = 1.0 / uparea_1d
+
+ # Resolve each distinct output-point location ("where": Gauges, Sites,
+ # LakeSites, Catchments, ...) only ONCE. The resolved map is used read-only by
+ # TimeoutputTimeseries.__init__, so sharing one object per `where` is safe.
+ outpoints_by_where = {}
+
for tss in report_time_serie_act:
where = report_time_serie_act[tss].where
- outpoints = binding[where]
- if where == "Catchments":
- outpoints = decompress(outpoints)
+ if where in outpoints_by_where:
+ outpoints = outpoints_by_where[where]
else:
- coord = binding[where].split() # could be gauges, sites, lakeSites etc.
- if len(coord) % 2 == 0:
- outpoints = valuecell(self.var.MaskMap, coord, outpoints)
+ outpoints = binding[where]
+ if where == "Catchments":
+ outpoints = decompress(outpoints)
else:
- try:
- outpoints = loadmap(where, pcr=True)
- outpoints = ifthen(outpoints != 0, outpoints)
- # this is necessary if netcdf maps are loaded !! otherwise strange dis.tss
- except Exception as e:
- msg = "Setting output points\n {}".format(str(e))
- raise LisfloodFileError(outpoints, msg)
+ coord = binding[where].split() # could be gauges, sites, lakeSites etc.
+ if len(coord) % 2 == 0:
+ outpoints = valuecell(self.var.MaskMap, coord, outpoints)
+ else:
+ try:
+ outpoints = loadmap(where, pcr=True)
+ outpoints = ifthen(outpoints != 0, outpoints)
+ # this is necessary if netcdf maps are loaded !! otherwise strange dis.tss
+ except Exception as e:
+ msg = "Setting output points\n {}".format(str(e))
+ raise LisfloodFileError(outpoints, msg)
+ outpoints_by_where[where] = outpoints
if option['MonteCarlo']:
if os.path.exists(os.path.split(binding[tss])[0]):
@@ -537,8 +1051,22 @@ def initial(self):
msg = "Checking output timeseries \n"
raise LisfloodFileError(str(binding[tss]), msg)
- # initialise output objects
- self.output_maps = OutputMapsFactory(self.var)
+ # initialise output objects. Threaded writer is opt-in via the
+ # 'outputThreads' option. With a single map output there is nothing to
+ # parallelise, so the (cheap) threaded factory is discarded and the
+ # synchronous one kept. OutputMapsChunks-driven mid-run flushes are
+ # handled by whichever factory is selected here.
+ if option['outputThreads']:
+ factory = OutputMapsFactoryThreads(self.var)
+ if len(factory.output_maps) > 1:
+ self.output_maps = factory
+ else:
+ # dispose of the idle pool and keep the synchronous factory
+ factory.thread_pool.close()
+ factory.thread_pool.join()
+ self.output_maps = OutputMapsFactory(self.var)
+ else:
+ self.output_maps = OutputMapsFactory(self.var)
def dynamic(self):
""" dynamic part of the output module
@@ -551,6 +1079,7 @@ def dynamic(self):
settings = LisSettings.instance()
option = settings.options
flags = settings.flags
+ maskinfo = MaskInfo.instance()
report_time_serie_act = settings.report_timeseries
if not(option['InitLisfloodwithoutSplit']):
@@ -563,23 +1092,89 @@ def dynamic(self):
pass
sys.stdout.write("\n")
+ # earthkit/numpy fast path (opt-in useEarthkitHydro): the
+ # how=='total' series (the upstream-accumulation family) all share
+ # the same network and node_weights and differ only by field, so
+ # collect their fields in one pass and issue a SINGLE batched
+ # upstream accumulation for the whole step instead of one traversal
+ # per series. The batch axis is a pure outer vectorization, so each
+ # per-series result is bit-identical to the scalar path.
+ _fused_changed = {}
+ if option['useEarthkitHydro'] and earthkit_available():
+ _fused_fields = []
+ _fused_keys = []
+ for tss in report_time_serie_act:
+ _how = report_time_serie_act[tss].operation[0] if len(report_time_serie_act[tss].operation) else ''
+ if _how == 'total':
+ _what = 'self.var.' + report_time_serie_act[tss].output_var
+ _x_1d = eval(_what)
+ _fused_fields.append(np.asarray(_x_1d, dtype=np.float64))
+ _fused_keys.append(tss)
+ if _fused_fields:
+ _net = self._ek_network()
+ _stacked = np.stack(_fused_fields)
+ _tot = ek_upstream_sum_batch(_net, _stacked,
+ node_weights=self.var.EkPixelArea)
+ _mp = model_precision()
+ for _k, _key in enumerate(_fused_keys):
+ _fused_changed[_key] = (_tot[_k] * self.var.EkInvUpArea).astype(
+ _mp, copy=False)
+
for tss in report_time_serie_act:
# report time series
what = 'self.var.' + report_time_serie_act[tss].output_var
how = report_time_serie_act[tss].operation[0] if len(report_time_serie_act[tss].operation) else ''
- if how == 'mapmaximum':
- changed = compressArray(mapmaximum(decompress(eval(what))))
- what = 'changed'
- if how == 'total':
- changed = compressArray(catchmenttotal(decompress(eval(what)) * self.var.PixelAreaPcr, self.var.Ldd) * self.var.InvUpArea)
- what = 'changed'
- self.var.Tss[tss].sample(decompress(eval(what)))
+ # earthkit/numpy fast path (opt-in useEarthkitHydro): operate on
+ # the 1D compressed arrays directly, never building 2D PCRaster
+ # maps (kills the decompress/numpy2pcr/pcr2numpy round-trips and
+ # the single-threaded PCRaster engine reductions). The historical
+ # PCRaster statements are kept verbatim in the else branches.
+ if option['useEarthkitHydro'] and earthkit_available():
+ if how == 'mapmaximum':
+ x_1d = eval(what)
+ with np.errstate(invalid='ignore'):
+ changed = np.full(maskinfo.info.mapC, np.nanmax(x_1d),
+ dtype=np.float64)
+ what = 'changed'
+ elif how == 'total':
+ # distribute the pre-batched upstream sum for this series
+ changed = _fused_changed[tss]
+ what = 'changed'
+ else:
+ if how == 'mapmaximum':
+ changed = compressArray(mapmaximum(decompress(eval(what))))
+ what = 'changed'
+ if how == 'total':
+ changed = compressArray(catchmenttotal(decompress(eval(what)) * self.var.PixelAreaPcr, self.var.Ldd) * self.var.InvUpArea)
+ what = 'changed'
+ tss_obj = self.var.Tss[tss]
+ value_1d = eval(what)
+ # Fast path: when every gauge is a single cell, the inherited
+ # .sample()'s per-zone areaaverage equals the gauge-cell value,
+ # so read it straight from the 1D array and skip the
+ # decompress -> numpy2pcr full-map conversion (~50-136 ms/call on
+ # a global grid). Only for Scalar series (the areaaverage branch);
+ # a 1D numpy array implies Scalar/Directional here. Falls back to
+ # the exact PCRaster path otherwise (e.g. multi-cell gauge zones,
+ # non-spatial gauges, or non-ndarray values).
+ if getattr(tss_obj, '_compressed_sampling_ok', False) and isinstance(value_1d, np.ndarray):
+ tss_obj.sample_compressed(value_1d)
+ elif option['useEarthkitHydro'] and earthkit_available() and \
+ getattr(tss_obj, '_ek_zone_codes', None) is not None and \
+ isinstance(value_1d, np.ndarray):
+ # multi-cell gauge zones: numpy zone-mean (equivalent of the
+ # .sample() -> pcraster.areaaverage fallback), still without
+ # any 2D map round-trip
+ from .earthkit_network import ek_sample_zone_means
+ ek_sample_zone_means(tss_obj, value_1d)
+ else:
+ tss_obj.sample(decompress(value_1d))
# ************************************************************
# ***** WRITING RESULTS: MAPS ******************************
# ************************************************************
- self.output_maps.write()
+ self.output_maps.write()
# update local output steps
cdfflags = CDFFlags.instance()
diff --git a/src/lisflood/global_modules/parallelization.py b/src/lisflood/global_modules/parallelization.py
new file mode 100644
index 00000000..ffd22d19
--- /dev/null
+++ b/src/lisflood/global_modules/parallelization.py
@@ -0,0 +1,740 @@
+"""
+Centralized thread-pool governance for LISFLOOD.
+
+Several numerical libraries used by LISFLOOD each spin up their own thread
+pool sized to the host core count by default:
+
+ * ``numba`` - the ``@njit(parallel=True)`` routing/soil kernels (``prange``).
+ * BLAS/OpenMP - backing ``numpy``/``scipy`` linear algebra (e.g. the SVD in
+ the soil-hydraulics ``least_squares`` fit during initialization).
+
+BLAS (Basic Linear Algebra Subprograms) is the low-level library that numpy and
+scipy call under the hood for vector/matrix maths (dot products, matrix
+multiply, SVD, least-squares, ...). The optimized implementations (OpenBLAS,
+Intel MKL, BLIS) are multi-threaded and by default use one thread per CPU core.
+Because the actual backend depends on how numpy was built, its thread count is
+controlled through several implementation-specific environment variables
+(``OMP_NUM_THREADS``, ``OPENBLAS_NUM_THREADS``, ``MKL_NUM_THREADS``,
+``BLIS_NUM_THREADS``), which is why they are all set below.
+
+On the domain sizes LISFLOOD typically runs (a single catchment has a
+comparatively narrow river network and a modest pixel count), these pools do
+not have enough independent work per call to amortize thread fork/join and
+synchronization. Worse, they are *nested*: a numba-parallel section may call
+into BLAS, and each library independently tries to use every core. The result
+is heavy oversubscription - measured as ~1.5M context switches and ~5x more CPU
+time than wall time on a benchmark catchment, while actually running *slower*
+than a single-threaded execution.
+
+This module reads a small set of settings and applies consistent, explicit
+thread limits to both pools: numba (``set_num_threads``) and the BLAS/OpenMP
+backend. It is the single place that governs thread counts;
+``Lisflood_initial`` simply calls :func:`configure_parallelism` once at startup.
+
+Thread counts affect performance only, never results: LISFLOOD discharge output
+is bitwise-identical across thread configurations (verified on the test and Ob
+benchmark catchments). Defaults are therefore chosen for speed on typical
+single-catchment runs, and every knob can be raised by the user for large
+domains.
+
+Settings (all optional; read via ``binding.get`` so older settings files keep
+working):
+
+ * ``numCPUs_parallelNumba`` - existing setting; land-phase numba thread
+ count. absent/"auto" => auto policy (all
+ cores on large domains, serial below
+ NUMBA_AUTO_MIN_PIXELS); "all"/0 => all cores;
+ N => N threads.
+ * ``numCPUs_routingNumba`` - routing-substep numba thread cap. absent/
+ "auto" => clamp(round(host/2), 8, 16);
+ "all"/0 => all cores (no cap); N => N threads.
+ Applied only around the routing substep loop
+ (small per-substep payload collapses above a
+ modest OpenMP worker count) and restored to
+ the land-phase count afterwards; never raises
+ the pool above the land count.
+ * ``numCPUs_BLAS`` - BLAS/OpenMP thread count for numpy/scipy.
+ absent/"auto" => 1 (flat serial default on all
+ domain sizes; the only BLAS consumer is the
+ cold soil-init scalar least_squares, idle in
+ the dynamic loop); "all"/0 => all cores; N => N.
+ * ``numCPUs_soilInit`` - soilInit thread count for soil initialization.
+ Default: 0 (all cores).
+ * ``numCPUs_earthkit`` - thread count for the earthkit-hydro upstream-
+ accumulation pool (a native pool inside the
+ Rust extension that reads its size once at
+ first use via ``RAYON_NUM_THREADS`` and
+ otherwise sizes itself to the host, thereby
+ oversubscribing the node against the routing
+ kernel). Only the opt-in earthkit reporting
+ path builds this pool. Default: 1 (capped);
+ 0/"all"/"auto"/absent => uncapped (all cores).
+A value of 0 (or empty/"all") means "use all available cores". For BLAS,
+"auto" (or an absent setting) is a flat serial default (1) on all domain sizes.
+
+OMP_WAIT_POLICY is NOT set by this module. It is an energy lever (it trades idle
+OpenMP worker spin for sleep at roughly equal wall time), not a speed lever, so
+the default runtime behaviour is left untouched. An operator who wants to cut
+CPU-hours at equal wall can export ``OMP_WAIT_POLICY=passive`` in the job
+environment before launching LISFLOOD.
+"""
+
+import os
+import numpy as np
+
+# threadpoolctl can limit already-initialized BLAS/OpenMP pools at runtime,
+# which is more robust than relying on environment variables being set before
+# numpy is imported. It is an optional dependency: if unavailable we fall back
+# to environment variables (best-effort).
+try:
+ import threadpoolctl
+ _HAVE_THREADPOOLCTL = True
+except ImportError: # pragma: no cover - environment dependent
+ _HAVE_THREADPOOLCTL = False
+
+try:
+ from numba import set_num_threads as _numba_set_num_threads
+ from numba import config as _numba_config
+ # get_num_threads returns the number of threads Numba will actually use.
+ from numba import get_num_threads as _numba_get_num_threads
+ _HAVE_NUMBA_THREADCTL = True
+except ImportError: # pragma: no cover – defensive
+ _HAVE_NUMBA_THREADCTL = False
+
+
+def _openmp_layer_available():
+ """Return True if numba's OpenMP threading backend can be loaded.
+
+ Probes the backend module directly (no parallel kernel compile), so it is
+ cheap and side-effect free. Returns False if numba or the OpenMP pool is
+ unavailable.
+ """
+ try:
+ # Importing the backend module is enough to know the shared library is
+ # present; it does not start the pool.
+ from numba.np.ufunc import omppool # noqa: F401
+ return True
+ except Exception: # pragma: no cover - environment dependent
+ return False
+
+
+def _lock_and_verify_layer(intended):
+ """Force numba to initialize its threading layer NOW (while we are still
+ before any model parallel kernel) by running a trivial prange kernel, then
+ return the layer numba actually locked. numba fixes the layer on first
+ parallel use and ignores later changes, so locking it here deterministically
+ prevents a stray earlier kernel (or the TBB-preferring default) from winning.
+ Returns the actual locked layer string, or ``intended`` if the probe/readback
+ is unavailable."""
+ try:
+ import numpy as _np
+ from numba import njit as _njit, prange as _prange, threading_layer as _tl
+
+ @_njit(parallel=True, cache=False)
+ def _probe(a):
+ s = 0.0
+ for i in _prange(a.size):
+ s += a[i]
+ return s
+
+ _probe(_np.ones(16))
+ return str(_tl())
+ except Exception: # pragma: no cover - defensive; fall back to intent
+ return intended
+
+
+def select_numba_threading_layer():
+ """Pin numba's threading layer to OpenMP when it is safe to do so.
+
+ numba's ``default`` layer resolution prefers TBB over OpenMP when TBB is
+ installed. On LISFLOOD workloads OpenMP is as fast or faster (measured:
+ workqueue ~27% slower than OpenMP on the Congo benchmark; TBB at best on
+ par), and an accidentally-installed TBB passenger package would otherwise
+ silently take over. To make the layer deterministic we pin OpenMP, but
+ **only** when:
+
+ * the user has NOT set ``NUMBA_THREADING_LAYER`` (explicit choice wins),
+ * numba is importable, and
+ * the OpenMP backend is actually available.
+
+ If the env var is set, or OpenMP is unavailable, we leave numba's own
+ resolution untouched (it will fall back to whatever layer it can load).
+
+ Returns the layer string that is now in effect for reporting, or None when
+ numba is unavailable.
+ """
+ if not _HAVE_NUMBA_THREADCTL:
+ return None
+
+ # Respect an explicit user override via the environment. numba reads
+ # NUMBA_THREADING_LAYER at import; mirror it into the live config so the
+ # value we report matches what numba will use, and honor it as-is.
+ env_layer = os.environ.get("NUMBA_THREADING_LAYER")
+ if env_layer:
+ try:
+ _numba_config.THREADING_LAYER = env_layer.strip().lower()
+ except Exception: # pragma: no cover - defensive
+ pass
+ return str(_numba_config.THREADING_LAYER)
+
+ # No explicit choice: pin OpenMP when its backend is available. numba's
+ # default resolution prefers TBB when TBB is installed, but OpenMP is as
+ # fast or faster on LISFLOOD workloads and avoids TBB's fork fragility.
+ # This must run before the FIRST parallel kernel executes
+ # (configure_parallelism is called once in Lisflood_initial.__init__,
+ # before any module initial()/dynamic()), because numba locks the
+ # threading layer on first parallel use and ignores later changes.
+ intended = str(_numba_config.THREADING_LAYER)
+ if _openmp_layer_available():
+ try:
+ _numba_config.THREADING_LAYER = "omp"
+ except Exception: # pragma: no cover - defensive
+ pass
+ # Belt-and-braces: also export the env var so any numba (re)read or a
+ # forked worker that re-imports numba sees the same choice.
+ os.environ.setdefault("NUMBA_THREADING_LAYER", "omp")
+ intended = "omp"
+
+ # Lock the layer NOW (before any model parallel kernel) and read back what
+ # numba actually selected. This is the ground truth, not the intent: if a
+ # stray earlier parallel call or a build quirk already locked another layer
+ # (e.g. TBB, numba's default when TBB is installed), we detect it here
+ # instead of silently running on the wrong layer.
+ actual = _lock_and_verify_layer(intended)
+ if actual != intended:
+ try:
+ import warnings
+ from .errors import LisfloodWarning
+ warnings.warn(LisfloodWarning(
+ "numba threading layer is '{}' but '{}' was requested; the layer "
+ "was already locked before it could be set. Set "
+ "NUMBA_THREADING_LAYER={} in the environment before launching to "
+ "force it.".format(actual, intended, intended)))
+ except Exception: # pragma: no cover - defensive
+ pass
+ return actual
+
+# Keep a reference to any threadpoolctl limiter so the applied BLAS limits are
+# not garbage-collected (and thus reverted) during the model run.
+_blas_limiter = None
+
+# Last-applied effective thread counts, so the startup banner can report the
+# same numbers that were actually applied (see get_effective_parallelism).
+_effective = None
+
+# Resolved model floating-point dtype (np.float64 default). Set once at startup
+# by configure_parallelism from the `modelPrecision` setting, and read by the
+# array-creation funnels (MaskInfo.in_zero/in_one, add1.compressArray/decompress/
+# loadmap_base) so that when modelPrecision=float32 the compressed state arrays
+# are *born* float32 (avoiding the mixed-dtype upcast trap). float64 keeps every
+# funnel bit-identical to today.
+_model_dtype = np.float64
+
+
+def get_model_dtype():
+ """Return the resolved model dtype (np.float64 by default; np.float32 when
+ modelPrecision=float32 was applied at startup)."""
+ return _model_dtype
+
+
+def set_model_dtype(dtype):
+ """Set the process-wide model dtype. Called by configure_parallelism; exposed
+ for tests. Only float32/float64 are meaningful."""
+ global _model_dtype
+ _model_dtype = np.dtype(dtype).type if not isinstance(dtype, type) else dtype
+ return _model_dtype
+
+
+def get_effective_parallelism():
+ """Return the last-applied effective thread counts, or None if parallelism
+ has not been configured yet.
+
+ The dict is keyed 'numba', 'routing_numba', 'blas', 'soilinit',
+ 'earthkit', 'numba_threading_layer', and 'host' (host core count). 'numba'
+ is the concrete land-phase numba count (the restore target after the routing
+ cap); 'routing_numba' is the applied routing-substep cap (an int), or None
+ when the routing pool is left at all cores.
+ """
+ return _effective
+
+
+def get_routing_numba_threads():
+ """Return the applied routing-substep numba cap (an int), or None when the
+ routing pool is left uncapped (explicit all/0) or parallelism has not been
+ configured. The routing substep window uses this as the capped count."""
+ if _effective is None:
+ return None
+ return _effective.get("routing_numba")
+
+
+def get_land_numba_threads():
+ """Return the concrete land-phase numba count published at startup, i.e. the
+ count the routing-substep window restores to on exit. None when parallelism
+ has not been configured or numba thread control is unavailable."""
+ if _effective is None:
+ return None
+ return _effective.get("numba")
+
+
+def set_numba_threads(n):
+ """Thin, guarded wrapper over ``numba.set_num_threads``.
+
+ Applies ``n`` only when numba thread control is available and
+ ``0 < n <= NUMBA_NUM_THREADS`` (the same guard the startup path uses). A
+ no-op when numba is unavailable or ``n`` is None/out of range, so the
+ routing-substep window never raises a new failure mode. Thread count affects
+ scheduling only, never results.
+ """
+ if not _HAVE_NUMBA_THREADCTL:
+ return
+ if n is None:
+ return
+ if 0 < n <= _numba_config.NUMBA_NUM_THREADS:
+ _numba_set_num_threads(n)
+
+
+def _parse_thread_count(value):
+ """Interpret a settings value as a thread count.
+
+ Returns an int >= 1, or None meaning "use all available cores".
+ Accepts ints, numeric strings, empty string, and the literal "all".
+ """
+ if value is None:
+ return None
+ if isinstance(value, str):
+ value = value.strip().lower()
+ if value in ("", "all", "auto"):
+ return None
+ try:
+ value = int(value)
+ except ValueError:
+ return None
+ try:
+ value = int(value)
+ except (TypeError, ValueError):
+ return None
+ if value <= 0:
+ return None # 0 / negative => all cores
+ return value
+
+
+def _host_cpu_count():
+ """
+ Return the number of logical CPUs that the current job is permitted to use.
+
+ The routine recognises two common HPC workload managers:
+
+ * **SLURM** – uses the environment variables
+ `SLURM_TASKS_PER_NODE` (tasks per node) and `SLURM_CPUS_PER_TASK`
+ (CPUs allocated to each task). If only `SLURM_NTASKS` is defined
+ (single‑node jobs without `--ntasks-per-node`), it is used as a fallback.
+
+ * **PBS** – uses `PBS_NP` (total number of allocated CPUs) or,
+ when a node‑wise layout is requested, the combination of
+ `PBS_NUM_NODES` and `PBS_NUM_PPN` (processes per node).
+
+ If none of the above variables are present, the function returns
+ `os.cpu_count()` (the total logical cores on the node).
+
+ The return value is always an ``int`` ≥ 1.
+ """
+ # --------------------------------------------------------------
+ # 1. SLURM – most detailed information available
+ # --------------------------------------------------------------
+ try:
+ # SLURM may expose a comma‑separated list when heterogeneous
+ # allocations are used (e.g. "28(x2),14").
+ tasks_per_node = os.getenv("SLURM_TASKS_PER_NODE")
+ cpus_per_task = os.getenv("SLURM_CPUS_PER_TASK")
+ ntasks = os.getenv("SLURM_NTASKS") # fallback
+
+ if tasks_per_node:
+ # Keep the first numeric entry before any '(' or ','.
+ first_entry = tasks_per_node.split(',')[0].split('(')[0]
+ tasks_per_node = int(first_entry)
+ cpus_per_task = int(cpus_per_task) if cpus_per_task else 1
+ return max(1, tasks_per_node * cpus_per_task)
+
+ if ntasks:
+ # When only SLURM_NTASKS is set (e.g. a simple `srun` without
+ # explicit per‑node control) we assume one CPU per task.
+ return max(1, int(ntasks))
+ except Exception:
+ # Any parsing problem – fall through to the next detector.
+ pass
+
+ # --------------------------------------------------------------
+ # 2. PBS
+ # --------------------------------------------------------------
+ try:
+ # PBS_NP provides the total number of processors allocated to the job.
+ pbs_np = os.getenv("PBS_NP")
+ if pbs_np:
+ return max(1, int(pbs_np))
+
+ # When PBS_NP is not set, the classic layout variables can be used.
+ # PBS_NUM_NODES = number of allocated nodes
+ # PBS_NUM_PPN = processes (CPUs) per node
+ num_nodes = os.getenv("PBS_NUM_NODES")
+ ppn = os.getenv("PBS_NUM_PPN")
+ if num_nodes and ppn:
+ return max(1, int(num_nodes) * int(ppn))
+ except Exception:
+ pass
+
+ # --------------------------------------------------------------
+ # 3. Fallback – physical core count of the node
+ # --------------------------------------------------------------
+ return os.cpu_count() or 1
+
+
+def _resolve(binding, key, default):
+ """Read a thread-count setting, falling back to a default when absent."""
+ if binding is not None and key in binding and binding.get(key) not in (None, ""):
+ return _parse_thread_count(binding.get(key))
+ return default
+
+
+# Below this many valid pixels, numba thread-parallelism (prange over the
+# river network's topological orders) does not have enough independent work
+# per batch to amortize fork/join, and running serially is faster. Measured on
+# the Ob benchmark (~6,900 pixels): serial was faster than any threaded config.
+# Only used when the user leaves numba on "auto" (numCPUs_parallelNumba=0).
+NUMBA_AUTO_MIN_PIXELS = 50000
+
+
+def resolve_numba_threads(binding, num_pixels=None):
+ """Decide the numba thread count.
+
+ * If the user set ``numCPUs_parallelNumba`` explicitly (>=1), honor it.
+ * If it is 0/"auto" (the historical default meaning "all cores") and we
+ know the domain size, auto-tune: use 1 thread for small domains where
+ threading is pure overhead, otherwise all cores.
+ * If the domain size is unknown, keep the historical "all cores" behavior.
+
+ Returns an int >= 1, or None meaning "all available cores".
+ """
+ explicit = _resolve(binding, "numCPUs_parallelNumba", None)
+ if explicit is not None:
+ return explicit
+ # auto mode
+ if num_pixels is not None and num_pixels < NUMBA_AUTO_MIN_PIXELS:
+ return 1
+ return None
+
+
+# Routing-kernel numba cap bounds. The per-substep routing kernels have a small
+# payload (one topological traversal of the river network per substep), so
+# beyond a modest OpenMP worker count the fork/join cost dominates and wall time
+# grows. The auto policy therefore caps the numba pool around the routing
+# substep loop to clamp(round(host/2), 8, 16): half the host cores, but never
+# fewer than 8 nor more than 16.
+ROUTING_NUMBA_MIN_THREADS = 8
+ROUTING_NUMBA_MAX_THREADS = 16
+
+
+def _routing_numba_auto(host):
+ """Auto policy value: clamp(round(host/2), MIN, MAX)."""
+ if host is None:
+ host = _host_cpu_count()
+ k = int(round(host / 2))
+ return min(max(k, ROUTING_NUMBA_MIN_THREADS), ROUTING_NUMBA_MAX_THREADS)
+
+
+def resolve_routing_numba_threads(binding, host=None):
+ """Decide the routing-kernel numba thread count (unified semantics).
+
+ Controlled by the optional ``numCPUs_routingNumba`` setting:
+
+ * absent / "" / "auto" -> the auto policy
+ ``clamp(round(host/2), ROUTING_NUMBA_MIN_THREADS,
+ ROUTING_NUMBA_MAX_THREADS)`` (half the host cores, floored at 8, capped
+ at 16). ``round`` is Python banker's rounding, but the documented
+ anchors (host 16->8, 28->14, 56->16, 112->16) are all even so the
+ half-case never arises.
+ * "all" / "0" -> None, meaning "all cores" (no routing-specific
+ cap; the land-phase numba count is left to stand).
+ * a positive integer N -> that fixed count.
+ * a malformed token -> the auto policy (fail safe).
+
+ ``host`` defaults to :func:`_host_cpu_count` when not supplied. Thread count
+ affects scheduling only, never results. Returns an int >= 1 (the auto clamp
+ result or an explicit N), or None meaning "all cores".
+ """
+ raw = binding.get("numCPUs_routingNumba") if binding is not None else None
+ if isinstance(raw, str):
+ raw = raw.strip().lower()
+
+ if raw in (None, "", "auto"):
+ return _routing_numba_auto(host)
+
+ # "all" / "0" -> all cores (None); positive int N -> N.
+ if raw in ("all", "0", 0):
+ return None
+ parsed = _parse_thread_count(raw)
+ if parsed is None:
+ # Unparseable token (e.g. "abc"): fail safe to the auto policy.
+ return _routing_numba_auto(host)
+ return parsed
+
+
+def resolve_pool_threads(binding, key, num_pixels=None):
+ """Decide the thread count for a shared library pool (BLAS).
+
+ This pool works on per-call arrays that are small relative to the whole
+ domain, and it coexists with (and nests inside) the numba routing pool. A
+ full-core BLAS pool therefore oversubscribes the node against the
+ numba OpenMP threads rather than adding useful throughput. The behaviour
+ is:
+
+ * If the user set a concrete number, honor it verbatim (e.g. ``1``, written
+ by calibration templates to force a single-core instance; ``0``/``"all"``
+ => unconditionally all cores regardless of domain size).
+ * If the setting is **absent** or set to ``"auto"``, the default is a flat
+ ``1`` (serial) on ALL domain sizes. The per-call arrays are small and the
+ nested pools would merely contend with the numba routing pool for cores;
+ a single-threaded pool is the measured best on both small catchments and
+ large/global domains. ``num_pixels`` is accepted for signature
+ compatibility (the land-numba resolver still uses
+ ``NUMBA_AUTO_MIN_PIXELS``) but no longer influences this default.
+
+ Returns an int >= 1, or None meaning "all available cores".
+ """
+ raw = binding.get(key) if binding is not None else None
+ if isinstance(raw, str):
+ raw = raw.strip().lower()
+
+ auto = raw in (None, "", "auto")
+ if not auto:
+ # Explicit concrete number / "all" / "0": honor verbatim.
+ return _parse_thread_count(raw)
+
+ # auto (absent or "auto") -> flat serial default on all domain sizes.
+ return 1
+
+
+def resolve_soilinit_workers(binding, real_cpu_count):
+ """Number of worker processes for the ColdStart soil-init solve.
+
+ Controlled by the dedicated 'numCPUs_soilInit' setting (process-based
+ parallelism for the per-pixel scipy least_squares solve; unrelated to the
+ numba/BLAS thread pools). Values: a positive integer, or
+ 0 / "all" / "auto" for all available cores. Defaults to 0 (all cores) when
+ unset; the solve is bit-identical regardless of the worker count.
+ """
+ explicit = _resolve(binding, "numCPUs_soilInit", None)
+ if explicit is not None:
+ return explicit
+ return real_cpu_count if real_cpu_count is not None else _host_cpu_count()
+
+
+def resolve_earthkit_threads(binding):
+ """Thread count for the earthkit-hydro upstream-accumulation pool.
+
+ That pool is a native (rayon) pool inside the earthkit-hydro Rust
+ extension, sized once at first use from the ``RAYON_NUM_THREADS``
+ environment variable and otherwise to the host core count. It is only built
+ on the opt-in earthkit reporting path. Controlled by ``numCPUs_earthkit``:
+
+ * absent / "" / "auto" -> 1 (DEFAULT: single-threaded accumulation pool).
+ * a positive integer -> cap the pool to that many threads.
+ * 0 / "all" -> None, meaning "all cores" (leave uncapped).
+
+ The accumulation is a depth-bounded topological traversal with little work
+ per node; a bounded pool avoids oversubscribing the node against the routing
+ kernel's OpenMP threads. Unlike :func:`resolve_pool_threads`, the default is
+ a flat 1 and is NOT domain-size-aware: the contention is about pool
+ coexistence with the routing kernel, not array size. Thread count affects
+ scheduling only, never the result (the traversal order is deterministic and
+ independent of the pool size).
+
+ Returns an int >= 1, or None meaning "all cores / leave uncapped".
+ """
+ raw = binding.get("numCPUs_earthkit") if binding is not None else None
+ if isinstance(raw, str):
+ raw = raw.strip().lower()
+ if raw in (None, "", "auto"):
+ return 1 # default: single-threaded rayon
+ return _parse_thread_count(raw) # int>0 => cap; 0/"all" => None
+
+
+def resolve_model_dtype(binding):
+ """Decide the floating-point dtype for LISFLOOD model state & kernels.
+
+ Controlled by the optional ``modelPrecision`` setting (a string). This is a
+ PRECISION/perf lever, separate from the thread-count knobs. It governs the
+ whole model: the compressed state arrays (via the add1/MaskInfo creation
+ funnels) and the kinematic-wave routing kernel.
+
+ * absent / "" / "float64" / "f64" / "double" -> np.float64 (DEFAULT).
+ The historical, bit-identical path. The default MUST stay float64 so that
+ unchanged settings files reproduce today's output bit-for-bit (Task 8.2).
+ * "float32" / "f32" / "single" -> np.float32.
+ Opt-in faster path (halves memory traffic on the bandwidth-bound
+ elementwise module bodies and the routing kernel; results stay within the
+ project reference tolerance, not bit-identical).
+ * any unknown value -> np.float64 (fail safe:
+ never silently degrade precision on a typo).
+
+ Returns a numpy dtype (np.float64 or np.float32).
+ """
+ raw = binding.get("modelPrecision") if binding is not None else None
+ if isinstance(raw, str):
+ raw = raw.strip().lower()
+ if raw in ("float32", "f32", "single"):
+ return np.float32
+ return np.float64
+
+
+# Backwards-compatible alias (the flag was briefly called routingPrecision).
+def resolve_routing_dtype(binding):
+ """Deprecated name for :func:`resolve_model_dtype`; also accepts the old
+ ``routingPrecision`` key for settings files written against the earlier
+ name. ``modelPrecision`` takes precedence when both are present."""
+ if binding is not None:
+ mp = binding.get("modelPrecision")
+ if mp not in (None, ""):
+ return resolve_model_dtype(binding)
+ raw = binding.get("routingPrecision")
+ if isinstance(raw, str):
+ raw = raw.strip().lower()
+ if raw in ("float32", "f32", "single"):
+ return np.float32
+ return np.float64
+
+
+def configure_parallelism(binding, num_pixels=None):
+ """Apply thread-pool limits for numba and BLAS/OpenMP.
+
+ numba's thread count is decided here (including the small-domain auto-tune)
+ and applied via ``numba.set_num_threads``; it governs the soilloop and the
+ kinematic-wave/MCT routing kernels.
+
+ Parameters
+ ----------
+ binding : dict-like
+ LISFLOOD settings binding. May be None (defaults are used).
+ num_pixels : int or None
+ Number of valid land pixels in the domain, used for numba auto-tune.
+
+ Returns
+ -------
+ dict
+ The effective thread counts, keyed 'numba', 'routing_numba',
+ 'blas', 'soilinit', 'earthkit', 'numba_threading_layer', and 'host'.
+ 'numba' is the concrete land-phase count (the routing cap's restore
+ target); 'routing_numba' is the applied routing-substep cap (an int), or
+ None when the routing pool is left at all cores. For the thread-count
+ keys, None means "all cores".
+ """
+ global _blas_limiter, _effective
+
+ # --- model floating-point precision (state + kernels) ---
+ # Resolve once and publish so the array-creation funnels emit this dtype.
+ global _model_dtype
+ _model_dtype = resolve_model_dtype(binding)
+
+ # --- numba threading layer ---
+ # Pin OpenMP when the user has not overridden NUMBA_THREADING_LAYER and the
+ # OpenMP backend is available (prevents an installed TBB from silently
+ # taking priority; OpenMP is as fast or faster on LISFLOOD workloads). Must
+ # happen before any parallel kernel compiles, i.e. here at startup.
+ effective_threading_layer = select_numba_threading_layer()
+
+ # --- numba (soilloop + kinematic-wave/MCT routing kernels) ---
+ # None means "leave numba at its default" (all cores); a concrete N caps it.
+ numba_threads = resolve_numba_threads(binding, num_pixels)
+
+ if _HAVE_NUMBA_THREADCTL:
+ # If the user asked for a concrete value, try to set it; otherwise
+ # leave Numba at its default (which may already be “all cores”).
+ if numba_threads is not None:
+ # Numba cannot be asked for more threads than its own hard limit.
+ if 0 < numba_threads <= _numba_config.NUMBA_NUM_THREADS:
+ _numba_set_num_threads(numba_threads)
+ # If the request is larger than the limit we simply ignore it –
+ # Numba will keep its previous (default) setting.
+ # *** query the **actual** number of threads Numba will use ***
+ effective_numba = _numba_get_num_threads()
+ else:
+ effective_numba = None # Numba not available
+
+ # BLAS: when the user does not set it, the default is a flat serial 1 on
+ # all domain sizes. The per-call arrays are small and the pool nests inside
+ # (and contends with) the numba routing pool, so a full-core pool only
+ # oversubscribes the node rather than adding throughput. An explicit value
+ # in the settings is always honored verbatim.
+ blas_threads = resolve_pool_threads(binding, "numCPUs_BLAS", num_pixels)
+
+ host = _host_cpu_count()
+
+ # --- BLAS / OpenMP (numpy, scipy) ---
+ if blas_threads is not None:
+ target = max(1, min(blas_threads, host))
+ else:
+ target = host
+ effective_blas = target
+ # Set env vars too (helps subprocesses and any pools not yet created).
+ for var in ("OMP_NUM_THREADS", "OPENBLAS_NUM_THREADS",
+ "MKL_NUM_THREADS", "VECLIB_MAXIMUM_THREADS",
+ "BLIS_NUM_THREADS"):
+ os.environ[var] = str(target)
+ if _HAVE_THREADPOOLCTL:
+ try:
+ # limit_num_threads returns a controller object; keep a ref so
+ # the limits persist for the lifetime of the run.
+ _blas_limiter = threadpoolctl.threadpool_limits(
+ limits=target, user_api="blas")
+ except Exception: # pragma: no cover - defensive
+ _blas_limiter = None
+
+ soilinit_workers = resolve_soilinit_workers(binding, host)
+
+ # --- earthkit-hydro accumulation pool (rayon, native Rust extension) ---
+ # The rayon global pool reads RAYON_NUM_THREADS ONCE, when it is built on
+ # the first upstream accumulation in the dynamic loop, and ignores later
+ # changes. It must therefore be set here at startup, before the first
+ # earthkit call. Capping it stops the accumulation pool from oversubscribing
+ # the node against the routing kernel's OpenMP threads. The value is a
+ # validated integer string (no user string is interpolated unvalidated, and
+ # no shell is involved). Thread count affects scheduling only, never results.
+ earthkit_threads = resolve_earthkit_threads(binding)
+ if earthkit_threads is not None:
+ effective_earthkit = max(1, min(int(earthkit_threads), host))
+ os.environ["RAYON_NUM_THREADS"] = str(effective_earthkit)
+ else:
+ # Uncapped: leave RAYON_NUM_THREADS untouched so an operator who set it
+ # directly in the environment keeps control (mirrors the handling of
+ # NUMBA_THREADING_LAYER above).
+ effective_earthkit = None
+
+ # --- routing-substep numba cap ---
+ # The routing substep loop applies a lower numba count than the land phase
+ # (small per-substep payload collapses above a modest OpenMP worker count)
+ # and restores the land count afterwards. The land-phase numba count is the
+ # restore target (effective_numba). INVARIANT: the routing cap never RAISES
+ # the numba pool above the land count -- the apply-time min() clamps it to
+ # the land count (so on a small domain where the land policy chose serial,
+ # the window is a no-op), to the host core count, and to numba's hard limit.
+ raw_routing = resolve_routing_numba_threads(binding, host)
+ if raw_routing is None:
+ routing_applied = None # explicit all/0: no cap; land stands
+ else:
+ bounds = [raw_routing, host]
+ if effective_numba is not None:
+ bounds.append(effective_numba)
+ if _HAVE_NUMBA_THREADCTL:
+ bounds.append(_numba_config.NUMBA_NUM_THREADS)
+ routing_applied = min(bounds)
+
+ effective = {
+ "numba": effective_numba,
+ "routing_numba": routing_applied,
+ "blas": effective_blas,
+ "soilinit": soilinit_workers,
+ "earthkit": effective_earthkit,
+ "numba_threading_layer": effective_threading_layer,
+ "host": host,
+ }
+ _effective = effective
+ return effective
diff --git a/src/lisflood/global_modules/settings.py b/src/lisflood/global_modules/settings.py
index d60a93bd..2d62169c 100755
--- a/src/lisflood/global_modules/settings.py
+++ b/src/lisflood/global_modules/settings.py
@@ -68,7 +68,7 @@ def __call__(cls, *args, **kwargs):
new_init_args = []
for a in init_args:
if isinstance(a[1], np.ndarray):
- new_init_args.append((a[0], a[1].tostring()))
+ new_init_args.append((a[0], a[1].tobytes()))
else:
new_init_args.append(a)
key = (cls, frozenset(new_init_args))
@@ -102,7 +102,7 @@ def __call__(cls, *args, **kwargs):
new_init_args = []
for a in init_args:
if isinstance(a[1], np.ndarray):
- new_init_args.append((a[0], a[1].tostring()))
+ new_init_args.append((a[0], a[1].tobytes()))
else:
new_init_args.append(a)
key = (task_id, cls, str(new_init_args))
@@ -144,23 +144,26 @@ def get_flag(self, out_type, frequency):
if out_type == 'end':
flag = 0
elif out_type == 'steps':
- if frequency == 'all':
+ # 'daily' is a temporal-aggregation frequency (MapOutputAggregated);
+ # it manages its own period boundary and always-True checkpoint, so
+ # it reuses the 'all' step counter rather than needing its own flag.
+ if frequency in ('all', 'daily'):
flag = 1
elif frequency == 'monthly':
flag = 3
elif frequency == 'yearly':
flag = 4
else:
- raise ValueError(f'Output frequency {frequency} not recognised! Valid values are: (all, monthly, yearly)')
+ raise ValueError(f'Output frequency {frequency} not recognised! Valid values are: (all, daily, monthly, yearly)')
elif out_type == 'all':
- if frequency == 'all':
+ if frequency in ('all', 'daily'):
flag = 2
elif frequency == 'monthly':
flag = 5
elif frequency == 'yearly':
flag = 6
else:
- raise ValueError(f'Output frequency {frequency} not recognised! Valid values are: (all, monthly, yearly)')
+ raise ValueError(f'Output frequency {frequency} not recognised! Valid values are: (all, daily, monthly, yearly)')
else:
raise ValueError(f'Output type {out_type} not recognised! Valid values are: (end, steps, all)')
@@ -200,14 +203,18 @@ def update(self, var):
self.inc(6)
def frequency_check(self, var, frequency):
- if frequency == 'all':
+ # 'daily' is a temporal-aggregation frequency handled entirely by
+ # MapOutputAggregated (always-True checkpoint + own day boundary), so it
+ # never reaches here via the normal all/steps path; treat it like 'all'
+ # for defensiveness rather than raising.
+ if frequency in ('all', 'daily'):
check = True
elif frequency == 'monthly':
check = var.monthend
elif frequency == 'yearly':
check = var.yearend
else:
- raise ValueError(f'Output frequency {frequency} not recognised! Valid values are: (all, monthly, yearly)')
+ raise ValueError(f'Output frequency {frequency} not recognised! Valid values are: (all, daily, monthly, yearly)')
return check
@@ -245,12 +252,29 @@ def __init__(self, mask, maskmap):
self.info = self.Info(mask, mask.shape, self.flat, self.flat.shape, self.mask_compressed.shape, self.mask_all)
self._in_zero = np.zeros(self.mask_compressed.shape)
self._in_one = np.ones(self.mask_compressed.shape)
+ # dtype of the cached templates; rebuilt lazily if the model dtype changes
+ self._template_dtype = self._in_zero.dtype
self.maskmap = maskmap
+ def _sync_template_dtype(self):
+ """Rebuild the zero/one templates if the model precision (modelPrecision)
+ differs from their current dtype. float64 (default) is a no-op, so the
+ historical behaviour is bit-identical; float32 makes in_zero/in_one emit
+ float32 templates (half the memory traffic, and so the state arrays they
+ seed are born float32)."""
+ from .parallelization import get_model_dtype
+ want = np.dtype(get_model_dtype())
+ if want != self._template_dtype:
+ self._in_zero = np.zeros(self.mask_compressed.shape, dtype=want)
+ self._in_one = np.ones(self.mask_compressed.shape, dtype=want)
+ self._template_dtype = want
+
def in_zero(self):
+ self._sync_template_dtype()
return self._in_zero.copy()
def in_one(self):
+ self._sync_template_dtype()
return self._in_one.copy()
def __iter__(self):
@@ -616,6 +640,10 @@ def _options(settingsfile_dom):
# overwrite defaults
options.update(option_setting)
options['nonInit'] = not options['InitLisflood']
+ # Total Water Storage output is only meaningful when at least one
+ # storage/TWS map output is requested. Derived flag lets restrictoption
+ # (pure AND) express "repTWSMaps OR repStorageMaps".
+ options['repTWS'] = options['repTWSMaps'] or options['repStorageMaps']
return options
def _filter_steps(self, user_settings):
@@ -818,6 +846,7 @@ def __init__(self, model):
header = "\n\n ========================== LISFLOOD Simulation Information and Setting =============================\n"
msg = ''
settings = LisSettings.instance()
+ binding = settings.binding
option = settings.options
out_dir = settings.output_dir
ens_members = settings.ens_members[0]
@@ -841,6 +870,39 @@ def __init__(self, model):
msg += "\t[X] The simulation output as specified in the settings file can be found in {}\n".format(out_dir)
msg += "\t[X] Activated modules: {}\n".format(activated_options)
msg += "\t[X] Report options: {}\n".format(activated_rep)
+ # Thread-pool governance (numba/BLAS) plus the soil-init worker
+ # process count. Affects performance only, not results. "all" means every
+ # available core is used.
+ from .parallelization import get_effective_parallelism
+ _par = get_effective_parallelism()
+ if _par is not None:
+ msg += "\t[X] Parallelization: numba={}, routingNumba={}, BLAS={}, soilInit={}, earthkit={}, numbaLayer={} (host cores={})\n".format(
+ _par.get('numba'), _par.get('routing_numba'),
+ _par.get('blas'), _par.get('soilinit'),
+ _par.get('earthkit'),
+ _par.get('numba_threading_layer'), _par.get('host'))
+ # Routing precision (opt-in float32; default float64 == bit-identical).
+ try:
+ from .parallelization import resolve_model_dtype
+ import numpy as _np
+ _rdt = resolve_model_dtype(binding)
+ if _rdt == _np.float32:
+ msg += "\t[X] Model precision: float32\n"
+ else:
+ msg += "\t[X] Model precision: float64\n"
+ except Exception:
+ pass
+ # Packing and aggregation info
+ if binding.get('OutputPacking', 'False') == 'True':
+ msg += "\t[X] Output Packing: int16 scale/offset enabled\n"
+ if binding.get('OutputPackingDischarge', 'False') == 'True':
+ msg += "\t[X] Output Packing (Discharge): int16 log10 encoding enabled\n"
+ for agg_key in ['OutputDailyMean', 'OutputDailySum',
+ 'OutputMonthlyMean', 'OutputMonthlySum',
+ 'OutputYearlyMean', 'OutputYearlySum']:
+ agg_val = binding.get(agg_key, '').strip()
+ if agg_val:
+ msg += "\t[X] {}: {}\n".format(agg_key, agg_val)
self._msg = '{}{}'.format(header, msg)
def __str__(self):
diff --git a/src/lisflood/global_modules/zusatz.py b/src/lisflood/global_modules/zusatz.py
index 190324d7..c5236e4f 100755
--- a/src/lisflood/global_modules/zusatz.py
+++ b/src/lisflood/global_modules/zusatz.py
@@ -33,6 +33,7 @@
from .decorators import counted
from .settings import datetoint, LisSettings, MaskInfo, MaskAreaInfo
from .errors import LisfloodError, LisfloodFileError, LisfloodWarning
+from .earthkit_network import compute_gauge_zone_codes
MAX_READ_TRIALS = 100 # max number of trial allowed re-read an input file: to avoid crashes due to temporary network interruptions
@@ -364,21 +365,111 @@ def __init__(self, tssFilename, model, idMap=None, noHeader=False):
self._sampleAddresses = [-9999 for i in range(self._ncodesId)]
# init with the left/top cell - could also be 0 but then you have to catch it in
# the sample routine and put an exeption in
- # number of cells in map
- nrCells = pcraster.clone().nrRows() * pcraster.clone().nrCols()
- for cell in range(1, nrCells + 1):
- if (pcraster.cellvalue(self._spatialId, cell)[1]):
- # get point code from outlets map for pixel cell
- outlet_code = pcraster.cellvalue(self._spatialId, cell)[0]
- # get index of the point code in the sorted list of outlets codes
- outlet_idx = np.where(codesId == outlet_code)[0][0]
- # store point code
- self._codesId[outlet_idx] = outlet_code
- # store outlets location (cell)
- self._sampleAddresses[outlet_idx] = cell
+ #
+ # Vectorized replacement for the former per-cell loop:
+ # for cell in range(1, nrCells + 1):
+ # if pcraster.cellvalue(self._spatialId, cell)[1]:
+ # outlet_code = pcraster.cellvalue(self._spatialId, cell)[0]
+ # outlet_idx = np.where(codesId == outlet_code)[0][0]
+ # self._codesId[outlet_idx] = outlet_code
+ # self._sampleAddresses[outlet_idx] = cell
+ # which called pcraster.cellvalue twice for EVERY cell of the (full,
+ # uncropped) clone grid, once per time-series object.
+ # Optimized version to avoid the big loop:
+ if self._spatialId.isSpatial():
+ # 1-based row-major linear index for every grid cell
+ flat_codes = outletsmapnp.ravel(order='C')
+ flat_index = np.arange(1, flat_codes.size + 1, dtype=np.int64)
+ valid = np.isfinite(flat_codes) & (flat_codes > 0)
+ codes_valid = flat_codes[valid].astype(codesId.dtype)
+ index_valid = flat_index[valid]
+ # map each gauge code to its position in the sorted-unique codesId
+ code_to_idx = {code: i for i, code in enumerate(codesId)}
+ for code, cell in zip(codes_valid, index_valid):
+ outlet_idx = code_to_idx[code]
+ # keep the last (largest linear index) occurrence, as the
+ # original ascending-cell loop did
+ self._codesId[outlet_idx] = code
+ self._sampleAddresses[outlet_idx] = int(cell)
self._spatialIdGiven = True
+ # --- Fast 1D sampling support (see sample_compressed) -------------
+ # The inherited .sample() builds a full 2D PCRaster map (via
+ # decompress -> numpy2pcr, ~50-136 ms/call on a global grid) only so
+ # pcraster.areaaverage(expression, spatialId) can average the value
+ # over each gauge zone and read it at the gauge cell. When every
+ # gauge id occupies a SINGLE cell, that zone-average is exactly the
+ # value at that cell, so we can read it directly from the 1D
+ # compressed state array and skip the map conversion entirely.
+ # We precompute, per gauge (in _codesId order), the index into the
+ # 1D compressed array; and a flag that is True only when all gauge
+ # zones are single-cell (otherwise callers must use the PCRaster
+ # path to preserve the areaaverage/areamajority semantics).
+ self._compressed_sampling_ok = False
+ self._sample_compressed_idx = None
+ if self._spatialId.isSpatial():
+ try:
+ maskinfo = MaskInfo.instance()
+ maskflat = maskinfo.info.maskflat # True = masked/outside domain
+ gauge_flat = outletsmapnp.ravel(order='C')
+ # codes at the valid (compressed) cells, in compressed order
+ gauge_compressed = gauge_flat[~maskflat]
+ # Only iterate over gauge cells (codes > 0), not ALL valid
+ # pixels. On a global grid ncomp is millions but gauges are
+ # only thousands — the old per-pixel Python loop was O(ncomp)
+ # × n_series and caused ~200 s of init overhead.
+ gauge_mask = np.isfinite(gauge_compressed) & (gauge_compressed > 0)
+ gauge_positions = np.nonzero(gauge_mask)[0] # ascending compressed indices
+ gauge_codes_at = gauge_compressed[gauge_positions]
+ # count cells per code and keep LAST (largest) compressed index
+ code_to_comp = {}
+ counts = {}
+ for pos, c in zip(gauge_positions, gauge_codes_at):
+ counts[c] = counts.get(c, 0) + 1
+ code_to_comp[c] = int(pos)
+ all_single_cell = all(v == 1 for v in counts.values()) and len(counts) > 0
+ # also require every reported gauge code to be present in the
+ # compressed (in-domain) cells
+ idx = []
+ ok = all_single_cell
+ for code in self._codesId:
+ if code in code_to_comp:
+ idx.append(code_to_comp[code])
+ else:
+ ok = False
+ idx.append(-1)
+ if ok:
+ self._sample_compressed_idx = np.asarray(idx, dtype=np.int64)
+ self._compressed_sampling_ok = True
+ else:
+ self._sample_compressed_idx = None
+ self._compressed_sampling_ok = False
+ except Exception:
+ self._compressed_sampling_ok = False
+ self._sample_compressed_idx = None
+
+ # --- earthkit/numpy zone-mean support (useEarthkitHydro) -----
+ # Zone codes over the compressed cells (0 = not a gauge cell),
+ # used by earthkit_network.ek_sample_zone_means as the numpy
+ # equivalent of the inherited .sample()'s
+ # pcraster.areaaverage(expr, spatialId) fallback for multi-cell
+ # gauge zones (no 2D map round-trip). Checked against the
+ # engine in tests/test_earthkit_network.py.
+ try:
+ if self._spatialId.isSpatial():
+ maskinfo_ek = MaskInfo.instance()
+ self._ek_zone_codes, self._ek_n_codes = (
+ compute_gauge_zone_codes(outletsmapnp,
+ maskinfo_ek.info.maskflat,
+ self._ncodesId))
+ else:
+ self._ek_zone_codes = None
+ self._ek_n_codes = 0
+ except Exception:
+ self._ek_zone_codes = None
+ self._ek_n_codes = 0
+
nrCols = self._ncodesId
self._sampleValues = [
[Decimal("NaN")] * nrCols for _ in [0] * nrRows]
@@ -399,6 +490,44 @@ def firstout(self,expression):
value = Decimal("NaN")
return value
+ def sample_compressed(self, array_1d):
+ """Fast equivalent of .sample() that reads gauge values directly from the
+ 1D compressed state array, avoiding the decompress -> numpy2pcr full-map
+ conversion.
+
+ Only valid when self._compressed_sampling_ok is True, i.e. every gauge
+ id occupies a single cell (so the inherited .sample()'s per-zone
+ pcraster.areaaverage reduces to that cell's value). The caller
+ (output.py) is responsible for checking _compressed_sampling_ok and
+ falling back to .sample(decompress(...)) otherwise, and for using this
+ path only for Scalar/Directional (areaaverage) series.
+
+ Values are cast to float32 before storing, matching PCRaster's Scalar
+ precision so the %14g tss formatting is identical to the .sample() path.
+ Invalid / NaN values are stored as Decimal('NaN') (written as 1e31),
+ exactly as .sample() does for an invalid cellvalue.
+ """
+ arrayRowPos = self._userModel.currentTimeStep() - self._userModel.firstTimeStep()
+ # tss header datatype: the .sample() path records the expression's
+ # PCRaster dataType; for the Scalar series routed here it is 'VALUESCALE.Scalar'.
+ if self._spatialDatatype is None:
+ self._spatialDatatype = 'VALUESCALE.Scalar'
+ vals = np.asarray(array_1d, dtype=np.float32)
+ row = self._sampleValues[arrayRowPos]
+ # Gather all gauge values and test finiteness with two vectorized ops,
+ # then do the lean per-gauge assignment. The loop stays because row[col]
+ # must hold a Python float (valid) or Decimal('NaN') (invalid) so that
+ # _writeTssFile can distinguish them (1e31 vs %14g) - exactly as the
+ # inherited .sample() stores them.
+ gathered = vals[self._sample_compressed_idx]
+ finite = np.isfinite(gathered)
+ nan = Decimal("NaN")
+ for col in range(self._ncodesId):
+ row[col] = float(gathered[col]) if finite[col] else nan
+ # Write the tss on the final step, mirroring the inherited .sample().
+ if self._userModel.currentTimeStep() == self._userModel.nrTimeSteps():
+ self._writeTssFile()
+
#####################################################################################################################
# TOOLS TO OPEN/READ INPUT FILES ITERATIVELY, IN CASE OF NETWORK TEMPORARILY DOWN
diff --git a/src/lisflood/hydrological_modules/kinematic_wave_parallel.py b/src/lisflood/hydrological_modules/kinematic_wave_parallel.py
index 1f4d5fd2..4331c786 100755
--- a/src/lisflood/hydrological_modules/kinematic_wave_parallel.py
+++ b/src/lisflood/hydrological_modules/kinematic_wave_parallel.py
@@ -36,7 +36,6 @@
import numpy as np
import pandas as pd
-import numexpr as nx
from ..global_modules.errors import LisfloodWarning
@@ -116,11 +115,19 @@ def topoDistFromSea(downstream_lookup, upstream_lookup):
class kinematicWave:
""""""
- def __init__(self, compressed_encoded_ldd, land_mask, alpha_channel, beta, space_delta, time_delta, alpha_floodplains=None, flagnancheck=False):
+ def __init__(self, compressed_encoded_ldd, land_mask, alpha_channel, beta, space_delta, time_delta, alpha_floodplains=None, flagnancheck=False, routing_dtype=np.float64):
""""""
# variable to avoid printing repeated warning messages
self.kinematic_wave_warning_printed=False
self.flagnancheck=flagnancheck
+ # Routing precision for the kinematic numba kernel. float64 (default) is the
+ # historical, bit-identical path. float32 halves the kernel's memory traffic
+ # at the cost of a dtype-appropriate Newton tolerance (opt-in via the
+ # `routingPrecision` setting). The MCT path and all reductions/coupling stay
+ # float64; this flag only governs the narrow kinematic-kernel boundary.
+ self.routing_dtype = np.dtype(routing_dtype)
+ self.newton_tol = (kwpt.NEWTON_TOL if self.routing_dtype == np.float64
+ else kwpt.NEWTON_TOL_FLOAT32)
# Parameters for the solution of the discretised Kinematic wave continuity equation
self.alpha_channel = alpha_channel
self.space_delta = space_delta
@@ -172,9 +179,19 @@ def _setRoutingOrders(self):
# array containing all pixels organised by computation order
def kinematicWaveRouting(self, discharge_avg, discharge, specific_lateral_inflow, section="main_channel"):
- """Kinematic wave routing: wrapper around kinematic_wave_parallel_tools.kinematicWave"""
- # Lateral inflow (m3 s-1)
- lateral_inflow = nx.evaluate("q * dx", local_dict={"q": specific_lateral_inflow, "dx": self.space_delta})
+ """Kinematic wave routing: wrapper around kinematic_wave_parallel_tools.kinematicWave
+
+ When ``routing_dtype`` is float64 (the default) this is the original,
+ bit-identical path. When it is float32 the Newton solve runs in float32
+ over float32 copies of the dynamic arrays, and the results are written
+ back into the caller's float64 ``discharge``/``discharge_avg`` arrays in
+ place (narrow boundary: everything outside the kernel stays float64).
+ """
+ # Lateral inflow (m3 s-1). space_delta may be a per-pixel array or a
+ # scalar cell size, so this broadcast multiply stays on the host; only the
+ # heavier Newton constant term (with the ** beta power) is folded into the
+ # kernel's parallel loop below.
+ lateral_inflow = specific_lateral_inflow * self.space_delta
# Choose between main channel and floodplain routing
if section == "main_channel":
a_dx_div_dt = self.a_dx_div_dt_channel
@@ -186,13 +203,46 @@ def kinematicWaveRouting(self, discharge_avg, discharge, specific_lateral_inflow
a_dx = self.a_dx_floodplains
else:
raise Exception("The section parameter must be either 'main_channel' or 'floodplain'!")
- # Constant term in f(x) evaluation for Newton-Raphson method
- local_dict = {"a_dx_div_dt": a_dx_div_dt, "Qold": discharge, "b": self.beta, "lateral_inflow": lateral_inflow}
- constant = nx.evaluate("a_dx_div_dt * Qold ** b + lateral_inflow", local_dict=local_dict)
- # Solve the Kinematic wave equation
- kwpt.kinematicRouting(discharge_avg, discharge, lateral_inflow, constant, self.upstream_lookup,\
- self.num_upstream_pixels, self.pixels_ordered, self.order_start_stop,\
- self.inv_time_delta,self.beta, self.inv_beta, self.b_minus_1, a_dx_div_dt, b_a_dx_div_dt, a_dx)
+
+ if self.routing_dtype == np.float64:
+ # Historical path - bit-identical. The Newton tolerance is passed
+ # explicitly (== the former module constant NEWTON_TOL = 1e-12). The
+ # Newton f(x) constant term is now derived inside the kernel's parallel
+ # loop (from a_dx_div_dt, discharge and lateral_inflow), so it runs on
+ # the governed routing pool; the per-pixel arithmetic and operator
+ # order are unchanged, keeping the result identical.
+ # Solve the Kinematic wave equation
+ kwpt.kinematicRouting(discharge_avg, discharge, lateral_inflow, self.upstream_lookup,\
+ self.num_upstream_pixels, self.pixels_ordered, self.order_start_stop,\
+ self.inv_time_delta, self.beta, self.inv_beta, self.b_minus_1,\
+ a_dx_div_dt, b_a_dx_div_dt, a_dx, kwpt.NEWTON_TOL)
+ else:
+ # --- float32 narrow-boundary path (opt-in) ---
+ dt = self.routing_dtype
+ # float32 copies of the dynamic arrays; the kernel mutates these in place.
+ discharge32 = discharge.astype(dt)
+ discharge_avg32 = discharge_avg.astype(dt)
+ lateral_inflow32 = lateral_inflow.astype(dt)
+ # Constants coherent in dtype so numba does not upcast to float64.
+ a_dx_div_dt32 = a_dx_div_dt.astype(dt)
+ b_a_dx_div_dt32 = b_a_dx_div_dt.astype(dt)
+ a_dx32 = a_dx.astype(dt)
+ # Solve the Kinematic wave equation. The Newton constant term is folded
+ # into the kernel; passing float32 a_dx_div_dt32/discharge32 keeps the
+ # ** beta power in float32, matching the former float32 precompute.
+ kwpt.kinematicRouting(discharge_avg32, discharge32, lateral_inflow32, self.upstream_lookup,\
+ self.num_upstream_pixels, self.pixels_ordered, self.order_start_stop,\
+ dt.type(self.inv_time_delta), dt.type(self.beta), dt.type(self.inv_beta),\
+ dt.type(self.b_minus_1), a_dx_div_dt32, b_a_dx_div_dt32, a_dx32,\
+ dt.type(self.newton_tol))
+ # Cast results back into the caller's float64 arrays in place, so the rest
+ # of routing.dynamic (storage recompute, MCT, mass balance, coupling) sees
+ # float64 exactly as before.
+ discharge[:] = discharge32
+ discharge_avg[:] = discharge_avg32
+
+ # NaN/Inf check runs for BOTH precision paths (the float64 path previously
+ # ran it unconditionally; keep that behaviour).
if self.flagnancheck:
if self.kinematic_wave_warning_printed==False:
if np.all(np.isfinite(discharge))==False:
diff --git a/src/lisflood/hydrological_modules/kinematic_wave_parallel_tools.py b/src/lisflood/hydrological_modules/kinematic_wave_parallel_tools.py
index 5f11b94a..de90ae43 100755
--- a/src/lisflood/hydrological_modules/kinematic_wave_parallel_tools.py
+++ b/src/lisflood/hydrological_modules/kinematic_wave_parallel_tools.py
@@ -23,7 +23,13 @@
-NEWTON_TOL = 1e-12 # Max allowed error in iterative solution
+NEWTON_TOL = 1e-12 # Max allowed error in iterative solution (float64 default)
+# float32 cannot represent a 1e-12 absolute tolerance (machine eps ~1.2e-7), so the
+# Newton solve would never satisfy fabs(error)<=1e-12 and would burn MAX_ITERS on
+# every pixel. The tolerance is therefore passed in per call (newton_tol) and chosen
+# by the dtype at the wrapper: 1e-12 for float64 (unchanged, bit-identical), a
+# float32-appropriate value otherwise. See kinematic_wave_parallel.py.
+NEWTON_TOL_FLOAT32 = 1e-6 # Max allowed error in iterative solution (float32 path)
MAX_ITERS = 3000 # Max number of iterations
MIN_DISCHARGE = 1e-30 # Minimum allowed discharge value to avoid numerical instability
@@ -32,15 +38,14 @@
# ROUTING FUNCTIONS
# -------------------------------------------------------------------------------------------------
@njit(parallel=True, fastmath=False, cache=True)
-def kinematicRouting(discharge_avg, discharge, lateral_inflow, constant, upstream_lookup,\
+def kinematicRouting(discharge_avg, discharge, lateral_inflow, upstream_lookup,\
num_upstream_pixels, ordered_pixels, start_stop, inv_time_delta, beta, inv_beta,\
- b_minus_1, a_dx_div_dt, b_a_dx_div_dt, a_dx):
+ b_minus_1, a_dx_div_dt, b_a_dx_div_dt, a_dx, newton_tol):
"""
This function performs the kinematic wave routing algorithm to simulate the movement of water through a network of interconnected channels.
:param discharge_avg:
:param discharge:
:param lateral_inflow:
- :param constant:
:param upstream_lookup:
:param num_upstream_pixels:
:param ordered_pixels:
@@ -63,9 +68,16 @@ def kinematicRouting(discharge_avg, discharge, lateral_inflow, constant, upstrea
for index in prange(first, last): # this is a parallel loop
# Solve the kinematic wave for the current pixel
pix = ordered_pixels[index]
+ # Derive the Newton f(x) constant term per pixel inside this parallel
+ # loop so the elementwise power/multiply/add runs on this kernel's
+ # governed pool instead of a serial host pass. Operator order
+ # (** before *, then +) matches the former numpy precompute, and
+ # constant_pix reads the pre-solve discharge[pix] (the old-timestep
+ # value) because solve1Pixel mutates discharge[pix] in place below.
+ constant_pix = a_dx_div_dt[pix] * discharge[pix] ** beta + lateral_inflow[pix]
discharge_before = discharge[pix]
- if solve1Pixel(pix, discharge, constant[pix], upstream_lookup,\
- num_upstream_pixels, a_dx_div_dt[pix], b_a_dx_div_dt[pix], beta, inv_beta, b_minus_1):
+ if solve1Pixel(pix, discharge, constant_pix, upstream_lookup,\
+ num_upstream_pixels, a_dx_div_dt[pix], b_a_dx_div_dt[pix], beta, inv_beta, b_minus_1, newton_tol):
solve1PixelAvg(pix, discharge_avg, discharge_before, discharge[pix], lateral_inflow[pix], upstream_lookup,\
num_upstream_pixels, inv_time_delta, beta, a_dx[pix])
@@ -74,7 +86,7 @@ def kinematicRouting(discharge_avg, discharge, lateral_inflow, constant, upstrea
def solve1Pixel(
pix, discharge, constant,
upstream_lookup, num_upstream_pixels, a_dx_div_dt,
- b_a_dx_div_dt, beta, inv_beta, b_minus_1):
+ b_a_dx_div_dt, beta, inv_beta, b_minus_1, newton_tol):
"""
Te Chow et al. 1988 - Applied Hydrology - Chapter 9.6
:param pix:
@@ -96,7 +108,7 @@ def solve1Pixel(
upstream_inflow += discharge[upstream_lookup[pix,ups_ix]]
# If old discharge, upstream inflow and lateral inflow are below accuracy: set discharge to 0 and exit
- if upstream_inflow <= NEWTON_TOL:
+ if upstream_inflow <= newton_tol:
discharge[pix] = 0
return True
@@ -114,16 +126,16 @@ def solve1Pixel(
count = 0
previous_estimate = -1.0
while count < MAX_ITERS:
- if fabs(error) <= NEWTON_TOL or dis == previous_estimate:
+ if fabs(error) <= newton_tol or dis == previous_estimate:
break
previous_estimate = dis
factor = (1 + b_a_dx_div_dt * dis**b_minus_1)
- dis = max(dis - error / factor, NEWTON_TOL)
+ dis = max(dis - error / factor, newton_tol)
error = dis + a_dx_div_dt * dis**beta - upstream_inflow
count += 1
# If iterations converge to NEWTON_TOL, set value to 0
- if dis == NEWTON_TOL:
+ if dis == newton_tol:
discharge[pix] = 0
return True
diff --git a/src/lisflood/hydrological_modules/lakes.py b/src/lisflood/hydrological_modules/lakes.py
index 72c4b768..5c148d04 100755
--- a/src/lisflood/hydrological_modules/lakes.py
+++ b/src/lisflood/hydrological_modules/lakes.py
@@ -22,7 +22,7 @@
import pcraster
from ..global_modules.errors import LisfloodWarning
-from ..global_modules.add1 import loadmap, compressArray, decompress
+from ..global_modules.add1 import loadmap, compressArray, decompress, lookup_table_values
from ..global_modules.settings import LisSettings, MaskInfo
from . import HydroModule
@@ -75,7 +75,6 @@ def initial(self):
settings.build_reportedmaps_dicts()
return
# break if no lakes
- LakeSitePcr = loadmap('LakeSites', pcr=True) # moved here to use the caching feature during calibration
if option['simulateLakes'] and not option['InitLisflood']:
@@ -86,26 +85,42 @@ def initial(self):
# Add lake locations to structures map (used to modify LddChan
# and to calculate LddStructuresChan)
- # PCRaster part
+ # numpy replacement of the PCRaster part
# -----------------------
- LakeSitePcr = pcraster.ifthen((pcraster.defined(LakeSitePcr) & pcraster.boolean(decompress(self.var.IsChannel))), LakeSitePcr)
- IsStructureLake = pcraster.boolean(LakeSitePcr)
+ # IsStructureLake was pcraster.boolean(LakeSitePcr), i.e. True where
+ # a lake site is present in the channel network. LakeSitesC is
+ # already zeroed outside the channel network (above), so the
+ # compressed boolean is simply LakeSitesC > 0.
+ IsStructureLakeC = LakeSitesC > 0
# additional structure map only for lakes to calculate water balance
- # self.var.IsUpsOfStructureLake = pcraster.downstream(self.var.LddKinematic, pcraster.cover(IsStructureLake, 0))
- self.var.IsUpsOfStructureLake = pcraster.downstream(self.var.LddChan, pcraster.cover(IsStructureLake, 0))
+ # (was: pcraster.downstream(LddChan, cover(IsStructureLake, 0))):
+ # each cell takes the boolean value of its downstream neighbour.
+ # PCRaster semantics at pit cells (no downstream neighbour): the
+ # result is the pit cell's OWN value (verified against pcraster.
+ # downstream on a synthetic d8 network). Uses the unmodified
+ # LddChan topology of self.var.downstruct (pits = sentinel
+ # mapC[0]), computed in routing.initial() before lakes.
# Get all pixels just upstream of lakes
# -----------------------
+ n_cells = maskinfo.info.mapC[0]
+ downs = self.var.downstruct
+ is_pit = downs >= n_cells
+ self.var.IsUpsOfStructureLakeC = np.where(
+ is_pit, IsStructureLakeC, IsStructureLakeC[np.minimum(downs, n_cells - 1)])
- LakeArea = pcraster.lookupscalar(str(binding['TabLakeArea']), LakeSitePcr)
- LakeAreaC = compressArray(LakeArea)
- self.var.LakeAreaCC = np.compress(LakeSitesC > 0, LakeAreaC)
-
- self.var.LakeSitesC2 = LakeSitesC # additional var
+ # Direct per-point table lookup on the lake ids (LakeSitesCC),
+ # replacing the full-grid pcraster.lookupscalar+compressArray+
+ # np.compress (~25 s/call on the global domain).
+ self.var.LakeAreaCC = lookup_table_values(str(binding['TabLakeArea']), self.var.LakeSitesCC)
+
+ self.var.LakeSitesC2 = LakeSitesC # additional var
# Surface area of each lake [m2]
- LakeA = pcraster.lookupscalar(str(binding['TabLakeA']), LakeSitePcr)
- LakeAC = compressArray(LakeA) * loadmap('LakeMultiplier')
- self.var.LakeACC = np.compress(LakeSitesC > 0, LakeAC)
+ # LakeMultiplier is a (full-domain) calibration parameter map, so
+ # extract its per-point values before multiplying, matching the
+ # original full-domain multiply-then-compress.
+ lake_multiplier_cc = np.compress(LakeSitesC > 0, loadmap('LakeMultiplier'))
+ self.var.LakeACC = lookup_table_values(str(binding['TabLakeA']), self.var.LakeSitesCC) * lake_multiplier_cc
# Lake parameter A (suggested value equal to outflow width in [m])
# multiplied with the calibration parameter LakeMultiplier
@@ -113,9 +128,7 @@ def initial(self):
LakeInitialLevelValue = loadmap('LakeInitialLevelValue')
if np.max(LakeInitialLevelValue) == -9999:
# 'cold' start
- LakeAvNetInflowEstimate = pcraster.lookupscalar(str(binding['TabLakeAvNetInflowEstimate']), LakeSitePcr)
- LakeAvNetC = compressArray(LakeAvNetInflowEstimate)
- self.var.LakeAvNetCC = np.compress(LakeSitesC > 0, LakeAvNetC)
+ self.var.LakeAvNetCC = lookup_table_values(str(binding['TabLakeAvNetInflowEstimate']), self.var.LakeSitesCC)
LakeStorageIniM3CC = self.var.LakeAreaCC * np.sqrt(self.var.LakeAvNetCC / self.var.LakeACC)
# Initial lake storage [m3] S1 based on: S = LakeArea * H = LakeArea * sqrt(Q/a)
self.var.LakeLevelCC = LakeStorageIniM3CC / self.var.LakeAreaCC
diff --git a/src/lisflood/hydrological_modules/reservoir.py b/src/lisflood/hydrological_modules/reservoir.py
index 27d54235..62a017de 100755
--- a/src/lisflood/hydrological_modules/reservoir.py
+++ b/src/lisflood/hydrological_modules/reservoir.py
@@ -22,11 +22,10 @@
import warnings
-from pcraster.operations import ifthen, boolean, defined, lookupscalar
import numpy as np
from ..global_modules.settings import LisSettings, MaskInfo
-from ..global_modules.add1 import loadmap, compressArray, decompress, makenumpy
+from ..global_modules.add1 import loadmap, compressArray, makenumpy, lookup_table_values
from ..global_modules.errors import LisfloodWarning
from . import HydroModule
@@ -95,7 +94,8 @@ def initial(self):
if option['simulateReservoirs']:
# load reservoir locations and keep only those on the channel network
reservoirs = loadmap('ReservoirSites') # moved here to use the caching feature during calibration
- reservoirs[(reservoirs < 1) | (self.var.IsChannel == 0)] = 0
+ # ~(>= 1) also zeroes NaN, keeping ReservoirIndex aligned with ReservoirSitesCC
+ reservoirs[~(reservoirs >= 1) | (self.var.IsChannel == 0)] = 0
self.var.ReservoirSitesC = reservoirs
self.var.ReservoirSitesCC = np.compress(reservoirs > 0, reservoirs)
self.var.ReservoirIndex = np.nonzero(reservoirs)[0]
@@ -108,7 +108,6 @@ def initial(self):
# rebuild lists of reported files with simulateReservoirs and repsimulateReservoirs = False
settings.build_reportedmaps_dicts()
return
- ReservoirSitePcr = loadmap('ReservoirSites', pcr=True) # moved here to use the caching feature during calibration
if option['simulateReservoirs'] and not option['InitLisflood']:
@@ -120,20 +119,22 @@ def initial(self):
self.var.IsStructureChan = np.where(self.var.ReservoirSitesC > 0, np.bool(1), self.var.IsStructureChan)
# Add reservoir locations to structures map (used to modify LddChan
# and to calculate LddStructuresChan)
-
- self.var.ReservoirSites = ReservoirSitePcr
- # filter out reservoirs that are not part of the channel network
- # (following logic of 'old' code the inflow into these reservoirs is
- # always zero, so either change this or leave them out!)
- ReservoirSitePcr = ifthen((defined(ReservoirSitePcr) & boolean(decompress(self.var.IsChannel))), ReservoirSitePcr)
-
+
# RESERVOIR CHARACTERISTICS
# reservoir storage capacity [m3]
- total_storage = lookupscalar(str(binding['ReservoirTotalStorage']), ReservoirSitePcr)
- total_storage = compressArray(total_storage)
- self.var.TotalReservoirStorageM3C = np.where(np.isnan(total_storage), 0, total_storage)
- self.var.TotalReservoirStorageM3CC = np.compress(self.var.ReservoirSitesC > 0, self.var.TotalReservoirStorageM3C)
+ # Direct numpy table lookup on the reservoir ids (ReservoirSitesCC),
+ # replacing the full-grid pcraster.lookupscalar + compressArray +
+ # np.compress (~30 s/call on the global domain). lookup_table_values
+ # returns NaN for ids absent from the table, matching lookupscalar's
+ # missing value (then zeroed below as before).
+ total_storage = lookup_table_values(str(binding['ReservoirTotalStorage']), self.var.ReservoirSitesCC)
+ # per-point array (same order as ReservoirSitesCC / ReservoirIndex)
+ self.var.TotalReservoirStorageM3CC = np.where(np.isnan(total_storage), 0, total_storage)
+ # full-domain compressed form (0 outside reservoir cells), still used
+ # by waterabstraction. Scatter the per-point values at reservoir cells.
+ self.var.TotalReservoirStorageM3C = maskinfo.in_zero()
+ np.put(self.var.TotalReservoirStorageM3C, self.var.ReservoirIndex, self.var.TotalReservoirStorageM3CC)
# reservoir catchment area [m2]
catchment_area = loadmap('UpAreaTrans')
@@ -143,23 +144,24 @@ def initial(self):
# MODEL PARAMETERS
# flood storage limit (fraction of total storage [-])
+ # txt branch: direct per-point table lookup (replaces full-grid
+ # lookupscalar+compress+compress). non-txt branch unchanged.
if str(binding['ReservoirFloodStorage']).endswith('txt'):
- flood_storage = lookupscalar(str(binding['ReservoirFloodStorage']), ReservoirSitePcr)
- flood_storage = compressArray(flood_storage)
+ self.var.FloodStorageLimit = lookup_table_values(str(binding['ReservoirFloodStorage']), self.var.ReservoirSitesCC)
else:
flood_storage = loadmap('ReservoirFloodStorage')
flood_storage = makenumpy(flood_storage)
- self.var.FloodStorageLimit = np.compress(self.var.ReservoirSitesC > 0, flood_storage)
+ self.var.FloodStorageLimit = np.compress(self.var.ReservoirSitesC > 0, flood_storage)
# factor of the flood outflow
if str(binding['ReservoirFloodOutflowFactor']).endswith('txt'):
- factor_outflow = lookupscalar(str(binding['ReservoirFloodOutflowFactor']), ReservoirSitePcr)
- factor_outflow = compressArray(factor_outflow)
+ factor_outflow = lookup_table_values(str(binding['ReservoirFloodOutflowFactor']), self.var.ReservoirSitesCC)
+ factor_outflow = np.where(factor_outflow <= 0, 0.3, factor_outflow)
else:
factor_outflow = loadmap('ReservoirFloodOutflowFactor')
factor_outflow = makenumpy(factor_outflow)
- factor_outflow = np.where(factor_outflow <= 0, 0.3, factor_outflow)
- factor_outflow = np.compress(self.var.ReservoirSitesC > 0, factor_outflow)
+ factor_outflow = np.where(factor_outflow <= 0, 0.3, factor_outflow)
+ factor_outflow = np.compress(self.var.ReservoirSitesC > 0, factor_outflow)
# STORAGE LIMITS
@@ -174,19 +176,11 @@ def initial(self):
# RELEASE ATTRIBUTES
# minimum reservoir outflow [m3/s]
- MinReservoirOutflow = lookupscalar(str(binding['ReservoirMinOutflow']), ReservoirSitePcr)
- MinReservoirOutflowC = compressArray(MinReservoirOutflow)
- self.var.MinReservoirOutflow = np.compress(self.var.ReservoirSitesC > 0, MinReservoirOutflowC)
-
+ self.var.MinReservoirOutflow = lookup_table_values(str(binding['ReservoirMinOutflow']), self.var.ReservoirSitesCC)
# normal outflow [m3/s]
- normal_outflow = lookupscalar(str(binding['ReservoirNormalOutflow']), ReservoirSitePcr)
- normal_outflow = compressArray(normal_outflow)
- self.var.NormalReservoirOutflow = np.compress(self.var.ReservoirSitesC > 0, normal_outflow)
-
+ self.var.NormalReservoirOutflow = lookup_table_values(str(binding['ReservoirNormalOutflow']), self.var.ReservoirSitesCC)
# flood-control outflow [m3/s]
- flood_outflow = lookupscalar(str(binding['ReservoirFloodOutflow']), ReservoirSitePcr)
- flood_outflow = compressArray(flood_outflow)
- flood_outflow = np.compress(self.var.ReservoirSitesC > 0, flood_outflow)
+ flood_outflow = lookup_table_values(str(binding['ReservoirFloodOutflow']), self.var.ReservoirSitesCC)
self.var.FloodReservoirOutflow = np.maximum(self.var.NormalReservoirOutflow, factor_outflow * flood_outflow)
# INITIAL CONDITIONS
diff --git a/src/lisflood/hydrological_modules/routing.py b/src/lisflood/hydrological_modules/routing.py
index 9288d59c..7ce9e4de 100644
--- a/src/lisflood/hydrological_modules/routing.py
+++ b/src/lisflood/hydrological_modules/routing.py
@@ -34,6 +34,7 @@
from ..global_modules.settings import LisSettings, MaskInfo
from ..global_modules.errors import LisfloodWarning
from ..global_modules.add1 import loadmap, loadmap_base, compressArray, decompress
+from ..global_modules.parallelization import resolve_model_dtype
from . import HydroModule
@@ -548,9 +549,13 @@ def initialKinematicWave(self):
# Initialise parallel kinematic wave router: main channel-only routing if self.var.ChannelAlpha2 is None; else split-routing(main channel + floodplains)
# Initialization includes LDD for kinematic routing
maskinfo = MaskInfo.instance()
+ # Routing precision (opt-in float32); default resolves to float64 = unchanged.
+ self.var.model_dtype = resolve_model_dtype(settings.binding)
+ self.var.routing_dtype = self.var.model_dtype
self.river_router = kinematicWave(compressArray(self.var.LddKinematic), ~maskinfo.info.mask, self.var.ChannelAlpha,
self.var.Beta, self.var.ChanLength, self.var.DtRouting,
- alpha_floodplains=self.var.ChannelAlpha2, flagnancheck=flags['nancheck'])
+ alpha_floodplains=self.var.ChannelAlpha2, flagnancheck=flags['nancheck'],
+ routing_dtype=self.var.routing_dtype)
# ************************************************************
# ***** WATER BALANCE ************
@@ -581,7 +586,7 @@ def initialKinematicWave(self):
if option['simulateLakes']:
self.var.StorageStepINIT += self.var.LakeStorageIniM3
# DisStructure += np.where(compressArray(self.var.IsUpsOfStructureLake), 0.5 * self.var.ChanQ * self.var.DtRouting, 0) #cm22-1
- DisStructure += np.where(compressArray(self.var.IsUpsOfStructureLake), 0.5 * self.var.ChanQAvgDt * self.var.DtRouting, 0)
+ DisStructure += np.where(self.var.IsUpsOfStructureLakeC, 0.5 * self.var.ChanQAvgDt * self.var.DtRouting, 0)
self.var.DischargeM3StructuresIni = np.take(np.bincount(self.var.Catchments, weights=DisStructure), self.var.Catchments)
else:
# self.var.StorageStepINIT= self.var.ChanM3Kin+self.var.Chan2M3Kin-self.var.Chan2M3Start
@@ -611,7 +616,7 @@ def initialMCT(self):
maskinfo = MaskInfo.instance()
# self.var.IsChannelKinematicPcr = (self.var.IsChannelPcr == 1) & (self.var.IsChannelMCTPcr == 0) #pcr
- # self.var.IsChannelKinematic = np.bool8(compressArray(self.var.IsChannelKinematicPcr)) #np
+ # self.var.IsChannelKinematic = np.bool(compressArray(self.var.IsChannelKinematicPcr)) #np
# # Identify channel pixels where Kinematic wave is used instead of MCT
self.var.LddMCT = lddmask(self.var.LddChan, self.var.IsChannelMCTPcr) #pcr
@@ -1032,8 +1037,8 @@ def dynamic(self, NoRoutingExecuted):
# Legacy
# TotalCrossSectionArea = np.maximum(self.var.ChanM3Kin * self.var.InvChanLength, 0.01)
- TotalCrossSectionArea = np.maximum(self.var.ChanM3 * self.var.InvChanLength, 0.01)
- self.var.FlowVelocity = np.minimum(self.var.ChanQKin/TotalCrossSectionArea, 0.36*self.var.ChanQKin**0.24)
+ tmpLocalVarTotalCrossSectionArea = np.maximum(self.var.ChanM3 * self.var.InvChanLength, 0.01) # added prefix tmpLocalVar to avoid confusion with self.var.TotalCrossSectionArea
+ self.var.FlowVelocity = np.minimum(self.var.ChanQKin/tmpLocalVarTotalCrossSectionArea, 0.36*self.var.ChanQKin**0.24)
# Channel velocity (m/s); dividing Q (m3/s) by CrossSectionArea (m2)
# avoid extreme velocities by using the Wollheim 2006 equation
# assume 0.1 for upstream areas (outside ChanLdd)
@@ -1044,6 +1049,12 @@ def dynamic(self, NoRoutingExecuted):
# if flow is slow, Traveltime=DtSec then TravelDistance=PixelLength
# maximum set to 30km/day for 5km cell, is at DtSec/Traveltime=6, is at Traveltime 1 (each worker runs the exact same loop over a
+ contiguous chunk). Falls back to serial for small arrays, num_workers<=1, or
+ when already running inside a forked/daemon worker (nested pools are unsafe;
+ this is the lisflood-calibration case, where each LISFLOOD instance must stay
+ single-process)."""
+ n = x0.shape[0]
+ serial = (num_workers is None or num_workers <= 1 or n < 2 * num_workers
+ or _running_in_child_process())
+ if serial:
+ return _solve_saturation_chunk((x0, seep, ksat, genu_inv_m, genu_m))
+
+ # split into a few chunks per worker to balance load; use an explicit 'fork'
+ # context (fast, no re-import) and BLAS already pinned to 1 thread per worker.
+ n_chunks = min(n, num_workers * 4)
+ idx_chunks = np.array_split(np.arange(n), n_chunks)
+ tasks = [(x0[ix], seep[ix], ksat[ix], genu_inv_m[ix], genu_m[ix]) for ix in idx_chunks]
+ ctx = multiprocessing.get_context('fork')
+ with ctx.Pool(num_workers) as pool:
+ parts = pool.map(_solve_saturation_chunk, tasks)
+ solved = np.empty(n)
+ for ix, part in zip(idx_chunks, parts):
+ solved[ix] = part
+ return solved
+
def pressure2SoilMoistureFun(residual_sm, sat_sm, GenuA, GenuN, GenuM):
"""Generate a function to compute soil moisture values corresponding to characteristic pressure head levels [cm].
IT WON'T BE NEEDED ANYMORE ONCE DATA STRUCTURES WILL ALSO HAVE A "layer" DIMENSION, TO MERGE LAYER-SPECIFIC VARIABLES (1a, 1b, 2)"""
@@ -287,6 +342,12 @@ def splitlanduse(array1, array2=None, array3=None):
self.var.SeepTopToSubBAv[1] = loadmap('SeepTopToSubBAverageForestMap')
self.var.SeepTopToSubBAv[2] = loadmap('SeepTopToSubBAverageIrrigationMap')
+ # number of worker processes for the ColdStart layer-2 saturation solve
+ # (per-pixel least_squares); set via 'numCPUs_soilInit' (default 0 = all cores).
+ from ..global_modules.parallelization import get_effective_parallelism
+ _par = get_effective_parallelism()
+ soilinit_workers = _par.get('soilinit')
+
for veg, luse in self.var.VEGETATION_LANDUSE.items():
iveg = self.var.vegetation.index(veg)
iluse = self.var.SOIL_USES.index(luse)
@@ -307,13 +368,17 @@ def splitlanduse(array1, array2=None, array3=None):
if check_prerun_results < 0.0:
warnings.warn(LisfloodWarning('WARNING: soil moisture end state for bottom layer OR average fluxes not provided/erroneous. Soil moisture states are initialized at field capacity'))
else:
- SOLVED = ((ThetaInit2Value[iveg] * self.var.SoilDepth2[iluse])-self.var.WRes2[iluse])/(self.var.WS2[iluse]-self.var.WRes2[iluse])
- for ii in np.arange(len(ThetaInit2Value[iveg])):
- data = []
- analyticalcheckzero = []
- data=(self.var.SeepTopToSubBAv[iluse][ii], self.var.KSat2[iluse][ii], self.var.GenuInvM2[iluse][ii], self.var.GenuM2[iluse][ii])
- analyticalcheckzero = least_squares(function_estimate,((ThetaInit2Value[iveg][ii] * self.var.SoilDepth2[iluse][ii])-self.var.WRes2[iluse][ii])/(self.var.WS2[iluse][ii]-self.var.WRes2[iluse][ii]),bounds=(self.var.WFC2[iluse][ii]*0,self.var.WFC2[iluse][ii]*0+1.0), gtol = 10e-12, args = data)
- SOLVED[ii]=analyticalcheckzero.x
+ # x0 for each pixel = initial saturation degree from ThetaInit2
+ x0 = ((ThetaInit2Value[iveg] * self.var.SoilDepth2[iluse]) - self.var.WRes2[iluse]) / (self.var.WS2[iluse] - self.var.WRes2[iluse])
+ # Solve K(s)=SeepTopToSubBAv per pixel (same scipy least_squares as
+ # before, bit-identical), parallelized over processes for speed.
+ SOLVED = solve_saturation_degree(
+ np.asarray(x0, dtype=float),
+ np.asarray(self.var.SeepTopToSubBAv[iluse], dtype=float),
+ np.asarray(self.var.KSat2[iluse], dtype=float),
+ np.asarray(self.var.GenuInvM2[iluse], dtype=float),
+ np.asarray(self.var.GenuM2[iluse], dtype=float),
+ num_workers=soilinit_workers)
ini_2 = SOLVED*(self.var.WS2[iluse]-self.var.WRes2[iluse])+self.var.WRes2[iluse]
self.var.W2[iveg] = np.where(self.var.PoreSpaceNotZero2[iluse], ini_2, 0)
diff --git a/src/lisflood/hydrological_modules/soilloop.py b/src/lisflood/hydrological_modules/soilloop.py
index 5d3962e4..88673124 100644
--- a/src/lisflood/hydrological_modules/soilloop.py
+++ b/src/lisflood/hydrological_modules/soilloop.py
@@ -18,7 +18,6 @@
from nine import range
import numpy as np
-import numexpr as nx
from numba import njit, prange, vectorize
from builtins import min, max
# from . import HydroModule
@@ -75,6 +74,67 @@ def potentialTranspiration(TranspirMax, TaInterception):
return np.maximum(TranspirMax - TaInterception, 0)
+@njit(parallel=True, fastmath=False, cache=True)
+def soilWaterStressAndTranspiration(swdf, WFC1, WWP1, WFC1a, WWP1a, WFC1b, WWP1b,
+ W1, W1a, W1b, potential_transpiration,
+ isFrozenSoil, RWS, Ta):
+ """Per-pixel soil-water-stress and actual-transpiration computation for one
+ prescribed vegetation fraction.
+
+ This is a fused, parallel rewrite of the per-vegetation numpy block in
+ ``dynamic_canopy`` (critical moisture -> stress factor RWS -> actual
+ transpiration Ta -> layer-wise abstraction from W1a/W1b). It reproduces the
+ original arithmetic exactly, pixel by pixel, to avoid allocating ~30
+ intermediate full-domain arrays per vegetation type.
+
+ ``swdf`` (soil water depletion fraction) is precomputed by the caller in
+ numpy to preserve the original float32 precision of the ETRef term.
+
+ RWS, Ta, W1a, W1b, W1 are updated in place (1D pixel arrays for this veg).
+ """
+ num_pix = W1.shape[0]
+ for pix in prange(num_pix):
+ # Critical soil moisture per layer. NOTE: the wcrit1a/wcrit1b formula is
+ # also computed in dynamic_canopy (for the Irrigated WFilla/WFillb); keep
+ # the two in sync if this changes.
+ one_minus_swdf = 1.0 - swdf[pix]
+ wcrit1 = one_minus_swdf * (WFC1[pix] - WWP1[pix]) + WWP1[pix]
+ wcrit1a = one_minus_swdf * (WFC1a[pix] - WWP1a[pix]) + WWP1a[pix]
+ wcrit1b = one_minus_swdf * (WFC1b[pix] - WWP1b[pix]) + WWP1b[pix]
+
+ # Transpiration reduction factor (in case of water stress)
+ # if WCrit1 = WWP1, RWS is 1, there is no water stress in that case
+ denom = wcrit1 - WWP1[pix]
+ rws = (W1[pix] - WWP1[pix]) / denom if denom > 0 else 1.0
+ rws = min(max(rws, 0.0), 1.0)
+ RWS[pix] = rws
+
+ transpirable_water = max(W1[pix] - WWP1[pix], 0.0)
+ # actual transpiration based on both layers 1a and 1b
+ ta = 0.0 if isFrozenSoil[pix] else min(rws * potential_transpiration[pix], transpirable_water)
+ Ta[pix] = ta
+ # transpiration is 0 when soil is frozen
+ # calculate distribution where to take Ta from:
+ # 1st: above wCrit from layer 1a
+ # 2nd: above Wcrit from layer 1b
+ # 3rd: distribute take off according to soil moisture availability below wcrit
+ wc1a = max(W1a[pix] - wcrit1a, 0.0)
+ wc1b = max(W1b[pix] - wcrit1b, 0.0)
+ ta1a = min(ta, wc1a)
+ rest_ta = max(ta - ta1a, 0.0)
+ ta1b = min(rest_ta, wc1b)
+ rest_ta = max(rest_ta - ta1b, 0.0)
+ avail_1a = max(W1a[pix] - ta1a - WWP1a[pix], 0.0)
+ avail_1b = max(W1b[pix] - ta1b - WWP1b[pix], 0.0)
+ avail_tot = avail_1a + avail_1b
+ if avail_tot > 0:
+ ta1a += (avail_1a / avail_tot) * rest_ta
+ ta1b += (avail_1b / avail_tot) * rest_ta
+ W1a[pix] -= ta1a
+ W1b[pix] -= ta1b
+ W1[pix] = W1a[pix] + W1b[pix]
+
+
@njit(parallel=True, fastmath=False, cache=True)
def soilColumnsWaterBalance(index_landuse_all, is_irrigated, is_paddy_irrig, paddy_inactive, DtDay,
AvailableWaterForInfiltration, Rain, SnowMelt,
@@ -529,8 +589,8 @@ def dynamic_canopy(self):
# ***********************************************************************************************************************
# ***** POTENTIAL INTERCEPTION EVAPORATION ******************************************************************************
# ***********************************************************************************************************************
- one_minus_LAITerm = nx.evaluate('1. - LAITerm', local_dict={'LAITerm': self.var.LAITerm.sel(vegetation=self.var.prescribed_vegetation).values})
- TaInterceptionMax = nx.evaluate('EWRef * one_minus_LAITerm', global_dict={'EWRef': self.var.EWRef[None]})
+ one_minus_LAITerm = 1. - self.var.LAITerm.sel(vegetation=self.var.prescribed_vegetation).values
+ TaInterceptionMax = self.var.EWRef[None] * one_minus_LAITerm
## SG if self.settings.option.get('cropsEPIC'):
if option['cropsEPIC']:
TaInterceptionMax = np.vstack((TaInterceptionMax, self.var.crop_module.potential_interception_evaporation.values))
@@ -547,8 +607,7 @@ def dynamic_canopy(self):
# ***********************************************************************************************************************
# ***** POTENTIAL PLANT TRANSPIRATION ***********************************************************************************
# ***********************************************************************************************************************
- gl_dict = {'CropCoef': self.var.CropCoef[self.index_landuse_prescr], 'ETRef': self.var.ETRef[None]}
- TranspirMax = nx.evaluate('CropCoef * ETRef * one_minus_LAITerm', global_dict=gl_dict)
+ TranspirMax = self.var.CropCoef[self.index_landuse_prescr] * self.var.ETRef[None] * one_minus_LAITerm
if option['cropsEPIC']:
TranspirMax = np.vstack((TranspirMax, self.var.crop_module.potential_crop_ET.values))
# maximum transpiration rate ([mm] per timestep)
@@ -565,68 +624,54 @@ def dynamic_canopy(self):
for veg in self.var.prescribed_vegetation:
iveg, ilanduse, landuse = self.var.get_landuse_and_indexes_from_vegetation_epic(veg)
- swdf = 1 / (0.76 + 1.5 * np.minimum(0.1 * self.var.ETRef * self.var.InvDtDay, 1.0)) - 0.10 * (5 - self.var.CropGroupNumber.values[ilanduse])
# soil water depletion fraction (easily available soil water)
# Van Diepen et al., 1988: WOFOST 6.0, p.87
# to avoid a strange behaviour of the p-formula's, ETRef is set to a maximum of
# 10 mm/day. Thus, p will range from 0.15 to 0.45 at ETRef eq 10 and
# CropGroupNumber 1-5
+ swdf = 1 / (0.76 + 1.5 * np.minimum(0.1 * self.var.ETRef * self.var.InvDtDay, 1.0)) - 0.10 * (5 - self.var.CropGroupNumber.values[ilanduse])
swdf = np.where(self.var.CropGroupNumber.values[ilanduse] <= 2.5, swdf + (np.minimum(0.1 * self.var.ETRef * self.var.InvDtDay, 1.0) - 0.6) / (
self.var.CropGroupNumber.values[ilanduse] * (self.var.CropGroupNumber.values[ilanduse] + 3)), swdf)
# correction for crop groups 1 and 2 (Van Diepen et al, 1988)
swdf = np.maximum(np.minimum(swdf, 1.0), 0)
# p is between 0 and 1
- WCrit1 = ((1 - swdf) * (self.var.WFC1.values[ilanduse] - self.var.WWP1.values[ilanduse])) + self.var.WWP1.values[ilanduse]
- WCrit1a = ((1 - swdf) * (self.var.WFC1a.values[ilanduse] - self.var.WWP1a.values[ilanduse])) + self.var.WWP1a.values[ilanduse]
- WCrit1b = ((1 - swdf) * (self.var.WFC1b.values[ilanduse] - self.var.WWP1b.values[ilanduse])) + self.var.WWP1b.values[ilanduse]
- # critical moisture amount ([mm] water slice) for all layers
- if option['wateruse']:
- if landuse == "Irrigated":
- #CR: using the original xarray variables WPF3a and WPF3a to keep WFilla and WFillb as xarrays
- #N.B: WPF3a and WPF3b values are not changed in this function, so I can use the original
- self.var.WFilla = np.minimum(WCrit1a, self.var.WPF3a.values[ilanduse])
- self.var.WFillb = np.minimum(WCrit1b, self.var.WPF3b.values[ilanduse])
- # if water use is calculated, get the filling of the soil layer for either pF3 or WCrit1
- # that is the amount of water the soil gets filled by water from irrigation
- # bc the divisor can have 0 -> this calculation is done first and raise a warning - zero encountered - even if it is catched afterwards
- self.var.RWS.values[iveg] = np.where((WCrit1 - self.var.WWP1.values[ilanduse]) > 0,\
- (self.var.W1.values[ilanduse] - self.var.WWP1.values[ilanduse]) / (WCrit1 - self.var.WWP1.values[ilanduse]), 1)
- # Transpiration reduction factor (in case of water stress)
- # if WCrit1 = WWP1, RWS is zero there is no water stress in that case
- self.var.RWS.values[iveg] = np.maximum(np.minimum(self.var.RWS.values[iveg], 1), 0)
+
+ if option['wateruse'] and landuse == "Irrigated":
+ # WFilla/WFillb (irrigation target filling) is consumed later by
+ # the water-abstraction module, so it must still be computed here.
+ # NOTE: the critical-moisture formula below is intentionally
+ # mirrored inside soilWaterStressAndTranspiration (wcrit1a/wcrit1b),
+ # which needs it for every vegetation fraction. Here it is only
+ # needed for the Irrigated fraction's WFilla/WFillb. If you
+ # change this formula, update the kernel too.
+ WCrit1a = ((1 - swdf) * (self.var.WFC1a.values[ilanduse] - self.var.WWP1a.values[ilanduse])) + self.var.WWP1a.values[ilanduse]
+ WCrit1b = ((1 - swdf) * (self.var.WFC1b.values[ilanduse] - self.var.WWP1b.values[ilanduse])) + self.var.WWP1b.values[ilanduse]
+ #CR: using the original xarray variables WPF3a and WPF3a to keep WFilla and WFillb as xarrays
+ #N.B: WPF3a and WPF3b values are not changed in this function, so I can use the original
+ self.var.WFilla = np.minimum(WCrit1a, self.var.WPF3a.values[ilanduse])
+ self.var.WFillb = np.minimum(WCrit1b, self.var.WPF3b.values[ilanduse])
+
+ # Fused per-pixel computation of soil water stress (RWS), actual
+ # transpiration (Ta) and the layer-wise abstraction from W1a/W1b.
+ # Updates RWS/Ta/W1a/W1b/W1 rows in place (parallel over pixels).
+ # Soil properties (WFC*/WWP*) are indexed per land-use class
+ # (ilanduse); soil-moisture state and outputs (W1/W1a/W1b/Ta/RWS)
+ # are indexed per vegetation (iveg), matching how each is allocated.
+ soilWaterStressAndTranspiration(
+ swdf,
+ self.var.WFC1.values[ilanduse], self.var.WWP1.values[ilanduse],
+ self.var.WFC1a.values[ilanduse], self.var.WWP1a.values[ilanduse],
+ self.var.WFC1b.values[ilanduse], self.var.WWP1b.values[ilanduse],
+ self.var.W1.values[iveg], self.var.W1a.values[iveg], self.var.W1b.values[iveg],
+ self.var.potential_transpiration[iveg], self.var.isFrozenSoil,
+ self.var.RWS.values[iveg], self.var.Ta.values[iveg])
+
# Transpiration reduction factor (in case of water stress)
if option['repStressDays']:
self.var.SoilMoistureStressDays.values[iveg] = np.where(self.var.RWS.values[iveg] < 1, self.var.DtDay, 0)
# Count number of days with soil water stress, RWS is between 0 and 1
# no reduction of Transpiration at RWS=1, at RWS=0 there is no Transpiration at all
- transpirable_water = np.maximum(self.var.W1.values[ilanduse] - self.var.WWP1.values[ilanduse], 0)
- self.var.Ta.values[iveg] = np.minimum(self.var.RWS.values[iveg] * self.var.potential_transpiration[iveg], transpirable_water)
- # actual transpiration based on both layers 1a and 1b
- self.var.Ta.values[iveg] = np.where(self.var.isFrozenSoil, 0, self.var.Ta.values[iveg])
- # transpiration is 0 when soil is frozen
- # calculate distribution where to take Ta from:
- # 1st: above wCrit from layer 1a
- # 2nd: above Wcrit from layer 1b
- # 3rd: distribute take off according to soil moisture availability below wcrit
- wc1a = np.maximum(self.var.W1a.values[ilanduse] - WCrit1a, 0) # unstressed water availability from layer 1a without stress (above critical soil moisture)
- wc1b = np.maximum(self.var.W1b.values[ilanduse] - WCrit1b, 0) # (same as above but for layer 1b)
- Ta1a = np.minimum(self.var.Ta.values[iveg], wc1a) # temporary transpiration from layer 1a (<= unstressed layer 1a availability)
- restTa = np.maximum(self.var.Ta.values[iveg] - Ta1a, 0) # transpiration left after layer 1a unstressed water has been abstracted
- Ta1b = np.minimum(restTa, wc1b) # temporary transpiration from layer 1b (<= unstressed layer 1b availability)
- restTa = np.maximum(restTa - Ta1b, 0) # transpiration left after layers 1a and 1b unstressed water have been abstracted
- stressed_availability_1a = np.maximum(self.var.W1a.values[ilanduse] - Ta1a - self.var.WWP1a.values[ilanduse], 0) #|
- stressed_availability_1b = np.maximum(self.var.W1b.values[ilanduse] - Ta1b - self.var.WWP1b.values[ilanduse], 0) #|
- stressed_availability_tot = stressed_availability_1a + stressed_availability_1b #|> distribution of abstractions of
- available = stressed_availability_tot > 0 #|> soil moisture below the critical value
- fraction_rest_1a = np.where(available, stressed_availability_1a / stressed_availability_tot, 0) #|> proportionally to each root-zone layer (1a and 1b)
- fraction_rest_1b = np.where(available, stressed_availability_1b / stressed_availability_tot, 0) #|> "stressed" availability
- Ta1a += fraction_rest_1a * restTa #|
- Ta1b += fraction_rest_1b * restTa #|
- self.var.W1a.values[ilanduse] -= Ta1a
- self.var.W1b.values[ilanduse] -= Ta1b
- self.var.W1.values[iveg] = self.var.W1a.values[ilanduse] + self.var.W1b.values[ilanduse]
-
def dynamic_soil(self):
settings = LisSettings.instance()
@@ -636,7 +681,7 @@ def dynamic_soil(self):
""" Dynamic part of the soil/vegetation loop describing soil processes (after canopy ones have been processed):
"""
# Maximum evaporation from a shaded soil surface in [mm] per time step
- ESMax = nx.evaluate('ESRef * LAITerm', local_dict={'ESRef': self.var.ESRef[None], 'LAITerm': self.var.LAITerm.sel(vegetation=self.var.prescribed_vegetation).values})
+ ESMax = self.var.ESRef[None] * self.var.LAITerm.sel(vegetation=self.var.prescribed_vegetation).values
### SG if self.settings.option.get('cropsEPIC'):
if option['cropsEPIC']:
ESMax = np.vstack((ESMax, self.var.crop_module.potential_undercanopy_evaporation.values))
diff --git a/src/lisflood/hydrological_modules/structures.py b/src/lisflood/hydrological_modules/structures.py
index 44dbeaa8..0dffa42f 100755
--- a/src/lisflood/hydrological_modules/structures.py
+++ b/src/lisflood/hydrological_modules/structures.py
@@ -16,10 +16,11 @@
"""
from __future__ import absolute_import, print_function
-from pcraster import downstream, boolean, cover, lddrepair, ifthenelse
+import numpy as np
+from pcraster import ldd, nominal
from ..global_modules.add1 import decompress, compressArray
-from ..global_modules.settings import LisSettings
+from ..global_modules.settings import LisSettings, MaskInfo
class structures(object):
@@ -57,27 +58,43 @@ def initial(self):
option = settings.options
if not option['InitLisflood']:
# not done in Init Lisflood
- IsUpsOfStructureKinematic = downstream( #pcr map
- self.var.LddKinematic,
- cover(boolean(decompress(self.var.IsStructureKinematic)), boolean(0))
- )
- # Downstream assigns to result the expression value of the neighbouring downstream cell
- # Over is used to cover missing values on an expression with values taken from one or more different expression(s)
- # Decompress is numpy2pcr
- # Find location of pixels immediately upstream of a structure on the LddKinematic
-
- IsUpsOfStructureChan = downstream( #pcr map
- self.var.LddChan,
- cover(boolean(decompress(self.var.IsStructureChan)), boolean(0))
- )
- # Find location of pixels immediately upstream of a structure on the LddChan
-
- self.var.IsUpsOfStructureKinematicC = compressArray(IsUpsOfStructureKinematic) #np compressed array
+ maskinfo = MaskInfo.instance()
+ n_cells = maskinfo.info.mapC[0]
+ downs = self.var.downstruct
+ is_pit = downs >= n_cells
+
+ # Cell-immediately-upstream-of-structure mask: each cell takes the
+ # structure boolean of its downstream neighbour. Pit cells (no
+ # downstream neighbour, flagged by the sentinel index) keep their own
+ # value, matching PCRaster downstream pit semantics. downstruct is the
+ # compressed downstream-cell index for the channel LDD, which equals
+ # the kinematic LDD at this point in initialisation (the MCT channel
+ # split happens later), so the same index serves both gathers.
+ IsStructureKinematicC = np.asarray(self.var.IsStructureKinematic).astype(bool)
+ IsStructureChanC = np.asarray(self.var.IsStructureChan).astype(bool)
+
+ self.var.IsUpsOfStructureKinematicC = np.where( #np compressed array
+ is_pit, IsStructureKinematicC,
+ IsStructureKinematicC[np.minimum(downs, n_cells - 1)])
# Location of pixels immediately upstream of a structure on the LddKinematic
- self.var.IsUpsOfStructureChanC = compressArray(IsUpsOfStructureChan) #np compressed array
+ self.var.IsUpsOfStructureChanC = np.where( #np compressed array
+ is_pit, IsStructureChanC,
+ IsStructureChanC[np.minimum(downs, n_cells - 1)])
# Location of pixels immediately upstream of a structure on the LddChan
- self.var.LddKinematic = lddrepair(ifthenelse(IsUpsOfStructureKinematic, 5, self.var.LddKinematic)) #pcr map
+ # Insert a pit (ldd=5) at each cell immediately upstream of a
+ # structure. On a valid LDD this is equivalent to
+ # lddrepair(ifthenelse(mask, 5, Ldd)): adding a pit cannot create a
+ # loop or an off-map pointer, so LDD re-validation is a no-op. The
+ # maps are rebuilt as PCRaster LDD-typed fields because later modules
+ # (initialMCT/initialKinematicWave) consume them as LDDs; decompress
+ # yields a scalar field, so an explicit ldd(nominal(...)) cast
+ # restores the LDD type the consumers require.
+ LddKinematicC = compressArray(self.var.LddKinematic)
+ LddChanC = compressArray(self.var.LddChan)
+ LddKinematicC[self.var.IsUpsOfStructureKinematicC] = 5
+ LddChanC[self.var.IsUpsOfStructureChanC] = 5
+ self.var.LddKinematic = ldd(nominal(decompress(LddKinematicC))) #pcr map
# Update LddKinematic by adding a pit in the pixel immediately upstream of a structure
- self.var.LddChan = lddrepair(ifthenelse(IsUpsOfStructureChan, 5, self.var.LddChan)) #pcr map
+ self.var.LddChan = ldd(nominal(decompress(LddChanC))) #pcr map
# Update LddChan by adding a pit in the pixel immediately upstream of a structure
diff --git a/src/lisflood/hydrological_modules/waterabstraction.py b/src/lisflood/hydrological_modules/waterabstraction.py
index 4676b400..97635e51 100644
--- a/src/lisflood/hydrological_modules/waterabstraction.py
+++ b/src/lisflood/hydrological_modules/waterabstraction.py
@@ -14,9 +14,10 @@
import warnings
from pcraster import boolean, nominal, ifthen, defined, areamaximum, downstream, cover, lddrepair, ifthenelse, upstream, \
- scalar, accuflux, celllength, windowtotal, areaaverage
+ scalar, accuflux, celllength, windowtotal, areaaverage, pcr2numpy
from pcraster.operators import pcrDiv
import numpy as np
+from numba import njit, prange
from netCDF4 import Dataset
from ..global_modules.add1 import loadmap, decompress, compressArray, readnetcdf, readmapsparse
@@ -25,6 +26,294 @@
from ..global_modules.netcdf import xarray_reader
from ..global_modules.errors import LisfloodError
+try:
+ from ..global_modules.earthkit_network import (
+ earthkit_available, ek_upstream_sum,
+ )
+except ImportError: # pragma: no cover - earthkit-hydro not installed
+ earthkit_available = lambda: False
+ ek_upstream_sum = None
+
+
+@njit(parallel=True, fastmath=False, cache=True)
+def computeLivestockDemand(LivestockDemandMM, LivestockConsumptiveUseFraction,
+ FractionGroundwaterUsed, FractionSurfaceWaterUseDomLivInd,
+ FractionNonConventionalWaterUsed, MMtoM3):
+ """Per-pixel livestock water demand (LIVESTOCK section of waterabstraction.dynamic).
+
+ Numba-parallel port of the original elementwise numpy block; the arithmetic
+ is line-for-line identical (array expressions, auto-parallelized by numba's
+ parallel=True). Governed by the numCPUs_parallelNumba thread setting
+ (numba.set_num_threads, applied centrally at startup).
+ """
+ consumption_required_livestock_MM = LivestockDemandMM * LivestockConsumptiveUseFraction
+ consumption_GW_livestock_MM = consumption_required_livestock_MM * FractionGroundwaterUsed
+ consumption_SW_required_livestock_MM = consumption_required_livestock_MM * FractionSurfaceWaterUseDomLivInd
+ # consumption_NC_required_livestock_MM = consumption_required_livestock_MM * FractionNonConventionalWaterUsed # (dead local in the original; never consumed downstream)
+ abstraction_required_livestock_M3 = LivestockDemandMM * MMtoM3
+ abstraction_GW_livestock_M3 = FractionGroundwaterUsed * abstraction_required_livestock_M3
+ abstraction_NC_livestock_M3 = FractionNonConventionalWaterUsed * abstraction_required_livestock_M3
+ abstraction_SW_required_livestock_M3 = abstraction_required_livestock_M3 - abstraction_GW_livestock_M3 - abstraction_NC_livestock_M3
+ return (consumption_GW_livestock_MM, consumption_SW_required_livestock_MM,
+ abstraction_required_livestock_M3, abstraction_GW_livestock_M3,
+ abstraction_SW_required_livestock_M3)
+
+
+@njit(parallel=True, fastmath=False, cache=True)
+def computeDomesticDemand(DomesticDemandMM, DomesticWaterSavingConstant, leak_demand_fraction,
+ DomesticConsumptiveUseFraction, FractionGroundwaterUsed,
+ FractionSurfaceWaterUseDomLivInd, FractionNonConventionalWaterUsed, MMtoM3):
+ """Per-pixel domestic water demand (DOMESTIC section of waterabstraction.dynamic).
+
+ Numba-parallel port of the original elementwise numpy block; the arithmetic
+ is line-for-line identical (array expressions, auto-parallelized by numba's
+ parallel=True). Governed by the numCPUs_parallelNumba thread setting.
+ """
+ demand_reduced_domestic_MM = DomesticDemandMM * DomesticWaterSavingConstant
+ leakage_domestic_MM = leak_demand_fraction * demand_reduced_domestic_MM # Leakage in mm per day
+ abstraction_required_domestic_MM = demand_reduced_domestic_MM + leakage_domestic_MM
+ abstraction_required_domestic_M3 = abstraction_required_domestic_MM * MMtoM3
+ consumption_required_domestic_MM = demand_reduced_domestic_MM * DomesticConsumptiveUseFraction
+ consumption_GW_domestic_MM = consumption_required_domestic_MM * FractionGroundwaterUsed
+ consumption_SW_required_domestic_MM = consumption_required_domestic_MM * FractionSurfaceWaterUseDomLivInd
+ abstraction_GW_domestic_M3 = FractionGroundwaterUsed * abstraction_required_domestic_M3
+ abstraction_NC_domestic_M3 = FractionNonConventionalWaterUsed * abstraction_required_domestic_M3
+ abstraction_SW_required_domestic_M3 = abstraction_required_domestic_M3 - abstraction_GW_domestic_M3 - abstraction_NC_domestic_M3
+ return (abstraction_required_domestic_M3, consumption_GW_domestic_MM,
+ consumption_SW_required_domestic_MM, abstraction_GW_domestic_M3,
+ abstraction_SW_required_domestic_M3)
+
+
+@njit(parallel=True, fastmath=False, cache=True)
+def computeIndustryDemand(IndustrialDemandMM, IndustryConsumptiveUseFraction, FractionGroundwaterUsed,
+ FractionSurfaceWaterUseDomLivInd, FractionNonConventionalWaterUsed, MMtoM3):
+ """Per-pixel industrial water demand (INDUSTRY section of waterabstraction.dynamic).
+
+ Numba-parallel port of the original elementwise numpy block; the arithmetic
+ is line-for-line identical (array expressions, auto-parallelized by numba's
+ parallel=True). Governed by the numCPUs_parallelNumba thread setting.
+ """
+ abstraction_required_industry_M3 = IndustrialDemandMM * MMtoM3
+ consumption_required_industry_MM = IndustrialDemandMM * IndustryConsumptiveUseFraction
+ consumption_GW_industry_MM = consumption_required_industry_MM * FractionGroundwaterUsed
+ consumption_SW_required_industry_MM = consumption_required_industry_MM * FractionSurfaceWaterUseDomLivInd
+ abstraction_GW_industry_M3 = FractionGroundwaterUsed * abstraction_required_industry_M3
+ abstraction_NC_industry_M3 = FractionNonConventionalWaterUsed * abstraction_required_industry_M3
+ abstraction_SW_required_industry_M3 = abstraction_required_industry_M3 - abstraction_GW_industry_M3 - abstraction_NC_industry_M3
+ return (abstraction_required_industry_M3, consumption_GW_industry_MM,
+ consumption_SW_required_industry_MM, abstraction_GW_industry_M3,
+ abstraction_SW_required_industry_M3)
+
+
+@njit(parallel=True, fastmath=False, cache=True)
+def computeEnergyDemand(EnergyDemandMM, EnergyConsumptiveUseFraction, MMtoM3):
+ """Per-pixel energy water demand (ENERGY section of waterabstraction.dynamic).
+
+ Numba-parallel port of the original elementwise numpy block; the arithmetic
+ is line-for-line identical (array expressions, auto-parallelized by numba's
+ parallel=True). Governed by the numCPUs_parallelNumba thread setting.
+ All energy abstraction is taken from surface water.
+ """
+ consumption_required_energy_MM = EnergyDemandMM * EnergyConsumptiveUseFraction
+ abstraction_SW_required_energy_M3 = EnergyDemandMM * MMtoM3
+ return consumption_required_energy_MM, abstraction_SW_required_energy_M3
+
+def _as_pixel_array(x, template):
+ """Return ``x`` as a float64 array matching ``template``'s shape.
+
+ Some bindings (e.g. IrrigationMult or the efficiency maps) can be scalar
+ values instead of per-pixel maps; the original numpy code multiplied by
+ them directly. Multiplying by a scalar is bit-identical to multiplying by
+ a uniform array of the same value, so scalars are broadcast here to keep
+ the numba kernels array-only. Arrays are passed through unmodified.
+ """
+ a = np.asarray(x, dtype=np.float64)
+ if a.ndim == 0:
+ return np.full(template.shape, float(a), np.float64)
+ return a
+
+
+@njit(parallel=True, fastmath=False, cache=True)
+def computeIrrigationDemand(RWS_iveg, potential_transpiration_iveg, W1_iveg, WWP1_iluse,
+ SoilFraction_iveg, isFrozenSoil, IrrigationMult,
+ IrrigationEfficiency, ConveyanceEfficiency, MMtoM3):
+ """Per-pixel irrigation water demand (§6.1 of waterabstraction.dynamic).
+
+ Numba-parallel port of the original elementwise numpy block, restricted to
+ the Irrigated_prescribed vegetation index (the caller passes the already
+ gathered sub-arrays). Arithmetic is line-for-line identical; the Ta update
+ round-trip through self.var.Ta / self.var.W1 (float64 in the original) is
+ preserved bit-for-bit by the caller, which writes the kernel results back
+ into the same float64 arrays before reading them again.
+ """
+ Ta_iveg = np.minimum(RWS_iveg * potential_transpiration_iveg,
+ W1_iveg - WWP1_iluse)
+ for p in prange(Ta_iveg.size):
+ if Ta_iveg[p] < 0.0:
+ Ta_iveg[p] = 0.0
+ demand_irrigation_MM = (potential_transpiration_iveg - Ta_iveg) * SoilFraction_iveg
+ consumption_required_irrigation_MM = np.where(isFrozenSoil, 0.0,
+ demand_irrigation_MM * IrrigationMult)
+ abstraction_required_irrigation_MM = np.where(
+ (IrrigationEfficiency * ConveyanceEfficiency) > 0,
+ consumption_required_irrigation_MM / (IrrigationEfficiency * ConveyanceEfficiency), 0.0)
+ abstraction_required_irrigation_M3 = np.maximum(abstraction_required_irrigation_MM * MMtoM3, 0.0)
+ return Ta_iveg, consumption_required_irrigation_MM, abstraction_required_irrigation_MM, \
+ abstraction_required_irrigation_M3
+
+@njit(parallel=True, fastmath=False, cache=True)
+def computeIrrigationReUsePartition(PotentialIrrigationWaterReUseM3Annual, ActualAccumulatedReUsedWaterM3,
+ PotentialIrrigationWaterReUseM3Daily, abstraction_required_irrigation_M3,
+ consumption_required_irrigation_MM, GWfed_fraction_irrigation, MMtoM3):
+ """Re-used water partition and SW/GW split of irrigation demand (§6.2-§6.4).
+
+ Numba-parallel port of the original elementwise numpy block; arithmetic is
+ line-for-line identical (np.minimum / np.maximum / np.where on contiguous
+ float64 arrays). The caller updates the running ActualAccumulatedReUsedWaterM3
+ state (per-pixel accumulator reset on the first calendar day) from the
+ kernel's reuse output.
+ """
+ WaterAvailableForReUseM3 = np.minimum(np.maximum(
+ PotentialIrrigationWaterReUseM3Annual - ActualAccumulatedReUsedWaterM3, 0.0),
+ PotentialIrrigationWaterReUseM3Daily)
+ abstraction_Reuse_irrigation_M3 = np.minimum(WaterAvailableForReUseM3, abstraction_required_irrigation_M3)
+ fraction_irrigation_SwGw = 1.0 - np.where(abstraction_required_irrigation_M3 > 0,
+ abstraction_Reuse_irrigation_M3 / abstraction_required_irrigation_M3, 0.0)
+ abstraction_SwGw_required_irrigation_M3 = fraction_irrigation_SwGw * abstraction_required_irrigation_M3
+ consumption_SwGw_required_irrigation_MM = fraction_irrigation_SwGw * consumption_required_irrigation_MM
+ abstraction_GW_required_irrigation_M3 = GWfed_fraction_irrigation * abstraction_SwGw_required_irrigation_M3
+ abstraction_SW_required_irrigation_M3 = np.maximum(
+ abstraction_SwGw_required_irrigation_M3 - abstraction_GW_required_irrigation_M3, 0.0)
+ consumption_GW_required_irrigation_MM = GWfed_fraction_irrigation * consumption_required_irrigation_MM
+ consumption_SW_required_irrigation_MM = np.maximum(
+ consumption_SwGw_required_irrigation_MM - consumption_GW_required_irrigation_MM, 0.0)
+ consumption_GW_actual_irrigation_MM = consumption_GW_required_irrigation_MM
+ consumption_GW_actual_irrigation_M3 = consumption_GW_required_irrigation_MM * MMtoM3
+ return (WaterAvailableForReUseM3, abstraction_Reuse_irrigation_M3,
+ abstraction_SwGw_required_irrigation_M3, abstraction_GW_required_irrigation_M3,
+ abstraction_SW_required_irrigation_M3, consumption_GW_required_irrigation_MM,
+ consumption_SW_required_irrigation_MM, consumption_GW_actual_irrigation_MM,
+ consumption_GW_actual_irrigation_M3, consumption_SwGw_required_irrigation_MM)
+
+
+@njit(parallel=True, fastmath=False, cache=True)
+def computeChannelAbstraction(AreaTotalAvailableWaterFromChannelsM3,
+ areatotal_withdrawal_CH_required_M3, InvNoRoutSteps,
+ PixelAvailableWaterFromChannelsM3):
+ """§10.3-§10.4 channel withdrawal against the water-region available water.
+
+ Numba-parallel port of the original elementwise numpy block; arithmetic is
+ line-for-line identical. The caller passes AreaTotalAvailableWaterFromChannelsM3
+ already gathered with np.take(np.bincount(...), ...) (the regional reduction
+ stays in the driver, so the summation order is unchanged) and the
+ e-flow-clamped per-pixel available water PixelAvailableWaterFromChannelsM3.
+ """
+ areatotal_withdrawal_CH_actual_M3 = np.minimum(AreaTotalAvailableWaterFromChannelsM3,
+ areatotal_withdrawal_CH_required_M3)
+ FractionAbstractedFromChannels = np.where(AreaTotalAvailableWaterFromChannelsM3 > 0,
+ np.minimum(areatotal_withdrawal_CH_actual_M3 / AreaTotalAvailableWaterFromChannelsM3, 1.0), 0.0)
+ withdrawal_CH_actual_M3 = FractionAbstractedFromChannels * PixelAvailableWaterFromChannelsM3
+ withdrawal_CH_actual_M3_routStep = withdrawal_CH_actual_M3 * InvNoRoutSteps
+ return (AreaTotalAvailableWaterFromChannelsM3, areatotal_withdrawal_CH_actual_M3,
+ FractionAbstractedFromChannels, withdrawal_CH_actual_M3,
+ withdrawal_CH_actual_M3_routStep)
+
+
+@njit(parallel=True, fastmath=False, cache=True)
+def computeMetFraction(at_actual_M3, at_required_M3):
+ """Fraction of the water-region requirement actually met (§12.1/§12.2 pattern).
+
+ Numba-parallel port of the repeated
+ ``np.minimum(np.where(required > 0, actual / required, 0.), 1.)`` idiom;
+ arithmetic identical, evaluated per pixel.
+ """
+ return np.minimum(np.where(at_required_M3 > 0, at_actual_M3 / at_required_M3, 0.0), 1.0)
+
+
+@njit(parallel=True, fastmath=False, cache=True)
+def computeMetFractionSW(FractionAbstractedByLakesReservoirs, fraction_met_CH):
+ """Fraction of the sector requirement met from lakes/reservoirs + channels (§13).
+
+ Numba-parallel port of the repeated
+ ``np.minimum(FractionAbstractedByLakesReservoirs + fraction_met_CH *
+ (1 - FractionAbstractedByLakesReservoirs), 1.)`` idiom.
+ """
+ return np.minimum(FractionAbstractedByLakesReservoirs + fraction_met_CH * (1.0 - FractionAbstractedByLakesReservoirs), 1.0)
+
+
+@njit(parallel=True, fastmath=False, cache=True)
+def replaceNaNWithZero(a):
+ """Numba-parallel equivalent of np.where(np.isnan(a), 0, a).
+
+ Bit-identical to the original elementwise numpy expression (NaN test and
+ replacement per pixel, no arithmetic).
+ """
+ out = a.copy()
+ for p in prange(out.size):
+ if np.isnan(out[p]):
+ out[p] = 0.0
+ return out
+
+
+@njit(parallel=True, fastmath=False, cache=True)
+def computeGroundwaterAbstractionUpdate(LZ, abstraction_GW_actual_M3, M3toMM,
+ IrriLossCUM, abstraction_GW_noReturn_M3,
+ consumption_GW_noReturn_M3, InvNoRoutSteps):
+ """Groundwater mass-balance update and return flow (§8.2-§8.3).
+
+ Numba-parallel port of the original elementwise numpy block; arithmetic is
+ line-for-line identical. Mutates LZ (lower-zone storage, mm) and IrriLossCUM
+ (irrigation loss accounting, m3) in place and returns the per-routing-step
+ return flow to the channel, with the original < 1e-18 clamp (which preserves
+ NaN, as in the numpy np.where).
+ """
+ returnflow = np.empty(LZ.size, np.float64)
+ for p in prange(LZ.size):
+ LZ[p] -= abstraction_GW_actual_M3[p] * M3toMM[p]
+ IrriLossCUM[p] += abstraction_GW_actual_M3[p]
+ rf = (abstraction_GW_noReturn_M3[p] - consumption_GW_noReturn_M3[p]) * InvNoRoutSteps
+ returnflow[p] = 0.0 if rf < 1e-18 else rf
+ return returnflow
+
+
+@njit(parallel=True, fastmath=False, cache=True)
+def computeNoReturnSectorMet(fraction_met_SW_noReturn, abstraction_SW_required_energy_M3,
+ abstraction_SW_required_domestic_M3, abstraction_SW_required_livestock_M3,
+ abstraction_SW_required_industry_M3, consumption_required_energy_MM,
+ consumption_GW_domestic_MM, consumption_SW_required_domestic_MM,
+ consumption_GW_livestock_MM, consumption_SW_required_livestock_MM,
+ consumption_GW_industry_MM, consumption_SW_required_industry_MM):
+ """§13.2-§14 fan-out of the region-met fraction over the no-return sectors.
+
+ Numba-parallel port of the original elementwise numpy block; arithmetic is
+ line-for-line identical (products and sums over contiguous float64 arrays).
+ Inputs abstraction_SW_required_*_M3 and consumption_GW/SW_required_*_MM are
+ the outputs of the demand kernels above.
+ """
+ n = fraction_met_SW_noReturn.size
+ abstraction_SW_actual_energy_M3 = np.empty(n, np.float64)
+ abstraction_SW_actual_domestic_M3 = np.empty(n, np.float64)
+ abstraction_SW_actual_livestock_M3 = np.empty(n, np.float64)
+ abstraction_SW_actual_industry_M3 = np.empty(n, np.float64)
+ consumption_actual_energy_MM = np.empty(n, np.float64)
+ consumption_actual_domestic_MM = np.empty(n, np.float64)
+ consumption_actual_livestock_MM = np.empty(n, np.float64)
+ consumption_actual_industry_MM = np.empty(n, np.float64)
+ for p in prange(n):
+ f = fraction_met_SW_noReturn[p]
+ abstraction_SW_actual_energy_M3[p] = abstraction_SW_required_energy_M3[p] * f
+ abstraction_SW_actual_domestic_M3[p] = abstraction_SW_required_domestic_M3[p] * f
+ abstraction_SW_actual_livestock_M3[p] = abstraction_SW_required_livestock_M3[p] * f
+ abstraction_SW_actual_industry_M3[p] = abstraction_SW_required_industry_M3[p] * f
+ consumption_actual_energy_MM[p] = consumption_required_energy_MM[p] * f
+ consumption_actual_domestic_MM[p] = consumption_GW_domestic_MM[p] + consumption_SW_required_domestic_MM[p] * f
+ consumption_actual_livestock_MM[p] = consumption_GW_livestock_MM[p] + consumption_SW_required_livestock_MM[p] * f
+ consumption_actual_industry_MM[p] = consumption_GW_industry_MM[p] + consumption_SW_required_industry_MM[p] * f
+ return (abstraction_SW_actual_energy_M3, abstraction_SW_actual_domestic_M3,
+ abstraction_SW_actual_livestock_M3, abstraction_SW_actual_industry_M3,
+ consumption_actual_energy_MM, consumption_actual_domestic_MM,
+ consumption_actual_livestock_MM, consumption_actual_industry_MM)
+
class waterabstraction(HydroModule):
"""
@@ -295,29 +584,23 @@ def dynamic(self):
# ***** LIVESTOCK ********************************************
# ************************************************************
- consumption_required_livestock_MM = self.var.LivestockDemandMM * self.var.LivestockConsumptiveUseFraction
- consumption_GW_livestock_MM = consumption_required_livestock_MM * self.var.FractionGroundwaterUsed
- consumption_SW_required_livestock_MM = consumption_required_livestock_MM * self.FractionSurfaceWaterUseDomLivInd
- consumption_NC_required_livestock_MM = consumption_required_livestock_MM * self.var.FractionNonConventionalWaterUsed
- abstraction_required_livestock_M3 = self.var.LivestockDemandMM * self.var.MMtoM3
- abstraction_GW_livestock_M3 = self.var.FractionGroundwaterUsed * abstraction_required_livestock_M3
- abstraction_NC_livestock_M3 = self.var.FractionNonConventionalWaterUsed * abstraction_required_livestock_M3
- abstraction_SW_required_livestock_M3 = abstraction_required_livestock_M3 - abstraction_GW_livestock_M3 - abstraction_NC_livestock_M3
+ (consumption_GW_livestock_MM, consumption_SW_required_livestock_MM,
+ abstraction_required_livestock_M3, abstraction_GW_livestock_M3,
+ abstraction_SW_required_livestock_M3) = computeLivestockDemand(
+ self.var.LivestockDemandMM, self.var.LivestockConsumptiveUseFraction,
+ self.var.FractionGroundwaterUsed, self.FractionSurfaceWaterUseDomLivInd,
+ self.var.FractionNonConventionalWaterUsed, self.var.MMtoM3)
# ************************************************************
# ***** DOMESTIC *********************************************
# ************************************************************
- demand_reduced_domestic_MM = self.var.DomesticDemandMM * self.var.DomesticWaterSavingConstant
- leakage_domestic_MM = self.leak_demand_fraction * demand_reduced_domestic_MM # Leakage in mm per day
- abstraction_required_domestic_MM = demand_reduced_domestic_MM + leakage_domestic_MM
- abstraction_required_domestic_M3 = abstraction_required_domestic_MM * self.var.MMtoM3
- consumption_required_domestic_MM = demand_reduced_domestic_MM * self.var.DomesticConsumptiveUseFraction
- consumption_GW_domestic_MM = consumption_required_domestic_MM * self.var.FractionGroundwaterUsed
- consumption_SW_required_domestic_MM = consumption_required_domestic_MM * self.FractionSurfaceWaterUseDomLivInd
- abstraction_GW_domestic_M3 = self.var.FractionGroundwaterUsed * abstraction_required_domestic_M3
- abstraction_NC_domestic_M3 = self.var.FractionNonConventionalWaterUsed * abstraction_required_domestic_M3
- abstraction_SW_required_domestic_M3 = abstraction_required_domestic_M3 - abstraction_GW_domestic_M3 - abstraction_NC_domestic_M3
+ (abstraction_required_domestic_M3, consumption_GW_domestic_MM,
+ consumption_SW_required_domestic_MM, abstraction_GW_domestic_M3,
+ abstraction_SW_required_domestic_M3) = computeDomesticDemand(
+ self.var.DomesticDemandMM, self.var.DomesticWaterSavingConstant, self.leak_demand_fraction,
+ self.var.DomesticConsumptiveUseFraction, self.var.FractionGroundwaterUsed,
+ self.FractionSurfaceWaterUseDomLivInd, self.var.FractionNonConventionalWaterUsed, self.var.MMtoM3)
# ************************************************************
# ***** INDUSTRY *********************************************
@@ -327,20 +610,18 @@ def dynamic(self):
# self.var.IndustrialConsumptiveUseMM = self.var.IndustrialAbstractionMM * self.var.IndustryConsumptiveUseFraction ## QUESTION ABOVE ###
# Industrial Water Demand (mm per day) ## QUESTION ABOVE ###
# WaterReUseFraction: Fraction of water re-used in industry (e.g. 50% = 0.5 = half of the water is re-used, used twice (baseline=0, maximum=1) ## QUESTION ABOVE ###
- abstraction_required_industry_M3 = self.var.IndustrialDemandMM * self.var.MMtoM3
- consumption_required_industry_MM = self.var.IndustrialDemandMM * self.var.IndustryConsumptiveUseFraction
- consumption_GW_industry_MM = consumption_required_industry_MM * self.var.FractionGroundwaterUsed
- consumption_SW_required_industry_MM = consumption_required_industry_MM * self.FractionSurfaceWaterUseDomLivInd
- abstraction_GW_industry_M3 = self.var.FractionGroundwaterUsed * abstraction_required_industry_M3
- abstraction_NC_industry_M3 = self.var.FractionNonConventionalWaterUsed * abstraction_required_industry_M3
- abstraction_SW_required_industry_M3 = abstraction_required_industry_M3 - abstraction_GW_industry_M3 - abstraction_NC_industry_M3
+ (abstraction_required_industry_M3, consumption_GW_industry_MM,
+ consumption_SW_required_industry_MM, abstraction_GW_industry_M3,
+ abstraction_SW_required_industry_M3) = computeIndustryDemand(
+ self.var.IndustrialDemandMM, self.var.IndustryConsumptiveUseFraction, self.var.FractionGroundwaterUsed,
+ self.FractionSurfaceWaterUseDomLivInd, self.var.FractionNonConventionalWaterUsed, self.var.MMtoM3)
# ************************************************************
# ***** ENERGY ***********************************************
# ************************************************************
# EnergyConsumptiveUseMM is the amount that evaporates etc
- consumption_required_energy_MM = self.var.EnergyDemandMM * self.var.EnergyConsumptiveUseFraction
- abstraction_SW_required_energy_M3 = self.var.EnergyDemandMM* self.var.MMtoM3
+ consumption_required_energy_MM, abstraction_SW_required_energy_M3 = computeEnergyDemand(
+ self.var.EnergyDemandMM, self.var.EnergyConsumptiveUseFraction, self.var.MMtoM3)
# all taken from surface water
# ************************************************************
@@ -354,37 +635,40 @@ def dynamic(self):
irr_pre = "Irrigated_prescribed" # It applies only to prescribed irrigation fraction (EPIC simulates specific irrigated crops, if any)
ivegIrrigatedPrescribed, iluseIrrigatedPrescribed, _ = self.var.get_landuse_and_indexes_from_vegetation_GLOBAL(irr_pre)
- self.var.Ta.values[ivegIrrigatedPrescribed] = np.maximum(np.minimum(self.var.RWS.values[ivegIrrigatedPrescribed] * self.var.potential_transpiration.values[ivegIrrigatedPrescribed],
- self.var.W1.values[ivegIrrigatedPrescribed] - self.var.WWP1.values[iluseIrrigatedPrescribed]), maskinfo.in_zero())
- demand_irrigation_MM = (self.var.potential_transpiration.values[ivegIrrigatedPrescribed] - self.var.Ta.values[ivegIrrigatedPrescribed]) * self.var.SoilFraction.values[ivegIrrigatedPrescribed]
- demand_irrigation_MM = np.where(self.var.isFrozenSoil, maskinfo.in_zero(),
- demand_irrigation_MM)
- consumption_required_irrigation_MM = demand_irrigation_MM * self.var.IrrigationMult
- self.var.abstraction_required_irrigation_MM = np.where((self.var.IrrigationEfficiency * self.var.ConveyanceEfficiency) > 0,
- consumption_required_irrigation_MM / (self.var.IrrigationEfficiency * self.var.ConveyanceEfficiency), maskinfo.in_zero())
- self.var.abstraction_required_irrigation_M3 = np.maximum(self.var.abstraction_required_irrigation_MM * self.var.MMtoM3,
- maskinfo.in_zero())
+ # Per-pixel irrigation demand on the Irrigated_prescribed vegetation fraction (numba-parallel)
+ (Ta_iveg, consumption_required_irrigation_MM,
+ self.var.abstraction_required_irrigation_MM,
+ self.var.abstraction_required_irrigation_M3) = computeIrrigationDemand(
+ self.var.RWS.values[ivegIrrigatedPrescribed],
+ self.var.potential_transpiration.values[ivegIrrigatedPrescribed],
+ self.var.W1.values[ivegIrrigatedPrescribed],
+ self.var.WWP1.values[iluseIrrigatedPrescribed],
+ self.var.SoilFraction.values[ivegIrrigatedPrescribed],
+ self.var.isFrozenSoil,
+ _as_pixel_array(self.var.IrrigationMult, self.var.RWS.values[ivegIrrigatedPrescribed]),
+ _as_pixel_array(self.var.IrrigationEfficiency, self.var.RWS.values[ivegIrrigatedPrescribed]),
+ _as_pixel_array(self.var.ConveyanceEfficiency, self.var.RWS.values[ivegIrrigatedPrescribed]),
+ self.var.MMtoM3)
+ # Ta update: write the clamped value back; the demand above was
+ # already computed from this same value inside the kernel, and
+ # downstream readers (soil, §15.2) see it exactly as before
+ self.var.Ta.values[ivegIrrigatedPrescribed] = Ta_iveg
if option["cropsEPIC"]: # Add pixel-averaged EPIC gross irrigation requirement to LISFLOOD (perscribed_irrigation fraction) irrigation demand
consumption_required_irrigation_MM += (self.var.crop_module.NIR_mm + self.var.crop_module.irrigation_losses_atmosphere).values
self.var.abstraction_required_irrigation_M3 += self.var.crop_module.GIR_m3.values # [mm]
# 6.2 Amount of treated wastewater used for irrigation
if self.var.CalendarDay == 1:
self.ActualAccumulatedReUsedWaterM3 = maskinfo.in_zero()
- WaterAvailableForReUseM3 = np.minimum(np.maximum(self.PotentialIrrigationWaterReUseM3Annual - self.ActualAccumulatedReUsedWaterM3, 0), self.PotentialIrrigationWaterReUseM3Daily) # potential ReUsed water amount (m3) available as irrigation
- self.var.abstraction_Reuse_irrigation_M3 = np.minimum(WaterAvailableForReUseM3, self.var.abstraction_required_irrigation_M3) # actual ReUsed water amount (m3) for irrigation
+ # 6.2-6.4 Re-used water partition and SW/GW split of the irrigation demand (numba-parallel)
+ (WaterAvailableForReUseM3, self.var.abstraction_Reuse_irrigation_M3,
+ abstraction_SwGw_required_irrigation_M3, abstraction_GW_required_irrigation_M3,
+ abstraction_SW_required_irrigation_M3, consumption_GW_required_irrigation_MM,
+ consumption_SW_required_irrigation_MM, consumption_GW_actual_irrigation_MM,
+ consumption_GW_actual_irrigation_M3, consumption_SwGw_required_irrigation_MM) = computeIrrigationReUsePartition(
+ self.PotentialIrrigationWaterReUseM3Annual, self.ActualAccumulatedReUsedWaterM3,
+ self.PotentialIrrigationWaterReUseM3Daily, self.var.abstraction_required_irrigation_M3,
+ consumption_required_irrigation_MM, self.GWfed_fraction_irrigation, self.var.MMtoM3)
self.ActualAccumulatedReUsedWaterM3 += self.var.abstraction_Reuse_irrigation_M3 # added to the accumulated annual reuse
-
-
- fraction_irrigation_SwGw = 1. - np.where(self.var.abstraction_required_irrigation_M3 > 0,
- self.var.abstraction_Reuse_irrigation_M3 / self.var.abstraction_required_irrigation_M3, 0.)
- abstraction_SwGw_required_irrigation_M3 = fraction_irrigation_SwGw * self.var.abstraction_required_irrigation_M3
- consumption_SwGw_required_irrigation_MM = fraction_irrigation_SwGw * consumption_required_irrigation_MM
-
- # 6.3 Partition irrigation required abstraction and consumption among ground- and surface-water resources after removing re-used water input
- abstraction_GW_required_irrigation_M3 = self.GWfed_fraction_irrigation * abstraction_SwGw_required_irrigation_M3
- abstraction_SW_required_irrigation_M3 = np.maximum(abstraction_SwGw_required_irrigation_M3 - abstraction_GW_required_irrigation_M3, 0)
- consumption_GW_required_irrigation_MM = self.GWfed_fraction_irrigation * consumption_required_irrigation_MM
- consumption_SW_required_irrigation_MM = np.maximum(consumption_SwGw_required_irrigation_MM - consumption_GW_required_irrigation_MM, 0)
# 6.4 Regulation of groundwater abstraction for irrigation (if crop module and option are active)
if option["cropsEPIC"] and option['regulate_GW_irrigation_abstraction']:
@@ -420,13 +704,13 @@ def dynamic(self):
# Dec 2024: the current code does not allow horizontal flow in the groundwater domain, meaning that there is no replenishing from neighboring pixels. Furthermore, LZ can be inefinitely negative. These feature can lead to spurious values of return flow and to excessive depletion of some groundwater pixels. In order to diminish these issues, as a temporary solution, the line below impose the abstraction of the consumptive use only.
abstraction_GW_noReturn_M3 = consumption_GW_noReturn_M3
self.var.abstraction_GW_actual_M3 = abstraction_GW_noReturn_M3 + abstraction_GW_actual_irrigation_M3
- # 8.2 Groundwater lower zone mass balance update
- self.var.LZ -= self.var.abstraction_GW_actual_M3 * self.var.M3toMM
- self.var.IrriLossCUM += self.var.abstraction_GW_actual_M3 # irrigation local losses accounting (CHECK) ### STEF!!! self.var.TotalAbstractionFromGroundwaterM3
- # Abstraction is taken from lower groundwater zone. For mass balance calculation also summed up in IrrilossCUM (in M3)
- # 8.3 Return flow to channel (per routing time step) from groundwater users not storing water (all except irrigation)
- self.var.returnflow_GwAbs2Channel_M3_routStep = (abstraction_GW_noReturn_M3 - consumption_GW_noReturn_M3) * self.var.InvNoRoutSteps
- self.var.returnflow_GwAbs2Channel_M3_routStep = np.where(self.var.returnflow_GwAbs2Channel_M3_routStep<1e-18,0.0,self.var.returnflow_GwAbs2Channel_M3_routStep) # this check avoids spurious numerical results.
+ # 8.2-8.3 Groundwater lower zone mass balance update and return flow to channel
+ # (numba-parallel, updates LZ and IrriLossCUM in place; identical arithmetic and
+ # the same < 1e-18 clamp, which preserves NaN values exactly like the np.where)
+ self.var.returnflow_GwAbs2Channel_M3_routStep = computeGroundwaterAbstractionUpdate(
+ self.var.LZ, self.var.abstraction_GW_actual_M3, self.var.M3toMM,
+ self.var.IrriLossCUM, abstraction_GW_noReturn_M3, consumption_GW_noReturn_M3,
+ self.var.InvNoRoutSteps)
# ***********************************************************************
# ***** ABSTRACTION SUPPLIED BY NONCONVENTIONAL SOURCES (DESALINATION) **
@@ -440,15 +724,13 @@ def dynamic(self):
if option['simulateReservoirs'] and not(option['InitLisflood']):
PotentialAbstractionFromReservoirsM3 = np.minimum(0.02 * self.var.ReservoirStorageM3,
0.01 * self.var.TotalReservoirStorageM3C) * self.var.DtDay # self.var.DtDay is required when DtSec is not 86400
- PotentialAbstractionFromReservoirsM3 = np.where(np.isnan(PotentialAbstractionFromReservoirsM3), 0,
- PotentialAbstractionFromReservoirsM3)
+ PotentialAbstractionFromReservoirsM3 = replaceNaNWithZero(PotentialAbstractionFromReservoirsM3)
else:
PotentialAbstractionFromReservoirsM3 = maskinfo.in_zero()
if option['simulateLakes'] and not(option['InitLisflood']):
PotentialAbstractionFromLakesM3 = 0.10 * self.var.LakeStorageM3 * self.var.DtDay # self.var.DtDay is required when DtSec is not 86400
- PotentialAbstractionFromLakesM3 = np.where(np.isnan(PotentialAbstractionFromLakesM3), 0,
- PotentialAbstractionFromLakesM3)
+ PotentialAbstractionFromLakesM3 = replaceNaNWithZero(PotentialAbstractionFromLakesM3)
else:
PotentialAbstractionFromLakesM3 = maskinfo.in_zero()
@@ -509,13 +791,17 @@ def dynamic(self):
self.var.AreaTotalAvailableWaterFromChannelsM3 = np.maximum(
np.take(np.bincount(self.var.WUseRegionC, weights=PixelAvailableWaterFromChannelsM3),
self.var.WUseRegionC),maskinfo.in_zero())
- # 10.3 Actual channel withdrawal
- self.var.areatotal_withdrawal_CH_actual_M3 = np.minimum(self.var.AreaTotalAvailableWaterFromChannelsM3,
- areatotal_withdrawal_CH_required_M3)
- self.var.FractionAbstractedFromChannels = np.where(self.var.AreaTotalAvailableWaterFromChannelsM3 > 0, np.minimum(
- self.var.areatotal_withdrawal_CH_actual_M3 / self.var.AreaTotalAvailableWaterFromChannelsM3, 1), 0)
- self.var.withdrawal_CH_actual_M3 = self.var.FractionAbstractedFromChannels * PixelAvailableWaterFromChannelsM3 # daily channel abstraction
- self.var.withdrawal_CH_actual_M3_routStep = self.var.withdrawal_CH_actual_M3 * self.var.InvNoRoutSteps # channel abstraction per routing time step
+ # 10.2-10.4 Actual channel withdrawal against the water-region available water
+ # (numba-parallel; the regional np.bincount reduction stays in the driver so
+ # every regional summation keeps its original order)
+ (self.var.AreaTotalAvailableWaterFromChannelsM3,
+ self.var.areatotal_withdrawal_CH_actual_M3,
+ self.var.FractionAbstractedFromChannels,
+ self.var.withdrawal_CH_actual_M3,
+ self.var.withdrawal_CH_actual_M3_routStep) = computeChannelAbstraction(
+ self.var.AreaTotalAvailableWaterFromChannelsM3,
+ areatotal_withdrawal_CH_required_M3, self.var.InvNoRoutSteps,
+ PixelAvailableWaterFromChannelsM3)
self.var.withdrawal_CH_actual_Region_M3 = np.take(np.bincount(self.var.WUseRegionC, weights=self.var.withdrawal_CH_actual_M3),self.var.WUseRegionC) ## QUESTION! ## ONLY FOR REPORTING?
self.var.wateruseCum += self.var.withdrawal_CH_actual_M3 # summing up for water balance calculation
@@ -539,16 +825,16 @@ def dynamic(self):
areatotal_abstraction_CH_required_irrigation_M3 = np.take(np.bincount(self.var.WUseRegionC, weights= abstraction_CH_required_irrigation_M3), self.var.WUseRegionC)
irrabs_minus_shortage_ATCHM3 = areatotal_abstraction_CH_required_irrigation_M3 - self.var.areatotal_shortage_SW_M3
areatotal_abstraction_CH_actual_irrigation_M3 = np.maximum(irrabs_minus_shortage_ATCHM3, 0.)
- fraction_met_CH_irrigation = np.minimum(np.where(areatotal_abstraction_CH_required_irrigation_M3 > 0,
- areatotal_abstraction_CH_actual_irrigation_M3 / areatotal_abstraction_CH_required_irrigation_M3, 0.), 1.)
+ fraction_met_CH_irrigation = computeMetFraction(areatotal_abstraction_CH_actual_irrigation_M3,
+ areatotal_abstraction_CH_required_irrigation_M3)
abstraction_CH_actual_irrigation_M3 = abstraction_CH_required_irrigation_M3 * fraction_met_CH_irrigation
# 12.2 Actual channel abstractions for the sectors without return flows (ene, dom, liv, ind), accounting for shortage where needed
withdrawal_CH_required_noReturn_M3 = consumption_SW_required_noReturn_M3 * (1 - FractionAbstractedByLakesReservoirs)
areatotal_withdrawal_CH_required_noReturn_M3 = np.take(np.bincount(self.var.WUseRegionC, weights=withdrawal_CH_required_noReturn_M3), self.var.WUseRegionC)
areatotal_shortage_CH_beyondIrrigation_M3 = np.maximum(-irrabs_minus_shortage_ATCHM3, 0.)
areatotal_withdrawal_CH_actual_noReturn_M3 = np.maximum(areatotal_withdrawal_CH_required_noReturn_M3 - areatotal_shortage_CH_beyondIrrigation_M3, 0.)
- fraction_met_CH_noReturn = np.minimum(np.where(areatotal_withdrawal_CH_required_noReturn_M3 > 0,
- areatotal_withdrawal_CH_actual_noReturn_M3 / areatotal_withdrawal_CH_required_noReturn_M3, 0.), 1.)
+ fraction_met_CH_noReturn = computeMetFraction(areatotal_withdrawal_CH_actual_noReturn_M3,
+ areatotal_withdrawal_CH_required_noReturn_M3)
## TEST
#aux = (self.var.abstraction_GW_actual_M3.mean(), self.var.LakeAbstractionM3.mean(), self.var.ReservoirAbstractionM3.mean(),
# self.var.withdrawal_CH_actual_M3.mean())
@@ -556,7 +842,16 @@ def dynamic(self):
# 12.4 Bookkeeping for over-all water balance, and repwateruseGauges and repwateruseSites
self.var.cumulated_CH_withdrawal += self.var.withdrawal_CH_actual_M3 # bookkeeping for over-all water balance
if (option['repwateruseGauges']) or (option['repwateruseSites']):
- self.var.WUseSumM3 = compressArray(accuflux(self.var.Ldd, decompress(self.var.withdrawal_CH_actual_M3)*self.var.InvDtSec))
+ if option['useEarthkitHydro'] and earthkit_available():
+ # the shared bridge cache already keys the network on the
+ # Ldd, so no per-module cache is needed
+ from ..global_modules.earthkit_network import get_earthkit_network
+ net = get_earthkit_network(pcr2numpy(self.var.Ldd, 0))
+ self.var.WUseSumM3 = ek_upstream_sum(
+ net,
+ np.asarray(self.var.withdrawal_CH_actual_M3, dtype=np.float64) * self.var.InvDtSec)
+ else:
+ self.var.WUseSumM3 = compressArray(accuflux(self.var.Ldd, decompress(self.var.withdrawal_CH_actual_M3)*self.var.InvDtSec))
# ********************************************************************************************
# 13. Actual surface water abstractions (except prescribed paddy rice)
@@ -564,22 +859,25 @@ def dynamic(self):
# 13.1 Irrigation
abstraction_SW_actual_irrigation_M3 = abstraction_SW_required_irrigation_M3 * FractionAbstractedByLakesReservoirs + abstraction_CH_actual_irrigation_M3
self.var.areatotal_abstraction_SW_actual_irrigation_M3 = np.take(np.bincount(self.var.WUseRegionC, weights=abstraction_SW_actual_irrigation_M3), self.var.WUseRegionC)
- fraction_met_SW_irrigation = np.minimum(FractionAbstractedByLakesReservoirs + fraction_met_CH_irrigation * (1 - FractionAbstractedByLakesReservoirs), 1.)
- # 13.2 Other uses for which return flows are not simulated
- fraction_met_SW_noReturn = np.minimum(FractionAbstractedByLakesReservoirs + fraction_met_CH_noReturn * (1 - FractionAbstractedByLakesReservoirs), 1.)
- abstraction_SW_actual_energy_M3 = abstraction_SW_required_energy_M3 * fraction_met_SW_noReturn
- abstraction_SW_actual_domestic_M3 = abstraction_SW_required_domestic_M3 * fraction_met_SW_noReturn
- abstraction_SW_actual_livestock_M3 = abstraction_SW_required_livestock_M3 * fraction_met_SW_noReturn
- abstraction_SW_actual_industry_M3 = abstraction_SW_required_industry_M3 * fraction_met_SW_noReturn
+ fraction_met_SW_irrigation = computeMetFractionSW(FractionAbstractedByLakesReservoirs, fraction_met_CH_irrigation)
+ # 13.2 Other uses for which return flows are not simulated (numba-parallel fan-out)
+ fraction_met_SW_noReturn = computeMetFractionSW(FractionAbstractedByLakesReservoirs, fraction_met_CH_noReturn)
+ (abstraction_SW_actual_energy_M3, abstraction_SW_actual_domestic_M3,
+ abstraction_SW_actual_livestock_M3, abstraction_SW_actual_industry_M3,
+ consumption_actual_energy_MM, consumption_actual_domestic_MM,
+ consumption_actual_livestock_MM, consumption_actual_industry_MM) = computeNoReturnSectorMet(
+ fraction_met_SW_noReturn,
+ abstraction_SW_required_energy_M3, abstraction_SW_required_domestic_M3,
+ abstraction_SW_required_livestock_M3, abstraction_SW_required_industry_M3,
+ consumption_required_energy_MM, consumption_GW_domestic_MM, consumption_SW_required_domestic_MM,
+ consumption_GW_livestock_MM, consumption_SW_required_livestock_MM,
+ consumption_GW_industry_MM, consumption_SW_required_industry_MM)
# ********************************************************************************************
# 14. Actual consumptions from surface water and groundwater bodies (for WeiC, WeiP)
# ********************************************************************************************
self.var.consumption_actual_irrigation_MM = consumption_GW_actual_irrigation_MM + consumption_SW_required_irrigation_MM * fraction_met_SW_irrigation
- consumption_actual_energy_MM = consumption_required_energy_MM * fraction_met_SW_noReturn
- consumption_actual_domestic_MM = consumption_GW_domestic_MM + consumption_SW_required_domestic_MM * fraction_met_SW_noReturn
- consumption_actual_livestock_MM = consumption_GW_livestock_MM + consumption_SW_required_livestock_MM * fraction_met_SW_noReturn
- consumption_actual_industry_MM = consumption_GW_industry_MM + consumption_SW_required_industry_MM * fraction_met_SW_noReturn
+
self.var.consumption_SwGw_actual_M3 = (self.var.consumption_actual_irrigation_MM + consumption_actual_energy_MM + consumption_actual_domestic_MM +\
consumption_actual_livestock_MM + consumption_actual_industry_MM) * self.var.MMtoM3 +\
self.var.PaddyRiceWaterAbstractionFromSurfaceWaterM3
diff --git a/src/lisflood/hydrological_modules/waterbalance.py b/src/lisflood/hydrological_modules/waterbalance.py
index 551b5001..c50a0216 100755
--- a/src/lisflood/hydrological_modules/waterbalance.py
+++ b/src/lisflood/hydrological_modules/waterbalance.py
@@ -102,7 +102,7 @@ def initial(self):
# CM
if option['simulateLakes']:
# DisStructure += np.where(compressArray(self.var.IsUpsOfStructureLake), 0.5 * self.var.ChanQ * self.var.DtRouting, 0)
- DisStructure += np.where(compressArray(self.var.IsUpsOfStructureLake), 0.5 * self.var.ChanQAvgDt * self.var.DtRouting, 0)
+ DisStructure += np.where(self.var.IsUpsOfStructureLakeC, 0.5 * self.var.ChanQAvgDt * self.var.DtRouting, 0)
# DisStructure += cover(ifthen(self.var.IsUpsOfStructureLake,
# 0.5 * self.var.ChanQ * self.var.DtRouting), scalar(0.0))
@@ -141,11 +141,7 @@ def dynamic(self):
option = settings.options
maskinfo = MaskInfo.instance()
- if (not(option['InitLisflood'])) and option['repTotalWaterStorageMaps']:
- ChannelStoredM3 = self.storage_channel(option)
- HillslopeStoredM3 = self.storage_hillslope()
- self.var.TotalWaterStorageMM = (ChannelStoredM3 + HillslopeStoredM3) * self.var.M3toMM
-
+
if (not(option['InitLisflood'])) and option['repMBTs']:
# ************************************************************
diff --git a/src/lisflood/hydrological_modules/waterstorage.py b/src/lisflood/hydrological_modules/waterstorage.py
new file mode 100644
index 00000000..32e41053
--- /dev/null
+++ b/src/lisflood/hydrological_modules/waterstorage.py
@@ -0,0 +1,207 @@
+"""
+
+Copyright 2019 European Union
+
+Licensed under the EUPL, Version 1.2 or as soon they will be approved by the European Commission subsequent versions of the EUPL (the "Licence");
+
+You may not use this work except in compliance with the Licence.
+You may obtain a copy of the Licence at:
+
+https://joinup.ec.europa.eu/sites/default/files/inline-files/EUPL%20v1_2%20EN(1).txt
+
+Unless required by applicable law or agreed to in writing, software distributed under the Licence is distributed on an "AS IS" basis,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the Licence for the specific language governing permissions and limitations under the Licence.
+
+"""
+from __future__ import absolute_import, print_function
+
+import warnings
+# warnings.formatwarning = lambda msg, args, *kwargs: f'{msg}\n'
+
+import numpy as np
+from ..global_modules.add1 import loadmap
+from ..global_modules.settings import LisSettings, MaskInfo
+from ..global_modules.errors import LisfloodWarning
+from . import HydroModule
+
+
+class waterstorage(HydroModule):
+ """
+ # ************************************************************
+ # ***** WATER STORAGE *************************************
+ # ************************************************************
+ # Sum up water storage in individual compartements
+ """
+ input_files_keys = {'all': []}
+ module_name = 'WaterStorage'
+
+ def __init__(self, waterstorage_variable):
+ self.var = waterstorage_variable
+
+# --------------------------------------------------------------------------
+# --------------------------------------------------------------------------
+
+ def initial(self):
+ """ initial part of the water storage module
+ """
+
+ # ************************************************************
+ # ***** WATER STORAGE INIT
+ # ************************************************************
+ settings = LisSettings.instance()
+ option = settings.options
+
+ # load water map and separate into maps of lake and reservoir distribution
+ if (not(option['InitLisflood'])) and (option['repStorageMaps'] or option['repTWSMaps']):
+
+ # Precompute lake extent masks and areas (static, don't change during simulation)
+ self.var.lake_extent_masks = {} # dict: lake_id -> pixel indices
+ self.var.lake_extent_areas = {} # dict: lake_id -> total grid area [m2]
+ if option['simulateLakes']:
+ self.var.LakeDistribution = loadmap('LakeExtent')
+ lake_ids = np.unique(self.var.LakeDistribution[~np.isnan(self.var.LakeDistribution)]).astype(int)
+ number_of_lakeIDs = len(lake_ids[lake_ids != 0])
+ if (number_of_lakeIDs == 0):
+ warnings.warn(LisfloodWarning('LakeExtent map contains no lake IDs. Please check consistency between LakeExtent map and model domain.'))
+ if self.var.LakeSitesCC.size != number_of_lakeIDs:
+ warnings.warn(LisfloodWarning('Number of lake IDs in map LakeExtent ('+str(number_of_lakeIDs)+') not equal to number of lake sites defined in map LakeSites ('+str(self.var.LakeSitesCC.size)+').'))
+ lake_extent = self.var.LakeDistribution
+ valid_lake_ids = set(self.var.LakeSitesCC.astype(int)) # IDs that have outlets in domain
+ for n in np.unique(lake_extent[~np.isnan(lake_extent)]).astype(int):
+ if n != 0 and n in valid_lake_ids:
+ mask = np.nonzero(lake_extent == n)
+ self.var.lake_extent_masks[n] = mask
+ self.var.lake_extent_areas[n] = np.nansum(self.var.PixelArea[mask])
+ elif n != 0:
+ warnings.warn(LisfloodWarning(
+ f'Lake ID {n} found in LakeExtent map but has no outlet in the domain. Skipping.'))
+
+ # Precompute reservoir extent masks and areas (static)
+ self.var.reservoir_extent_masks = {}
+ self.var.reservoir_extent_areas = {}
+ if option['simulateReservoirs']:
+ self.var.ReservoirDistribution = loadmap('ReservoirExtent')
+ res_ids = np.unique(self.var.ReservoirDistribution[~np.isnan(self.var.ReservoirDistribution)]).astype(int)
+ number_of_reservoirIDs = len(res_ids[res_ids != 0])
+ if (number_of_reservoirIDs == 0):
+ warnings.warn(LisfloodWarning('ReservoirExtent map contains no reservoir IDs. Please check consistency between ReservoirExtent map and model domain.'))
+ if self.var.ReservoirSitesCC.size != number_of_reservoirIDs:
+ warnings.warn(LisfloodWarning('Number of reservoir IDs in map ReservoirExtent ('+str(number_of_reservoirIDs)+') not equal to number of reservoir sites defined in map ReservoirSites ('+str(self.var.ReservoirSitesCC.size)+').'))
+ reservoir_extent = self.var.ReservoirDistribution
+ valid_res_ids = set(self.var.ReservoirSitesCC.astype(int))
+ for n in np.unique(reservoir_extent[~np.isnan(reservoir_extent)]).astype(int):
+ if n != 0 and n in valid_res_ids:
+ mask = np.nonzero(reservoir_extent == n)
+ self.var.reservoir_extent_masks[n] = mask
+ self.var.reservoir_extent_areas[n] = np.nansum(self.var.PixelArea[mask])
+ elif n != 0:
+ warnings.warn(LisfloodWarning(
+ f'Reservoir ID {n} found in ReservoirExtent map but has no outlet in the domain. Skipping.'))
+
+# --------------------------------------------------------------------------
+# --------------------------------------------------------------------------
+
+ def dynamic(self):
+ """ dynamic part of the water storage module
+ """
+ settings = LisSettings.instance()
+ option = settings.options
+ maskinfo = MaskInfo.instance()
+
+
+ if (not(option['InitLisflood'])) and (option['repStorageMaps'] or option['repTWSMaps']):
+
+ # ************************************************************
+ # ***** WATER STORAGE
+ # ************************************************************
+
+ # river water storage [m3]
+ tws_riverM3 = self.var.TotalCrossSectionArea * self.var.ChanLength
+
+ # [m3] -> [m]
+ tws_riverM = tws_riverM3 / self.var.PixelArea
+
+ # overlandflow water storage [m3]
+ tws_oflowM3 = self.var.OFM3Direct + self.var.OFM3Forest + self.var.OFM3Other
+
+ # [m3] -> [m]
+ tws_oflowM = tws_oflowM3 / self.var.PixelArea
+
+ # lake water storage [m3]
+ tws_lakeM3 = np.zeros(tws_riverM3.shape, dtype=np.float32)
+ # lake water storage [m]
+ tws_lakeM = np.zeros(tws_riverM.shape, dtype=np.float32)
+
+ if option['simulateLakes']:
+ LakeArea = maskinfo.in_zero()
+ np.put(LakeArea, self.var.LakeIndex, self.var.LakeAreaCC)
+ tws_lakeM3 = self.var.LakeLevel * LakeArea
+
+ # [m3] -> [m] distribute lake/river/oflow over lake areas
+ for n, lake_mask in self.var.lake_extent_masks.items():
+ grid_area_lake = self.var.lake_extent_areas[n]
+ tws_lakeM[lake_mask] = tws_lakeM3[self.var.LakeSitesC2==n] / grid_area_lake
+ tws_riverM[lake_mask] = np.nansum(tws_riverM3[lake_mask]) / grid_area_lake
+ tws_oflowM[lake_mask] = np.nansum(tws_oflowM3[lake_mask]) / grid_area_lake
+
+ # reservoir water storage [m3]
+ tws_reservoirM3 = np.zeros(tws_riverM3.shape, dtype=np.float32)
+ # reservoir water storage [m]
+ tws_reservoirM = np.zeros(tws_riverM.shape, dtype=np.float32)
+
+ if option['simulateReservoirs']:
+ TotalReservoirStorage = maskinfo.in_zero()
+ np.put(TotalReservoirStorage, self.var.ReservoirIndex, self.var.TotalReservoirStorageM3CC)
+ tws_reservoirM3 = self.var.ReservoirFill * TotalReservoirStorage
+
+ # [m3] -> [m] distribute reservoir/river/oflow over reservoir areas
+ for n, reservoir_mask in self.var.reservoir_extent_masks.items():
+ grid_area_reservoir = self.var.reservoir_extent_areas[n]
+ tws_reservoirM[reservoir_mask] = tws_reservoirM3[self.var.ReservoirSitesC==n] / grid_area_reservoir
+ tws_riverM[reservoir_mask] = np.nansum(tws_riverM3[reservoir_mask]) / grid_area_reservoir
+ tws_oflowM[reservoir_mask] = np.nansum(tws_oflowM3[reservoir_mask]) / grid_area_reservoir
+
+ # soil water storage [mm] -> [m]
+ tws_soil1M = ((self.var.Theta1a[0] * self.var.SoilDepth1a[0]) * self.var.OtherFraction +
+ (self.var.Theta1a[1] * self.var.SoilDepth1a[1]) * self.var.ForestFraction +
+ (self.var.Theta1a[2] * self.var.SoilDepth1a[2]) * self.var.IrrigationFraction
+ ) / 1000
+
+ tws_soil2M = ((self.var.Theta1b[0] * self.var.SoilDepth1b[0]) * self.var.OtherFraction +
+ (self.var.Theta1b[1] * self.var.SoilDepth1b[1]) * self.var.ForestFraction +
+ (self.var.Theta1b[2] * self.var.SoilDepth1b[2]) * self.var.IrrigationFraction
+ ) / 1000
+
+ tws_soil3M = ((self.var.Theta2[0] * self.var.SoilDepth2[0]) * self.var.OtherFraction +
+ (self.var.Theta2[1] * self.var.SoilDepth2[1]) * self.var.ForestFraction +
+ (self.var.Theta2[2] * self.var.SoilDepth2[2]) * self.var.IrrigationFraction
+ ) / 1000
+
+ tws_soilM = tws_soil1M + tws_soil2M + tws_soil3M
+
+ # groundwater storage [mm] -> [m], (uz,uzf,uzi,lz,fracforest,fracirrigated,fracother)
+ tws_groundwaterM = ( self.var.UZ[0] * self.var.OtherFraction +
+ self.var.UZ[1] * self.var.ForestFraction +
+ self.var.UZ[2] * self.var.IrrigationFraction +
+ self.var.LZ
+ ) / 1000
+
+ # snow storage [m]
+ tws_snowM = self.var.SnowCover / 1000
+
+ # cumulative interception storage [mm] and cumulative depression storage [mm] -> [m]
+ tws_cumM = ( self.var.CumInterception[0] * self.var.OtherFraction +
+ self.var.CumInterception[1] * self.var.ForestFraction +
+ self.var.CumInterception[2] * self.var.IrrigationFraction +
+ self.var.CumInterSealed * self.var.DirectRunoffFraction
+ ) / 1000
+
+ # total water storage maps
+ self.var.riverstor = tws_riverM + tws_oflowM
+ self.var.lakestor = tws_lakeM + tws_reservoirM
+ self.var.soilstor = tws_soilM
+ self.var.gwstor = tws_groundwaterM
+ self.var.snowstor = tws_snowM
+ self.var.cumstor = tws_cumM
+ self.var.twsstor = tws_riverM + tws_oflowM + tws_lakeM + tws_reservoirM + tws_soilM + tws_groundwaterM + tws_snowM + tws_cumM
\ No newline at end of file
diff --git a/src/lisfloodSettings_reference.xml b/src/lisfloodSettings_reference.xml
index 2c7c7832..a967e3fb 100644
--- a/src/lisfloodSettings_reference.xml
+++ b/src/lisfloodSettings_reference.xml
@@ -68,6 +68,15 @@ You can use builtin path variables in this template and reference to other paths
+ # option to write map output asynchronously on a thread pool, overlapping
+ # NetCDF disk I/O with the model computation of the next time step
+ # (helps map-heavy runs; default off = historical synchronous writing)
+
+
+ # option to prefetch the next NetCDF forcing time chunk in a background
+ # thread while the current one is consumed (needs NetCDFTimeChunks > 1)
+
+
# option to Report PaddyRiceDebug information in CSV file for debugging Paddy Rice Water Abstraction issue on dry Channels
@@ -165,7 +174,10 @@ You can use builtin path variables in this template and reference to other paths
-
+
+
+
+
@@ -211,13 +223,44 @@ You can use builtin path variables in this template and reference to other paths
+
+
+ The option "OutputPacking" enables CF-convention scale_factor/add_offset packing
+ of output maps into int16 (2 bytes per value instead of 4 or 8).
+ - "False" (default): write raw floating-point values (dtype from OutputMapsDataType)
+ - "True": pack into signed 16-bit integers using per-variable scale/offset
+ Readers automatically unpack using: value = packed * scale_factor + add_offset
+ Note: State/end maps used for warm starts are NEVER packed (always full precision).
+
+
+
+
+
+ The option "OutputPackingDischarge" enables LOGARITHMIC int16 packing of the
+ DischargeMaps output only. It is COMPLETELY INDEPENDENT of "OutputPacking":
+ discharge is excluded from the linear scale/offset packing because its global
+ value range spans several orders of magnitude, which a linear int16 encoding
+ cannot represent without unacceptable precision loss.
+ - "False" (default): discharge written as raw floating-point values
+ - "True": pack discharge into signed 16-bit integers in log10 space,
+ giving near-constant RELATIVE precision across the full range.
+ Encoding (custom, NON-CF attributes written on the variable):
+ stored = round((log10(q) - packing_log_min) / packing_log_scale) + packing_int_offset
+ q = 10 ** (packing_log_min + packing_log_scale * (stored - packing_int_offset))
+ Resolved range: 0.01 to 500000 m3/s. Discharge below 0.01 m3/s (and zero,
+ negative or missing values) is stored as _FillValue; values above 500000 are
+ clipped. Because this encoding is non-CF, standard readers will NOT auto-unpack
+ it: use the packing_* attributes to invert the transform.
+
+
+
**************************************************************
PARALLELISATION WITH NUMBA (USED IN ROUTING AND SOILLOOP)
**************************************************************
-
+
!-- Parallelisation of using Numba runtime compiled library .
The option "numCPUs_parallelNumba" may take the following values:
@@ -225,7 +268,82 @@ You can use builtin path variables in this template and reference to other paths
(if NUMBA_NUM_THREADS is not set, will take the number of CPU cores determined by python's multiprocessing.cpu_count())
- "1" : serial execution (not parallel)
- "2", "3", ... : manual setting of the number of parallel threads.
- (if exceeding NUMBA_NUM_THREADS, the value is set to NUMBA_NUM_THREADS) -->
+ (if exceeding NUMBA_NUM_THREADS, the value is set to NUMBA_NUM_THREADS)
+ Value legend (shared by every numCPUs_* setting): absent or "auto" => the
+ automatic policy (the default); "all" or "0" => all available cores;
+ a positive integer => that fixed count. This key governs the LAND-PHASE
+ Numba kernels (soil/vegetation loop, water abstraction, surface routing);
+ its "auto" default is domain-size-aware (serial on small catchments where
+ thread fork/join dominates, all cores on large/global domains). -->
+
+
+
+
+
+ !-- Number of Numba threads used specifically for the channel-routing
+ SUBSTEP kernels (kinematic-wave / Muskingum-Cunge), which run many times
+ per model step. Each substep kernel has a small per-call payload, so
+ beyond a modest thread count the OpenMP fork/join cost dominates and more
+ threads make routing slower. This is capped INDEPENDENTLY of the
+ land-phase pool (numCPUs_parallelNumba), which keeps using all cores.
+ Values:
+ - "auto" (default) : host-relative cap -- about half the cores,
+ clamped to the 8..16 band, and never above
+ the land-phase count
+ - "all" / "0" : all available cores (no routing-specific cap)
+ - "1", "2", ... : manual thread count for the routing kernels
+ Scheduling only: the routing result is identical for any thread count. -->
+
+
+
+
+
+ !-- Number of threads used by the BLAS/OpenMP backend of numpy/scipy.
+ In LISFLOOD, BLAS is used only by the cold-start soil-hydraulics
+ least-squares solve (a scalar per-pixel fit with no matrix algebra) and
+ is idle during the dynamic loop, so extra BLAS threads do not speed the
+ model up; they only oversubscribe against the soil-init process pool.
+ Values (shared legend): absent or "auto" => the automatic policy default,
+ which is a single thread (1); "all" or "0" => all available cores;
+ a positive integer => that fixed count.
+ Leave at "auto" (=1); set a higher value only if a specific BLAS-bound
+ step is known to benefit. -->
+
+
+
+
+
+ !-- Number of worker PROCESSES used for the cold-start soil-layer-2
+ saturation solve (a per-pixel scipy least-squares that can dominate
+ initialization time on large/global domains). This is process-based
+ parallelism, independent of the numba/BLAS thread pools; results
+ are identical regardless of the value. Values:
+ - "0" / "all" / "auto": use all available cores (default)
+ - "1" : serial
+ - "2", "3", ... : manual process count
+ Set this to "1" when an outer tool already parallelizes LISFLOOD runs
+ (e.g. lisflood-calibration runs one LISFLOOD per DEAP individual): each
+ instance must stay single-process to avoid oversubscription. As a
+ safeguard, this parallelism auto-disables when LISFLOOD detects it is
+ running inside a forked/daemon worker. -->
+
+
+
+
+
+ !-- Floating-point precision of the LISFLOOD model state and the
+ kinematic-wave routing kernel. PERFORMANCE/precision lever (separate from the thread settings).
+ Values:
+ - "float64" (default) : historical path; bit-identical to previous
+ LISFLOOD versions. Use for calibration and any
+ bitwise-reproducibility need.
+ - "float32" : faster routing kernel (halves its memory
+ traffic). NOT bit-identical but stays within the
+ LISFLOOD reference tolerance (atol=1e-4,
+ rtol=1e-3) on tested catchments. The
+ Muskingum-Cunge-Todini (MCT) routing and all
+ mass-balance reductions remain float64.
+ An unknown value falls back to "float64" (never silently degrades). -->
@@ -344,6 +462,47 @@ You can use builtin path variables in this template and reference to other paths
+
+
+ Semicolon-separated list of variable names to output as daily mean
+ (useful with sub-daily time steps; one value per calendar day).
+ Variables listed here are excluded from normal ReportSteps output.
+ Example: "DischargeMaps;Theta1Maps;UZMaps"
+ Leave empty to disable. Requires netCDF output.
+
+
+
+
+ See OutputDailyMean
+
+
+
+
+ Semicolon-separated list of variable names to output as monthly mean.
+ Variables listed here are excluded from normal ReportSteps output.
+ Example: "DischargeMaps;Theta1Maps;UZMaps"
+ Leave empty to disable.
+
+
+
+
+
+ See OutputMonthlyMean
+
+
+
+
+
+ See OutputMonthlyMean
+
+
+
+
+
+ See OutputMonthlyMean
+
+
+
@@ -1662,7 +1821,27 @@ You can use builtin path variables in this template and reference to other paths
+
+
+ **************************************************************
+ TWS OPTION
+ **************************************************************
+
+
+
+
+ Map containing the lake ID for each pixel of its extent
+
+
+
+
+
+ Map containing the reservoir ID for each pixel of its extent
+
+
+
+
@@ -1830,6 +2009,10 @@ You can use builtin path variables in this template and reference to other paths
-->
+
+
+
+
@@ -1839,6 +2022,8 @@ You can use builtin path variables in this template and reference to other paths
+
+
@@ -1914,7 +2099,48 @@ You can use builtin path variables in this template and reference to other paths
Number of days used for internal spin-up (fluxes computations during prerun)
-
+
+
+
+ Semicolon-separated list of variable names to output as daily mean
+ (useful with sub-daily time steps; one value per calendar day).
+ Variables listed here are excluded from normal ReportSteps output.
+ Example: "DischargeMaps;Theta1Maps;UZMaps"
+ Leave empty to disable. Requires netCDF output.
+
+
+
+
+ See OutputDailyMean
+
+
+
+
+ Semicolon-separated list of variable names to output as monthly mean.
+ Variables listed here are excluded from normal ReportSteps output.
+ Example: "DischargeMaps;Theta1Maps;UZMaps"
+ Leave empty to disable.
+
+
+
+
+
+ See OutputMonthlyMean
+
+
+
+
+
+ See OutputMonthlyMean
+
+
+
+
+
+ See OutputMonthlyMean
+
+
+
@@ -3277,6 +3503,59 @@ You can use builtin path variables in this template and reference to other paths
+
+
+ Reported transpiration maps, weighted sum over the fractions of each pixel [mm]
+
+
+
+
+
+ Reported total water storage storage [m]
+
+
+
+
+
+ Reported lake and reservoir storage [m]
+
+
+
+
+
+ Reported river storage [m]
+
+
+
+
+
+ Reported soil storage [m]
+
+
+
+
+
+ Reported groundwater storage [m]
+
+
+
+
+
+ Reported snow storage [m]
+
+
+
+
+
+ Reported cumulative interception and depression storage [m]
+
+
+
+
+
+ Reported riverflow momentum for HAM motion term [kgm/s]
+
+
@@ -3673,12 +3952,6 @@ You can use builtin path variables in this template and reference to other paths
-
-
- Reported total water storage [mm]
-
-
-
Reported days since last rain
@@ -4083,7 +4356,19 @@ You can use builtin path variables in this template and reference to other paths
- Reported percolation from upper to lower zone [mm]
+ Reported percolation from upper to lower zone, forest fraction [mm]
+
+
+
+
+
+ Reported percolation from upper to lower zone, other fraction [mm]
+
+
+
+
+
+ Reported percolation from upper to lower zone, irrigation fraction [mm]
@@ -4531,6 +4816,11 @@ You can use builtin path variables in this template and reference to other paths
+
+
+ Reported total water storage upstream average at gauges [m]
+
+
@@ -5160,8 +5450,8 @@ You can use builtin path variables in this template and reference to other paths
**************************************************************
TABLES WITH TOPSOIL SOIL PHYSICAL PARAMETERS (HYPRES)
- Each parameter is defined for upper (1a and 1b) and
- lower (2) soil layers
+ Each parameter is defined for upper (1 and 2) and
+ lower (3) soil layers
**************************************************************
@@ -5265,7 +5555,7 @@ You can use builtin path variables in this template and reference to other paths
**************************************************************
TABLES WITH TOPSOIL SOIL PHYSICAL PARAMETERS (HYPRES) for Forest
- Each parameter is defined for 1a and 1b
+ Each parameter is defined for 1 and 2
Normal parameter is taken for the lower soil layer
**************************************************************
@@ -5447,6 +5737,14 @@ You can use builtin path variables in this template and reference to other paths
reservoir routine switches to flood control mode.
+
+
+
+ Map containing the reservoir ID for each pixel of its spatial extent.
+ Required when simulateReservoirs is active and repTWSMaps or repStorageMaps or repStateUpsGauges is enabled.
+
+
+
@@ -5493,6 +5791,14 @@ You can use builtin path variables in this template and reference to other paths
+
+
+ Map containing the lake ID for each pixel of its spatial extent.
+ Required when simulateLakes is active and repTWSMaps or repStorageMaps or repStateUpsGauges is enabled.
+
+
+
+
**************************************************************
Input tables
diff --git a/tests/conftest.py b/tests/conftest.py
index a8a3de2f..67c52f05 100755
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -10,6 +10,21 @@ def pytest_addoption(parser):
'If missing, those tests are not executing')
+def pytest_configure(config):
+ # useEarthkitHydro defaults to True, so by default every end-to-end test
+ # exercises the earthkit-hydro reporting path (with automatic PCRaster
+ # fallback when earthkit-hydro is not installed). To cover the PCRaster
+ # path, run the suite a second time with:
+ # LISFLOOD_TEST_EARTHKIT_OFF=1 pytest tests/
+ # which flips the default to False before any settings file is parsed.
+ import os
+ if os.environ.get('LISFLOOD_TEST_EARTHKIT_OFF') == '1':
+ from lisflood.global_modules import default_options as _do
+ assert 'useEarthkitHydro' in _do.default_options
+ _do.default_options['useEarthkitHydro'] = False
+
+
+
@pytest.fixture(scope='class', autouse=True)
def options(request):
options = dict()
diff --git a/tests/data/LF_ETRS89_UseCase/maps/lake_extent.nc b/tests/data/LF_ETRS89_UseCase/maps/lake_extent.nc
new file mode 100644
index 00000000..d35af82f
Binary files /dev/null and b/tests/data/LF_ETRS89_UseCase/maps/lake_extent.nc differ
diff --git a/tests/data/LF_ETRS89_UseCase/maps/res_extent.nc b/tests/data/LF_ETRS89_UseCase/maps/res_extent.nc
new file mode 100644
index 00000000..d633c630
Binary files /dev/null and b/tests/data/LF_ETRS89_UseCase/maps/res_extent.nc differ
diff --git a/tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/Cumstor.nc b/tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/Cumstor.nc
new file mode 100644
index 00000000..32af7218
Binary files /dev/null and b/tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/Cumstor.nc differ
diff --git a/tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/FlowMomentum.nc b/tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/FlowMomentum.nc
new file mode 100644
index 00000000..898b9ace
Binary files /dev/null and b/tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/FlowMomentum.nc differ
diff --git a/tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/tws.nc b/tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/GWstor.nc
similarity index 64%
rename from tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/tws.nc
rename to tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/GWstor.nc
index 56b3724a..074cae87 100644
Binary files a/tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/tws.nc and b/tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/GWstor.nc differ
diff --git a/tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/Lakestor.nc b/tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/Lakestor.nc
new file mode 100644
index 00000000..5ac62235
Binary files /dev/null and b/tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/Lakestor.nc differ
diff --git a/tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/Riverstor.nc b/tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/Riverstor.nc
new file mode 100644
index 00000000..21efc749
Binary files /dev/null and b/tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/Riverstor.nc differ
diff --git a/tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/Snowstor.nc b/tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/Snowstor.nc
new file mode 100644
index 00000000..c921db19
Binary files /dev/null and b/tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/Snowstor.nc differ
diff --git a/tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/Soilstor.nc b/tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/Soilstor.nc
new file mode 100644
index 00000000..3ff1844c
Binary files /dev/null and b/tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/Soilstor.nc differ
diff --git a/tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/TotalWS.nc b/tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/TotalWS.nc
new file mode 100644
index 00000000..e33b93c4
Binary files /dev/null and b/tests/data/LF_ETRS89_UseCase/reference/output_reference_6h/TotalWS.nc differ
diff --git a/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/Cumstor.nc b/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/Cumstor.nc
new file mode 100644
index 00000000..ef99041e
Binary files /dev/null and b/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/Cumstor.nc differ
diff --git a/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/FlowMomentum.nc b/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/FlowMomentum.nc
new file mode 100644
index 00000000..a83463d2
Binary files /dev/null and b/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/FlowMomentum.nc differ
diff --git a/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/GWstor.nc b/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/GWstor.nc
new file mode 100644
index 00000000..7f415dc4
Binary files /dev/null and b/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/GWstor.nc differ
diff --git a/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/Lakestor.nc b/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/Lakestor.nc
new file mode 100644
index 00000000..a09910fe
Binary files /dev/null and b/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/Lakestor.nc differ
diff --git a/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/Riverstor.nc b/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/Riverstor.nc
new file mode 100644
index 00000000..3cf2895b
Binary files /dev/null and b/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/Riverstor.nc differ
diff --git a/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/Snowstor.nc b/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/Snowstor.nc
new file mode 100644
index 00000000..30cf7cb9
Binary files /dev/null and b/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/Snowstor.nc differ
diff --git a/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/Soilstor.nc b/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/Soilstor.nc
new file mode 100644
index 00000000..07f8bdee
Binary files /dev/null and b/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/Soilstor.nc differ
diff --git a/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/TotalWS.nc b/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/TotalWS.nc
new file mode 100644
index 00000000..531f89e7
Binary files /dev/null and b/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/TotalWS.nc differ
diff --git a/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/tws.nc b/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/tws.nc
deleted file mode 100644
index 2a2911f7..00000000
Binary files a/tests/data/LF_ETRS89_UseCase/reference/output_reference_daily/tws.nc and /dev/null differ
diff --git a/tests/data/LF_ETRS89_UseCase/settings/base.xml b/tests/data/LF_ETRS89_UseCase/settings/base.xml
index a2dab4ad..86d2fb71 100644
--- a/tests/data/LF_ETRS89_UseCase/settings/base.xml
+++ b/tests/data/LF_ETRS89_UseCase/settings/base.xml
@@ -41,7 +41,6 @@
-
@@ -65,6 +64,9 @@
+
+
+
@@ -240,8 +242,9 @@ The option "numCPUs_parallelNumba" may take the following values:
-
+
+ was 6000..9999
Time steps at which to write model state maps (i.e. only
those maps that would be needed to define initial conditions
for succeeding model run)
@@ -1521,6 +1524,30 @@ The option "numCPUs_parallelNumba" may take the following values:
+
+
+
+
+
+ **************************************************************
+ TWS OPTION
+ **************************************************************
+
+
+
+
+ Map containing the lake ID for each pixel of its extent
+
+
+
+
+
+ Map containing the reservoir ID for each pixel of its extent
+
+
+
+
+
@@ -3418,12 +3445,63 @@ LFBINDING: MORE LOW-LEVEL CONTROL OVER MODEL IN- AND OUTPUT
-
+
+
+ Reported total water storage storage [m]
+ added by GFZ
+
+
+
+
+
+ Reported lake and reservoir storage [m]
+ added by GFZ
+
+
+
+
+
+ Reported river storage [m]
+ added by GFZ
+
+
+
+
+
+ Reported soil storage [m]
+ added by GFZ
+
+
+
+
+
+ Reported groundwater storage [m]
+ added by GFZ
+
+
+
+
- Reported total water storage [mm]
+ Reported snow storage [m]
+ added by GFZ
+
+
+ Reported cumulative interception and depression storage storage [m]
+ added by GFZ
+
+
+
+
+
+ Reported riverflow momentum for HAM motion term [kgm/s]
+ added by GFZ
+
+
+
+
Reported days since last rain
@@ -5233,6 +5311,30 @@ LFBINDING: MORE LOW-LEVEL CONTROL OVER MODEL IN- AND OUTPUT
+
+
+
+
+
+ **************************************************************
+ TWS OPTION
+ **************************************************************
+
+
+
+
+ Map containing the lake ID for each pixel of its extent
+
+
+
+
+
+ Map containing the reservoir ID for each pixel of its extent
+
+
+
+
+
diff --git a/tests/data/LF_ETRS89_UseCase/settings/cold.xml b/tests/data/LF_ETRS89_UseCase/settings/cold.xml
index 737c1d12..d41d6257 100644
--- a/tests/data/LF_ETRS89_UseCase/settings/cold.xml
+++ b/tests/data/LF_ETRS89_UseCase/settings/cold.xml
@@ -100,7 +100,10 @@
-
+
+
+
+
@@ -1562,6 +1565,29 @@ threshold value below which there is no outflow to the channel
+
+
+
+
+ **************************************************************
+ TWS OPTION
+ **************************************************************
+
+
+
+
+ Map containing the lake ID for each pixel of its extent
+
+
+
+
+
+ Map containing the reservoir ID for each pixel of its extent
+
+
+
+
+
@@ -4234,7 +4260,12 @@ LFBINDING: MORE LOW-LEVEL CONTROL OVER MODEL IN- AND OUTPUT
Days since last rain [days]
-
+
+
+
+ Reported total water storage upstream average at gauges [m]
+
+
@@ -5300,6 +5331,29 @@ LFBINDING: MORE LOW-LEVEL CONTROL OVER MODEL IN- AND OUTPUT
+
+
+
+
+ **************************************************************
+ TWS OPTION
+ **************************************************************
+
+
+
+
+ Map containing the lake ID for each pixel of its extent
+
+
+
+
+
+ Map containing the reservoir ID for each pixel of its extent
+
+
+
+
+
diff --git a/tests/data/LF_ETRS89_UseCase/settings/full.xml b/tests/data/LF_ETRS89_UseCase/settings/full.xml
index f1da1c11..cd743674 100644
--- a/tests/data/LF_ETRS89_UseCase/settings/full.xml
+++ b/tests/data/LF_ETRS89_UseCase/settings/full.xml
@@ -55,7 +55,10 @@
-
+
+
+
+
@@ -2100,8 +2103,31 @@
+
+
+
+
+
+
+ **************************************************************
+ TWS OPTION
+ **************************************************************
+
+
+
+
+ Map containing the lake ID for each pixel of its extent
+
+
+
+
+
+ Map containing the reservoir ID for each pixel of its extent
+
+
+
@@ -6581,6 +6607,12 @@ LFBINDING: MORE LOW-LEVEL CONTROL OVER MODEL IN- AND OUTPUT
+
+
+
+ Reported total water storage upstream average at gauges [m]
+
+
@@ -8389,8 +8421,31 @@ LFBINDING: MORE LOW-LEVEL CONTROL OVER MODEL IN- AND OUTPUT
+
+
+
+
+
+
+ **************************************************************
+ TWS OPTION
+ **************************************************************
+
+
+
+
+ Map containing the lake ID for each pixel of its extent
+
+
+
+
+
+ Map containing the reservoir ID for each pixel of its extent
+
+
+
diff --git a/tests/data/LF_ETRS89_UseCase/settings/inflow.xml b/tests/data/LF_ETRS89_UseCase/settings/inflow.xml
index 0b55b8a8..2f71a149 100644
--- a/tests/data/LF_ETRS89_UseCase/settings/inflow.xml
+++ b/tests/data/LF_ETRS89_UseCase/settings/inflow.xml
@@ -61,7 +61,10 @@
-
+
+
+
+
diff --git a/tests/data/LF_ETRS89_UseCase/settings/mct_cold.xml b/tests/data/LF_ETRS89_UseCase/settings/mct_cold.xml
index 34bf434e..bbe89c51 100644
--- a/tests/data/LF_ETRS89_UseCase/settings/mct_cold.xml
+++ b/tests/data/LF_ETRS89_UseCase/settings/mct_cold.xml
@@ -163,7 +163,10 @@ You can use builtin path variables in this template and reference to other paths
-
+
+
+
+
@@ -3580,12 +3583,6 @@ LFBINDING: MORE LOW-LEVEL CONTROL OVER MODEL IN- AND OUTPUT
-
-
- Reported total water storage [mm]
-
-
-
Reported days since last rain
diff --git a/tests/data/LF_ETRS89_UseCase/settings/mct_cold_for_results_generation.xml b/tests/data/LF_ETRS89_UseCase/settings/mct_cold_for_results_generation.xml
index d0b754b1..4749693d 100644
--- a/tests/data/LF_ETRS89_UseCase/settings/mct_cold_for_results_generation.xml
+++ b/tests/data/LF_ETRS89_UseCase/settings/mct_cold_for_results_generation.xml
@@ -160,7 +160,10 @@ You can use builtin path variables in this template and reference to other paths
-
+
+
+
+
@@ -3530,12 +3533,6 @@ LFBINDING: MORE LOW-LEVEL CONTROL OVER MODEL IN- AND OUTPUT
-
-
- Reported total water storage [mm]
-
-
-
Reported days since last rain
diff --git a/tests/data/LF_ETRS89_UseCase/settings/mct_inflow.xml b/tests/data/LF_ETRS89_UseCase/settings/mct_inflow.xml
index b524d4b5..cf897d80 100644
--- a/tests/data/LF_ETRS89_UseCase/settings/mct_inflow.xml
+++ b/tests/data/LF_ETRS89_UseCase/settings/mct_inflow.xml
@@ -160,7 +160,10 @@ You can use builtin path variables in this template and reference to other paths
-
+
+
+
+
@@ -3583,12 +3586,6 @@ LFBINDING: MORE LOW-LEVEL CONTROL OVER MODEL IN- AND OUTPUT
-
-
- Reported total water storage [mm]
-
-
-
Reported days since last rain
diff --git a/tests/data/LF_ETRS89_UseCase/settings/mct_inflow_test.xml b/tests/data/LF_ETRS89_UseCase/settings/mct_inflow_test.xml
index 34b65929..1530db36 100644
--- a/tests/data/LF_ETRS89_UseCase/settings/mct_inflow_test.xml
+++ b/tests/data/LF_ETRS89_UseCase/settings/mct_inflow_test.xml
@@ -160,7 +160,10 @@ You can use builtin path variables in this template and reference to other paths
-
+
+
+
+
@@ -3583,12 +3586,6 @@ LFBINDING: MORE LOW-LEVEL CONTROL OVER MODEL IN- AND OUTPUT
-
-
- Reported total water storage [mm]
-
-
-
Reported days since last rain
diff --git a/tests/data/LF_ETRS89_UseCase/settings/mct_warm.xml b/tests/data/LF_ETRS89_UseCase/settings/mct_warm.xml
index fac126b3..76315d4f 100644
--- a/tests/data/LF_ETRS89_UseCase/settings/mct_warm.xml
+++ b/tests/data/LF_ETRS89_UseCase/settings/mct_warm.xml
@@ -162,7 +162,10 @@ You can use builtin path variables in this template and reference to other paths
-
+
+
+
+
@@ -3537,12 +3540,6 @@ LFBINDING: MORE LOW-LEVEL CONTROL OVER MODEL IN- AND OUTPUT
-
-
- Reported total water storage [mm]
-
-
-
Reported days since last rain
diff --git a/tests/data/LF_ETRS89_UseCase/settings/prerun.xml b/tests/data/LF_ETRS89_UseCase/settings/prerun.xml
index 994e10b9..bc12a8e5 100644
--- a/tests/data/LF_ETRS89_UseCase/settings/prerun.xml
+++ b/tests/data/LF_ETRS89_UseCase/settings/prerun.xml
@@ -60,7 +60,9 @@
-
+
+
+
diff --git a/tests/data/LF_ETRS89_UseCase/settings/warm.xml b/tests/data/LF_ETRS89_UseCase/settings/warm.xml
index 992da1a8..191919c2 100644
--- a/tests/data/LF_ETRS89_UseCase/settings/warm.xml
+++ b/tests/data/LF_ETRS89_UseCase/settings/warm.xml
@@ -86,7 +86,10 @@
-
+
+
+
+
diff --git a/tests/data/LF_MCT_UseCase/settings/mct_cold.xml b/tests/data/LF_MCT_UseCase/settings/mct_cold.xml
index 6c545239..caf44fc3 100644
--- a/tests/data/LF_MCT_UseCase/settings/mct_cold.xml
+++ b/tests/data/LF_MCT_UseCase/settings/mct_cold.xml
@@ -163,7 +163,10 @@ You can use builtin path variables in this template and reference to other paths
-
+
+
+
+
@@ -3693,12 +3696,6 @@ LFBINDING: MORE LOW-LEVEL CONTROL OVER MODEL IN- AND OUTPUT
-
-
- Reported total water storage [mm]
-
-
-
Reported days since last rain
diff --git a/tests/data/LF_MCT_UseCase/settings/mct_cold_for_results_generation.xml b/tests/data/LF_MCT_UseCase/settings/mct_cold_for_results_generation.xml
index 82f62c65..1eec4877 100644
--- a/tests/data/LF_MCT_UseCase/settings/mct_cold_for_results_generation.xml
+++ b/tests/data/LF_MCT_UseCase/settings/mct_cold_for_results_generation.xml
@@ -161,7 +161,10 @@ You can use builtin path variables in this template and reference to other paths
-
+
+
+
+
@@ -3690,12 +3693,6 @@ LFBINDING: MORE LOW-LEVEL CONTROL OVER MODEL IN- AND OUTPUT
-
-
- Reported total water storage [mm]
-
-
-
Reported days since last rain
diff --git a/tests/data/LF_MCT_UseCase/settings/mct_inflow.xml b/tests/data/LF_MCT_UseCase/settings/mct_inflow.xml
index 3d003e9c..f8934451 100644
--- a/tests/data/LF_MCT_UseCase/settings/mct_inflow.xml
+++ b/tests/data/LF_MCT_UseCase/settings/mct_inflow.xml
@@ -161,7 +161,10 @@ You can use builtin path variables in this template and reference to other paths
-
+
+
+
+
@@ -3695,12 +3698,6 @@ LFBINDING: MORE LOW-LEVEL CONTROL OVER MODEL IN- AND OUTPUT
-
-
- Reported total water storage [mm]
-
-
-
Reported days since last rain
diff --git a/tests/data/LF_MCT_UseCase/settings/mct_inflow_test.xml b/tests/data/LF_MCT_UseCase/settings/mct_inflow_test.xml
index d4afb104..d4663f71 100644
--- a/tests/data/LF_MCT_UseCase/settings/mct_inflow_test.xml
+++ b/tests/data/LF_MCT_UseCase/settings/mct_inflow_test.xml
@@ -160,7 +160,10 @@ You can use builtin path variables in this template and reference to other paths
-
+
+
+
+
@@ -3588,12 +3591,6 @@ LFBINDING: MORE LOW-LEVEL CONTROL OVER MODEL IN- AND OUTPUT
-
-
- Reported total water storage [mm]
-
-
-
Reported days since last rain
diff --git a/tests/data/LF_MCT_UseCase/settings/mct_prerun_test_only.xml b/tests/data/LF_MCT_UseCase/settings/mct_prerun_test_only.xml
index e9ce35dc..eb097a9c 100644
--- a/tests/data/LF_MCT_UseCase/settings/mct_prerun_test_only.xml
+++ b/tests/data/LF_MCT_UseCase/settings/mct_prerun_test_only.xml
@@ -162,7 +162,10 @@ You can use builtin path variables in this template and reference to other paths
-
+
+
+
+
@@ -3688,12 +3691,6 @@ LFBINDING: MORE LOW-LEVEL CONTROL OVER MODEL IN- AND OUTPUT
-
-
- Reported total water storage [mm]
-
-
-
Reported days since last rain
diff --git a/tests/data/LF_MCT_UseCase/settings/mct_warm.xml b/tests/data/LF_MCT_UseCase/settings/mct_warm.xml
index 0daf0e67..029694aa 100644
--- a/tests/data/LF_MCT_UseCase/settings/mct_warm.xml
+++ b/tests/data/LF_MCT_UseCase/settings/mct_warm.xml
@@ -162,7 +162,10 @@ You can use builtin path variables in this template and reference to other paths
-
+
+
+
+
@@ -3536,12 +3539,6 @@ LFBINDING: MORE LOW-LEVEL CONTROL OVER MODEL IN- AND OUTPUT
-
-
- Reported total water storage [mm]
-
-
-
Reported days since last rain
diff --git a/tests/data/LF_MCT_UseCase/settings/mct_warm_endmaps.xml b/tests/data/LF_MCT_UseCase/settings/mct_warm_endmaps.xml
index 80f39f53..876d2cbf 100644
--- a/tests/data/LF_MCT_UseCase/settings/mct_warm_endmaps.xml
+++ b/tests/data/LF_MCT_UseCase/settings/mct_warm_endmaps.xml
@@ -162,7 +162,10 @@ You can use builtin path variables in this template and reference to other paths
-
+
+
+
+
@@ -3536,12 +3539,6 @@ LFBINDING: MORE LOW-LEVEL CONTROL OVER MODEL IN- AND OUTPUT
-
-
- Reported total water storage [mm]
-
-
-
Reported days since last rain
diff --git a/tests/data/LF_lat_lon_UseCase/prerun_lat_lon.xml b/tests/data/LF_lat_lon_UseCase/prerun_lat_lon.xml
index 607d038b..4679cb35 100644
--- a/tests/data/LF_lat_lon_UseCase/prerun_lat_lon.xml
+++ b/tests/data/LF_lat_lon_UseCase/prerun_lat_lon.xml
@@ -80,7 +80,9 @@
-
+
+
+
diff --git a/tests/data/LF_lat_lon_UseCase/run_lat_lon.xml b/tests/data/LF_lat_lon_UseCase/run_lat_lon.xml
index 749c515e..3a3a7bb6 100644
--- a/tests/data/LF_lat_lon_UseCase/run_lat_lon.xml
+++ b/tests/data/LF_lat_lon_UseCase/run_lat_lon.xml
@@ -81,7 +81,9 @@
-
+
+
+
diff --git a/tests/test_add1_helpers.py b/tests/test_add1_helpers.py
new file mode 100644
index 00000000..09dafc38
--- /dev/null
+++ b/tests/test_add1_helpers.py
@@ -0,0 +1,108 @@
+"""
+Unit tests for pure helper functions in lisflood.global_modules.add1 that are
+not exercised by the end-to-end suite.
+
+Scope is limited to functions with real branching / edge-case logic where a
+regression could slip through silently:
+
+ * generateName - 8.3 "DOS style" timestep filename builder with several
+ validation branches.
+ * takeClosest - bisect-based "closest left value" lookup with boundary
+ behavior.
+ * read_tss_header - .tss header parser with two format branches (with header
+ vs bare table), used by the inflow module.
+
+These are characterization tests: they pin the current behavior so a future
+refactor is caught.
+"""
+
+import os
+
+import pytest
+
+from lisflood.global_modules.add1 import generateName, takeClosest, read_tss_header
+from lisflood.global_modules.errors import LisfloodError
+
+
+class TestGenerateName:
+ """8.3-style name: , total 12 chars incl. the dot,
+ directory preserved."""
+
+ def test_pads_timestep_into_8_3_format(self):
+ # 'dis' (3) + timestep 5 -> zero-filled to 8 chars before the dot
+ assert generateName("dis", 5) == "dis00000.005"
+
+ def test_large_timestep_fills_available_width(self):
+ # space = 11 - (len('dis') + len('12345')) = 3 zeros -> 'dis00012345',
+ # then split into 8.3 -> 'dis00012.345'
+ assert generateName("dis", 12345) == "dis00012.345"
+
+ def test_directory_component_is_preserved(self):
+ result = generateName(os.path.join("out", "dis"), 5)
+ assert result == os.path.join("out", "dis00000.005")
+
+ def test_extension_in_name_is_rejected(self):
+ with pytest.raises(LisfloodError):
+ generateName("dis.map", 5)
+
+ def test_empty_filename_is_rejected(self):
+ with pytest.raises(LisfloodError):
+ generateName("", 5)
+
+ def test_name_longer_than_8_chars_is_rejected(self):
+ with pytest.raises(LisfloodError):
+ generateName("longname9", 5)
+
+ def test_negative_timestep_is_rejected(self):
+ with pytest.raises(LisfloodError):
+ generateName("dis", -1)
+
+
+class TestTakeClosest:
+ """Returns the closest left (<=) value in a sorted list; clamps at both
+ ends."""
+
+ def test_exact_match_returns_left_neighbor(self):
+ # bisect_left on an exact match points at it, so 'before' is returned
+ assert takeClosest([0, 10, 20, 30], 20) == 10
+
+ def test_between_values_returns_lower(self):
+ assert takeClosest([0, 10, 20, 30], 17) == 10
+
+ def test_below_range_clamps_to_first(self):
+ assert takeClosest([0, 10, 20], -5) == 0
+
+ def test_above_range_clamps_to_last(self):
+ assert takeClosest([0, 10, 20], 999) == 20
+
+
+class TestReadTssHeader:
+ """Parses a LISFLOOD .tss file header. Two branches: a 'timeseries' header
+ (returns the listed outlet IDs) or a bare table (returns 1..N-1 column
+ indices)."""
+
+ def _write(self, tmp_path, content):
+ f = tmp_path / "series.tss"
+ f.write_text(content)
+ return str(f)
+
+ def test_with_timeseries_header_returns_outlet_ids(self, tmp_path):
+ # header format: 'timeseries ...', then a count, a title line, then
+ # (count-1) outlet id lines, then data.
+ content = (
+ "timeseries scalar\n"
+ "3\n"
+ "timestep\n"
+ "101\n"
+ "202\n"
+ "1 1.0 2.0\n"
+ )
+ path = self._write(tmp_path, content)
+ assert read_tss_header(path) == [101, 202]
+
+ def test_bare_table_returns_column_indices(self, tmp_path):
+ # no 'timeseries' keyword: first row has N whitespace columns, header
+ # is the progressive indices 1..N-1 (first column is the timestep).
+ content = "1 3.14 2.71 9.9\n2 1.0 2.0 3.0\n"
+ path = self._write(tmp_path, content)
+ assert read_tss_header(path) == [1, 2, 3]
diff --git a/tests/test_caching.py b/tests/test_caching.py
index 816ae5c5..4fbfef79 100644
--- a/tests/test_caching.py
+++ b/tests/test_caching.py
@@ -10,7 +10,7 @@
from lisflood.global_modules.decorators import Cache
from lisflood.global_modules.settings import LisSettings
-from .test_utils import setoptions, mk_path_out, ETRS89TestCase
+from .test_utils import setoptions, mk_path_out, ETRS89TestCase, worker_out
class TestCaching(ETRS89TestCase):
@@ -18,9 +18,14 @@ class TestCaching(ETRS89TestCase):
settings_file = os.path.join(case_dir, 'settings', 'full.xml')
path_out = os.path.join(case_dir, 'out')
if not os.path.exists(path_out):
- os.mkdir(path_out)
- out_dir_a = os.path.join(case_dir, 'out', 'a')
- out_dir_b = os.path.join(case_dir, 'out', 'b')
+ os.makedirs(path_out, exist_ok=True)
+ # Output dirs isolated per xdist worker (see worker_out): 'out//a' in
+ # parallel, 'out/a' serially, so this file's 'out/a' does not clobber the
+ # identically named dir used by test_chunking / test_water_abstraction.
+ out_dir_a = os.path.join(case_dir, worker_out('a'))
+ out_dir_b = os.path.join(case_dir, worker_out('b'))
+ pathout_a = '$(PathRoot)/' + worker_out('a')
+ pathout_b = '$(PathRoot)/' + worker_out('b')
def test_caching_24h(self):
dt_sec = 86400
@@ -38,7 +43,7 @@ def run_lisflood_caching(self, dt_sec, test_extract=False):
settings_a = setoptions(self.settings_file,
vars_to_set={'StepStart': '30/07/2016 06:00', 'StepEnd': '01/08/2016 06:00',
- 'DtSec': dt_sec, 'PathOut': '$(PathRoot)/out/a',
+ 'DtSec': dt_sec, 'PathOut': self.pathout_a,
'MapsCaching': 'True'})
mk_path_out(self.out_dir_a)
lisfloodexe(settings_a)
@@ -59,7 +64,7 @@ def run_lisflood_caching(self, dt_sec, test_extract=False):
settings_b = setoptions(self.settings_file,
vars_to_set={'StepStart': '30/07/2016 06:00', 'StepEnd': '01/08/2016 06:00',
- 'DtSec': dt_sec, 'PathOut': '$(PathRoot)/out/b',
+ 'DtSec': dt_sec, 'PathOut': self.pathout_b,
'MapsCaching': 'True'})
mk_path_out(self.out_dir_b)
lisfloodexe(settings_b)
@@ -130,7 +135,7 @@ def run(self, dt_sec, step_start, step_end):
print('Cache size is {}'.format(cache_size_a))
print('Items found: {}'.format(cache_found_a))
- assert cache_found_a == 2 # here we used Gauges map 3 times (for DisTS, ChanqTS and ChanqAvgDtTS)
+ assert cache_found_a == 0 # now Gauges map is not used multiple times anymore
lisfloodexe(settings)
@@ -141,7 +146,7 @@ def run(self, dt_sec, step_start, step_end):
Cache.info()
- assert cache_found_b == cache_size_b + 4 # add 4 for the 2 additional calls of Gauges map
+ assert cache_found_b == cache_size_b # now Gauges map is not used multiple times anymore
assert cache_size_a == cache_size_b
self.compare_reference('dis', check='map', step_length=dt_sec)
diff --git a/tests/test_chunking.py b/tests/test_chunking.py
index 16592f5c..5f78d90c 100644
--- a/tests/test_chunking.py
+++ b/tests/test_chunking.py
@@ -9,16 +9,23 @@
from lisflood.main import lisfloodexe
from lisflood.global_modules.settings import LisSettings
-from .test_utils import setoptions, mk_path_out, ETRS89TestCase
+from .test_utils import setoptions, mk_path_out, ETRS89TestCase, worker_out
class TestChunking(ETRS89TestCase):
case_dir = os.path.join(os.path.dirname(__file__), 'data', 'LF_ETRS89_UseCase')
settings_file = os.path.join(case_dir, 'settings', 'full.xml')
- out_dir_a = os.path.join(case_dir, 'out', 'a')
- out_dir_b = os.path.join(case_dir, 'out', 'b')
- out_dir_c = os.path.join(case_dir, 'out', 'c')
- out_dir_d = os.path.join(case_dir, 'out', 'd')
+ # Output dirs are isolated per xdist worker (worker_out -> 'out//a' in
+ # parallel, 'out/a' serially) so concurrent test files do not clobber each
+ # other's shared 'out/a'..'out/d'.
+ out_dir_a = os.path.join(case_dir, worker_out('a'))
+ out_dir_b = os.path.join(case_dir, worker_out('b'))
+ out_dir_c = os.path.join(case_dir, worker_out('c'))
+ out_dir_d = os.path.join(case_dir, worker_out('d'))
+ pathout_a = '$(PathRoot)/' + worker_out('a')
+ pathout_b = '$(PathRoot)/' + worker_out('b')
+ pathout_c = '$(PathRoot)/' + worker_out('c')
+ pathout_d = '$(PathRoot)/' + worker_out('d')
def test_chunking_24h(self):
dt_sec = 86400
@@ -32,7 +39,7 @@ def run_lisflood_chunking(self, dt_sec):
settings_a = setoptions(self.settings_file,
vars_to_set={'StepStart': '30/07/2016 06:00', 'StepEnd': '01/09/2016 06:00',
- 'DtSec': dt_sec, 'PathOut': '$(PathRoot)/out/a',
+ 'DtSec': dt_sec, 'PathOut': self.pathout_a,
'NetCDFTimeChunks': '1'})
mk_path_out(self.out_dir_a)
lisfloodexe(settings_a)
@@ -41,7 +48,7 @@ def run_lisflood_chunking(self, dt_sec):
settings_b = setoptions(self.settings_file,
vars_to_set={'StepStart': '30/07/2016 06:00', 'StepEnd': '01/09/2016 06:00',
- 'DtSec': dt_sec, 'PathOut': '$(PathRoot)/out/b',
+ 'DtSec': dt_sec, 'PathOut': self.pathout_b,
'NetCDFTimeChunks': '10'})
mk_path_out(self.out_dir_b)
lisfloodexe(settings_b)
@@ -50,7 +57,7 @@ def run_lisflood_chunking(self, dt_sec):
settings_c = setoptions(self.settings_file,
vars_to_set={'StepStart': '30/07/2016 06:00', 'StepEnd': '01/09/2016 06:00',
- 'DtSec': dt_sec, 'PathOut': '$(PathRoot)/out/c',
+ 'DtSec': dt_sec, 'PathOut': self.pathout_c,
'NetCDFTimeChunks': 'auto'})
mk_path_out(self.out_dir_c)
lisfloodexe(settings_c)
@@ -59,7 +66,7 @@ def run_lisflood_chunking(self, dt_sec):
settings_d = setoptions(self.settings_file,
vars_to_set={'StepStart': '30/07/2016 06:00', 'StepEnd': '01/09/2016 06:00',
- 'DtSec': dt_sec, 'PathOut': '$(PathRoot)/out/d',
+ 'DtSec': dt_sec, 'PathOut': self.pathout_d,
'NetCDFTimeChunks': '-1'})
mk_path_out(self.out_dir_d)
lisfloodexe(settings_d)
@@ -72,3 +79,70 @@ def teardown_method(self):
shutil.rmtree(self.out_dir_b, ignore_errors=True)
shutil.rmtree(self.out_dir_c, ignore_errors=True)
shutil.rmtree(self.out_dir_d, ignore_errors=True)
+
+
+class TestChunkingPrefetch(ETRS89TestCase):
+ """Bit-identity gate for the opt-in NetCDFTimeChunksPrefetch background
+ chunk prefetch: outputs must be identical to the synchronous path for the
+ same NetCDFTimeChunks setting."""
+
+ case_dir = os.path.join(os.path.dirname(__file__), 'data', 'LF_ETRS89_UseCase')
+ settings_file = os.path.join(case_dir, 'settings', 'full.xml')
+ # Output dirs are isolated per xdist worker so concurrent test files do
+ # not clobber each other's shared 'out/a'..'out/d'.
+ out_dir_a = os.path.join(case_dir, worker_out('pa'))
+ out_dir_b = os.path.join(case_dir, worker_out('pb'))
+ out_dir_c = os.path.join(case_dir, worker_out('pc'))
+ pathout_a = '$(PathRoot)/' + worker_out('pa')
+ pathout_b = '$(PathRoot)/' + worker_out('pb')
+ pathout_c = '$(PathRoot)/' + worker_out('pc')
+
+ def test_prefetch_24h(self):
+ self.run_lisflood_prefetch(86400)
+
+ @pytest.mark.slow
+ def test_prefetch_6h(self):
+ self.run_lisflood_prefetch(21600)
+
+ def run_lisflood_prefetch(self, dt_sec):
+
+ # reference: chunked load, synchronous (historical behaviour)
+ settings_a = setoptions(self.settings_file,
+ vars_to_set={'StepStart': '30/07/2016 06:00', 'StepEnd': '01/09/2016 06:00',
+ 'DtSec': dt_sec, 'PathOut': self.pathout_a,
+ 'NetCDFTimeChunks': '10'})
+ mk_path_out(self.out_dir_a)
+ lisfloodexe(settings_a)
+ comparator = NetCDFComparator(settings_a.maskpath, array_equal=True)
+ out_a = settings_a.output_dir
+
+ # same chunking WITH background prefetch: must be bit-identical
+ settings_b = setoptions(self.settings_file,
+ opts_to_set='NetCDFTimeChunksPrefetch',
+ vars_to_set={'StepStart': '30/07/2016 06:00', 'StepEnd': '01/09/2016 06:00',
+ 'DtSec': dt_sec, 'PathOut': self.pathout_b,
+ 'NetCDFTimeChunks': '10'})
+ mk_path_out(self.out_dir_b)
+ lisfloodexe(settings_b)
+ out_b = settings_b.output_dir
+ comparator.compare_dirs(out_a, out_b)
+
+ # edge case: prefetch enabled with NetCDFTimeChunks=1 (a chunk per
+ # step): the scheduling window is never hit, so the loader falls
+ # back to the synchronous path
+ settings_c = setoptions(self.settings_file,
+ opts_to_set='NetCDFTimeChunksPrefetch',
+ vars_to_set={'StepStart': '30/07/2016 06:00', 'StepEnd': '01/09/2016 06:00',
+ 'DtSec': dt_sec, 'PathOut': self.pathout_c,
+ 'NetCDFTimeChunks': '1'})
+ mk_path_out(self.out_dir_c)
+ lisfloodexe(settings_c)
+ out_c = settings_c.output_dir
+ comparator.compare_dirs(out_a, out_c)
+
+ def teardown_method(self):
+ print('Cleaning directories')
+ shutil.rmtree(self.out_dir_a, ignore_errors=True)
+ shutil.rmtree(self.out_dir_b, ignore_errors=True)
+ shutil.rmtree(self.out_dir_c, ignore_errors=True)
+
diff --git a/tests/test_dates_steps.py b/tests/test_dates_steps.py
index 0171ece7..0f5b93c2 100644
--- a/tests/test_dates_steps.py
+++ b/tests/test_dates_steps.py
@@ -7,7 +7,7 @@
from lisflood.main import lisfloodexe
-from .test_utils import setoptions, mk_path_out
+from .test_utils import setoptions, mk_path_out, worker_out
class TestStepsDates():
@@ -21,9 +21,9 @@ def test_dates_before_1970(self):
'repSnowMaps', 'repLZMaps', 'repUZMaps'],
vars_to_set={'StepStart': '01/01/1951 06:00', 'StepEnd': '05/01/1951 06:00',
'CalendarDayStart': '01/01/1900 00:00', 'DtSec': '21600',
- 'PathOut': '$(PathRoot)/out/0', 'PathMeteo': '$(PathRoot)/meteo_1950'}
+ 'PathOut': '$(PathRoot)/' + worker_out('0'), 'PathMeteo': '$(PathRoot)/meteo_1950'}
)
- mk_path_out('data/LF_ETRS89_UseCase/out/0')
+ mk_path_out(os.path.join('data/LF_ETRS89_UseCase', worker_out('0')))
lisfloodexe(settings)
assert settings.step_start_int == 74510
assert settings.step_end_int == 74526
@@ -36,16 +36,16 @@ def test_dates_steps_daily(self):
opts_to_set=['repStateMaps', 'repEndMaps', 'repDischargeMaps',
'repSnowMaps', 'repLZMaps', 'repUZMaps'],
vars_to_set={'StepStart': '30/07/2016 06:00', 'StepEnd': '01/08/2016 06:00',
- 'PathOut': '$(PathRoot)/out/1'}
+ 'PathOut': '$(PathRoot)/' + worker_out('1')}
)
- mk_path_out('data/LF_ETRS89_UseCase/out/1')
+ mk_path_out(os.path.join('data/LF_ETRS89_UseCase', worker_out('1')))
lisfloodexe(settings_a)
settings_b = setoptions(self.settings_files['full'],
opts_to_set=['repStateMaps', 'repEndMaps', 'repDischargeMaps',
'repSnowMaps', 'repLZMaps', 'repUZMaps'],
vars_to_set={'StepStart': 6057, 'StepEnd': 6059,
- 'PathOut': '$(PathRoot)/out/2'})
- mk_path_out('data/LF_ETRS89_UseCase/out/2')
+ 'PathOut': '$(PathRoot)/' + worker_out('2')})
+ mk_path_out(os.path.join('data/LF_ETRS89_UseCase', worker_out('2')))
lisfloodexe(settings_b)
@@ -67,17 +67,17 @@ def test_dates_steps_6h(self):
'repSnowMaps', 'repLZMaps', 'repUZMaps'],
vars_to_set={'StepStart': '30/07/2016 06:00', 'StepEnd': '31/07/2016 18:00',
'DtSec': '21600',
- 'PathOut': '$(PathRoot)/out/1'}
+ 'PathOut': '$(PathRoot)/' + worker_out('1')}
)
- mk_path_out('data/LF_ETRS89_UseCase/out/1')
+ mk_path_out(os.path.join('data/LF_ETRS89_UseCase', worker_out('1')))
lisfloodexe(settings_a)
settings_b = setoptions(self.settings_files['full'],
opts_to_set=['repStateMaps', 'repEndMaps', 'repDischargeMaps',
'repSnowMaps', 'repLZMaps', 'repUZMaps'],
vars_to_set={'StepStart': 24225, 'StepEnd': 24231,
'DtSec': '21600',
- 'PathOut': '$(PathRoot)/out/2'})
- mk_path_out('data/LF_ETRS89_UseCase/out/2')
+ 'PathOut': '$(PathRoot)/' + worker_out('2')})
+ mk_path_out(os.path.join('data/LF_ETRS89_UseCase', worker_out('2')))
lisfloodexe(settings_b)
@@ -94,9 +94,10 @@ def test_dates_steps_6h(self):
comparator.compare_dirs(out_a, out_b)
def teardown_method(self):
- path_out_0 = os.path.join(os.path.dirname(__file__), 'data/LF_ETRS89_UseCase/out/0')
- path_out_a = os.path.join(os.path.dirname(__file__), 'data/LF_ETRS89_UseCase/out/1')
- path_out_b = os.path.join(os.path.dirname(__file__), 'data/LF_ETRS89_UseCase/out/2')
+ base = os.path.join(os.path.dirname(__file__), 'data/LF_ETRS89_UseCase')
+ path_out_0 = os.path.join(base, worker_out('0'))
+ path_out_a = os.path.join(base, worker_out('1'))
+ path_out_b = os.path.join(base, worker_out('2'))
for folder in (path_out_0, path_out_a, path_out_b):
if os.path.exists(folder):
shutil.rmtree(folder)
diff --git a/tests/test_earthkit_network.py b/tests/test_earthkit_network.py
new file mode 100644
index 00000000..1d35a9f9
--- /dev/null
+++ b/tests/test_earthkit_network.py
@@ -0,0 +1,227 @@
+"""
+Unit tests for the earthkit-hydro bridge (lisflood.global_modules.
+earthkit_network) behind the useEarthkitHydro reporting path (default True).
+
+The end-to-end suite cannot reliably catch a regression here: if the bridge
+breaks or earthkit-hydro is missing, the PCRaster fallback silently takes
+over. These tests pin the two
+contracts the bridge must uphold:
+
+ 1. Accumulation - a RiverNetwork built by the bridge from a numpy Ldd must
+ reproduce PCRaster accuflux on the in-domain (compressed) cells.
+ 2. Zone means - ek_sample_zone_means must reproduce the inherited
+ TimeoutputTimeseries.sample() tss values for multi-cell gauge zones
+ (the pcraster.areaaverage(expr, spatialId) path), including
+ Decimal('NaN') handling and the final-step file write.
+
+Characterization tests: they pin current behaviour so an earthkit-hydro
+upgrade that breaks the bridge is caught by the suite, not by a silently
+different run. No settings files or test datasets are required.
+"""
+
+import types
+from decimal import Decimal
+
+import numpy as np
+import pcraster
+import pytest
+
+from lisflood.global_modules.earthkit_network import (
+ compute_gauge_zone_codes,
+ earthkit_available,
+ ek_sample_zone_means,
+ ek_upstream_sum,
+ ek_upstream_sum_batch,
+ ek_zone_mean_at_gauges,
+ get_earthkit_network,
+)
+
+requires_earthkit = pytest.mark.skipif(
+ not earthkit_available(), reason='earthkit-hydro not installed')
+
+
+def east_flow_ldd(nrows=10, ncols=10):
+ """Sound Ldd: every cell flows east (pcr keypad code 6), last column pits (5)."""
+ ldd = np.zeros((nrows, ncols), dtype=np.int32)
+ for r in range(nrows):
+ for c in range(ncols):
+ ldd[r, c] = 6 if c < ncols - 1 else 5
+ return ldd
+
+
+def compress_like_lisflood(array_2d, ldd_2d):
+ """Flatten to LISFLOOD's compressed order (row-major valid d8 cells)."""
+ valid = (ldd_2d > 0) & (ldd_2d <= 9)
+ return np.asarray(array_2d, dtype=np.float64).ravel(order='C')[valid.ravel()]
+
+
+class TestRiverNetworkAccumulation:
+
+ @requires_earthkit
+ def test_upstream_sum_matches_pcraster_accuflux(self):
+ pcraster.setclone(10, 10, 1.0, 0.0, 0.0)
+ ldd = east_flow_ldd()
+ field = np.random.default_rng(42).random((10, 10))
+ ldd_pcr = pcraster.ldd(pcraster.numpy2pcr(pcraster.Nominal, ldd, 0))
+ acc_pcr = pcraster.pcr2numpy(
+ pcraster.accuflux(ldd_pcr,
+ pcraster.numpy2pcr(pcraster.Scalar, field, np.nan)),
+ np.nan)
+ net = get_earthkit_network(ldd)
+ acc_ek = ek_upstream_sum(net, compress_like_lisflood(field, ldd))
+ # PCRaster Scalar accumulates in float32, earthkit in float64: allow
+ # float32-level rounding differences.
+ np.testing.assert_allclose(
+ acc_ek, compress_like_lisflood(acc_pcr, ldd), rtol=1e-6, atol=1e-9)
+
+ @requires_earthkit
+ def test_network_is_cached_per_ldd(self):
+ pcraster.setclone(8, 8, 1.0, 0.0, 0.0)
+ ldd = east_flow_ldd(8, 8)
+ assert get_earthkit_network(ldd) is get_earthkit_network(ldd.copy())
+
+
+class TestUpstreamSumBatch:
+ """The batched helper must match stacked per-row scalar calls bit-for-bit
+ and reject a mis-shaped (non-2D / wrong-axis) input."""
+
+ @requires_earthkit
+ def test_batch_matches_per_row_bit_identical(self):
+ pcraster.setclone(10, 10, 1.0, 0.0, 0.0)
+ ldd = east_flow_ldd()
+ net = get_earthkit_network(ldd)
+ rng = np.random.default_rng(7)
+ n_nodes = ek_upstream_sum(net, compress_like_lisflood(
+ np.ones((10, 10)), ldd)).shape[0]
+ weights = rng.random(n_nodes)
+ fields = [rng.random(n_nodes) for _ in range(5)]
+ per_row = np.stack(
+ [ek_upstream_sum(net, f, node_weights=weights) for f in fields])
+ batched = ek_upstream_sum_batch(net, np.stack(fields),
+ node_weights=weights)
+ assert np.array_equal(batched, per_row)
+
+ @requires_earthkit
+ def test_batch_matches_per_row_unweighted(self):
+ pcraster.setclone(10, 10, 1.0, 0.0, 0.0)
+ ldd = east_flow_ldd()
+ net = get_earthkit_network(ldd)
+ rng = np.random.default_rng(11)
+ n_nodes = ek_upstream_sum(net, compress_like_lisflood(
+ np.ones((10, 10)), ldd)).shape[0]
+ fields = [rng.random(n_nodes) for _ in range(3)]
+ per_row = np.stack([ek_upstream_sum(net, f) for f in fields])
+ batched = ek_upstream_sum_batch(net, np.stack(fields))
+ assert np.array_equal(batched, per_row)
+
+ @requires_earthkit
+ def test_non_2d_input_raises(self):
+ pcraster.setclone(10, 10, 1.0, 0.0, 0.0)
+ ldd = east_flow_ldd()
+ net = get_earthkit_network(ldd)
+ n_nodes = ek_upstream_sum(net, compress_like_lisflood(
+ np.ones((10, 10)), ldd)).shape[0]
+ with pytest.raises(ValueError):
+ ek_upstream_sum_batch(net, np.ones(n_nodes))
+
+ @requires_earthkit
+ def test_wrong_axis_layout_raises(self):
+ # node axis must be last; a (n_nodes, n_series) array is rejected by
+ # the node_weights broadcast (trailing-axis mismatch).
+ pcraster.setclone(10, 10, 1.0, 0.0, 0.0)
+ ldd = east_flow_ldd()
+ net = get_earthkit_network(ldd)
+ n_nodes = ek_upstream_sum(net, compress_like_lisflood(
+ np.ones((10, 10)), ldd)).shape[0]
+ weights = np.ones(n_nodes)
+ transposed = np.ones((n_nodes, 3))
+ with pytest.raises(ValueError):
+ ek_upstream_sum_batch(net, transposed, node_weights=weights)
+
+
+class TestZoneMeans:
+
+ def test_zone_mean_matches_numpy_oracle(self):
+ field = np.array([1.0, 2.0, 3.0, 4.0, 10.0, 20.0, 5.0])
+ # non-contiguous codes on purpose (5 and 17)
+ codes = np.array([5, 5, 0, 17, 17, 0, 5], dtype=np.int64)
+ means = ek_zone_mean_at_gauges(None, field, codes, 2)
+ assert means[5] == pytest.approx(8.0 / 3.0)
+ assert means[17] == pytest.approx(7.0)
+
+ def test_zone_mean_nan_cells_are_propagated(self):
+ means = ek_zone_mean_at_gauges(None, np.array([1.0, np.nan]),
+ np.array([1, 1], dtype=np.int64), 1)
+ assert np.isnan(means[1])
+
+ def test_compute_gauge_zone_codes_maps_nan_to_zero(self):
+ gauge_map = np.array([[1.0, np.nan], [3.0, 3.0]])
+ maskflat = np.array([False, False, True, False])
+ zone_codes, n_codes = compute_gauge_zone_codes(gauge_map, maskflat, 3)
+ np.testing.assert_array_equal(zone_codes, [1, 0, 3])
+ assert n_codes == 3
+
+
+
+class TestEkSampleZoneMeans:
+ """tss-row equivalence with the PCRaster .sample() path."""
+
+ @staticmethod
+ def _fake_tss(zone_codes, n_codes, codes_id, n_rows=3, final_step=3):
+ """Minimal TimeoutputTimeseries stand-in with the zone attributes
+ prepared in __init__ (compute_gauge_zone_codes) and a write recorder."""
+ tss = types.SimpleNamespace(
+ _userModel=types.SimpleNamespace(
+ currentTimeStep=lambda: tss.current_step[0],
+ firstTimeStep=lambda: 1,
+ nrTimeSteps=lambda: final_step),
+ _spatialDatatype=None,
+ _sampleValues=[[None] * len(codes_id) for _ in range(n_rows)],
+ _codesId=codes_id,
+ _ncodesId=len(codes_id),
+ _ek_zone_codes=zone_codes,
+ _ek_n_codes=n_codes,
+ current_step=[1],
+ written=[],
+ )
+ tss._writeTssFile = lambda: tss.written.append(tss.current_step[0])
+ return tss
+
+ @requires_earthkit
+ def test_row_matches_pcraster_areaaverage(self):
+ # zone 5 at flat cells 0,1; zone 17 at flat cell 5 of a 2x3 domain
+ # (all in-mask, so the compressed field covers all 6 cells);
+ # exactly representable values so the float32 casts agree bit-for-bit.
+ gauge_map = np.array([[5.0, 5.0, np.nan], [np.nan, np.nan, 17.0]])
+ zone_codes, n_codes = compute_gauge_zone_codes(
+ gauge_map, np.zeros(6, dtype=bool), 2)
+ tss = self._fake_tss(zone_codes, n_codes, [5, 17])
+ ek_sample_zone_means(tss, np.array([2.0, 6.0, 0.0, 0.0, 0.0, 10.0]))
+ np.testing.assert_array_equal(tss._sampleValues[0], [4.0, 10.0])
+ assert tss._spatialDatatype == 'VALUESCALE.Scalar'
+
+ @requires_earthkit
+ def test_nan_values_become_decimal_nan(self):
+ gauge_map = np.array([[5.0, np.nan], [np.nan, 17.0]])
+ zone_codes, n_codes = compute_gauge_zone_codes(
+ gauge_map, np.zeros(4, dtype=bool), 2)
+ tss = self._fake_tss(zone_codes, n_codes, [5, 17])
+ ek_sample_zone_means(tss, np.array([np.nan, 0.0, 0.0, 8.0]))
+ # Decimal('NaN') compares unequal to itself (IEEE rules), which is why
+ # _writeTssFile treats it as a sentinel: test via is_nan(), not ==.
+ assert isinstance(tss._sampleValues[0][0], Decimal)
+ assert tss._sampleValues[0][0].is_nan()
+ assert tss._sampleValues[0][1] == 8.0
+
+ @requires_earthkit
+ def test_tss_written_only_on_final_step(self):
+ gauge_map = np.array([[5.0, np.nan], [np.nan, 17.0]])
+ zone_codes, n_codes = compute_gauge_zone_codes(
+ gauge_map, np.zeros(4, dtype=bool), 2)
+ tss = self._fake_tss(zone_codes, n_codes, [5, 17])
+ for step, vals in [(1, [2.0, 0.0, 0.0, 8.0]),
+ (2, [4.0, 0.0, 0.0, 8.0]),
+ (3, [6.0, 0.0, 0.0, 8.0])]:
+ tss.current_step[0] = step
+ ek_sample_zone_means(tss, np.array(vals, dtype=np.float64))
+ assert tss.written == [3]
diff --git a/tests/test_mct_results.py b/tests/test_mct_results.py
index 011ffabb..76e33fca 100644
--- a/tests/test_mct_results.py
+++ b/tests/test_mct_results.py
@@ -17,11 +17,26 @@ class TestTSSResults():
case_dir = os.path.join(os.path.dirname(__file__), 'data', 'LF_MCT_UseCase')
+ def _mk_tracked_out(self, path):
+ """Create an output folder and record it so the teardown removes only
+ the folders this test created (never the shared 'out/' directory or
+ folders created by other tests). Returns the actually-created path,
+ which under pytest-xdist is worker-scoped (e.g. 'out/gw0/...'); callers
+ must use this return value for PathOut so the created dir and the
+ LISFLOOD output dir stay in sync."""
+ if not hasattr(self, '_created_out_dirs'):
+ self._created_out_dirs = []
+ created = mk_path_out(path)
+ self._created_out_dirs.append(created)
+ return created
+
def run_mct(self, date_start, date_end, dtsec, dtsec_chan, type):
- mk_path_out(os.path.join(self.case_dir, 'out'))
+ os.makedirs(os.path.join(self.case_dir, 'out'), exist_ok=True)
# generate lisflood results
out_path_ref = os.path.join(self.case_dir, 'reference', 'output_reference_mct_'+type)
- self.out_path_run = os.path.join(self.case_dir, 'out', 'output_mct_'+type)
+ # Create the (worker-scoped) output dir first and use the returned path for
+ # PathOut so the created dir and the LISFLOOD output dir match under xdist.
+ self.out_path_run = self._mk_tracked_out(os.path.join(self.case_dir, 'out', 'output_mct_'+type))
settings_file = os.path.join(self.case_dir, 'settings', 'mct_cold.xml')
if "_calib" in type:
opts_to_set = ['MCTRouting','simulateCalibrationPoints']
@@ -39,8 +54,6 @@ def run_mct(self, date_start, date_end, dtsec, dtsec_chan, type):
'DtSecChannel' : dtsec_chan, # single routing step
'PathOut': self.out_path_run})
-
- mk_path_out(self.out_path_run)
lisfloodexe(settings)
# set precision for the test
@@ -64,10 +77,10 @@ def run_mct(self, date_start, date_end, dtsec, dtsec_chan, type):
comparator.compare_files(reference, output_tss)
def run_mcts(self, date_start, date_end, dtsec, dtsec_chan, type):
- mk_path_out(os.path.join(self.case_dir, 'out'))
+ os.makedirs(os.path.join(self.case_dir, 'out'), exist_ok=True)
# generate lisflood results
out_path_ref = os.path.join(self.case_dir, 'reference', 'output_reference_mcts_'+type)
- self.out_path_run = os.path.join(self.case_dir, 'out', 'output_mcts_'+type)
+ self.out_path_run = self._mk_tracked_out(os.path.join(self.case_dir, 'out', 'output_mcts_'+type))
settings_file = os.path.join(self.case_dir, 'settings', 'mct_cold.xml')
if "_calib" in type:
opts_to_set = ['MCTRouting','SplitRouting','simulateCalibrationPoints']
@@ -85,7 +98,7 @@ def run_mcts(self, date_start, date_end, dtsec, dtsec_chan, type):
'DtSecChannel' : dtsec_chan, # single routing step
'PathOut': self.out_path_run})
- mk_path_out(self.out_path_run)
+ self._mk_tracked_out(self.out_path_run)
lisfloodexe(settings)
# set precisioon for the test
@@ -119,10 +132,10 @@ def run_mcts(self, date_start, date_end, dtsec, dtsec_chan, type):
comparator.compare_files(reference, output_tss)
def run_kin(self, date_start, date_end, dtsec, dtsec_chan, type):
- mk_path_out(os.path.join(self.case_dir, 'out'))
+ os.makedirs(os.path.join(self.case_dir, 'out'), exist_ok=True)
# generate lisflood results
out_path_ref = os.path.join(self.case_dir, 'reference', 'output_reference_kin_'+type)
- self.out_path_run = os.path.join(self.case_dir, 'out', 'output_kin_'+type)
+ self.out_path_run = self._mk_tracked_out(os.path.join(self.case_dir, 'out', 'output_kin_'+type))
settings_file = os.path.join(self.case_dir, 'settings', 'mct_cold.xml')
settings = setoptions(settings_file,
opts_to_unset=['MCTRouting',
@@ -134,7 +147,7 @@ def run_kin(self, date_start, date_end, dtsec, dtsec_chan, type):
'DtSecChannel' : dtsec_chan, # single routing step
'PathOut': self.out_path_run})
- mk_path_out(self.out_path_run)
+ self._mk_tracked_out(self.out_path_run)
lisfloodexe(settings)
# set precisioon for the test
@@ -158,10 +171,10 @@ def run_kin(self, date_start, date_end, dtsec, dtsec_chan, type):
comparator.compare_files(reference, output_tss)
def run_split(self, date_start, date_end, dtsec, dtsec_chan, type):
- mk_path_out(os.path.join(self.case_dir, 'out'))
+ os.makedirs(os.path.join(self.case_dir, 'out'), exist_ok=True)
# generate lisflood results
out_path_ref = os.path.join(self.case_dir, 'reference', 'output_reference_split_'+type)
- self.out_path_run = os.path.join(self.case_dir, 'out', 'output_split_'+type)
+ self.out_path_run = self._mk_tracked_out(os.path.join(self.case_dir, 'out', 'output_split_'+type))
settings_file = os.path.join(self.case_dir, 'settings', 'mct_cold.xml')
settings = setoptions(settings_file,
opts_to_set = ['SplitRouting'],
@@ -173,7 +186,6 @@ def run_split(self, date_start, date_end, dtsec, dtsec_chan, type):
'DtSecChannel' : dtsec_chan, # single routing step
'PathOut': self.out_path_run})
- mk_path_out(self.out_path_run)
lisfloodexe(settings)
# set precision for the test
@@ -207,9 +219,9 @@ def run_split(self, date_start, date_end, dtsec, dtsec_chan, type):
comparator.compare_files(reference, output_tss)
def run_prerun_mct_test(self, date_start, date_end, dtsec, dtsec_chan, type):
- mk_path_out(os.path.join(self.case_dir, 'out'))
+ os.makedirs(os.path.join(self.case_dir, 'out'), exist_ok=True)
# generate lisflood results
- path_out_a = os.path.join(self.case_dir, 'out', 'prerunMCTON_'+type)
+ path_out_a = self._mk_tracked_out(os.path.join(self.case_dir, 'out', 'prerunMCTON_'+type))
self.out_path_run = path_out_a
settings_file = os.path.join(self.case_dir, 'settings', 'mct_prerun_test_only.xml')
settings = setoptions(settings_file,
@@ -225,10 +237,10 @@ def run_prerun_mct_test(self, date_start, date_end, dtsec, dtsec_chan, type):
nc_comparator = NetCDFComparator(settings.maskpath, array_equal=True)
tss_comparator = TSSComparator(array_equal=True)
- mk_path_out(self.out_path_run)
+ self._mk_tracked_out(self.out_path_run)
lisfloodexe(settings)
- path_out_b = os.path.join(self.case_dir, 'out', 'prerunMCTOFF_'+type)
+ path_out_b = self._mk_tracked_out(os.path.join(self.case_dir, 'out', 'prerunMCTOFF_'+type))
self.out_path_run = path_out_b
settings_file = os.path.join(self.case_dir, 'settings', 'mct_prerun_test_only.xml')
settings = setoptions(settings_file,
@@ -242,7 +254,7 @@ def run_prerun_mct_test(self, date_start, date_end, dtsec, dtsec_chan, type):
'PathOut': self.out_path_run})
- mk_path_out(self.out_path_run)
+ self._mk_tracked_out(self.out_path_run)
lisfloodexe(settings)
@@ -251,9 +263,12 @@ def run_prerun_mct_test(self, date_start, date_end, dtsec, dtsec_chan, type):
def teardown_method(self):
print('Cleaning directories')
- out_path = os.path.join(self.case_dir, 'out')
- if os.path.exists(out_path) and os.path.isdir(out_path):
- shutil.rmtree(out_path, ignore_errors=True)
+ # Remove only the folders this test created, leaving the shared 'out/'
+ # directory and any folders created by other tests untouched.
+ for path in getattr(self, '_created_out_dirs', []):
+ if os.path.exists(path) and os.path.isdir(path):
+ shutil.rmtree(path, ignore_errors=True)
+ self._created_out_dirs = []
# do_not_run
diff --git a/tests/test_mct_results_slow.py b/tests/test_mct_results_slow.py
index 03f7c397..15673731 100644
--- a/tests/test_mct_results_slow.py
+++ b/tests/test_mct_results_slow.py
@@ -18,10 +18,12 @@ class TestTSSResults():
case_dir = os.path.join(os.path.dirname(__file__), 'data', 'LF_ETRS89_UseCase')
def run_mct(self, date_start, date_end, dtsec, dtsec_chan, type):
- mk_path_out(os.path.join(self.case_dir, 'out'))
+ os.makedirs(os.path.join(self.case_dir, 'out'), exist_ok=True)
# generate lisflood results
out_path_ref = os.path.join(self.case_dir, 'reference', 'output_reference_mct_'+type)
- self.out_path_run = os.path.join(self.case_dir, 'out', 'output_mct_'+type)
+ # Capture the (worker-scoped under xdist) path from mk_path_out and use it
+ # for PathOut so the created dir and the LISFLOOD output dir match.
+ self.out_path_run = mk_path_out(os.path.join(self.case_dir, 'out', 'output_mct_'+type))
settings_file = os.path.join(self.case_dir, 'settings', 'mct_cold.xml')
if "_calib" in type:
opt_to_set = ['MCTRouting', 'TransLoss', 'simulateCalibrationPoints']
@@ -83,10 +85,10 @@ def run_mct(self, date_start, date_end, dtsec, dtsec_chan, type):
comparator.compare_files(reference, output_tss)
def run_mcts(self, date_start, date_end, dtsec, dtsec_chan, type):
- mk_path_out(os.path.join(self.case_dir, 'out'))
+ os.makedirs(os.path.join(self.case_dir, 'out'), exist_ok=True)
# generate lisflood results
out_path_ref = os.path.join(self.case_dir, 'reference', 'output_reference_mcts_'+type)
- self.out_path_run = os.path.join(self.case_dir, 'out', 'output_mcts_'+type)
+ self.out_path_run = mk_path_out(os.path.join(self.case_dir, 'out', 'output_mcts_'+type))
settings_file = os.path.join(self.case_dir, 'settings', 'mct_cold.xml')
if "_calib" in type:
opt_to_set = ['MCTRouting', 'SplitRouting', 'simulateCalibrationPoints']
@@ -155,10 +157,10 @@ def run_mcts(self, date_start, date_end, dtsec, dtsec_chan, type):
comparator.compare_files(reference, output_tss)
def run_kin(self, date_start, date_end, dtsec, dtsec_chan, type):
- mk_path_out(os.path.join(self.case_dir, 'out'))
+ os.makedirs(os.path.join(self.case_dir, 'out'), exist_ok=True)
# generate lisflood results
out_path_ref = os.path.join(self.case_dir, 'reference', 'output_reference_kin_'+type)
- self.out_path_run = os.path.join(self.case_dir, 'out', 'output_kin_'+type)
+ self.out_path_run = mk_path_out(os.path.join(self.case_dir, 'out', 'output_kin_'+type))
settings_file = os.path.join(self.case_dir, 'settings', 'mct_cold.xml')
settings = setoptions(settings_file,
opts_to_set=['TransLoss'],
@@ -210,10 +212,10 @@ def run_kin(self, date_start, date_end, dtsec, dtsec_chan, type):
comparator.compare_files(reference, output_tss)
def run_split(self, date_start, date_end, dtsec, dtsec_chan, type):
- mk_path_out(os.path.join(self.case_dir, 'out'))
+ os.makedirs(os.path.join(self.case_dir, 'out'), exist_ok=True)
# generate lisflood results
out_path_ref = os.path.join(self.case_dir, 'reference', 'output_reference_split_'+type)
- self.out_path_run = os.path.join(self.case_dir, 'out', 'output_split_'+type)
+ self.out_path_run = mk_path_out(os.path.join(self.case_dir, 'out', 'output_split_'+type))
settings_file = os.path.join(self.case_dir, 'settings', 'mct_cold.xml')
settings = setoptions(settings_file,
opts_to_set = ['SplitRouting'],
@@ -269,8 +271,10 @@ def run_split(self, date_start, date_end, dtsec, dtsec_chan, type):
def teardown_method(self):
print('Cleaning directories')
- out_path = os.path.join(self.case_dir, 'out')
- if os.path.exists(out_path) and os.path.isdir(out_path):
+ # Remove only the subfolder this test created, not the shared 'out/'
+ # directory, so tests running afterwards still find 'out/' in place.
+ out_path = getattr(self, 'out_path_run', None)
+ if out_path and os.path.exists(out_path) and os.path.isdir(out_path):
shutil.rmtree(out_path, ignore_errors=True)
diff --git a/tests/test_mct_warmstart.py b/tests/test_mct_warmstart.py
index ee958c29..19d1eef2 100644
--- a/tests/test_mct_warmstart.py
+++ b/tests/test_mct_warmstart.py
@@ -28,7 +28,7 @@
from lisflood.main import lisfloodexe
-from .test_utils import setoptions, mk_path_out
+from .test_utils import setoptions, mk_path_out, worker_out
class TestWarmStart():
@@ -110,7 +110,9 @@ def run_warmstart_by_dtsec(self, mct_case, dt_sec, dt_sec_channel, step_end, ste
check_every = 13 # steps
- self.path_out_reference = os.path.join(self.case_dir, 'out', 'longrun_reference{}'.format(dt_sec))
+ # mk_path_out returns the (per-xdist-worker) scoped path; use it for PathOut
+ # so the created dir and the LISFLOOD output dir stay in sync under parallel runs.
+ self.path_out_reference = mk_path_out(os.path.join(self.case_dir, 'out', 'longrun_reference{}'.format(dt_sec)))
if mct_case == 'mct':
opts_to_set = ['repStateMaps','TransLoss']
@@ -141,8 +143,7 @@ def run_warmstart_by_dtsec(self, mct_case, dt_sec, dt_sec_channel, step_end, ste
'DtSec': dt_sec,
'DtSecChannel': dt_sec_channel,
'TransSub': 0.3})
- # ** execute
- mk_path_out(self.path_out_reference)
+ # ** execute (path already created by mk_path_out above)
lisfloodexe(settings_longrun)
@@ -150,7 +151,7 @@ def run_warmstart_by_dtsec(self, mct_case, dt_sec, dt_sec_channel, step_end, ste
run_number = 1
cold_start_step_end = step_start
- self.path_out = os.path.join(self.case_dir, 'out', 'run{}_{}'.format(dt_sec, run_number))
+ self.path_out = mk_path_out(os.path.join(self.case_dir, 'out', 'run{}_{}'.format(dt_sec, run_number)))
settings_coldstart = setoptions(self.settings_files['cold'],
opts_to_set=opts_to_set,
@@ -163,8 +164,7 @@ def run_warmstart_by_dtsec(self, mct_case, dt_sec, dt_sec_channel, step_end, ste
'DtSec': dt_sec,
'DtSecChannel': dt_sec_channel,
'TransSub': 0.3})
- # ** execute
- mk_path_out(self.path_out)
+ # ** execute (path already created by mk_path_out above)
lisfloodexe(settings_coldstart)
# warm run (2. single step warm start/stop with initial conditions from previous run)
@@ -188,7 +188,7 @@ def run_warmstart_by_dtsec(self, mct_case, dt_sec, dt_sec_channel, step_end, ste
while warm_step_start <= step_limit:
run_number += 1
path_init = prev_settings.output_dir
- self.path_out = (os.path.join(self.case_dir, 'out', 'run{}_{}'.format(dt_sec, run_number)))
+ self.path_out = mk_path_out(os.path.join(self.case_dir, 'out', 'run{}_{}'.format(dt_sec, run_number)))
settings_warmstart = setoptions(self.settings_files[warm_settings_file],
opts_to_set=opts_to_set,
@@ -203,8 +203,7 @@ def run_warmstart_by_dtsec(self, mct_case, dt_sec, dt_sec_channel, step_end, ste
'DtSec': dt_sec,
'DtSecChannel': dt_sec_channel,
'TransSub': 0.3})
- # ** execute
- mk_path_out(self.path_out)
+ # ** execute (path already created by mk_path_out above)
lisfloodexe(settings_warmstart)
# checking values at current timestep (using datetime)
@@ -232,7 +231,10 @@ def run_warmstart_by_dtsec(self, mct_case, dt_sec, dt_sec_channel, step_end, ste
def teardown_method(self):
print('Cleaning directories')
- folders_list = glob.glob(os.path.join(os.path.dirname(__file__), self.case_dir, 'out/run*')) + \
- glob.glob(os.path.join(os.path.dirname(__file__), self.case_dir, 'out/longrun_reference*'))
+ # Clean only this worker's output subtree (worker_out() -> 'out' serially,
+ # 'out/' under xdist) to avoid removing another worker's dirs.
+ base = os.path.join(os.path.dirname(__file__), self.case_dir, worker_out())
+ folders_list = glob.glob(os.path.join(base, 'run*')) + \
+ glob.glob(os.path.join(base, 'longrun_reference*'))
for folder in folders_list:
shutil.rmtree(folder)
\ No newline at end of file
diff --git a/tests/test_mct_warmstart_slow.py b/tests/test_mct_warmstart_slow.py
index baa6b3c8..c3c86428 100644
--- a/tests/test_mct_warmstart_slow.py
+++ b/tests/test_mct_warmstart_slow.py
@@ -28,7 +28,7 @@
from lisflood.main import lisfloodexe
-from .test_utils import setoptions, mk_path_out
+from .test_utils import setoptions, mk_path_out, worker_out
@pytest.mark.slow
@@ -115,11 +115,11 @@ def test_mct_warmstart_6h_withcalibpoints(self):
def run_warmstart_by_dtsec(self, mct_case, dt_sec, dt_sec_channel, step_end, step_start, calendar_day_start,report_steps='1..9999'):
- mk_path_out(os.path.join(self.case_dir, 'out'))
+ os.makedirs(os.path.join(self.case_dir, 'out'), exist_ok=True)
check_every = 13 # steps
- self.path_out_reference = os.path.join(self.case_dir, 'out', 'longrun_reference{}'.format(dt_sec))
+ self.path_out_reference = mk_path_out(os.path.join(self.case_dir, 'out', 'longrun_reference{}'.format(dt_sec)))
if mct_case == 'mct':
opts_to_set = ['repStateMaps','repDischargeMaps',
@@ -168,8 +168,7 @@ def run_warmstart_by_dtsec(self, mct_case, dt_sec, dt_sec_channel, step_end, ste
'DtSec': dt_sec,
'DtSecChannel': dt_sec_channel,
'TransSub': 0.3})
- # ** execute
- mk_path_out(self.path_out_reference)
+ # ** execute (path already created by mk_path_out above)
lisfloodexe(settings_longrun)
@@ -177,7 +176,7 @@ def run_warmstart_by_dtsec(self, mct_case, dt_sec, dt_sec_channel, step_end, ste
run_number = 1
cold_start_step_end = step_start
- self.path_out = os.path.join(self.case_dir, 'out', 'run{}_{}'.format(dt_sec, run_number))
+ self.path_out = mk_path_out(os.path.join(self.case_dir, 'out', 'run{}_{}'.format(dt_sec, run_number)))
settings_coldstart = setoptions(self.settings_files['cold'],
opts_to_set=opts_to_set,
@@ -190,8 +189,7 @@ def run_warmstart_by_dtsec(self, mct_case, dt_sec, dt_sec_channel, step_end, ste
'DtSec': dt_sec,
'DtSecChannel': dt_sec_channel,
'TransSub': 0.3})
- # ** execute
- mk_path_out(self.path_out)
+ # ** execute (path already created by mk_path_out above)
lisfloodexe(settings_coldstart)
# warm run (2. single step warm start/stop with initial conditions from previous run)
@@ -209,7 +207,7 @@ def run_warmstart_by_dtsec(self, mct_case, dt_sec, dt_sec_channel, step_end, ste
while warm_step_start <= step_limit:
run_number += 1
path_init = prev_settings.output_dir
- self.path_out = (os.path.join(self.case_dir, 'out', 'run{}_{}'.format(dt_sec, run_number)))
+ self.path_out = mk_path_out(os.path.join(self.case_dir, 'out', 'run{}_{}'.format(dt_sec, run_number)))
settings_warmstart = setoptions(self.settings_files['warm'],
opts_to_set=opts_to_set,
@@ -224,8 +222,7 @@ def run_warmstart_by_dtsec(self, mct_case, dt_sec, dt_sec_channel, step_end, ste
'DtSec': dt_sec,
'DtSecChannel': dt_sec_channel,
'TransSub': 0.3})
- # ** execute
- mk_path_out(self.path_out)
+ # ** execute (path already created by mk_path_out above)
lisfloodexe(settings_warmstart)
# checking values at current timestep (using datetime)
@@ -245,8 +242,11 @@ def run_warmstart_by_dtsec(self, mct_case, dt_sec, dt_sec_channel, step_end, ste
def teardown_method(self):
print('Cleaning directories')
- folders_list = glob.glob(os.path.join(os.path.dirname(__file__), self.case_dir, 'out/run*')) + \
- glob.glob(os.path.join(os.path.dirname(__file__), self.case_dir, 'out/longrun_reference*'))
+ # Clean only this worker's output subtree (worker_out() -> 'out' serially,
+ # 'out/' under xdist) to avoid removing another worker's dirs.
+ base = os.path.join(os.path.dirname(__file__), self.case_dir, worker_out())
+ folders_list = glob.glob(os.path.join(base, 'run*')) + \
+ glob.glob(os.path.join(base, 'longrun_reference*'))
for folder in folders_list:
shutil.rmtree(folder)
diff --git a/tests/test_options.py b/tests/test_options.py
index 451dc2af..00949042 100644
--- a/tests/test_options.py
+++ b/tests/test_options.py
@@ -104,7 +104,7 @@ def test_reservoirs_only(self, mocker):
mock_api.side_effect = self.dummyloadmap
mocker.patch('lisflood.hydrological_modules.reservoir.loadmap', new=mock_api)
lisfloodexe(settings)
- calls = [call('ReservoirSites'), call('ReservoirSites', pcr=True), call('UpAreaTrans'),
+ calls = [call('ReservoirSites'), call('UpAreaTrans'),
call('ReservoirFloodOutflowFactor'), call('ReservoirInitialFill')]
lisflood.hydrological_modules.reservoir.loadmap.assert_has_calls(calls)
diff --git a/tests/test_output_threads.py b/tests/test_output_threads.py
new file mode 100644
index 00000000..29d56b70
--- /dev/null
+++ b/tests/test_output_threads.py
@@ -0,0 +1,178 @@
+"""Task 3 (output writing) tests: opt-in threaded output writer ('outputThreads').
+
+Covers:
+- threaded writer produces byte-identical outputs to the synchronous writer
+ (array_equal comparison, so any float/NaN mismatch fails);
+- the OutputMapsChunks > report-steps flush path (partial chunk written at the
+ end of the run) is bit-identical between the two writer paths.
+"""
+import os
+import shutil
+import pytest
+
+from lisfloodutilities.compare.nc import NetCDFComparator
+
+from lisflood.main import lisfloodexe
+
+from .test_utils import setoptions, mk_path_out, ETRS89TestCase, worker_out
+
+
+class TestOutputThreads(ETRS89TestCase):
+ case_dir = os.path.join(os.path.dirname(__file__), 'data', 'LF_ETRS89_UseCase')
+ settings_file = os.path.join(case_dir, 'settings', 'full.xml')
+ # Output dirs isolated per xdist worker (worker_out -> 'out//a') so
+ # concurrent test files do not clobber each other's shared 'out/a'.
+ out_dir_sync = os.path.join(case_dir, worker_out('a'))
+ out_dir_threads = os.path.join(case_dir, worker_out('b'))
+ out_dir_sync_flush = os.path.join(case_dir, worker_out('c'))
+ out_dir_threads_flush = os.path.join(case_dir, worker_out('d'))
+ pathout_sync = '$(PathRoot)/' + worker_out('a')
+ pathout_threads = '$(PathRoot)/' + worker_out('b')
+ pathout_sync_flush = '$(PathRoot)/' + worker_out('c')
+ pathout_threads_flush = '$(PathRoot)/' + worker_out('d')
+
+ def _run(self, pathout, opts_to_set=None, **extra_vars):
+ """Run LISFLOOD on the full.xml usecase with the given PathOut and vars."""
+ settings = setoptions(self.settings_file, opts_to_set=opts_to_set,
+ vars_to_set={'PathOut': pathout, **extra_vars})
+ mk_path_out(os.path.join(self.case_dir, pathout.replace('$(PathRoot)/', '')))
+ lisfloodexe(settings)
+ return settings
+
+ def teardown_method(self):
+ for out_dir in (self.out_dir_sync, self.out_dir_threads,
+ self.out_dir_sync_flush, self.out_dir_threads_flush):
+ shutil.rmtree(out_dir, ignore_errors=True)
+
+ def test_threaded_output_bit_identical(self):
+ """Threaded writer must produce byte-identical outputs vs synchronous."""
+ # repDischargeMaps etc. on: report options gate which map outputs exist,
+ # so enable at least two map groups to genuinely exercise parallel writes.
+ map_opts = ['repDischargeMaps', 'repStateMaps']
+ settings_sync = self._run(self.pathout_sync, opts_to_set=map_opts,
+ StepStart='30/07/2016 06:00',
+ StepEnd='05/08/2016 06:00')
+ out_sync = settings_sync.output_dir
+
+ settings_thr = self._run(self.pathout_threads,
+ opts_to_set=map_opts + ['outputThreads'],
+ StepStart='30/07/2016 06:00',
+ StepEnd='05/08/2016 06:00')
+ out_thr = settings_thr.output_dir
+
+ comparator = NetCDFComparator(settings_thr.maskpath, array_equal=True)
+ comparator.compare_dirs(out_sync, out_thr)
+
+ def test_chunk_flush_partial_chunk_bit_identical(self):
+ """OutputMapsChunks > number of report steps: the end-of-run flush of the
+ partial chunk must be bit-identical between threaded and sync writers."""
+ map_opts = ['repDischargeMaps', 'repStateMaps']
+ settings_sync = self._run(self.pathout_sync_flush, opts_to_set=map_opts,
+ StepStart='30/07/2016 06:00',
+ StepEnd='05/08/2016 06:00', OutputMapsChunks='100')
+ out_sync = settings_sync.output_dir
+
+ settings_thr = self._run(self.pathout_threads_flush,
+ opts_to_set=map_opts + ['outputThreads'],
+ StepStart='30/07/2016 06:00',
+ StepEnd='05/08/2016 06:00', OutputMapsChunks='100')
+ out_thr = settings_thr.output_dir
+
+ comparator = NetCDFComparator(settings_thr.maskpath, array_equal=True)
+ comparator.compare_dirs(out_sync, out_thr)
+
+
+class TestPCRasterOutputThreads(ETRS89TestCase):
+ """PCRaster map output (netCDF off) must be identical with and without
+ 'outputThreads': same file names (step suffix taken from the staged step,
+ not from the live model clock) and same values."""
+ case_dir = os.path.join(os.path.dirname(__file__), 'data', 'LF_ETRS89_UseCase')
+ settings_file = os.path.join(case_dir, 'settings', 'full.xml')
+ out_sync = worker_out('pcr_a')
+ out_thr = worker_out('pcr_b')
+
+ @classmethod
+ def _short_names(cls):
+ """PCRaster map names must be <= 8 chars without extension: override every
+ map output name that is too long or carries an extension."""
+ import re
+ with open(cls.settings_file) as fh:
+ xml = fh.read()
+ names = {}
+ pattern = r' 8 or '.' in value):
+ names[name] = '$(PathOut)/m%03d' % i
+ return names
+
+ def _run(self, out, threaded, step_end='03/08/2016 06:00'):
+ vars_to_set = {'PathOut': '$(PathRoot)/' + out, 'DtSec': '86400',
+ 'StepStart': '29/07/2016 06:00', 'StepEnd': step_end}
+ vars_to_set.update(self._short_names())
+ opts = ['repDischargeMaps', 'repStateMaps', 'repEndMaps', 'repWIndex', 'repWaterUse']
+ if threaded:
+ opts.append('outputThreads')
+ settings = setoptions(self.settings_file, opts_to_set=opts,
+ opts_to_unset=['writeNetcdf', 'writeNetcdfStack'],
+ vars_to_set=vars_to_set)
+ path = os.path.join(self.case_dir, out)
+ shutil.rmtree(path, ignore_errors=True)
+ mk_path_out(path)
+ lisfloodexe(settings)
+ return path
+
+ @staticmethod
+ def _map_files(directory):
+ return sorted(f for f in os.listdir(directory)
+ if f[-4:].isdigit() or '.' in f and f.split('.')[-1].isdigit())
+
+ @classmethod
+ def _assert_dirs_equal(cls, sync_dir, thr_dir):
+ import numpy as np
+ from pcraster import readmap, pcr2numpy
+ maps_sync, maps_thr = cls._map_files(sync_dir), cls._map_files(thr_dir)
+ assert maps_sync, 'no PCRaster step maps produced'
+ assert maps_sync == maps_thr, set(maps_sync) ^ set(maps_thr)
+ for fname in maps_sync:
+ a = pcr2numpy(readmap(os.path.join(sync_dir, fname)), -9999)
+ b = pcr2numpy(readmap(os.path.join(thr_dir, fname)), -9999)
+ assert np.array_equal(a, b), f'{fname} differs between sync and threaded'
+
+ def teardown_method(self):
+ for out in (self.out_sync, self.out_thr):
+ shutil.rmtree(os.path.join(self.case_dir, out), ignore_errors=True)
+
+ def test_pcraster_threaded_matches_sync(self):
+ import numpy as np
+ from pcraster import readmap, pcr2numpy
+ sync_dir = self._run(self.out_sync, threaded=False)
+ thr_dir = self._run(self.out_thr, threaded=True)
+ maps_sync = sorted(f for f in os.listdir(sync_dir) if f[-4:].isdigit() or '.' in f and f.split('.')[-1].isdigit())
+ maps_thr = sorted(f for f in os.listdir(thr_dir) if f[-4:].isdigit() or '.' in f and f.split('.')[-1].isdigit())
+ assert maps_sync, 'no PCRaster step maps produced'
+ assert maps_sync == maps_thr, set(maps_sync) ^ set(maps_thr)
+ # Every reported step (first and last included) must be present for a per-step map.
+ assert any(f.startswith('dis') and f.endswith('.056') for f in maps_sync)
+ suffixes = {f[-3:] for f in maps_sync}
+ assert {'056', '057', '058', '059', '060', '061'} <= suffixes
+ # Monthly maps exist only at the month-end step (31/07/2016 = step 058),
+ # end-of-run maps only at the last step (061).
+ assert any(f.startswith('IrSh') and f.endswith('.058') for f in maps_sync)
+ assert not any(f.startswith('IrSh') and not f.endswith('.058') for f in maps_sync)
+ assert any(f.startswith('m006') and f.endswith('.061') for f in maps_sync)
+ for fname in maps_sync:
+ a = pcr2numpy(readmap(os.path.join(sync_dir, fname)), -9999)
+ b = pcr2numpy(readmap(os.path.join(thr_dir, fname)), -9999)
+ assert np.array_equal(a, b), f'{fname} differs between sync and threaded'
+
+ @pytest.mark.slow
+ def test_pcraster_threaded_stress_repeated(self):
+ """Longer run, threaded 3 times: every repetition must equal the
+ synchronous output (looks for nondeterministic races between workers
+ reporting concurrently)."""
+ step_end = '12/08/2016 06:00'
+ sync_dir = self._run(self.out_sync, threaded=False, step_end=step_end)
+ for _ in range(3):
+ thr_dir = self._run(self.out_thr, threaded=True, step_end=step_end)
+ self._assert_dirs_equal(sync_dir, thr_dir)
diff --git a/tests/test_parallelization.py b/tests/test_parallelization.py
new file mode 100644
index 00000000..452d48da
--- /dev/null
+++ b/tests/test_parallelization.py
@@ -0,0 +1,406 @@
+"""
+Unit tests for lisflood.global_modules.parallelization.
+
+Focus is the numba small‑domain auto‑tune decision that the performance change
+adds (explicit setting wins; below the pixel threshold run serial; at/above the
+threshold or unknown size use all cores). A couple of sentinel cases cover the
+"0 / unparseable -> all cores" fallback path.
+"""
+
+import os
+import pytest
+
+from lisflood.global_modules import parallelization as par
+
+
+@pytest.fixture(autouse=True)
+def reset_state(monkeypatch):
+ """
+ Ensure each test starts with a clean global state.
+ """
+ monkeypatch.setattr(par, "_effective", None, raising=False)
+ monkeypatch.setattr(par, "_blas_limiter", None, raising=False)
+ yield
+ # clean‑up after the test (in case a limiter was created)
+ monkeypatch.setattr(par, "_effective", None, raising=False)
+ monkeypatch.setattr(par, "_blas_limiter", None, raising=False)
+
+
+class TestEffectiveThreadCounts:
+ """Check that ``configure_parallelism`` reads all *numCPU* settings,
+ applies the limits, and stores the real numbers in the ``effective``
+ dictionary returned by the function."""
+
+ def _patch_host(self, monkeypatch, value):
+ """Force ``_host_cpu_count`` to return a deterministic core count."""
+ monkeypatch.setattr(par, "_host_cpu_count", lambda: value)
+
+ def test_explicit_settings_are_reflected(self, monkeypatch):
+ """All user‑specified thread numbers must appear unchanged in the
+ effective result (or be capped by the library’s hard limit)."""
+
+ # 1. deterministic host size – 8 logical CPUs
+ self._patch_host(monkeypatch, 8)
+
+ # 2. binding with explicit values for every pool
+ binding = {
+ "numCPUs_parallelNumba": "4",
+ "numCPUs_BLAS": "2",
+ "numCPUs_soilInit": "5",
+ }
+
+ eff = par.configure_parallelism(binding, num_pixels=100)
+
+ # ---- numba -------------------------------------------------
+ # If Numba is available we expect the value we set (capped by the
+ # library’s own maximum). When Numba is not present the entry is
+ # ``None`` – the test still passes.
+ if par._HAVE_NUMBA_THREADCTL:
+ # The library may silently cap the request, so we compare with the
+ # *actual* value stored in the effective dict.
+ assert eff["numba"] == par._numba_get_num_threads()
+ # The requested value must be ≥ the effective one and not exceed the
+ # hard limit.
+ assert 1 <= eff["numba"] <= 4
+ else:
+ assert eff["numba"] is None
+
+ # ---- BLAS --------------------------------------------------
+ assert eff["blas"] == 2 # exact value we asked for
+
+ # ---- soil‑init ---------------------------------------------
+ assert eff["soilinit"] == 5 # explicit value is taken verbatim
+
+ # ---- host --------------------------------------------------
+ assert eff["host"] == 8
+
+ def test_blas_all_cores_sentinel(self, monkeypatch):
+ """When the BLAS setting is a sentinel (0 / empty / “all”) the
+ effective BLAS thread count must expand to the host core count."""
+ # Force a known host size (e.g. 16 cores)
+ self._patch_host(monkeypatch, 16)
+
+ binding = {"numCPUs_BLAS": "0"} # sentinel meaning “all cores”
+
+ eff = par.configure_parallelism(binding, num_pixels=None)
+
+ # The module should translate the sentinel to the host count.
+ assert eff["blas"] == 16
+ # The host entry must also report the same value we patched.
+ assert eff["host"] == 16
+
+ def test_auto_and_missing_soilinit(self, monkeypatch):
+ """When the user leaves ``numCPUs_soilInit`` unset the function must
+ fall back to the (patched) host core count. The same holds for the
+ auto‑tune path of Numba."""
+
+ # 1. Force a known host size (12 logical CPUs)
+ self._patch_host(monkeypatch, 12)
+
+ # 2. Binding that only defines the *auto* value for Numba
+ binding = {
+ "numCPUs_parallelNumba": "0", # auto‑tune
+ # No entry for ``numCPUs_soilInit`` → fallback to host
+ }
+
+ # Small domain (below the threshold) forces serial Numba
+ eff = par.configure_parallelism(
+ binding,
+ num_pixels=par.NUMBA_AUTO_MIN_PIXELS - 10
+ )
+
+ # ---- numba -------------------------------------------------
+ # The auto‑tune logic should have forced a single thread.
+ if par._HAVE_NUMBA_THREADCTL:
+ assert eff["numba"] == 1
+ else:
+ assert eff["numba"] is None
+
+ # ---- soil‑init ---------------------------------------------
+ # Missing setting → default is now 0 (all cores) → host core count
+ assert eff["soilinit"] == 12
+
+ # ---- host --------------------------------------------------
+ assert eff["host"] == 12
+
+ def test_soilinit_auto_uses_all_cores_when_host_unknown(self, monkeypatch):
+ """``numCPUs_soilInit = 0`` (or ``"auto"``) must resolve to the host
+ core count even when the caller supplies ``real_cpu_count=None``."""
+
+ # Patch host count to a known value (6)
+ self._patch_host(monkeypatch, 6)
+
+ binding = {"numCPUs_soilInit": "0"} # auto / all cores
+
+ # Resolve directly via the helper – it should return the host size.
+ workers = par.resolve_soilinit_workers(binding, real_cpu_count=None)
+
+ assert workers == 6
+
+class TestEarthkitThreads:
+ """resolve_earthkit_threads + the RAYON_NUM_THREADS env-var application in
+ configure_parallelism. The earthkit accumulation pool (rayon, native) must
+ be capped by 'numCPUs_earthkit' so it stops oversubscribing against the
+ routing kernel. Default is 1; 0/"all"/"auto"/absent means uncapped (None).
+ This is a scheduling-only knob; it does not change any numerical result."""
+
+ def test_default_is_single_threaded(self):
+ assert par.resolve_earthkit_threads({}) == 1
+
+ def test_absent_binding_is_single_threaded(self):
+ assert par.resolve_earthkit_threads(None) == 1
+
+ def test_explicit_positive_value_caps(self):
+ assert par.resolve_earthkit_threads({"numCPUs_earthkit": "4"}) == 4
+
+ def test_zero_means_uncapped(self):
+ assert par.resolve_earthkit_threads({"numCPUs_earthkit": "0"}) is None
+
+ def test_all_means_uncapped(self):
+ assert par.resolve_earthkit_threads({"numCPUs_earthkit": "all"}) is None
+
+ def test_auto_is_single_threaded(self):
+ assert par.resolve_earthkit_threads({"numCPUs_earthkit": "auto"}) == 1
+
+ def test_configure_default_sets_rayon_to_one(self, monkeypatch):
+ monkeypatch.setattr(par, "_host_cpu_count", lambda: 16)
+ monkeypatch.delenv("RAYON_NUM_THREADS", raising=False)
+ eff = par.configure_parallelism({}, num_pixels=100)
+ assert eff["earthkit"] == 1
+ assert os.environ["RAYON_NUM_THREADS"] == "1"
+
+ def test_configure_explicit_value_is_reflected_and_applied(self, monkeypatch):
+ monkeypatch.setattr(par, "_host_cpu_count", lambda: 16)
+ monkeypatch.delenv("RAYON_NUM_THREADS", raising=False)
+ eff = par.configure_parallelism(
+ {"numCPUs_earthkit": "4"}, num_pixels=100)
+ assert eff["earthkit"] == 4
+ assert os.environ["RAYON_NUM_THREADS"] == "4"
+
+ def test_configure_caps_request_to_host(self, monkeypatch):
+ monkeypatch.setattr(par, "_host_cpu_count", lambda: 8)
+ monkeypatch.delenv("RAYON_NUM_THREADS", raising=False)
+ eff = par.configure_parallelism(
+ {"numCPUs_earthkit": "32"}, num_pixels=100)
+ assert eff["earthkit"] == 8
+ assert os.environ["RAYON_NUM_THREADS"] == "8"
+
+ def test_configure_uncapped_leaves_env_untouched(self, monkeypatch):
+ # 0/"all" => uncapped: do not overwrite an operator-set RAYON value.
+ monkeypatch.setattr(par, "_host_cpu_count", lambda: 16)
+ monkeypatch.setenv("RAYON_NUM_THREADS", "7")
+ eff = par.configure_parallelism(
+ {"numCPUs_earthkit": "0"}, num_pixels=100)
+ assert eff["earthkit"] is None
+ assert os.environ["RAYON_NUM_THREADS"] == "7"
+
+
+class TestNumbaAutoTune:
+ """resolve_numba_threads: the core decision the performance change adds.
+ Explicit setting wins; otherwise auto-tune on domain size."""
+
+ def test_explicit_value_wins_regardless_of_domain_size(self):
+ binding = {"numCPUs_parallelNumba": "4"}
+ assert par.resolve_numba_threads(binding, num_pixels=10) == 4
+ assert par.resolve_numba_threads(binding, num_pixels=10_000_000) == 4
+
+ def test_auto_runs_serial_below_threshold(self):
+ binding = {"numCPUs_parallelNumba": "0"} # auto
+ assert par.resolve_numba_threads(
+ binding, num_pixels=par.NUMBA_AUTO_MIN_PIXELS - 1) == 1
+
+ def test_auto_uses_all_cores_at_and_above_threshold(self):
+ binding = {"numCPUs_parallelNumba": "0"}
+ # boundary is exclusive: exactly at threshold is NOT "small"
+ assert par.resolve_numba_threads(
+ binding, num_pixels=par.NUMBA_AUTO_MIN_PIXELS) is None
+ assert par.resolve_numba_threads(
+ binding, num_pixels=par.NUMBA_AUTO_MIN_PIXELS + 1) is None
+
+ def test_auto_with_unknown_size_keeps_historical_all_cores(self):
+ # when domain size is unavailable we must not accidentally force serial
+ binding = {"numCPUs_parallelNumba": "0"}
+ assert par.resolve_numba_threads(binding, num_pixels=None) is None
+
+ def test_missing_setting_behaves_as_auto(self):
+ # older settings files without the key must still auto‑tune, not crash
+ assert par.resolve_numba_threads({}, num_pixels=10) == 1
+ assert par.resolve_numba_threads({}, num_pixels=None) is None
+
+
+class TestAllCoresSentinel:
+ """The '0 / unparseable -> use all cores (None)' fallback that the
+ auto‑tune relies on. One nominal and one degenerate case is enough."""
+
+ @pytest.mark.parametrize(
+ "value,expected",
+ [("2", 2), ("0", None), ("abc", None)]
+ )
+ def test_parse_thread_count(self, value, expected):
+ assert par._parse_thread_count(value) == expected
+
+class TestPoolThreadsAutoDefault:
+ """resolve_pool_threads (BLAS): flat serial (1) default on all
+ domain sizes when the setting is absent or "auto"; explicit values always
+ honored verbatim.
+
+ Measured basis: the BLAS per-call arrays are small and the pool
+ nests inside (and contends with) the numba routing pool, so a full-core pool
+ only oversubscribes the node. A single-threaded default is best on both
+ small catchments and large/global domains, while staying overridable
+ (calibration forces "1"; "0"/"all" forces all cores)."""
+
+ SMALL = par.NUMBA_AUTO_MIN_PIXELS - 1
+ LARGE = par.NUMBA_AUTO_MIN_PIXELS
+
+ def test_explicit_value_wins_regardless_of_domain_size(self):
+ for pix in (self.SMALL, self.LARGE, 10_000_000):
+ assert par.resolve_pool_threads(
+ {"numCPUs_BLAS": "2"}, "numCPUs_BLAS", num_pixels=pix) == 2
+
+ def test_explicit_one_forces_serial_even_on_large_domain(self):
+ # the calibration escape hatch: force single-core on a big domain
+ assert par.resolve_pool_threads(
+ {"numCPUs_BLAS": "1"}, "numCPUs_BLAS",
+ num_pixels=10_000_000) == 1
+
+ def test_explicit_zero_forces_all_cores_even_on_small_domain(self):
+ assert par.resolve_pool_threads(
+ {"numCPUs_BLAS": "0"}, "numCPUs_BLAS", num_pixels=self.SMALL) is None
+ assert par.resolve_pool_threads(
+ {"numCPUs_BLAS": "all"}, "numCPUs_BLAS", num_pixels=self.SMALL) is None
+
+ def test_auto_token_is_flat_one(self):
+ # flat serial default: "auto" -> 1 on BOTH small and large domains
+ assert par.resolve_pool_threads(
+ {"numCPUs_BLAS": "auto"}, "numCPUs_BLAS", num_pixels=self.SMALL) == 1
+ assert par.resolve_pool_threads(
+ {"numCPUs_BLAS": "auto"}, "numCPUs_BLAS", num_pixels=self.LARGE) == 1
+
+ def test_missing_setting_is_flat_one(self):
+ # old settings files without the key: serial on all domain sizes
+ assert par.resolve_pool_threads({}, "numCPUs_BLAS",
+ num_pixels=self.SMALL) == 1
+ assert par.resolve_pool_threads({}, "numCPUs_BLAS",
+ num_pixels=self.LARGE) == 1
+
+ def test_unknown_domain_size_stays_serial(self):
+ # The BLAS auto default is a flat serial 1 regardless of size.
+ assert par.resolve_pool_threads({}, "numCPUs_BLAS", num_pixels=None) == 1
+ assert par.resolve_pool_threads(
+ {"numCPUs_BLAS": "auto"}, "numCPUs_BLAS", num_pixels=None) == 1
+
+ def test_configure_parallelism_large_domain_stays_serial(self, monkeypatch):
+ monkeypatch.setattr(par, "_host_cpu_count", lambda: 16)
+ # no BLAS key -> auto; flat-1 default even on a large domain
+ eff = par.configure_parallelism({}, num_pixels=par.NUMBA_AUTO_MIN_PIXELS)
+ assert eff["blas"] == 1
+
+ def test_configure_parallelism_small_domain_stays_serial(self, monkeypatch):
+ monkeypatch.setattr(par, "_host_cpu_count", lambda: 16)
+ eff = par.configure_parallelism({}, num_pixels=100)
+ assert eff["blas"] == 1
+
+
+class TestRoutingNumbaResolver:
+ """resolve_routing_numba_threads: the new routing-substep cap policy.
+ auto/absent -> clamp(round(host/2), 8, 16); all/0 -> None (all cores);
+ N -> N; malformed -> auto policy. Scheduling-only; never changes results."""
+
+ @pytest.mark.parametrize(
+ "host,expected",
+ [(16, 8), (28, 14), (56, 16), (112, 16)],
+ )
+ def test_auto_token_is_clamped_half_host(self, host, expected):
+ assert par.resolve_routing_numba_threads(
+ {"numCPUs_routingNumba": "auto"}, host=host) == expected
+ # absent key resolves to the same auto policy
+ assert par.resolve_routing_numba_threads({}, host=host) == expected
+
+ def test_small_host_hits_clamp_floor(self):
+ # host 4 -> round(2) = 2 -> floored to 8 by the resolver. The host-level
+ # cap (never exceed host) is applied at apply time in
+ # configure_parallelism, asserted in TestRoutingNumbaConfigure.
+ assert par.resolve_routing_numba_threads(
+ {"numCPUs_routingNumba": "auto"}, host=4) == 8
+
+ def test_malformed_token_falls_back_to_auto(self):
+ assert par.resolve_routing_numba_threads(
+ {"numCPUs_routingNumba": "abc"}, host=28) == 14
+
+
+class TestRoutingNumbaConfigure:
+ """configure_parallelism: apply-time clamp and publication of the routing
+ cap, plus the get_routing_numba_threads / get_land_numba_threads accessors.
+ The applied cap is min(raw, NUMBA_NUM_THREADS, host, land_count) so it never
+ raises the pool above the land-phase count."""
+
+ def _patch_host(self, monkeypatch, value):
+ monkeypatch.setattr(par, "_host_cpu_count", lambda: value)
+
+ def _expected_applied(self, raw, host, land):
+ bounds = [raw, host]
+ if land is not None:
+ bounds.append(land)
+ if par._HAVE_NUMBA_THREADCTL:
+ bounds.append(par._numba_config.NUMBA_NUM_THREADS)
+ return min(bounds)
+
+ @pytest.mark.parametrize("host,raw", [(16, 8), (28, 14), (56, 16)])
+ def test_large_domain_applies_clamped_cap(self, monkeypatch, host, raw):
+ self._patch_host(monkeypatch, host)
+ # large domain -> land numba is all cores (effective_numba == host when
+ # numba is available; None otherwise).
+ eff = par.configure_parallelism({}, num_pixels=par.NUMBA_AUTO_MIN_PIXELS)
+ land = eff["numba"]
+ assert eff["routing_numba"] == self._expected_applied(raw, host, land)
+ # accessors echo the published values
+ assert par.get_routing_numba_threads() == eff["routing_numba"]
+ assert par.get_land_numba_threads() == eff["numba"]
+
+ def test_all_token_leaves_pool_uncapped(self, monkeypatch):
+ self._patch_host(monkeypatch, 56)
+ eff = par.configure_parallelism(
+ {"numCPUs_routingNumba": "all"}, num_pixels=par.NUMBA_AUTO_MIN_PIXELS)
+ assert eff["routing_numba"] is None
+ assert par.get_routing_numba_threads() is None
+
+ def test_explicit_integer_is_applied(self, monkeypatch):
+ self._patch_host(monkeypatch, 56)
+ eff = par.configure_parallelism(
+ {"numCPUs_routingNumba": "5"}, num_pixels=par.NUMBA_AUTO_MIN_PIXELS)
+ land = eff["numba"]
+ assert eff["routing_numba"] == self._expected_applied(5, 56, land)
+
+ def test_small_domain_cap_is_noop(self, monkeypatch):
+ # Small domain: the land policy chooses serial (1) when numba thread
+ # control is available, so the routing cap clamps to 1 (a no-op window).
+ self._patch_host(monkeypatch, 56)
+ eff = par.configure_parallelism({}, num_pixels=100)
+ if par._HAVE_NUMBA_THREADCTL:
+ assert eff["numba"] == 1
+ assert eff["routing_numba"] == 1
+ else:
+ # numba unavailable: land count is None, cap clamped to host/limit.
+ assert eff["routing_numba"] == min(16, 56)
+
+
+class TestSetNumbaThreadsGuard:
+ """Proves the land count is restored after the routing-substep window
+ closes (the cap is scoped, not global) — the behavioural invariant the
+ try/finally in Lisflood_dynamic relies on."""
+
+ def test_land_count_restored_after_routing_window(self):
+ # Simulate the scoped window: land -> routing_k -> land, and assert the
+ # numba-reported count returns to the land count (the cap is scoped).
+ if not par._HAVE_NUMBA_THREADCTL:
+ pytest.skip("numba thread control unavailable")
+ hard = par._numba_config.NUMBA_NUM_THREADS
+ land = min(2, hard)
+ routing_k = 1
+ par.set_numba_threads(land)
+ assert par._numba_get_num_threads() == land
+ par.set_numba_threads(routing_k)
+ assert par._numba_get_num_threads() == routing_k
+ par.set_numba_threads(land) # restore (the finally branch)
+ assert par._numba_get_num_threads() == land
diff --git a/tests/test_reported_maps.py b/tests/test_reported_maps.py
index 66bfb609..b1270ef8 100644
--- a/tests/test_reported_maps.py
+++ b/tests/test_reported_maps.py
@@ -15,7 +15,8 @@
class TestReportedMaps():
case_dir = os.path.join(os.path.dirname(__file__), 'data', 'LF_ETRS89_UseCase')
- mk_path_out(os.path.join(case_dir, 'out'))
+ # Ensure the shared 'out/' exists without wiping it (would clobber other tests).
+ os.makedirs(os.path.join(case_dir, 'out'), exist_ok=True)
settings_files = {
# full.xml of LF_ETRS89_UseCase has simulateLakes and repsimulateLakes off
diff --git a/tests/test_reported_steps.py b/tests/test_reported_steps.py
index 99bb776f..79a5c3e9 100755
--- a/tests/test_reported_steps.py
+++ b/tests/test_reported_steps.py
@@ -26,25 +26,31 @@
from lisflood.main import lisfloodexe
-from .test_utils import ETRS89TestCase, setoptions, mk_path_out
+from .test_utils import ETRS89TestCase, setoptions, mk_path_out, worker_out
class TestRepStepMaps(ETRS89TestCase):
case_dir = os.path.join(os.path.dirname(__file__), 'data', 'LF_ETRS89_UseCase')
settings_file = os.path.join(case_dir, 'settings', 'cold.xml')
out_dir = os.path.join(case_dir, 'out')
- out_dir_a = os.path.join(case_dir, 'out', 'a')
- out_dir_b = os.path.join(case_dir, 'out', 'b')
+ # Output dirs isolated per xdist worker (worker_out -> 'out//a' in
+ # parallel, 'out/a' serially) so this file's 'out/a','out/b' do not clobber
+ # the identically named dirs in other test files.
+ out_dir_a = os.path.join(case_dir, worker_out('a'))
+ out_dir_b = os.path.join(case_dir, worker_out('b'))
+ pathout_a = '$(PathRoot)/' + worker_out('a')
+ pathout_b = '$(PathRoot)/' + worker_out('b')
def test_reported_steps(self):
strReportStepA = 'starttime+10..endtime'
settings_a = setoptions(self.settings_file,
vars_to_set={'StepStart': '01/07/2016 00:00', 'StepEnd': '01/08/2016 00:00',
- 'PathOut': '$(PathRoot)/out/a',
+ 'PathOut': self.pathout_a,
'ReportSteps': strReportStepA,
})
- mk_path_out(self.out_dir)
+ # Ensure the shared 'out/' base exists without wiping it (other workers/tests use it).
+ os.makedirs(self.out_dir, exist_ok=True)
mk_path_out(self.out_dir_a)
lisfloodexe(settings_a)
@@ -59,7 +65,7 @@ def test_reported_steps(self):
settings_b = setoptions(self.settings_file,
vars_to_set={'StepStart': '01/07/2016 00:00', 'StepEnd': '01/08/2016 00:00',
- 'PathOut': '$(PathRoot)/out/b',
+ 'PathOut': self.pathout_b,
'ReportSteps': strReportStepB,
})
mk_path_out(self.out_dir_b)
diff --git a/tests/test_results.py b/tests/test_results.py
index 106bfd88..1746066b 100644
--- a/tests/test_results.py
+++ b/tests/test_results.py
@@ -30,7 +30,8 @@
@pytest.mark.slow
class TestCatch(ETRS89TestCase):
case_dir = os.path.join(os.path.dirname(__file__), 'data', 'LF_ETRS89_UseCase')
- mk_path_out(os.path.join(case_dir, 'out'))
+ # Ensure the shared 'out/' exists without wiping it (would clobber other tests).
+ os.makedirs(os.path.join(case_dir, 'out'), exist_ok=True)
modules_to_set = (
'SplitRouting',
@@ -64,7 +65,7 @@ def run(self, dt_sec, step_start, step_end):
opts_to_set=('repDischargeTs', 'repDischargeMaps',
"repThetaMaps", "repThetaForestMaps",
"repThetaIrrigationMaps", "repE2O2",
- "repTotalWaterStorageMaps") + self.modules_to_set,
+ "repTWSMaps", "repStorageMaps", "repFlowMomMaps") + self.modules_to_set,
opts_to_unset=opts_to_unset,
vars_to_set={'StepStart': step_start,
'StepEnd': step_end,
@@ -72,7 +73,7 @@ def run(self, dt_sec, step_start, step_end):
'PathOut': output_dir})
mk_path_out(output_dir)
lisfloodexe(settings)
-
+
def test_output_daily(self):
self.run('86400', '02/01/2016 06:00', '02/07/2016 06:00')
self.compare_reference('dis', check='map', step_length='86400')
@@ -85,7 +86,21 @@ def test_output_daily(self):
self.compare_reference('tha', check='map', step_length='86400')
self.compare_reference('thc', check='map', step_length='86400')
self.compare_reference('lz', check='map', step_length='86400')
- self.compare_reference('tws', check='map', step_length='86400')
+ self.compare_reference('TotalWS', check='map', step_length='86400')
+ self.compare_reference('GWstor', check='map', step_length='86400')
+ self.compare_reference('Soilstor', check='map', step_length='86400')
+ self.compare_reference('Riverstor', check='map', step_length='86400')
+ self.compare_reference('Lakestor', check='map', step_length='86400')
+ self.compare_reference('Snowstor', check='map', step_length='86400')
+ self.compare_reference('Cumstor', check='map', step_length='86400')
+ # FlowMomentum = area*length*1000*velocity reaches ~1e6-1e9 kgm/s. The
+ # comparator checks the absolute diff against zero (rtol collapses to 0
+ # against a zero target), so a tiny relative perturbation from float
+ # rounding in the numba water-demand kernels shows up as a large absolute
+ # diff. Measured max abs diff ~0.6 kgm/s daily / ~1.0 kgm/s 6h (max relative
+ # diff ~1.5e-5); atol=5.0 clears both with margin (still ~3e-6 relative to
+ # the field median) while catching any real regression.
+ self.compare_reference('FlowMomentum', check='map', step_length='86400', atol=5.0, rtol=1e-4)
def test_output_6h(self):
self.run('21600', '02/01/2016 06:00', '02/07/2016 06:00')
@@ -99,7 +114,17 @@ def test_output_6h(self):
self.compare_reference('tha', check='map', step_length='21600')
self.compare_reference('thc', check='map', step_length='21600')
self.compare_reference('lz', check='map', step_length='21600')
- self.compare_reference('tws', check='map', step_length='21600')
+ self.compare_reference('TotalWS', check='map', step_length='21600')
+ self.compare_reference('GWstor', check='map', step_length='21600')
+ self.compare_reference('Soilstor', check='map', step_length='21600')
+ self.compare_reference('Riverstor', check='map', step_length='21600')
+ self.compare_reference('Lakestor', check='map', step_length='21600')
+ self.compare_reference('Snowstor', check='map', step_length='21600')
+ self.compare_reference('Cumstor', check='map', step_length='21600')
+ # See test_output_daily: FlowMomentum magnitude (~1e6-1e9 kgm/s) makes the
+ # absolute-diff check hypersensitive; measured max abs diff ~1.0 kgm/s at 6h
+ # (relative ~1.5e-5), so atol=5.0 stays far below any real regression.
+ self.compare_reference('FlowMomentum', check='map', step_length='21600', atol=5.0, rtol=1e-4)
def test_initvars(self):
output_dir = mk_path_out(os.path.join(self.case_dir, 'out/test_results_initvars'))
diff --git a/tests/test_scale_offset_aggregation.py b/tests/test_scale_offset_aggregation.py
new file mode 100644
index 00000000..5fd65be0
--- /dev/null
+++ b/tests/test_scale_offset_aggregation.py
@@ -0,0 +1,1349 @@
+"""
+Tests for the scale/offset packing and temporal aggregation features.
+
+These tests verify that:
+1. Output maps produced with int16 scale/offset packing are equivalent to
+ float outputs within the expected quantization tolerance.
+2. Aggregated (monthly mean/sum) outputs match the result of aggregating
+ the daily time-step outputs after the fact.
+"""
+from __future__ import absolute_import
+
+import os
+import shutil
+import datetime
+
+import numpy as np
+import pytest
+from netCDF4 import Dataset
+
+from lisflood.main import lisfloodexe
+from lisflood.global_modules.default_options import default_options
+from lisflood.global_modules.output import (
+ PACK_MIN, PACK_MAX, PACK_FILL,
+ LOG_PACK_QMIN, LOG_PACK_QMAX, LOG_PACK_MIN, LOG_PACK_SCALE, LOG_PACK_ATTRS,
+ pack_discharge_log, unpack_discharge_log,
+)
+from lisflood.global_modules.settings import LisSettings
+
+
+# ---------------------------------------------------------------------------
+# Helpers
+# ---------------------------------------------------------------------------
+
+def _find_data_var(ds):
+ """Find the main data variable in a netCDF4 Dataset (the 3D variable)."""
+ for name in ds.variables:
+ var = ds.variables[name]
+ if len(var.dimensions) == 3:
+ return name
+ # fallback: return first variable that is not a dimension or projection
+ dims = set(ds.dimensions.keys())
+ skip = dims | {'laea', 'lambert_azimuthal_equal_area'}
+ for name in ds.variables:
+ if name not in skip:
+ return name
+ raise ValueError(f"No data variable found in {ds.filepath()}")
+
+_AGG_KEYS = ('OutputDailyMean', 'OutputDailySum', 'OutputMonthlyMean',
+ 'OutputMonthlySum', 'OutputYearlyMean', 'OutputYearlySum')
+
+
+def agg_vars(packing='False', **enabled):
+ """Settings overrides with every aggregation key blanked except `enabled`.
+
+ Blanking all keys guarantees that no aggregate leaks in from the base
+ settings file.
+ """
+ bad = set(enabled) - set(_AGG_KEYS)
+ assert not bad, f"unknown aggregation keys: {sorted(bad)}"
+ return {'OutputPacking': packing, **{k: enabled.get(k, '') for k in _AGG_KEYS}}
+
+
+def setoptions_with_new_vars(settings_file, opts_to_set=None, opts_to_unset=None,
+ vars_to_set=None, new_vars=None):
+ """Extended version of setoptions that can inject NEW textvar entries.
+
+ Parameters
+ ----------
+ settings_file : str
+ Path to XML settings template.
+ opts_to_set / opts_to_unset : list of str
+ Options to enable/disable.
+ vars_to_set : dict
+ Existing textvars to modify (name -> value).
+ new_vars : dict
+ New textvars to inject (name -> value). They are added to the first
+ section if not already present.
+ """
+ import uuid
+ from bs4 import BeautifulSoup
+ from lxml import etree
+ from lisflood.global_modules.settings import LisSettings, Singleton
+ from lisflood.global_modules.errors import LisfloodError
+
+ if isinstance(opts_to_set, str):
+ opts_to_set = [opts_to_set]
+ if isinstance(opts_to_unset, str):
+ opts_to_unset = [opts_to_unset]
+
+ opts_to_set = [] if opts_to_set is None else opts_to_set
+ opts_to_unset = [] if opts_to_unset is None else opts_to_unset
+ vars_to_set = {} if vars_to_set is None else vars_to_set
+ new_vars = {} if new_vars is None else new_vars
+
+ with open(settings_file) as filetocheck:
+ etree.parse(filetocheck)
+
+ with open(settings_file) as tpl:
+ soup = BeautifulSoup(tpl, 'lxml-xml')
+
+ # Set/unset options
+ lfoptions = soup.find('lfoptions')
+ for opt in opts_to_set:
+ tag = soup.find("setoption", {'name': opt})
+ if tag:
+ tag['choice'] = '1'
+ elif lfoptions:
+ # Option not in XML — inject it
+ new_tag = soup.new_tag("setoption", attrs={'name': opt, 'choice': '1'})
+ lfoptions.append(new_tag)
+ for opt in opts_to_unset:
+ tag = soup.find("setoption", {'name': opt})
+ if tag:
+ tag['choice'] = '0'
+ elif lfoptions:
+ new_tag = soup.new_tag("setoption", attrs={'name': opt, 'choice': '0'})
+ lfoptions.append(new_tag)
+
+ # Modify existing textvars
+ for textvar, value in vars_to_set.items():
+ for tag in soup.find_all("textvar", {'name': textvar}):
+ tag['value'] = value
+ break
+
+ # Inject new textvars (add to both and sections)
+ if new_vars:
+ lfuser = soup.find('lfuser')
+ lfbinding = soup.find('lfbinding')
+ for name, value in new_vars.items():
+ # Add/update in lfuser
+ if lfuser:
+ existing = lfuser.find("textvar", {'name': name})
+ if existing:
+ existing['value'] = value
+ else:
+ new_tag = soup.new_tag("textvar", attrs={'name': name, 'value': value})
+ lfuser.append(new_tag)
+ # Add/update in lfbinding (this is where the model reads bindings from)
+ if lfbinding:
+ existing = lfbinding.find("textvar", {'name': name})
+ if existing:
+ existing['value'] = value
+ else:
+ new_tag = soup.new_tag("textvar", attrs={'name': name, 'value': value})
+ lfbinding.append(new_tag)
+
+ # Write temporary settings file
+ uid = uuid.uuid4()
+ filename = os.path.join(os.path.dirname(settings_file),
+ './{}_{}.xml'.format(os.path.basename(settings_file), uid))
+ with open(filename, 'w') as dest:
+ dest.write(soup.prettify())
+ try:
+ Singleton._instances = {}
+ Singleton._current = {}
+ settings = LisSettings(filename)
+ options = settings.options
+ for opt in opts_to_set:
+ options[opt] = True
+ for opt in opts_to_unset:
+ options[opt] = False
+ except LisfloodError as e:
+ raise e
+ finally:
+ os.unlink(filename)
+ return settings
+
+
+def read_nc_variable(nc_path, var_name=None):
+ """Read the data variable from a NetCDF file.
+
+ Returns
+ -------
+ data : np.ndarray
+ The data array (auto-unscaled by netCDF4 library). Masked/fill values become NaN.
+ var_name : str
+ Name of the variable read.
+ """
+ with Dataset(nc_path, 'r') as ds:
+ if var_name is None:
+ var_name = _find_data_var(ds)
+ var = ds.variables[var_name]
+ data = var[:]
+ # Convert masked arrays to float with NaN for masked/fill values
+ if hasattr(data, 'filled'):
+ data = data.filled(np.nan).astype(np.float64)
+ else:
+ data = np.array(data, dtype=np.float64)
+ # Also treat legacy -9999 fill values as NaN
+ data[data == -9999.0] = np.nan
+ return data, var_name
+
+
+def get_nc_packing_info(nc_path, var_name=None):
+ """Get packing metadata from a NetCDF file.
+
+ Returns
+ -------
+ dict with keys: dtype, scale_factor, add_offset (or None if not packed)
+ """
+ with Dataset(nc_path, 'r') as ds:
+ ds.set_auto_maskandscale(False)
+ if var_name is None:
+ var_name = _find_data_var(ds)
+ var = ds.variables[var_name]
+ ncattrs = var.ncattrs()
+ info = {
+ 'dtype': var.dtype,
+ 'scale_factor': var.getncattr('scale_factor') if 'scale_factor' in ncattrs else None,
+ 'add_offset': var.getncattr('add_offset') if 'add_offset' in ncattrs else None,
+ }
+ return info
+
+
+# ---------------------------------------------------------------------------
+# Test class
+# ---------------------------------------------------------------------------
+
+@pytest.mark.slow
+class TestScaleOffsetPacking:
+ """Test that int16 scale/offset packing produces outputs equivalent to float."""
+
+ case_dir = os.path.join(os.path.dirname(__file__), 'data', 'LF_ETRS89_UseCase')
+ settings_file = os.path.join(case_dir, 'settings', 'full.xml')
+
+ # Variables to test: these have scale_factor/add_offset defined
+ # and use simple report options available in the test catchment.
+ # Theta1Maps: scale_factor=1.526e-5, add_offset=0.5 (range 0-1)
+ # Using repThetaMaps which produces Theta1Maps
+
+ def _run_lisflood(self, output_dir, packing=False, dt_sec='86400',
+ step_start='30/07/2016 06:00', step_end='05/08/2016 06:00',
+ opts_to_set=None, opts_to_unset=None, new_vars=None):
+ """Run lisflood with specified settings and return output path."""
+ full_out_dir = os.path.join(self.case_dir, 'out', output_dir)
+ if os.path.exists(full_out_dir):
+ shutil.rmtree(full_out_dir, ignore_errors=True)
+ os.makedirs(full_out_dir, exist_ok=True)
+
+ base_opts_set = list(opts_to_set or [])
+ base_opts_unset = list(opts_to_unset or [])
+ base_new_vars = dict(new_vars or {})
+
+ # Configure packing
+ if packing:
+ base_new_vars['OutputPacking'] = 'True'
+ else:
+ base_new_vars['OutputPacking'] = 'False'
+
+ settings = setoptions_with_new_vars(
+ self.settings_file,
+ opts_to_set=base_opts_set,
+ opts_to_unset=base_opts_unset,
+ vars_to_set={
+ 'StepStart': step_start,
+ 'StepEnd': step_end,
+ 'DtSec': dt_sec,
+ 'PathOut': full_out_dir,
+ },
+ new_vars=base_new_vars,
+ )
+ lisfloodexe(settings)
+ return full_out_dir
+
+ def test_packing_all_variables(self):
+ """All variables with scale_factor/add_offset should match float output within tolerance.
+
+ Dynamically discovers all packed variables from default_options and checks
+ every one that the model produces in this configuration.
+ """
+ # Enable as many report options as possible to produce packed variables
+ opts = [
+ 'repThetaMaps', 'repThetaForestMaps', 'repThetaIrrigationMaps',
+ 'repE2O2', 'repUZMaps', 'repGwPercUZLZMaps',
+ 'repSnowMaps', 'repSnowCoverMaps', 'repSnowMeltMaps',
+ 'repDischargeMaps', 'repSurfaceRunoffMaps',
+ 'repFastRunoffMaps', 'repInfiltrationMaps', 'repInterceptionMaps',
+ 'repTotalRunoffMaps', 'repWaterDepthMaps',
+ 'repESActMaps', 'repETActMaps', 'repETRefMaps', 'repEWIntMaps',
+ 'repTaMaps', 'repRainMaps', 'repPrefFlowMaps',
+ 'repSeepSubToGWMaps', 'repUZOutflowMaps',
+ ]
+
+ # Run without packing (float64 output)
+ out_float = self._run_lisflood(
+ 'test_packing_float',
+ packing=False,
+ opts_to_set=opts,
+ )
+
+ # Run with packing (int16 output)
+ out_packed = self._run_lisflood(
+ 'test_packing_int16',
+ packing=True,
+ opts_to_set=opts,
+ )
+
+ # Build lookup: all variables that have scale_factor defined
+ reportedmaps = default_options['reportedmaps']
+ packed_vars = {
+ name: rm for name, rm in reportedmaps.items()
+ if getattr(rm, 'scale_factor', None) is not None
+ }
+
+ checked_count = 0
+
+ for filename in sorted(os.listdir(out_packed)):
+ if not filename.endswith('.nc'):
+ continue
+
+ nc_float = os.path.join(out_float, filename)
+ nc_packed = os.path.join(out_packed, filename)
+
+ if not os.path.exists(nc_float):
+ continue
+
+ info_packed = get_nc_packing_info(nc_packed)
+ if info_packed['dtype'] != np.dtype('int16'):
+ continue # not packed, skip
+
+ # Get tolerance from the file's own scale_factor/add_offset attributes
+ scale_factor = info_packed['scale_factor']
+ add_offset = info_packed['add_offset']
+
+ data_float, _ = read_nc_variable(nc_float)
+ data_packed, _ = read_nc_variable(nc_packed)
+
+ valid = ~np.isnan(data_float) & ~np.isnan(data_packed)
+ if not valid.any():
+ continue
+
+ # Assert no valid float values are outside the packing range
+ pack_min = add_offset + scale_factor * PACK_MIN
+ pack_max = add_offset + scale_factor * PACK_MAX
+ out_of_range = valid & ((data_float < pack_min) | (data_float > pack_max))
+ assert not out_of_range.any(), \
+ f"{filename}: {out_of_range.sum()} values outside packing range " \
+ f"[{pack_min:.4g}, {pack_max:.4g}]. Scale/offset parameters need adjustment."
+
+ # Compare packed vs float within quantization tolerance
+ tolerance = scale_factor # full step tolerance
+ max_diff = np.max(np.abs(data_float[valid] - data_packed[valid]))
+ assert max_diff <= tolerance, \
+ f"Max difference {max_diff} exceeds tolerance {tolerance} for {filename}"
+ checked_count += 1
+
+ assert checked_count > 0, "No packed variables were checked"
+
+ @pytest.mark.parametrize('threaded', [False, True])
+ def test_packing_unpacked_map_not_corrupted(self, threaded):
+ """A reported map that carries no scale_factor/add_offset (e.g.
+ DischargeMaps 'dis') must be written as real float data when
+ OutputPacking=True, not replaced by the int16 fill value.
+
+ The on-disk variable dtype is chosen by the NetCDF header and the
+ per-step packing is chosen by the writer; both decisions must treat a
+ missing scale_factor/add_offset as "not packed". If they disagree for a
+ map without packing metadata, the writer packs into an int16 buffer that
+ is then written into a plain float variable as all-fill. This asserts
+ the written map holds real discharge instead.
+
+ Checked under both the synchronous and threaded writers, which share the
+ same packing decision in NetcdfStepsWriter.write().
+ """
+ # DischargeMaps ('dis') has scale_factor=None/add_offset=None.
+ assert getattr(default_options['reportedmaps']['DischargeMaps'],
+ 'scale_factor', None) is None, \
+ "test premise broken: DischargeMaps now carries packing metadata"
+
+ opts = ['repDischargeMaps']
+ thr = ['outputThreads'] if threaded else None
+
+ out_float = self._run_lisflood(
+ 'test_packing_unpacked_float', packing=False, opts_to_set=opts)
+ out_packed = self._run_lisflood(
+ 'test_packing_unpacked_int16', packing=True,
+ opts_to_set=(opts + (thr or [])))
+
+ fdis = os.path.join(out_float, 'dis.nc')
+ pdis = os.path.join(out_packed, 'dis.nc')
+ assert os.path.exists(fdis) and os.path.exists(pdis), \
+ f"dis.nc missing (float={os.path.exists(fdis)} packed={os.path.exists(pdis)})"
+
+ # Raw read of the packed file (no auto-unscale) to catch an all-fill map.
+ with Dataset(pdis, 'r') as ds:
+ vname = _find_data_var(ds)
+ raw = ds.variables[vname][:]
+ raw = np.asarray(raw.filled(raw.fill_value) if hasattr(raw, 'filled') else raw)
+ n_fill = int((raw == float(PACK_FILL)).sum())
+ assert n_fill != raw.size, \
+ f"dis.nc is entirely PACK_FILL ({PACK_FILL}): an unpacked map was " \
+ f"written through the packing branch"
+
+ # Physical values must match the non-packed float run (dis is unpacked,
+ # so the two should agree closely; allow a tiny float32 margin).
+ data_float, _ = read_nc_variable(fdis)
+ data_packed, _ = read_nc_variable(pdis)
+ valid = ~np.isnan(data_float) & ~np.isnan(data_packed)
+ assert valid.any(), "no jointly-valid dis cells to compare"
+ max_diff = np.max(np.abs(data_float[valid] - data_packed[valid]))
+ rel = max_diff / max(np.max(np.abs(data_float[valid])), 1e-9)
+ assert rel <= 1e-3, \
+ f"packed dis differs from float dis (max abs {max_diff:.4g}, rel {rel:.4g})"
+
+ def teardown_method(self):
+ """Clean up output directories."""
+ out_base = os.path.join(self.case_dir, 'out')
+ if os.path.exists(out_base):
+ for d in os.listdir(out_base):
+ if d.startswith('test_packing_'):
+ shutil.rmtree(os.path.join(out_base, d), ignore_errors=True)
+
+
+@pytest.mark.slow
+class TestTemporalAggregation:
+ """Test that temporal aggregation (monthly mean/sum) matches manual aggregation of daily outputs."""
+
+ case_dir = os.path.join(os.path.dirname(__file__), 'data', 'LF_ETRS89_UseCase')
+ settings_file = os.path.join(case_dir, 'settings', 'full.xml')
+
+ def _run_lisflood(self, output_dir, dt_sec='86400',
+ step_start='02/07/2016 06:00', step_end='01/09/2016 06:00',
+ opts_to_set=None, opts_to_unset=None, new_vars=None):
+ """Run lisflood with specified settings."""
+ full_out_dir = os.path.join(self.case_dir, 'out', output_dir)
+ if os.path.exists(full_out_dir):
+ shutil.rmtree(full_out_dir, ignore_errors=True)
+ os.makedirs(full_out_dir, exist_ok=True)
+
+ base_opts_set = list(opts_to_set or [])
+ base_opts_unset = list(opts_to_unset or [])
+ base_new_vars = dict(new_vars or {})
+
+ settings = setoptions_with_new_vars(
+ self.settings_file,
+ opts_to_set=base_opts_set,
+ opts_to_unset=base_opts_unset,
+ vars_to_set={
+ 'StepStart': step_start,
+ 'StepEnd': step_end,
+ 'DtSec': dt_sec,
+ 'PathOut': full_out_dir,
+ },
+ new_vars=base_new_vars,
+ )
+ lisfloodexe(settings)
+ return full_out_dir
+
+ @pytest.mark.parametrize("frequency,operation", [
+ ("monthly", "mean"),
+ ("monthly", "sum"),
+ ("yearly", "mean"),
+ ("yearly", "sum"),
+ ])
+ def test_temporal_aggregation(self, frequency, operation):
+ """Temporal aggregation should match manual aggregation of daily outputs.
+
+ Dynamically discovers all variables with scale_factor from default_options,
+ enables their report options, and verifies that the aggregated output
+ matches the manually computed aggregation from daily outputs.
+
+ The aggregation writes at period boundaries (month-end or year-end),
+ so only complete periods produce output.
+ """
+ # Choose time range: monthly needs ~2 months, yearly needs >1 year
+ if frequency == 'monthly':
+ step_start = '02/07/2016 06:00'
+ step_end = '01/09/2016 06:00'
+ else: # yearly
+ step_start = '02/01/2016 06:00'
+ step_end = '02/01/2017 06:00'
+ # Build list of all report options that produce packed variables
+ reportedmaps = default_options['reportedmaps']
+ packed_vars = {
+ name: rm for name, rm in reportedmaps.items()
+ if getattr(rm, "scale_factor", None) is not None and rm.all and not rm.steps
+ }
+ # Collect all report options needed
+ all_report_opts = set()
+ for rm in packed_vars.values():
+ all_report_opts.update(rm.all)
+ all_report_opts.discard('')
+ opts = list(all_report_opts)
+
+ # Build semicolon-separated list of all packed variable names for aggregation
+ agg_var_list = ';'.join(packed_vars.keys())
+
+ # Select the aggregation setting key
+ agg_setting = f'Output{frequency.capitalize()}{operation.capitalize()}'
+
+ # Run with daily all-steps output (no aggregation)
+ out_daily = self._run_lisflood(
+ f'test_agg_{frequency}_{operation}_daily',
+ step_start=step_start,
+ step_end=step_end,
+ opts_to_set=opts,
+ new_vars=agg_vars('False'),
+ )
+
+ # Run with temporal aggregation for all packed variables
+ out_agg = self._run_lisflood(
+ f'test_agg_{frequency}_{operation}_agg',
+ step_start=step_start,
+ step_end=step_end,
+ opts_to_set=opts,
+ new_vars=agg_vars('False', **{agg_setting: agg_var_list}),
+ )
+
+ # Aggregation function for manual computation
+ agg_func = np.nanmean if operation == 'mean' else np.nansum
+
+ checked_count = 0
+ for filename in sorted(os.listdir(out_agg)):
+ if not filename.endswith('.nc'):
+ continue
+
+ nc_daily_path = os.path.join(out_daily, filename)
+ nc_agg_path = os.path.join(out_agg, filename)
+
+ if not os.path.exists(nc_daily_path):
+ continue
+
+ # Load daily data
+ with Dataset(nc_daily_path, 'r') as ds:
+ time_var = ds.variables['time']
+ times = time_var[:]
+ time_units = time_var.units
+ calendar = time_var.calendar
+
+ from netCDF4 import num2date
+ dates = num2date(times, time_units, calendar)
+
+ var_name = _find_data_var(ds)
+ daily_data = ds.variables[var_name][:]
+
+ # Convert to float with NaN for masked values
+ if hasattr(daily_data, 'filled'):
+ daily_data = daily_data.filled(np.nan).astype(np.float64)
+ daily_data[daily_data == -9999.0] = np.nan
+
+ # Group daily data by period (month or year)
+ dt_day = 1
+ period_groups = {}
+ for i, d in enumerate(dates):
+ if frequency == 'monthly':
+ period_key = (d.year, d.month)
+ else:
+ period_key = d.year
+ if period_key not in period_groups:
+ period_groups[period_key] = []
+ period_groups[period_key].append(daily_data[i])
+
+ # Only keep completed periods (boundary reached)
+ completed_periods = set()
+ for i, d in enumerate(dates):
+ next_d = d + datetime.timedelta(days=dt_day)
+ if frequency == 'monthly':
+ if d.month != next_d.month:
+ completed_periods.add((d.year, d.month))
+ else:
+ if d.year != next_d.year:
+ completed_periods.add(d.year)
+
+ # Compute aggregation for each completed period
+ manual_agg = []
+ for key in sorted(completed_periods):
+ stack = np.array(period_groups[key])
+ manual_agg.append(agg_func(stack, axis=0))
+ manual_agg = np.array(manual_agg)
+
+ # Read aggregated output
+ data_agg, _ = read_nc_variable(nc_agg_path)
+
+ # Only compare if the aggregated output has fewer time steps than daily
+ # (confirms aggregation actually happened for this variable)
+ if data_agg.shape[0] >= daily_data.shape[0]:
+ continue
+
+ if data_agg.shape != manual_agg.shape:
+ continue
+
+ valid = ~np.isnan(manual_agg) & ~np.isnan(data_agg)
+ if not valid.any():
+ continue
+
+ # Compare: absolute tolerance for mean, relative for sum
+ if operation == 'mean':
+ max_diff = np.max(np.abs(data_agg[valid] - manual_agg[valid]))
+ assert max_diff < 1e-6, \
+ f"{filename}: {frequency} {operation} max difference {max_diff} exceeds tolerance 1e-6"
+ else:
+ max_rel_diff = np.max(np.abs(
+ (data_agg[valid] - manual_agg[valid]) /
+ np.where(manual_agg[valid] != 0, manual_agg[valid], 1.0)
+ ))
+ assert max_rel_diff < 1e-5, \
+ f"{filename}: {frequency} {operation} max relative difference {max_rel_diff} exceeds tolerance 1e-5"
+ checked_count += 1
+
+ assert checked_count > 0, f"No aggregated variables were checked for {frequency} {operation}"
+
+ @pytest.mark.parametrize("operation", ["mean", "sum"])
+ def test_daily_aggregation_subdaily(self, operation):
+ """Daily aggregation of a SUB-DAILY run must equal the manual daily
+ aggregation of the within-day steps.
+
+ Runs at DtSec=21600 (6-hourly, 4 steps/day) and checks that
+ OutputDaily{Mean,Sum} writes one slice per calendar day equal to the
+ mean/sum of that day's 4 sub-daily steps. This is the behaviour that is
+ impossible with the pre-existing monthly/yearly-only aggregation.
+ """
+ # ~3 days of 6-hourly steps; start at 00:00 so each day has its full 4 steps
+ step_start = '02/07/2016 00:00'
+ step_end = '05/07/2016 00:00'
+
+ reportedmaps = default_options['reportedmaps']
+ packed_vars = {
+ name: rm for name, rm in reportedmaps.items()
+ if getattr(rm, "scale_factor", None) is not None and rm.all and not rm.steps
+ }
+ all_report_opts = set()
+ for rm in packed_vars.values():
+ all_report_opts.update(rm.all)
+ all_report_opts.discard('')
+ opts = list(all_report_opts)
+
+ agg_var_list = ';'.join(packed_vars.keys())
+ agg_setting = 'OutputDailyMean' if operation == 'mean' else 'OutputDailySum'
+
+ # Sub-daily all-steps output (no aggregation) -> 4 slices/day reference
+ out_subdaily = self._run_lisflood(
+ f'test_agg_daily_{operation}_subdaily',
+ dt_sec='21600',
+ step_start=step_start,
+ step_end=step_end,
+ opts_to_set=opts,
+ new_vars=agg_vars('False'),
+ )
+
+ # Same run with daily aggregation enabled
+ out_agg = self._run_lisflood(
+ f'test_agg_daily_{operation}_agg',
+ dt_sec='21600',
+ step_start=step_start,
+ step_end=step_end,
+ opts_to_set=opts,
+ new_vars=agg_vars('False', **{agg_setting: agg_var_list}),
+ )
+
+ agg_func = np.nanmean if operation == 'mean' else np.nansum
+
+ checked_count = 0
+ for filename in sorted(os.listdir(out_agg)):
+ if not filename.endswith('.nc'):
+ continue
+ nc_sub_path = os.path.join(out_subdaily, filename)
+ nc_agg_path = os.path.join(out_agg, filename)
+ if not os.path.exists(nc_sub_path):
+ continue
+
+ with Dataset(nc_sub_path, 'r') as ds:
+ time_var = ds.variables['time']
+ times = time_var[:]
+ from netCDF4 import num2date
+ dates = num2date(times, time_var.units, time_var.calendar)
+ var_name = _find_data_var(ds)
+ sub_data = ds.variables[var_name][:]
+
+ if hasattr(sub_data, 'filled'):
+ sub_data = sub_data.filled(np.nan).astype(np.float64)
+ sub_data[sub_data == -9999.0] = np.nan
+
+ # Group sub-daily steps by calendar day
+ period_groups = {}
+ for i, d in enumerate(dates):
+ key = (d.year, d.month, d.day)
+ period_groups.setdefault(key, []).append(sub_data[i])
+
+ # Only days whose boundary (day change) is crossed within the run
+ dt_day = 0.25
+ completed = set()
+ for i, d in enumerate(dates):
+ next_d = d + datetime.timedelta(days=dt_day)
+ if d.day != next_d.day:
+ completed.add((d.year, d.month, d.day))
+
+ manual_agg = np.array([agg_func(np.array(period_groups[k]), axis=0)
+ for k in sorted(completed)])
+
+ data_agg, _ = read_nc_variable(nc_agg_path)
+
+ # Confirm aggregation actually reduced the step count
+ if data_agg.shape[0] >= sub_data.shape[0]:
+ continue
+ if data_agg.shape != manual_agg.shape:
+ continue
+
+ valid = ~np.isnan(manual_agg) & ~np.isnan(data_agg)
+ if not valid.any():
+ continue
+
+ if operation == 'mean':
+ max_diff = np.max(np.abs(data_agg[valid] - manual_agg[valid]))
+ assert max_diff < 1e-6, \
+ f"{filename}: daily {operation} max difference {max_diff} exceeds tolerance 1e-6"
+ else:
+ max_rel_diff = np.max(np.abs(
+ (data_agg[valid] - manual_agg[valid]) /
+ np.where(manual_agg[valid] != 0, manual_agg[valid], 1.0)
+ ))
+ assert max_rel_diff < 1e-5, \
+ f"{filename}: daily {operation} max relative difference {max_rel_diff} exceeds tolerance 1e-5"
+ checked_count += 1
+
+ assert checked_count > 0, f"No daily-aggregated variables were checked for {operation}"
+
+ @pytest.mark.parametrize("operation", ["mean", "sum"])
+ def test_daily_aggregation_subdaily_unpacked(self, operation):
+ """Daily aggregation must also work for UNPACKED variables.
+
+ The packed-variable tests select only reportedmaps entries that carry
+ scale_factor/add_offset. Variables WITHOUT packing metadata (e.g.
+ DischargeMaps, TavgMapsOut, FrostIndexMaps) go through a different
+ branch of the writer (is_packed / has_packing predicates), and their
+ aggregated output must (a) be produced, (b) stay unpacked float, and
+ (c) equal the manual daily mean/sum of the sub-daily steps.
+ """
+ # ~3 days of 6-hourly steps; start at 00:00 so each day has its full 4 steps
+ step_start = '02/07/2016 00:00'
+ step_end = '05/07/2016 00:00'
+
+ reportedmaps = default_options['reportedmaps']
+ # Curated set of unpacked ('all'-type) variables: core quantities whose
+ # report options are simple flags and whose output_var is always
+ # computed in the EFAS use case.
+ target_names = ('DischargeMaps', 'TavgMapsOut', 'FrostIndexMaps')
+ unpacked_vars = {
+ name: rm for name, rm in reportedmaps.items()
+ if getattr(rm, "scale_factor", None) is None and rm.all and not rm.steps
+ and name in target_names
+ }
+ assert set(unpacked_vars) == set(target_names), \
+ f"Expected unpacked target variables missing from reportedmaps: " \
+ f"{set(target_names) - set(unpacked_vars)}"
+
+ all_report_opts = set()
+ for rm in unpacked_vars.values():
+ all_report_opts.update(rm.all)
+ all_report_opts.discard('')
+ opts = list(all_report_opts)
+
+ agg_var_list = ';'.join(unpacked_vars.keys())
+ agg_setting = 'OutputDailyMean' if operation == 'mean' else 'OutputDailySum'
+
+ # Sub-daily all-steps output (no aggregation) -> 4 slices/day reference
+ out_subdaily = self._run_lisflood(
+ f'test_agg_daily_{operation}_unpacked_subdaily',
+ dt_sec='21600',
+ step_start=step_start,
+ step_end=step_end,
+ opts_to_set=opts,
+ new_vars=agg_vars('False'),
+ )
+
+ # Same run with daily aggregation enabled
+ out_agg = self._run_lisflood(
+ f'test_agg_daily_{operation}_unpacked_agg',
+ dt_sec='21600',
+ step_start=step_start,
+ step_end=step_end,
+ opts_to_set=opts,
+ new_vars=agg_vars('False', **{agg_setting: agg_var_list}),
+ )
+
+ checked_count = 0
+ for var_name in unpacked_vars:
+ checked_count += self._check_unpacked_daily_agg(
+ operation, var_name, out_subdaily, out_agg)
+
+ assert checked_count == len(unpacked_vars), \
+ f"Only {checked_count}/{len(unpacked_vars)} unpacked variables were checked for {operation}"
+
+ def _check_unpacked_daily_agg(self, operation, var_name, out_subdaily, out_agg):
+ """Compare one unpacked variable's aggregated output against the manual
+ daily aggregation of its sub-daily steps. Returns 1 when checked."""
+ from netCDF4 import num2date
+
+ filename = os.path.basename(LisSettings.instance().binding[var_name]) + '.nc'
+ nc_sub_path = os.path.join(out_subdaily, filename)
+ nc_agg_path = os.path.join(out_agg, filename)
+ assert os.path.exists(nc_agg_path), \
+ f"Aggregated output missing for unpacked variable {var_name}: {nc_agg_path}"
+ if not os.path.exists(nc_sub_path):
+ return 0
+
+ with Dataset(nc_sub_path, 'r') as ds:
+ time_var = ds.variables['time']
+ times = time_var[:]
+ dates = num2date(times, time_var.units, time_var.calendar)
+ var_name_nc = _find_data_var(ds)
+ sub_data = ds.variables[var_name_nc][:]
+
+ if hasattr(sub_data, 'filled'):
+ sub_data = sub_data.filled(np.nan).astype(np.float64)
+ sub_data[sub_data == -9999.0] = np.nan
+
+ # Group sub-daily steps by calendar day
+ period_groups = {}
+ for i, d in enumerate(dates):
+ key = (d.year, d.month, d.day)
+ period_groups.setdefault(key, []).append(sub_data[i])
+
+ # Only days whose boundary (day change) is crossed within the run
+ dt_day = 0.25
+ completed = set()
+ for i, d in enumerate(dates):
+ next_d = d + datetime.timedelta(days=dt_day)
+ if d.day != next_d.day:
+ completed.add((d.year, d.month, d.day))
+
+ agg_func = np.nanmean if operation == 'mean' else np.nansum
+ manual_agg = np.array([agg_func(np.array(period_groups[k]), axis=0)
+ for k in sorted(completed)])
+
+ data_agg, _ = read_nc_variable(nc_agg_path)
+
+ # Unpacked output must stay float (no int16 packing)
+ info = get_nc_packing_info(nc_agg_path)
+ assert info['dtype'] != np.dtype('int16'), \
+ f"{filename}: unpacked variable must NOT be int16-packed"
+
+ # Confirm aggregation actually reduced the step count
+ assert data_agg.shape[0] < sub_data.shape[0], \
+ f"{filename}: aggregation did not reduce the step count " \
+ f"({data_agg.shape[0]} >= {sub_data.shape[0]})"
+ assert data_agg.shape == manual_agg.shape, \
+ f"{filename}: aggregated shape {data_agg.shape} != manual {manual_agg.shape}"
+
+ valid = ~np.isnan(manual_agg) & ~np.isnan(data_agg)
+ assert valid.any(), f"{filename}: no valid values to compare"
+
+ if operation == 'mean':
+ max_diff = np.max(np.abs(data_agg[valid] - manual_agg[valid]))
+ assert max_diff < 1e-6, \
+ f"{filename}: daily {operation} (unpacked) max difference {max_diff} exceeds tolerance 1e-6"
+ else:
+ max_rel_diff = np.max(np.abs(
+ (data_agg[valid] - manual_agg[valid]) /
+ np.where(manual_agg[valid] != 0, manual_agg[valid], 1.0)
+ ))
+ assert max_rel_diff < 1e-5, \
+ f"{filename}: daily {operation} (unpacked) max relative difference {max_rel_diff} exceeds tolerance 1e-5"
+ return 1
+
+ @staticmethod
+ def _read_time_and_data(nc_path):
+ """Return (time values, units, calendar, data array) of a netCDF output."""
+ with Dataset(nc_path, 'r') as ds:
+ tvar = ds.variables['time']
+ tvals = np.asarray(tvar[:], dtype=np.float64)
+ units, calendar = tvar.units, tvar.calendar
+ data = ds.variables[_find_data_var(ds)][:]
+ return tvals, units, calendar, data
+
+ @pytest.mark.parametrize('threaded', [False, True])
+ def test_daily_run_monthly_aggregation_ends_on_period_boundary(self, threaded):
+ """Regression: a DAILY run ending exactly on a month+year boundary with
+ monthly aggregation must produce exactly one slice per month.
+
+ The aggregate flush decision is taken from the model calendar date. With
+ the threaded writer (outputThreads) write() runs on a worker thread while
+ the main thread has already advanced the clock, so reading the live date
+ shifts every period boundary by one step and produces an extra slice that
+ overflows the fixed time axis ("NetCDF: Index exceeds dimension bound").
+ The synchronous variant cannot trigger that race; the threaded variant is
+ the one that guards it.
+
+ The variable is reported BOTH per-step (repLZMaps) and as a monthly mean,
+ and packing is on.
+ """
+ out_dir = self._run_lisflood(
+ 'test_agg_daily_boundary_%s' % ('thr' if threaded else 'sync'),
+ dt_sec='86400',
+ step_start='02/01/2016 06:00',
+ step_end='31/12/2016 06:00', # ends on month + year boundary
+ opts_to_set=['repLZMaps', 'repE2O2'] + (['outputThreads'] if threaded else []),
+ new_vars=agg_vars('True', OutputMonthlyMean='LZMaps'),
+ )
+ nc_path = os.path.join(out_dir, 'lz.nc')
+ assert os.path.exists(nc_path), f"Output not found (run likely crashed): {nc_path}"
+
+ # 2016 is a leap year; Jan 2 .. Dec 31 contains all 12 month-ends.
+ tvals, units, calendar, data = self._read_time_and_data(nc_path)
+ assert tvals.size == 12, f"expected 12 monthly slices, got {tvals.size}"
+ assert data.shape[0] == tvals.size
+ assert np.all(np.isfinite(tvals)), "time axis has non-finite (unfilled) coordinates"
+ assert np.all(np.diff(tvals) > 0), "time stamps must be strictly increasing"
+ # Time stamps must be the 12 month-end dates (no shifted/duplicate stamps).
+ expected = [datetime.datetime(2016, m, d, 6, 0) for m, d in
+ zip(range(1, 13), [31, 29, 31, 30, 31, 30, 31, 31, 30, 31, 30, 31])]
+ from netCDF4 import num2date
+ got = num2date(tvals, units, calendar)
+ assert [(g.year, g.month, g.day) for g in got] == [(e.year, e.month, e.day) for e in expected]
+
+ @pytest.mark.parametrize('threaded', [False, True])
+ def test_daily_run_monthly_aggregation_ends_mid_period(self, threaded):
+ """A run ending mid-period writes completed periods only: the partial
+ final month is NOT written, and the time axis length equals the data
+ length."""
+ out_dir = self._run_lisflood(
+ 'test_agg_mid_period_%s' % ('thr' if threaded else 'sync'),
+ dt_sec='86400',
+ step_start='02/01/2016 06:00',
+ step_end='15/06/2016 06:00', # 5 completed months (Jan-May), June partial
+ opts_to_set=['repLZMaps', 'repE2O2'] + (['outputThreads'] if threaded else []),
+ new_vars=agg_vars('True', OutputMonthlyMean='LZMaps'),
+ )
+ nc_path = os.path.join(out_dir, 'lz.nc')
+ assert os.path.exists(nc_path), f"Output not found (run likely crashed): {nc_path}"
+ tvals, units, calendar, data = self._read_time_and_data(nc_path)
+ assert tvals.size == 5, f"expected 5 completed monthly slices, got {tvals.size}"
+ assert data.shape[0] == tvals.size
+ assert np.all(np.isfinite(tvals)), "time axis has non-finite (unfilled) coordinates"
+ from netCDF4 import num2date
+ got = num2date(tvals, units, calendar)
+ assert [(g.year, g.month) for g in got] == [(2016, m) for m in range(1, 6)]
+
+ def test_threaded_aggregation_equals_synchronous(self):
+ """Aggregated outputs (monthly mean + yearly sum) must be identical
+ between the threaded and synchronous writers."""
+ new_vars = agg_vars('False', OutputMonthlyMean='LZMaps', OutputMonthlySum='UZMaps')
+ results = {}
+ for label, extra in (('sync', []), ('thr', ['outputThreads'])):
+ out_dir = self._run_lisflood(
+ 'test_agg_eq_%s' % label,
+ dt_sec='86400',
+ step_start='02/01/2016 06:00',
+ step_end='15/06/2016 06:00',
+ opts_to_set=['repLZMaps', 'repUZMaps', 'repE2O2'] + extra,
+ new_vars=new_vars,
+ )
+ results[label] = {
+ name: self._read_time_and_data(os.path.join(out_dir, name))
+ for name in ('lz.nc', 'uz.nc')
+ if os.path.exists(os.path.join(out_dir, name))
+ }
+ assert results['sync'], "no aggregated output produced"
+ assert results['sync'].keys() == results['thr'].keys()
+ for name, (t_s, _, _, d_s) in results['sync'].items():
+ t_t, _, _, d_t = results['thr'][name]
+ assert np.array_equal(t_s, t_t), f"{name}: time axes differ"
+ assert d_s.shape == d_t.shape, f"{name}: shapes differ"
+ assert np.array_equal(np.ma.getmaskarray(d_s), np.ma.getmaskarray(d_t)), f"{name}: masks differ"
+ assert np.array_equal(np.ma.getdata(d_s), np.ma.getdata(d_t)), f"{name}: values differ"
+
+ @pytest.mark.parametrize('threaded', [False, True])
+ def test_legacy_monthly_output_time_axis(self, threaded):
+ """Legacy monthly outputs (ReportedMap monthly=True, selected through
+ repWIndex/repWaterUse + ReportSteps; no packing/aggregation settings)
+ must keep a fixed, pre-written time axis: finite month-end stamps and as
+ many time values as data slices."""
+ from netCDF4 import num2date
+ cal_start = datetime.datetime(1999, 12, 31, 6, 0)
+ start = datetime.datetime(2016, 1, 2, 6, 0)
+ end = datetime.datetime(2016, 12, 31, 6, 0)
+ # Step d is dated CalendarDayStart + (d-1)*DtDay.
+ first = (start - cal_start).days + 1
+ last = (end - cal_start).days + 1
+ out_dir = self._run_lisflood(
+ 'test_agg_legacy_%s' % ('thr' if threaded else 'sync'),
+ dt_sec='86400',
+ step_start=start.strftime('%d/%m/%Y %H:%M'),
+ step_end=end.strftime('%d/%m/%Y %H:%M'),
+ opts_to_set=['repWIndex', 'repWaterUse'] + (['outputThreads'] if threaded else []),
+ new_vars={'ReportSteps': '%d..%d' % (first, last)},
+ )
+ # Only the ReportedMaps flagged monthly=True use the month-end axis; other
+ # 'steps' outputs enabled by the same options are written every step.
+ binding = LisSettings.instance().binding
+ monthly_files = sorted({
+ os.path.basename(binding[key]) + '.nc'
+ for key, rm in default_options['reportedmaps'].items()
+ if getattr(rm, 'monthly', False) and key in binding
+ } & set(os.listdir(out_dir)))
+ assert monthly_files, "no legacy monthly output produced"
+ checked = 0
+ for fname in monthly_files:
+ tvals, units, calendar, data = self._read_time_and_data(os.path.join(out_dir, fname))
+ assert np.all(np.isfinite(tvals)) and np.all(tvals < 1e30), \
+ f"{fname}: time coordinate holds fill values: {tvals}"
+ assert data.shape[0] == tvals.size, f"{fname}: data slices != time coords"
+ got = num2date(tvals, units, calendar)
+ for g in got:
+ assert (g + datetime.timedelta(days=1)).month != g.month, \
+ f"{fname}: {g} is not a month-end"
+ checked += 1
+ assert checked > 0
+
+ def test_legacy_monthly_steps_threaded_matches_sync(self):
+ """Legacy monthly=True map outputs on MapOutputSteps with a sparse
+ ReportSteps (all steps but a few month-ends) must be identical (time axis,
+ values) with and without outputThreads. The month-end test uses the model clock, so a
+ live read on the worker thread would select the wrong steps."""
+ from netCDF4 import num2date
+ cal_start = datetime.datetime(1999, 12, 31, 6, 0)
+
+ def step_of(y, m, d):
+ return (datetime.datetime(y, m, d, 6, 0) - cal_start).days + 1
+
+ ends = [(1, 31), (2, 29), (3, 31)]
+ skipped = {(2, 29)}
+ # Dense selection (every step) except one month-end step,
+ # so a month-end flag read one step late selects a wrong step.
+ first, last = step_of(2016, 1, 2), step_of(2016, 3, 31)
+ skipped_steps = {step_of(2016, m, d) for m, d in skipped}
+ rep_steps = [s for s in range(first, last + 1) if s not in skipped_steps]
+ expected = [e for e in ends if e not in skipped]
+ results = {}
+ for label, extra in (('sync', []), ('thr', ['outputThreads'])):
+ out_dir = self._run_lisflood(
+ 'test_agg_legacy_sparse_%s' % label,
+ dt_sec='86400',
+ step_start='02/01/2016 06:00',
+ step_end='31/03/2016 06:00',
+ opts_to_set=['repWIndex', 'repWaterUse'] + extra,
+ new_vars={'ReportSteps': ','.join(str(s) for s in rep_steps)},
+ )
+ binding = LisSettings.instance().binding
+ monthly_files = sorted({
+ os.path.basename(binding[key]) + '.nc'
+ for key, rm in default_options['reportedmaps'].items()
+ if getattr(rm, 'monthly', False) and key in binding
+ } & set(os.listdir(out_dir)))
+ assert monthly_files, "no legacy monthly output produced"
+ results[label] = {f: self._read_time_and_data(os.path.join(out_dir, f))
+ for f in monthly_files}
+ assert results['sync'].keys() == results['thr'].keys()
+ for name, (t_s, units, cal, d_s) in results['sync'].items():
+ t_t, _, _, d_t = results['thr'][name]
+ got = [(g.month, g.day) for g in num2date(t_s, units, cal)]
+ assert got == expected, f"{name}: unexpected month-end selection {got}"
+ assert np.array_equal(t_s, t_t), f"{name}: time axes differ"
+ assert np.array_equal(np.ma.getmaskarray(d_s), np.ma.getmaskarray(d_t)), f"{name}: masks differ"
+ assert np.array_equal(np.ma.getdata(d_s), np.ma.getdata(d_t)), f"{name}: values differ"
+
+ def test_aggregation_sum_disables_packing(self):
+ """Sum-aggregated outputs should NOT use int16 packing even when OutputPacking=True.
+
+ Monthly/yearly sums can exceed the int16 range calibrated for daily values,
+ so packing is disabled for sum aggregates.
+ """
+ out_dir = self._run_lisflood(
+ 'test_agg_no_packing',
+ step_start='02/07/2016 06:00',
+ step_end='01/09/2016 06:00',
+ opts_to_set=['repThetaMaps', 'repE2O2'],
+ new_vars=agg_vars('True', OutputMonthlySum='Theta1Maps'),
+ )
+
+ nc_path = os.path.join(out_dir, 'tha.nc')
+ assert os.path.exists(nc_path), f"Output not found: {nc_path}"
+
+ info = get_nc_packing_info(nc_path)
+ # Sum-aggregated outputs should remain float (not packed)
+ assert info['dtype'] != np.dtype('int16'), \
+ "Sum-aggregated output should NOT be int16-packed"
+ assert info['scale_factor'] is None, \
+ "Sum-aggregated output should not have scale_factor attribute"
+
+ def test_daily_sum_keeps_packing(self):
+ """Daily sums stay within the per-step range (at most a few sub-daily
+ steps per day), so OutputDailySum keeps int16 packing with
+ OutputPacking=True, unlike monthly/yearly sums."""
+ out_dir = self._run_lisflood(
+ 'test_agg_daily_sum_packing',
+ dt_sec='21600',
+ step_start='02/07/2016 00:00',
+ step_end='05/07/2016 00:00',
+ opts_to_set=['repThetaMaps', 'repE2O2'],
+ new_vars=agg_vars('True', OutputDailySum='Theta1Maps'),
+ )
+
+ nc_path = os.path.join(out_dir, 'tha.nc')
+ assert os.path.exists(nc_path), f"Output not found: {nc_path}"
+
+ info = get_nc_packing_info(nc_path)
+ assert info['dtype'] == np.dtype('int16'), \
+ "Daily-sum output should stay int16-packed"
+ assert info['scale_factor'] is not None, \
+ "Daily-sum output should keep the scale_factor attribute"
+
+ @pytest.mark.parametrize('packing', ['False', 'True'])
+ @pytest.mark.parametrize('setting', ['OutputDailyMean', 'OutputDailySum'])
+ def test_threaded_daily_aggregation_equals_synchronous(self, setting, packing):
+ """Daily aggregates (sub-daily run) must be identical between the
+ threaded and synchronous writers, with and without packing."""
+ new_vars = agg_vars(packing, **{setting: 'LZMaps'})
+ results = {}
+ for label, extra in (('sync', []), ('thr', ['outputThreads'])):
+ out_dir = self._run_lisflood(
+ f'test_agg_daily_eq_{setting}_{packing}_{label}',
+ dt_sec='21600',
+ step_start='02/07/2016 00:00',
+ step_end='05/07/2016 00:00',
+ opts_to_set=['repLZMaps', 'repE2O2'] + extra,
+ new_vars=new_vars,
+ )
+ nc_path = os.path.join(out_dir, 'lz.nc')
+ assert os.path.exists(nc_path), f"Output not found: {nc_path}"
+ results[label] = self._read_time_and_data(nc_path)
+ t_s, _, _, d_s = results['sync']
+ t_t, _, _, d_t = results['thr']
+ assert d_s.shape[0] == 3, f"expected 3 daily slices, got {d_s.shape[0]}"
+ assert np.array_equal(t_s, t_t), "time axes differ"
+ assert d_s.shape == d_t.shape, "shapes differ"
+ assert np.array_equal(np.ma.getmaskarray(d_s), np.ma.getmaskarray(d_t)), "masks differ"
+ assert np.array_equal(np.ma.getdata(d_s), np.ma.getdata(d_t)), "values differ"
+
+ def teardown_method(self):
+ """Clean up output directories."""
+ out_base = os.path.join(self.case_dir, 'out')
+ if os.path.exists(out_base):
+ for d in os.listdir(out_base):
+ if d.startswith('test_agg_'):
+ shutil.rmtree(os.path.join(out_base, d), ignore_errors=True)
+
+
+class TestAggregationRequiresNetcdf:
+ """Temporal aggregation is netCDF-only: PCRaster output must be rejected
+ at setup with a clear error."""
+
+ case_dir = os.path.join(os.path.dirname(__file__), 'data', 'LF_ETRS89_UseCase')
+ settings_file = os.path.join(case_dir, 'settings', 'full.xml')
+ out_dir = os.path.join(case_dir, 'out', 'test_agg_pcraster_reject')
+
+ @pytest.mark.parametrize('option', ['OutputDailyMean', 'OutputMonthlySum', 'OutputYearlyMean'])
+ def test_aggregation_with_pcraster_output_rejected(self, option):
+ from lisflood.global_modules.errors import LisfloodError
+ os.makedirs(self.out_dir, exist_ok=True)
+ new_vars = agg_vars('False', **{option: 'LZMaps'})
+ settings = setoptions_with_new_vars(
+ self.settings_file,
+ opts_to_set=['repLZMaps', 'repE2O2'],
+ opts_to_unset=['writeNetcdf', 'writeNetcdfStack'],
+ vars_to_set={'StepStart': '02/01/2016 06:00', 'StepEnd': '05/01/2016 06:00',
+ 'DtSec': '86400', 'PathOut': self.out_dir},
+ new_vars=new_vars,
+ )
+ with pytest.raises(LisfloodError) as exc:
+ lisfloodexe(settings)
+ msg = str(exc.value)
+ assert option in msg and 'requires netCDF output' in msg
+
+ def teardown_method(self):
+ shutil.rmtree(self.out_dir, ignore_errors=True)
+
+
+# ---------------------------------------------------------------------------
+# Logarithmic discharge packing (OutputPackingDischarge)
+# ---------------------------------------------------------------------------
+
+class TestDischargeLogPackRoundtrip:
+ """Unit tests for the pure log10 pack/unpack functions.
+
+ These exercise pack_discharge_log / unpack_discharge_log directly (no model
+ run), so they are fast and run anywhere. They verify the three defining
+ properties of the encoding: (near) constant RELATIVE precision across the
+ whole resolved range, clipping of out-of-range values, and that
+ zero/negative/NaN/sub-minimum values map to the fill value (read back as
+ NaN).
+ """
+
+ def test_roundtrip_relative_precision(self):
+ """Values in [QMIN, QMAX] survive a pack/unpack round-trip within the
+ quantization step, i.e. roughly constant relative precision."""
+ # Sample the whole dynamic range geometrically (constant ratio spacing)
+ q = np.logspace(np.log10(LOG_PACK_QMIN), np.log10(LOG_PACK_QMAX), 2000)
+
+ packed = pack_discharge_log(q)
+ assert packed.dtype == np.int16
+ back = unpack_discharge_log(packed)
+
+ assert np.all(np.isfinite(back)), "all in-range values must round-trip to finite"
+
+ # One quantization step in log space is LOG_PACK_SCALE; the worst-case
+ # relative error of a single rounding step is therefore bounded by
+ # 10**(LOG_PACK_SCALE/2) - 1. Allow a hair of slack for float noise.
+ step_rel = 10.0 ** (LOG_PACK_SCALE / 2.0) - 1.0
+ rel_err = np.abs(back - q) / q
+ assert rel_err.max() <= step_rel * 1.05, \
+ f"max relative error {rel_err.max():.3e} exceeds one half-step {step_rel:.3e}"
+
+ def test_fill_for_nonpositive_and_subminimum(self):
+ """Zero, negatives, the -9999 sentinel, NaN, and values below QMIN all
+ pack to PACK_FILL and unpack to NaN."""
+ q = np.array([0.0, -1.0, -9999.0, np.nan, LOG_PACK_QMIN * 0.5], dtype=np.float64)
+ packed = pack_discharge_log(q)
+ assert np.all(packed == PACK_FILL), \
+ "non-positive / sub-minimum / NaN discharge must map to PACK_FILL"
+ back = unpack_discharge_log(packed)
+ assert np.all(np.isnan(back)), "fill values must unpack to NaN"
+
+ def test_clipping_above_qmax(self):
+ """Values above QMAX are clipped to the maximum code (not wrapped) and
+ decode to ~QMAX."""
+ q = np.array([LOG_PACK_QMAX, LOG_PACK_QMAX * 10.0, 1e9], dtype=np.float64)
+ packed = pack_discharge_log(q)
+ assert np.all(packed == PACK_MAX), "values >= QMAX must clip to PACK_MAX"
+ back = unpack_discharge_log(packed)
+ # Decoded value of PACK_MAX is exactly QMAX (to float precision)
+ assert np.allclose(back, LOG_PACK_QMAX, rtol=1e-6)
+
+ def test_endpoints_map_to_code_range(self):
+ """QMIN and QMAX map to the ends of the usable int16 range."""
+ packed = pack_discharge_log(np.array([LOG_PACK_QMIN, LOG_PACK_QMAX]))
+ assert packed[0] == PACK_MIN
+ assert packed[1] == PACK_MAX
+
+
+@pytest.mark.slow
+class TestDischargeLogPacking:
+ """Integration test: a model run with OutputPackingDischarge=True must write
+ discharge as an int16 variable carrying the custom log10 packing attributes,
+ which decode back to the float discharge within the expected relative
+ tolerance. OutputPackingDischarge is completely independent of OutputPacking.
+ """
+
+ case_dir = os.path.join(os.path.dirname(__file__), 'data', 'LF_ETRS89_UseCase')
+ settings_file = os.path.join(case_dir, 'settings', 'full.xml')
+
+ def _run_lisflood(self, output_dir, dt_sec='86400',
+ step_start='30/07/2016 06:00', step_end='05/08/2016 06:00',
+ opts_to_set=None, new_vars=None):
+ """Run lisflood and return the output directory."""
+ full_out_dir = os.path.join(self.case_dir, 'out', output_dir)
+ if os.path.exists(full_out_dir):
+ shutil.rmtree(full_out_dir, ignore_errors=True)
+ os.makedirs(full_out_dir, exist_ok=True)
+
+ settings = setoptions_with_new_vars(
+ self.settings_file,
+ opts_to_set=list(opts_to_set or []),
+ vars_to_set={
+ 'StepStart': step_start,
+ 'StepEnd': step_end,
+ 'DtSec': dt_sec,
+ 'PathOut': full_out_dir,
+ },
+ new_vars=dict(new_vars or {}),
+ )
+ lisfloodexe(settings)
+ return full_out_dir
+
+ def test_discharge_log_packing_roundtrip(self):
+ """dis.nc written with OutputPackingDischarge=True is int16 + log attrs,
+ decodes (via unpack_discharge_log) to the float-run discharge within the
+ encoding's relative precision, and leaves OTHER linearly-packable maps
+ untouched (the two packing schemes are independent)."""
+ opts = ['repDischargeMaps']
+
+ # Reference run: no packing at all (float discharge).
+ out_float = self._run_lisflood(
+ 'test_logpack_float',
+ opts_to_set=opts,
+ new_vars={'OutputPacking': 'False', 'OutputPackingDischarge': 'False'},
+ )
+
+ # Log-packed discharge, standard linear packing OFF (prove independence).
+ out_logpacked = self._run_lisflood(
+ 'test_logpack_int16',
+ opts_to_set=opts,
+ new_vars={'OutputPacking': 'False', 'OutputPackingDischarge': 'True'},
+ )
+
+ fdis = os.path.join(out_float, 'dis.nc')
+ pdis = os.path.join(out_logpacked, 'dis.nc')
+ assert os.path.exists(fdis) and os.path.exists(pdis), \
+ f"dis.nc missing (float={os.path.exists(fdis)} logpacked={os.path.exists(pdis)})"
+
+ # On-disk variable must be int16 and carry the custom log attributes,
+ # NOT the CF scale_factor/add_offset (which would trigger linear decode).
+ with Dataset(pdis, 'r') as ds:
+ ds.set_auto_maskandscale(False)
+ vname = _find_data_var(ds)
+ var = ds.variables[vname]
+ ncattrs = var.ncattrs()
+ assert var.dtype == np.dtype('int16'), \
+ f"log-packed discharge must be int16, got {var.dtype}"
+ assert var.getncattr('packing') == LOG_PACK_ATTRS['packing'], \
+ "log-packed discharge must carry packing='log10'"
+ for attr in ('packing_log_min', 'packing_log_scale', 'packing_int_offset',
+ 'packing_valid_min', 'packing_valid_max'):
+ assert attr in ncattrs, f"missing custom log-packing attribute '{attr}'"
+ assert 'scale_factor' not in ncattrs and 'add_offset' not in ncattrs, \
+ "log-packed discharge must NOT carry CF scale_factor/add_offset"
+ raw = np.asarray(var[:])
+
+ # The file must not be entirely fill: a real catchment has resolvable
+ # discharge somewhere.
+ assert int((raw == float(PACK_FILL)).sum()) != raw.size, \
+ "dis.nc is entirely PACK_FILL - no discharge was packed"
+
+ # Decode the raw int16 with the documented inverse transform and compare
+ # to the float run. Only cells that are resolvable in BOTH (float >= QMIN
+ # and not fill) are comparable; the log scheme intentionally drops
+ # discharge below QMIN to fill.
+ decoded = unpack_discharge_log(raw)
+ data_float, _ = read_nc_variable(fdis)
+
+ comparable = (~np.isnan(decoded) & ~np.isnan(data_float)
+ & (data_float >= LOG_PACK_QMIN) & (data_float <= LOG_PACK_QMAX))
+ assert comparable.any(), "no jointly-resolvable discharge cells to compare"
+
+ rel_err = np.abs(decoded[comparable] - data_float[comparable]) / data_float[comparable]
+ # One half-step of relative quantization error, with a small margin.
+ step_rel = 10.0 ** (LOG_PACK_SCALE / 2.0) - 1.0
+ assert rel_err.max() <= step_rel * 1.05, \
+ f"decoded discharge max relative error {rel_err.max():.3e} exceeds " \
+ f"the log-packing half-step {step_rel:.3e}"
+
+ # Sub-QMIN float discharge must have been stored as fill (dropped).
+ dropped = (~np.isnan(data_float)) & (data_float < LOG_PACK_QMIN)
+ if dropped.any():
+ assert np.all(raw[dropped] == PACK_FILL), \
+ "discharge below QMIN must be stored as PACK_FILL"
+
+ def test_independent_of_linear_packing(self):
+ """OutputPackingDischarge must not require OutputPacking: enabling only
+ the discharge log packing still produces an int16 log-packed dis.nc."""
+ out_dir = self._run_lisflood(
+ 'test_logpack_only',
+ opts_to_set=['repDischargeMaps'],
+ new_vars={'OutputPacking': 'False', 'OutputPackingDischarge': 'True'},
+ )
+ pdis = os.path.join(out_dir, 'dis.nc')
+ assert os.path.exists(pdis), f"dis.nc missing: {pdis}"
+ info = get_nc_packing_info(pdis)
+ assert info['dtype'] == np.dtype('int16'), \
+ "discharge log packing must apply even with OutputPacking=False"
+ assert info['scale_factor'] is None, \
+ "log-packed discharge must not use CF scale_factor"
+
+ def teardown_method(self):
+ """Clean up output directories."""
+ out_base = os.path.join(self.case_dir, 'out')
+ if os.path.exists(out_base):
+ for d in os.listdir(out_base):
+ if d.startswith('test_logpack_'):
+ shutil.rmtree(os.path.join(out_base, d), ignore_errors=True)
diff --git a/tests/test_subcatchments.py b/tests/test_subcatchments.py
index 027c7a94..ffdfc105 100644
--- a/tests/test_subcatchments.py
+++ b/tests/test_subcatchments.py
@@ -27,7 +27,7 @@
from lisflood.global_modules.settings import LisSettings
from lisflood.main import lisfloodexe
-from .test_utils import setoptions, mk_path_out
+from .test_utils import setoptions, mk_path_out, worker_out
@pytest.mark.slow
class TestSubcatchments():
@@ -112,7 +112,10 @@ def run_subcathmenttest_by_dtsec(self, dt_sec, step_end, step_start, report_step
nc_comparator.compare_dirs(path_out_subdomain, path_out_domain)
def teardown_method(self):
- folders_list = glob.glob(os.path.join(os.path.dirname(__file__), 'data/LF_ETRS89_UseCase/out/longrun_domain*')) + \
- glob.glob(os.path.join(os.path.dirname(__file__), 'data/LF_ETRS89_UseCase/out/longrun_subdomain*'))
+ # Clean only this worker's output subtree (worker_out() -> 'out' serially,
+ # 'out/' under xdist) to avoid removing another worker's dirs.
+ base = os.path.join(os.path.dirname(__file__), 'data/LF_ETRS89_UseCase', worker_out())
+ folders_list = glob.glob(os.path.join(base, 'longrun_domain*')) + \
+ glob.glob(os.path.join(base, 'longrun_subdomain*'))
for folder in folders_list:
shutil.rmtree(folder)
diff --git a/tests/test_utils.py b/tests/test_utils.py
index 1f48b7b2..b27083c8 100644
--- a/tests/test_utils.py
+++ b/tests/test_utils.py
@@ -68,12 +68,65 @@ def setoptions(settings_file, opts_to_set=None, opts_to_unset=None, vars_to_set=
return settings
+def xdist_worker_segment():
+ """Return a per-worker path segment for parallel (pytest-xdist) test runs,
+ or '' when running serially.
+
+ pytest-xdist runs each worker in its own process and exposes its id (e.g.
+ 'gw0', 'gw1', ...) via the PYTEST_XDIST_WORKER environment variable. Several
+ tests write to fixed output directories under a shared case dir (e.g.
+ 'LF_ETRS89_UseCase/out/a'); different test files reuse the same names, so
+ running them concurrently would clobber each other. Inserting the worker id
+ into the output path gives each worker its own tree ('out/gw0/a', ...).
+ When not under xdist the variable is unset and this returns '', so serial
+ runs keep the exact same paths as before (no behaviour change)."""
+ return os.environ.get('PYTEST_XDIST_WORKER', '')
+
+
+def worker_out(*subpath):
+ """Build an output-relative path that is isolated per xdist worker.
+
+ Returns e.g. 'out/gw0/a' under xdist, or 'out/a' when serial. Use this both
+ for the LISFLOOD ``PathOut`` setting (prefixed with ``$(PathRoot)/``) and
+ for the matching :func:`mk_path_out` filesystem path so they stay in sync."""
+ worker = xdist_worker_segment()
+ parts = ['out'] + ([worker] if worker else []) + list(subpath)
+ return os.path.join(*parts)
+
+
+def _insert_worker_segment(path):
+ """Insert the xdist worker id into a path right after its 'out' segment, so
+ parallel workers get separate output trees. Serial runs (worker id '') are
+ returned unchanged. Example: '.../LF_ETRS89_UseCase/out/test_results86400'
+ -> '.../LF_ETRS89_UseCase/out/gw0/test_results86400' under xdist."""
+ worker = xdist_worker_segment()
+ if not worker:
+ return path
+ parts = path.split(os.sep)
+ # find the last 'out' segment and insert the worker id after it, unless it
+ # is already worker-scoped (idempotent).
+ for i in range(len(parts) - 1, -1, -1):
+ if parts[i] == 'out':
+ if i + 1 < len(parts) and parts[i + 1] == worker:
+ break # already scoped
+ parts.insert(i + 1, worker)
+ break
+ return os.sep.join(parts)
+
+
def mk_path_out(p):
path_out = os.path.join(os.path.dirname(__file__), p)
+ # Under pytest-xdist, give each worker its own subtree beneath 'out/' so
+ # tests that write to identically named dirs (e.g. 'out/a', 'out/test_results86400')
+ # in different files do not clobber each other when run concurrently. Serial
+ # runs are unaffected (worker id is empty).
+ path_out = _insert_worker_segment(path_out)
if os.path.exists(path_out):
shutil.rmtree(path_out)
- if not os.path.exists(path_out):
- os.mkdir(path_out)
+ # Use makedirs (not mkdir) so a missing parent - e.g. when another test's
+ # teardown removed the shared 'out/' directory - does not raise
+ # FileNotFoundError. This makes tests independent of execution order.
+ os.makedirs(path_out)
return path_out
@@ -201,18 +254,102 @@ class ETRS89TestCase(object):
'tss': None,
},
},
- 'tws': {
- 'report_map': 'TotalWaterStorageMaps',
+ 'TotalWS': {
+ 'report_map': 'TWSMaps',
'report_tss': None,
'86400': {
- 'map': os.path.join(ref_dir, 'output_reference_daily/tws.nc'),
+ 'map': os.path.join(ref_dir, 'output_reference_daily/TotalWS.nc'),
'tss': None,
},
'21600': {
- 'map': os.path.join(ref_dir, 'output_reference_6h/tws.nc'),
+ 'map': os.path.join(ref_dir, 'output_reference_6h/TotalWS.nc'),
'tss': None,
},
- },
+ },
+ 'GWstor': {
+ 'report_map': 'GWSMaps',
+ 'report_tss': None,
+ '86400': {
+ 'map': os.path.join(ref_dir, 'output_reference_daily/GWstor.nc'),
+ 'tss': None,
+ },
+ '21600': {
+ 'map': os.path.join(ref_dir, 'output_reference_6h/GWstor.nc'),
+ 'tss': None,
+ },
+ },
+ 'Soilstor': {
+ 'report_map': 'SoilSMaps',
+ 'report_tss': None,
+ '86400': {
+ 'map': os.path.join(ref_dir, 'output_reference_daily/Soilstor.nc'),
+ 'tss': None,
+ },
+ '21600': {
+ 'map': os.path.join(ref_dir, 'output_reference_6h/Soilstor.nc'),
+ 'tss': None,
+ },
+ },
+ 'Riverstor': {
+ 'report_map': 'RiverSMaps',
+ 'report_tss': None,
+ '86400': {
+ 'map': os.path.join(ref_dir, 'output_reference_daily/Riverstor.nc'),
+ 'tss': None,
+ },
+ '21600': {
+ 'map': os.path.join(ref_dir, 'output_reference_6h/Riverstor.nc'),
+ 'tss': None,
+ },
+ },
+ 'Lakestor': {
+ 'report_map': 'LakeSMaps',
+ 'report_tss': None,
+ '86400': {
+ 'map': os.path.join(ref_dir, 'output_reference_daily/Lakestor.nc'),
+ 'tss': None,
+ },
+ '21600': {
+ 'map': os.path.join(ref_dir, 'output_reference_6h/Lakestor.nc'),
+ 'tss': None,
+ },
+ },
+ 'Snowstor': {
+ 'report_map': 'SnowSMaps',
+ 'report_tss': None,
+ '86400': {
+ 'map': os.path.join(ref_dir, 'output_reference_daily/Snowstor.nc'),
+ 'tss': None,
+ },
+ '21600': {
+ 'map': os.path.join(ref_dir, 'output_reference_6h/Snowstor.nc'),
+ 'tss': None,
+ },
+ },
+ 'Cumstor': {
+ 'report_map': 'CumSMaps',
+ 'report_tss': None,
+ '86400': {
+ 'map': os.path.join(ref_dir, 'output_reference_daily/Cumstor.nc'),
+ 'tss': None,
+ },
+ '21600': {
+ 'map': os.path.join(ref_dir, 'output_reference_6h/Cumstor.nc'),
+ 'tss': None,
+ },
+ },
+ 'FlowMomentum': {
+ 'report_map': 'FlowMomMaps',
+ 'report_tss': None,
+ '86400': {
+ 'map': os.path.join(ref_dir, 'output_reference_daily/FlowMomentum.nc'),
+ 'tss': None,
+ },
+ '21600': {
+ 'map': os.path.join(ref_dir, 'output_reference_6h/FlowMomentum.nc'),
+ 'tss': None,
+ },
+ },
'avgdis': {
'report_map': 'AvgDis',
'report_tss': None,
diff --git a/tests/test_warmstart.py b/tests/test_warmstart.py
index d122d969..6a7d2b2b 100644
--- a/tests/test_warmstart.py
+++ b/tests/test_warmstart.py
@@ -28,7 +28,7 @@
from lisflood.main import lisfloodexe
-from .test_utils import setoptions, mk_path_out
+from .test_utils import setoptions, mk_path_out, worker_out
@pytest.mark.slow
@@ -62,7 +62,7 @@ def run_warmstart_by_dtsec(self, dt_sec, step_end, step_start, report_steps='1..
case_dir = os.path.join(os.path.dirname(__file__), 'data', 'LF_ETRS89_UseCase')
out_dir = os.path.join(case_dir, 'out')
- mk_path_out(out_dir)
+ os.makedirs(out_dir, exist_ok=True)
# init
path_out_init = mk_path_out(os.path.join(out_dir,'init{}'.format(dt_sec)))
@@ -168,9 +168,13 @@ def run_warmstart_by_dtsec(self, dt_sec, step_end, step_start, report_steps='1..
def teardown_method(self):
- folders_list = glob.glob(os.path.join(os.path.dirname(__file__), 'data/LF_ETRS89_UseCase/out/run*')) + \
- glob.glob(os.path.join(os.path.dirname(__file__), 'data/LF_ETRS89_UseCase/out/longrun_reference*')) + \
- glob.glob(os.path.join(os.path.dirname(__file__), 'data/LF_ETRS89_UseCase/out/init*'))
+ # Clean only this worker's output subtree (worker_out() -> 'out' serially,
+ # 'out/' under xdist) so a parallel worker never removes another
+ # worker's run*/longrun_reference*/init* directories.
+ base = os.path.join(os.path.dirname(__file__), 'data/LF_ETRS89_UseCase', worker_out())
+ folders_list = glob.glob(os.path.join(base, 'run*')) + \
+ glob.glob(os.path.join(base, 'longrun_reference*')) + \
+ glob.glob(os.path.join(base, 'init*'))
for folder in folders_list:
shutil.rmtree(folder)
\ No newline at end of file
diff --git a/tests/test_water_abstraction.py b/tests/test_water_abstraction.py
index add136be..23fbf14e 100644
--- a/tests/test_water_abstraction.py
+++ b/tests/test_water_abstraction.py
@@ -5,7 +5,7 @@
from lisflood.main import lisfloodexe
-from .test_utils import setoptions, mk_path_out, ETRS89TestCase
+from .test_utils import setoptions, mk_path_out, ETRS89TestCase, worker_out
class TestWaterAbstraction(ETRS89TestCase):
@@ -20,16 +20,19 @@ def test_waterabstraction_6h(self):
def run_lisflood_waterabstraction(self, dt_sec):
case_dir = os.path.join(os.path.dirname(__file__), 'data', 'LF_ETRS89_UseCase')
out_dir = os.path.join(case_dir, 'out')
- mk_path_out(out_dir)
+ os.makedirs(out_dir, exist_ok=True)
settings_file = os.path.join(case_dir, 'settings', 'full.xml')
- out_dir_a = os.path.join(case_dir, 'out', 'a')
- out_dir_b = os.path.join(case_dir, 'out', 'b')
-
+ # Output dirs isolated per xdist worker (worker_out -> 'out//a' in
+ # parallel, 'out/a' serially) so this file's 'out/a','out/b' do not
+ # clobber the identically named dirs in test_chunking / test_caching.
+ out_dir_a = os.path.join(case_dir, worker_out('a'))
+ out_dir_b = os.path.join(case_dir, worker_out('b'))
+
settings_a = setoptions(settings_file,
opts_to_set=('TransientWaterDemandChange', 'useWaterDemandAveYear'),
vars_to_set={'StepStart': '30/07/2016 00:00', 'StepEnd': '01/08/2016 00:00',
- 'DtSec': dt_sec, 'PathOut': '$(PathRoot)/out/a',
+ 'DtSec': dt_sec, 'PathOut': '$(PathRoot)/' + worker_out('a'),
'PathWaterUse': '$(PathRoot)/maps/waterdemand'
})
mk_path_out(out_dir_a)
@@ -39,7 +42,7 @@ def run_lisflood_waterabstraction(self, dt_sec):
opts_to_set=('TransientWaterDemandChange'),
opts_to_unset=('useWaterDemandAveYear'),
vars_to_set={'StepStart': '30/07/2016 00:00', 'StepEnd': '01/08/2016 00:00',
- 'DtSec': dt_sec, 'PathOut': '$(PathRoot)/out/b',
+ 'DtSec': dt_sec, 'PathOut': '$(PathRoot)/' + worker_out('b'),
'PathWaterUse': '$(PathRoot)/maps/waterdemand19902019'})
mk_path_out(out_dir_b)
lisfloodexe(settings_b)
diff --git a/tox.ini b/tox.ini
deleted file mode 100644
index dc3c1fd7..00000000
--- a/tox.ini
+++ /dev/null
@@ -1,28 +0,0 @@
-[tox]
-
-envlist = py36,py37
-skip_missing_interpreters=True
-skipsdist=True
-
-[testenv]
-deps=
- pytest-sugar
- pytest-cov
- pytest-mock
- pytest-env
- pytest
-usedevelop=True
-
-conda_env=
- environment.yml
-conda_channels=
- conda-forge
-conda_install_args=
- --override-channels
-
-commands =
- pytest -x -l -ra --cov=lisflood --cov-config=.coveragerc tests/
-
-[flake8]
-ignore = E501
-exclude = .git,__pycache__,.eggs,.pytest_cache,.tox,build,dist