diff --git a/config/simdb.cfg b/config/simdb.cfg index 4d0f0162..4a6816f5 100644 --- a/config/simdb.cfg +++ b/config/simdb.cfg @@ -24,7 +24,7 @@ password = simdb database = simdb [validation] -path = ./validation +path = ./validation/iter_scenarios_validation.yaml auto_validate = True error_on_fail = True diff --git a/docs/maintenance_guide.md b/docs/maintenance_guide.md index e5664af2..475bfd5a 100644 --- a/docs/maintenance_guide.md +++ b/docs/maintenance_guide.md @@ -210,15 +210,16 @@ SSL_ENABLED = True ... ``` -Now create a validation schema in the application configuration directory, which can be located by using: +Now create a validation schema and configure its path. Set `validation.path` in the server config to point directly to your YAML schema file: +```ini +[validation] +path = /path/to/iter_scenarios_validation.yaml ``` -dirname "$(simdb config path)" -``` -In this directory, you should create a file ‘validation-schema.yaml’ specifying the validation schema. -Example of validation-schema.yaml: -``` +Example schema file: + +```yaml description: required: true type: string diff --git a/src/simdb/json.py b/src/simdb/json.py index 89301c88..fd35b655 100644 --- a/src/simdb/json.py +++ b/src/simdb/json.py @@ -49,4 +49,9 @@ def default(self, o: Any) -> Any: return {"_type": "uuid.UUID", "hex": o.hex} elif isinstance(o, enum.Enum): return o.value + elif isinstance(o, np.ndarray): + if np.issubdtype(o.dtype, np.number): + valid = o[~np.isnan(o)] if np.issubdtype(o.dtype, np.floating) else o + return {"min": float(valid.min()), "max": float(valid.max())} + return o.tolist() return super().default(o) diff --git a/src/simdb/validation/validator.py b/src/simdb/validation/validator.py index 2daa21ca..f9a3bb2b 100644 --- a/src/simdb/validation/validator.py +++ b/src/simdb/validation/validator.py @@ -1,6 +1,7 @@ import re +import warnings from pathlib import Path -from typing import Any, Dict, List, Optional, cast +from typing import Any, Dict, List, Optional, Union, cast import cerberus import numpy as np @@ -8,6 +9,7 @@ from simdb.config import Config, ConfigError from simdb.database.models.simulation import Simulation +from simdb.remote.models import RangeValue ValidatorBase = cast(Any, cerberus.Validator) @@ -65,7 +67,8 @@ def _compare(self, comparison, field, value, comparator: str, message: str): if comparison is None: return if isinstance(value, np.ndarray): - value = value[~np.isnan(value)] + if np.issubdtype(value.dtype, np.floating): + value = value[~np.isnan(value)] if value.size == 0: self._error(field, "Values in numpy array are NaN or empty") if not getattr(value, comparator)(comparison).all(): @@ -112,26 +115,32 @@ def _normalize_coerce_float(cls, value): def _normalize_coerce_numpy(cls, value): if isinstance(value, np.ndarray): return value + elif isinstance(value, dict) and "min" in value and "max" in value: + return np.array([value["min"], value["max"]], dtype=float) + elif isinstance(value, RangeValue): + return np.array([float(value.min), float(value.max)]) elif isinstance(value, str): return np.fromstring(value[1:-1], sep=" ") else: return np.array(value) -def _load_schema(path: Path): +def _load_schema(path: Union[Path, str]): + path = Path(path) if not path.exists(): - return [{}] + warnings.warn(f"Validation schema not found: {path}", stacklevel=2) + return {} - # load schema from file with path.open() as file: try: schema = yaml.load(file, Loader=yaml.SafeLoader) - return schema except yaml.YAMLError as err: raise LoadError( - f"Failed to read validation schema from file {file}" + f"Failed to read validation schema from file {path}" ) from err + return schema + class Validator: _validator: CustomValidator @@ -141,7 +150,7 @@ class Validator: def validation_schemas( cls, config: Config, simulation: Optional[Simulation], path=None ) -> List[Dict]: - root = Path( + configured_path = Path( str( config.get_option( "validation.path", default=str(config.config_directory) @@ -149,11 +158,19 @@ def validation_schemas( ) ) + if not configured_path.is_file(): + raise ConfigError( + f"validation.path '{configured_path}' is not a valid file. " + "Set validation.path to the full path of your validation " + "schema YAML file." + ) + default_schema_path = configured_path + paths = [] if path: paths.append(path) else: - paths.append(root / "validation-schema.yaml") + paths.append(default_schema_path) # Look for config sections like [validation "key=value"] and see if the # simulationhas metadata matching the given test. If matching, adding the diff --git a/validation/validation-schema.yaml b/validation/validation-schema.yaml deleted file mode 120000 index 94eadc88..00000000 --- a/validation/validation-schema.yaml +++ /dev/null @@ -1 +0,0 @@ -iter_scenarios_validation.yaml \ No newline at end of file