Add Pystp netcdf driver

pyistp/_impl.py

@@ -1,12 +1,24 @@
 from .drivers import current_driver, Driver
+
+
+def _driver_factory(file_or_buffer):
+    if isinstance(file_or_buffer, bytes):
+        magic = file_or_buffer[:4]
+    else:
+        with open(file_or_buffer, "rb") as f:
+            magic = f.read(4)
+    if magic in (b'\x89HDF', b'CDF'):
+        from .drivers.netcdf import Driver as NetCDFDriver
+        return NetCDFDriver(file_or_buffer)
+    return current_driver(file_or_buffer)
 from .data_variable import DataVariable
 from .support_data_variable import SupportDataVariable
 import re
 import numpy as np
 from typing import List, Optional
 import logging
 
-DEPEND_REGEX = re.compile("DEPEND_\\d")
+DEPEND_REGEX = re.compile("DEPEND_\\d", re.IGNORECASE)
 
 ISTP_NOT_COMPLIANT_W = "Non compliant ISTP file"
 
@@ -97,9 +109,9 @@ class ISTPLoaderImpl:
     def __init__(self, file=None, buffer=None, master_file=None, master_buffer=None):
         if file is not None:
             log.debug(f"Loading {file}")
-        self.cdf = current_driver(file or buffer)
+        self.cdf = _driver_factory(file or buffer)
         if master_file or master_buffer:
-            self.master_cdf = current_driver(master_file or master_buffer)
+            self.master_cdf = _driver_factory(master_file or master_buffer)
         else:
             self.master_cdf = self.cdf
         self.data_variables = []
@@ -116,7 +128,9 @@ def _update_data_vars_lis(self):
             self.data_variables = []
             for var in self.master_cdf.variables():
                 var_attrs = self.master_cdf.variable_attributes(var)
-                var_type = self.master_cdf.variable_attribute_value(var, 'VAR_TYPE')
+                # search for the VAR_TYPE attribute, regardless of its case
+                var_type_attr = next((a for a in var_attrs if a.upper() == 'VAR_TYPE'), None)
+                var_type = self.master_cdf.variable_attribute_value(var, var_type_attr) if var_type_attr else None
                 param_type = (self.master_cdf.variable_attribute_value(var,
                                                                        'PARAMETER_TYPE') or "").lower()  # another cluster CSA crap
                 if (var_type == 'data' or param_type == 'data') and not self.master_cdf.is_char(var):

pyistp/drivers/netcdf.py

@@ -0,0 +1,146 @@
+import netCDF4
+import numpy as np
+from typing import Any
+
+
+class Driver:
+    """NetCDF4 driver implementing the PyISTP Driver protocol."""
+
+    def __init__(self, file):
+        # Accept either a file path (str) or a bytes buffer
+        if isinstance(file, bytes):
+            self._ds = netCDF4.Dataset("in_memory.nc", memory=file)
+        else:
+            self._ds = netCDF4.Dataset(str(file), "r")
+
+    def variables(self):
+        return list(self._ds.variables.keys())
+
+    def has_variable(self, name):
+        return name in self._ds.variables
+
+    def variable_attributes(self, var):
+        if var not in self._ds.variables:
+            return []
+        return list(self._ds[var].ncattrs())
+
+    def variable_attribute_value(self, var, attr):
+        if var not in self._ds.variables:
+            return None
+        try:
+            return self._ds[var].getncattr(attr)
+        except AttributeError:
+            return None
+
+    def is_char(self, var):
+        if var not in self._ds.variables:
+            return False
+        return self._ds[var].dtype == str
+
+    def is_nrv(self, var):  # NOSONAR
+        # NRV concept does not exist in NetCDF4
+        return False
+
+    def shape(self, var):
+        return tuple(self._ds[var].shape)
+
+    def attributes(self):
+        return list(self._ds.ncattrs())
+
+    def attribute(self, key):
+        try:
+            return self._ds.getncattr(key)
+        except AttributeError:
+            return None
+
+    # Mapping from numpy dtype kinds to CDF type strings
+    _DTYPE_TO_CDF = {
+        'f4': 'CDF_FLOAT',
+        'f8': 'CDF_DOUBLE',
+        'i1': 'CDF_INT1',
+        'i2': 'CDF_INT2',
+        'i4': 'CDF_INT4',
+        'i8': 'CDF_INT8',
+        'u1': 'CDF_UINT1',
+        'u2': 'CDF_UINT2',
+        'u4': 'CDF_UINT4',
+        'S':  'CDF_CHAR',
+    }
+
+    # Milliseconds between CDF epoch (year 0000) and Unix epoch (1970-01-01)
+    _CDF_EPOCH_OFFSET_MS = 62_167_219_200_000
+
+    def _get_units(self, var):
+        v = self._ds[var]
+        for key in v.ncattrs():
+            if key.lower() == 'units':
+                return v.getncattr(key)
+        return None
+
+    def _is_cf_time(self, var):
+        """Return True if the variable uses CF time conventions
+        (units attribute containing 'since')."""
+        units = self._get_units(var)
+        return isinstance(units, str) and 'since' in units
+
+    def _is_cdf_epoch(self, var):
+        """Return True if the variable uses CDF_EPOCH convention
+        (float64, units='ms')."""
+        units = self._get_units(var)
+        return (isinstance(units, str)
+                and units.strip().lower() == 'ms'
+                and self._ds[var].dtype == np.float64)
+
+    def _cf_time_to_datetime64(self, var):
+        """Convert a CF time variable (units with 'since') to
+        datetime64[ns]."""
+        v = self._ds[var]
+        units = v.getncattr('units')
+        # netCDF4.num2date converts CF floats to cftime objects
+        dates: Any = netCDF4.num2date(
+            v[:], units, only_use_cftime_datetimes=False
+        )
+        # Convert to datetime64[ns] via ISO string representation
+        return np.array([np.datetime64(str(d), 'ns') for d in dates])
+
+    def _cdf_epoch_to_datetime64(self, var):
+        """Convert CDF_EPOCH (ms since year 0000) to datetime64[ns]."""
+        ms = np.array(self._ds[var][:], dtype=np.float64)
+        unix_ms = ms - self._CDF_EPOCH_OFFSET_MS
+        return (unix_ms * 1_000_000).astype('datetime64[ns]')
+
+    def _is_unix_ms_time(self, var):
+        units = self._get_units(var)
+        return isinstance(units, str) and units.strip().lower() == 'milliseconds'
+
+    def _unix_ms_time_to_datetime64(self, var):
+        """Convert ms since Unix epoch (1970-01-01) to datetime64[ns]."""
+        ms = np.array(self._ds[var][:], dtype=np.int64)
+        return (ms * 1_000_000).astype('datetime64[ns]')
+
+    def values(self, var, is_metadata_variable=False):  # NOSONAR
+        v = self._ds[var]
+        if self._is_cf_time(var):
+            return self._cf_time_to_datetime64(var)
+        if self._is_cdf_epoch(var):
+            return self._cdf_epoch_to_datetime64(var)
+        if self._is_unix_ms_time(var):
+            return self._unix_ms_time_to_datetime64(var)
+        if v.dtype == str:
+            # Native NetCDF4 string — return as numpy array of strings
+            raw = v[()]
+            if isinstance(raw, str):
+                raw = [raw]
+            return np.array(raw)
+        return np.array(v[:])
+
+    def cdf_type(self, var):
+        if self._is_cf_time(var) or self._is_unix_ms_time(var):
+            return 'CDF_TIME_TT2000'
+        if self._is_cdf_epoch(var):
+            return 'CDF_EPOCH'
+        v = self._ds[var]
+        if v.dtype == str:
+            return 'CDF_CHAR'
+        dtype_str = v.dtype.str.lstrip('<>=!')
+        return self._DTYPE_TO_CDF.get(dtype_str, f'CDF_UNKNOWN_{dtype_str}')

pyproject.toml

@@ -1,3 +1,3 @@
 [build-system]
 build-backend = "flit_core.buildapi"
 requires = ["flit_core"]
@@ -30,7 +30,7 @@
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
 ]
-dependencies = ['pycdfpp>=0.6.0']
+dependencies = ['pycdfpp>=0.6.0', 'netCDF4']
 [project.urls]
 homepage = "https://github.com/SciQLop/PyISTP"
 repository = "https://github.com/SciQLop/PyISTP"

requirements_dev.txt

@@ -8,3 +8,5 @@ coverage
 Sphinx
 twine
 ddt
+netCDF4
+pytest-cov