Support secure RemoteArray, RemoteStore, and RemoteCTable references

caterva2-server.sample.toml

@@ -7,7 +7,8 @@
 #
 # - listen: where the server listens to (a unix socket or a host/port) (default: localhost:8000)
 # - urlbase: the base url users will use to reach the server (default: http://localhost:8000)
-# - quota: if defined, it will limit the disk usage (default: 0, no limit)
+# - quota: limits apparent dataset bytes in public/shared/personal (default: 0, no limit)
+# - quota_work_bytes: separate quota-coordinated disk staging budget (default: "1G")
 # - maxusers: if defined, it will limit the number of users (default: 0, no limit)
 # - login: if true, users will need to authenticate (default: true)
 # - register: if true, users will be able to register (default: false)
@@ -25,11 +26,36 @@
 listen = "localhost:8000"
 urlbase = "http://localhost:8000"
 quota = "10G"
+# quota_work_bytes = "1G"
 maxusers = 5
 register = true  # allow users to register
 # publish_root = "s3://a-bucket/published"
 # peer_cache_quota = "1G"
 
+# Persisted RemoteArray objects are discoverable but cannot make outbound
+# requests by default. The runtime cache uses private sparse generations and
+# credential-free HTTPS is the supported source transport.
+# Every destination must be listed exactly; redirects, URL queries, private or
+# otherwise non-public destination addresses, and embedded expression
+# references are refused. MEMORY carriers are accepted under the same source
+# policy but execute without retained caching (same as NONE). DISK proxies cache
+# chunks in private runtime generations up to persisted max_cache_bytes (256 MiB
+# by default, or unbounded if None). With a customer quota, shared SQLite admission
+# permits DISK growth when capacity is available; denied fills still return data
+# without retention. See server docs for accounting exclusions, operational
+# headroom, and sparse lifecycle behavior.
+# [server.remote_proxy]
+# enabled = true
+# allowed_hosts = ["datasets.example.org", "objects.example.org:8443"]
+# timeout = 30
+# max_nbytes = 1073741824
+# max_rank = 16
+# max_chunks = 10000000
+# max_concurrency = 8
+# max_metadata_bytes = 16777216
+# max_nodes = 100000
+# cache_maintenance_seconds = 60
+
 # Mount a remote Caterva2 server's locally owned @public root as @labb.  This
 # activates the bundled C2Cache provider.  Repeat the table to mount more peers.
 # An optional per-peer cache_quota further bounds this peer within the pool.

caterva2/api_utils.py

@@ -152,8 +152,9 @@ def key_to_indices(key, ndim=None):
     return json.dumps(out, separators=(",", ":"))
 
 
-def get_download_url(path, urlbase):
-    return f"{urlbase}/api/download/{path}"
+def get_download_url(path, urlbase, *, include_cache=True):
+    url = f"{urlbase}/api/download/{path}"
+    return url if include_cache else f"{url}?include_cache=false"
 
 
 def get_handle_url(path, urlbase):

caterva2/client.py

@@ -300,7 +300,7 @@ def _toplevel_path(self):
             raise ValueError(f"Not supported for a member inside a container file: {self.path}")
         return self.path
 
-    def download(self, localpath=None):
+    def download(self, localpath=None, *, include_cache=True):
         """
         Downloads the file to storage.
 
@@ -309,6 +309,9 @@ def download(self, localpath=None):
         localpath : Path, optional
             The destination path for the downloaded file.  If not specified, the file will
             be downloaded to the current working directory.
+        include_cache : bool, optional
+            For a RemoteArray carrier, include its valid warm cache data.
+            Pass false to download a cold proxy without changing the server copy.
 
         Returns
         -------
@@ -329,6 +332,7 @@ def download(self, localpath=None):
         return self.client.download(
             self._toplevel_path(),
             localpath=localpath,
+            include_cache=include_cache,
         )
 
     def unfold(self):
@@ -528,10 +532,22 @@ def vlmeta(self):
         schunk_meta = self.meta.get("schunk", self.meta)
         return schunk_meta.get("vlmeta", {})
 
-    def get_download_url(self):
+    @property
+    def attrs(self):
+        """User attributes from cached metadata; changing them does not update the server.
+
+        Older servers without an ``attrs`` field fall back to :attr:`vlmeta`.
+        """
+        attrs = self.meta.get("attrs")
+        return self.vlmeta if attrs is None else attrs
+
+    def get_download_url(self, *, include_cache=True):
         """
         Retrieves the download URL for the file.
 
+        ``include_cache=False`` requests a cold RemoteArray carrier. It has no
+        effect on other file types.
+
         Returns
         -------
         str
@@ -546,7 +562,7 @@ def get_download_url(self):
         >>> file.get_download_url()
         'https://cat2.cloud/demo/api/fetch/example/ds-1d.b2nd'
         """
-        return api_utils.get_download_url(self.path, self.urlbase)
+        return api_utils.get_download_url(self.path, self.urlbase, include_cache=include_cache)
 
     def __getitem__(self, item):
         """
@@ -577,14 +593,13 @@ def slice(
             provided, each slice will be applied to the corresponding
             dimension.
         as_blosc2 : bool
-            If True (default), the result will be returned as a Blosc2 object
-            (either a `SChunk` or `NDArray`).  If False, it will be returned
-            as a NumPy array (equivalent to `self[key]`).
+            If True (default), return a Blosc2 object, including a CTable for
+            table requests. If False, return NumPy data or table row tuples.
 
         Returns
         -------
-        NDArray or SChunk or numpy.ndarray
-            A new Blosc2 object containing the requested slice.
+        NDArray or SChunk or CTable or numpy.ndarray or list
+            The requested slice; table requests return a CTable or row tuples.
 
         Examples
         --------
@@ -1337,19 +1352,19 @@ def get_slice(self, path, key=None, as_blosc2=True, field=None, ndim=None):
             dimension. If str, is interpreted as filter.
         as_blosc2 : bool
             If True (default), the result will be returned as a Blosc2 object
-            (either a `SChunk` or `NDArray`).  If False, it will be returned
-            as a NumPy array (equivalent to `self[key]`).
+            (including a CTable for table requests). If False, table rows are
+            returned as tuples, and other datasets as NumPy data.
         field: str
-            Shortcut to access a field in a structured array. If provided, `key` is ignored.
+            Select one field or table column after applying `key`.
         ndim: int
             How many dimensions the dataset has, where the caller knows.  Only an
             `Ellipsis` in the key needs it, and only one that is not its last
             entry: what it stands for is every dimension the key does not name.
 
         Returns
         -------
-        NDArray or SChunk or numpy.ndarray
-            A new Blosc2 object containing the requested slice.
+        NDArray or SChunk or CTable or numpy.ndarray or list
+            The requested slice or table rows.
 
         Examples
         --------
@@ -1369,52 +1384,36 @@ def get_slice(self, path, key=None, as_blosc2=True, field=None, ndim=None):
         if isinstance(path, Table):
             kind = "ctable"
         elif isinstance(path, File):
-            kind = None
+            kind = path.meta.get("kind")
         else:
             path_str = path.as_posix() if hasattr(path, "as_posix") else str(path)
-            kind = "ctable" if path_str.endswith(".b2z") else None
+            kind = self.get_info(path_str).get("kind")
         if isinstance(path, File):
             path = path.path
         urlbase, path = _format_paths(self.urlbase, path)
-        if field:  # blosc2 doesn't support indexing of multiple fields
-            return self._fetch_data(
-                path,
-                urlbase,
-                {"field": field},
-                auth_cookie=self.cookie,
-                as_blosc2=as_blosc2,
-                timeout=self.timeout,
-                kind=kind,
-            )
         if isinstance(key, str):
             # The key can still be a slice expression in string format (like for CLI utils)
             params = {"slice_": key} if _looks_like_slice(key) else {"filter": key}
-            return self._fetch_data(
-                path,
-                urlbase,
-                params=params,
-                auth_cookie=self.cookie,
-                as_blosc2=as_blosc2,
-                timeout=self.timeout,
-                kind=kind,
-            )
         else:
             # Coordinates go over as `indices` and are gathered by the server;
             # a plain box is a `slice_`, which says the same thing more cheaply
             indices = api_utils.key_to_indices(key, ndim)
             params = (
                 {"slice_": api_utils.slice_to_string(key, ndim)} if indices is None else {"indices": indices}
             )
-            # Fetch and return the data as a Blosc2 object / NumPy array
-            return self._fetch_data(
-                path,
-                urlbase,
-                params,
-                auth_cookie=self.cookie,
-                as_blosc2=as_blosc2,
-                timeout=self.timeout,
-                kind=kind,
-            )
+        if field is not None:
+            if "indices" in params:
+                raise IndexError("field cannot be combined with coordinate indices")
+            params["field"] = field
+        return self._fetch_data(
+            path,
+            urlbase,
+            params,
+            auth_cookie=self.cookie,
+            as_blosc2=as_blosc2,
+            timeout=self.timeout,
+            kind=kind,
+        )
 
     def get_chunk(self, path, nchunk):
         """
@@ -1489,7 +1488,7 @@ def _download_url(self, url, localpath, auth_cookie=None):
 
         return localpath
 
-    def download(self, dataset, localpath=None):
+    def download(self, dataset, localpath=None, *, include_cache=True):
         """
         Downloads a dataset to local storage.
 
@@ -1504,6 +1503,9 @@ def download(self, dataset, localpath=None):
         localpath : Path, optional
             Local path to save the downloaded dataset. Defaults to the current
             working directory if not specified.
+        include_cache : bool, optional
+            For a RemoteArray carrier, include its valid warm cache data.
+            Pass false to download a cold proxy without changing the server copy.
 
         Returns
         -------
@@ -1519,7 +1521,7 @@ def download(self, dataset, localpath=None):
         PosixPath('example/ds-2d-fields.b2nd')
         """
         urlbase, dataset = _format_paths(self.urlbase, dataset)
-        url = api_utils.get_download_url(dataset, urlbase)
+        url = api_utils.get_download_url(dataset, urlbase, include_cache=include_cache)
         localpath = pathlib.Path(localpath) if localpath else None
         if localpath is None:
             path = "." / pathlib.Path(dataset)
@@ -1913,6 +1915,20 @@ def unfold(self, remotepath):
         )
         return PurePosixPath(result)  # return path to top directory of dset
 
+    def refresh(self, path):
+        """Refresh a hosted RemoteStore or RemoteCTable carrier and return a fresh object.
+
+        Pass the carrier's ``.b2z`` path, including for a table inside a store.
+        This endpoint does not refresh RemoteArray references.
+        """
+        if isinstance(path, File):
+            path = path.path
+        _, formatted = _format_paths(self.urlbase, path)
+        if pathlib.PurePosixPath(formatted).suffix != ".b2z":
+            raise ValueError("Refresh requires a hosted RemoteStore or RemoteCTable .b2z carrier")
+        self._post(f"{self.urlbase}/api/refresh/{formatted}", auth_cookie=self.cookie, timeout=self.timeout)
+        return self.get(formatted)
+
     def remove(self, path):
         """
         Removes a dataset or the contents of a directory from a remote repository.

caterva2/clients/cli.py

@@ -173,6 +173,12 @@ def cmd_info(client, args, url):
         return
 
     # Helpers
+    def _print_attrs():
+        attrs = data.get("attrs")
+        if attrs is None:
+            attrs = (data.get("schunk") or data).get("vlmeta", {})
+        print("attrs: " + json.dumps(attrs, indent=2, ensure_ascii=False))
+
     def _human_bytes(n):
         if n is None:
             return "N/A"
@@ -219,6 +225,7 @@ def _filter_names(fl):
         print(f"cbytes : {_human_bytes(cbytes)}")
         print(f"ratio  : {nbytes / cbytes:.2f}x" if nbytes and cbytes else "ratio  : N/A")
         print(f"mtime  : {mtime}") if mtime is not None else print("mtime  : None")
+        _print_attrs()
         return
 
     # Extract fields
@@ -256,6 +263,7 @@ def _filter_names(fl):
         print(f"  filters: [{', '.join(fnames)}]")
     else:
         print("  filters: None")
+    _print_attrs()
 
 
 def _json_default(o):

caterva2/hdf5.py

@@ -371,9 +371,11 @@ def open_leaf(cls, h5file, dsetname):
         self.dset = h5file[dsetname] if dsetname else h5file
         b2args = b2args_from_h5dset(self.dset)
         self.b2arr = blosc2.empty(self.dset.shape or (), dtype=self.dset.dtype, **b2args)
+        for name, value in b2attrs_from_h5dset(self.dset).items():
+            self.b2arr.schunk.vlmeta.set_vlmeta(name, value, typesize=1)
         return self
 
-    def __init__(self, b2arr, h5file=None, dsetname=None):
+    def __init__(self, b2arr, h5file=None, dsetname=None, *, writer=None):
         if b2arr is not None:
             # The file has been opened already, so we just need to set the filename and dataset name
             self.dsetname = b2arr.vlmeta["_dsetname"]
@@ -426,7 +428,7 @@ def __init__(self, b2arr, h5file=None, dsetname=None):
             self.b2arr = blosc2.empty(
                 shape=shape,
                 dtype=dtype,
-                urlpath=urlpath,
+                urlpath=urlpath if writer is None else None,
                 mode="w",
                 **b2args,
             )
@@ -436,7 +438,7 @@ def __init__(self, b2arr, h5file=None, dsetname=None):
             del self.dset
             del self.fname
             del self.dsetname
-            if os.path.exists(urlpath):
+            if writer is None and os.path.exists(urlpath):
                 os.remove(urlpath)
             return
 
@@ -717,7 +719,7 @@ def serialize_h5_attrs_to_json(h5_attrs, indent=2):
     return json_str
 
 
-def create_hdf5_proxies(path: str | os.PathLike) -> Iterator[HDF5Proxy]:
+def create_hdf5_proxies(path: str | os.PathLike, *, writer=None) -> Iterator[HDF5Proxy]:
     """Create a generator of HDF5 proxies from the given HDF5 file."""
     attrs_dsetname = "!_attrs_.json.b2"  # the Blosc2 dataset name for the Group attributes in HDF5
     h5file = h5py.File(path, "r")
@@ -727,22 +729,31 @@ def create_hdf5_proxies(path: str | os.PathLike) -> Iterator[HDF5Proxy]:
     os.makedirs(dirname, exist_ok=True)
     jsonpath = os.path.join(dirname, attrs_dsetname)
     data = serialize_h5_attrs_to_json(h5file.attrs)
-    blosc2.SChunk(data=data.encode("utf-8"), urlpath=jsonpath, mode="w")
+    if writer is None:
+        blosc2.SChunk(data=data.encode("utf-8"), urlpath=jsonpath, mode="w")
+    else:
+        writer(jsonpath, blosc2.SChunk(data=data.encode("utf-8")).to_cframe())
 
     # Recursive function to visit all groups and datasets
     def visit_group(group):
         for name, obj in group.items():
             full_path = f"{group.name}/{name}".lstrip("/")
 
             if isinstance(obj, h5py.Dataset):
-                yield HDF5Proxy(None, h5file, full_path)
+                proxy = HDF5Proxy(None, h5file, full_path, writer=writer)
+                if writer is not None and hasattr(proxy, "b2arr"):
+                    writer(os.path.join(dirname, full_path + ".b2nd"), proxy.b2arr.to_cframe())
+                yield proxy
             if isinstance(obj, h5py.Group):
                 # Store HDF5 group attributes as JSON
                 groupname = dirname + "/" + full_path
                 os.makedirs(groupname, exist_ok=True)
                 jsonpath = os.path.join(groupname, attrs_dsetname)
                 data = serialize_h5_attrs_to_json(obj.attrs)
-                blosc2.SChunk(data=data.encode("utf-8"), urlpath=jsonpath, mode="w")
+                if writer is None:
+                    blosc2.SChunk(data=data.encode("utf-8"), urlpath=jsonpath, mode="w")
+                else:
+                    writer(jsonpath, blosc2.SChunk(data=data.encode("utf-8")).to_cframe())
 
                 # Recursively visit subgroups
                 yield from visit_group(obj)