Bundle C2Cache, add CTable (.b2z) and hierarchical container support, chunked writes, and fancy coordinate indexing

.dockerignore

@@ -0,0 +1,7 @@
+*
+!caterva2/
+!c2cache/
+!root-example/
+!pyproject.toml
+!README.md
+!docker/

.github/workflows/python-app-arm64.yml

@@ -2,15 +2,23 @@
 
 name: Python application (ubuntu, arm64)
 
+# Every commit on every branch, not just `main`: work lives on long-running
+# feature branches here, and a branch nobody runs the tests on is a branch whose
+# tests are found broken at merge time.  A pull request from a fork triggers
+# `pull_request` (a fork's pushes are not pushes to this repo), so both are kept.
 on:
   push:
-    branches: [ "main" ]
   pull_request:
-    branches: [ "main" ]
 
 permissions:
   contents: read
 
+# One run per branch: pushing again supersedes the run already going, which
+# matters most on the paid arm64 runner.  A run for `main` is never cancelled.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
 jobs:
   build:
     runs-on: ubicloud-standard-2-arm

.github/workflows/python-app.yml

@@ -3,15 +3,23 @@
 
 name: Python Intel (ubuntu, amd64)
 
+# Every commit on every branch, not just `main`: work lives on long-running
+# feature branches here, and a branch nobody runs the tests on is a branch whose
+# tests are found broken at merge time.  A pull request from a fork triggers
+# `pull_request` (a fork's pushes are not pushes to this repo), so both are kept.
 on:
   push:
-    branches: [ "main" ]
   pull_request:
-    branches: [ "main" ]
 
 permissions:
   contents: read
 
+# One run per branch: pushing again supersedes the run already going, which
+# matters most on the paid arm64 runner.  A run for `main` is never cancelled.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/main' }}
+
 jobs:
   build:
 

.gitignore

@@ -13,3 +13,7 @@ __pycache__
 # jupyterlite
 /.jupyterlite.doit.db
 /caterva2/services/static/jupyterlite/
+/jupyterlite-exts/.dev-jupyter-path/
+
+# container image exports (docker/caterva2-server/, see plans/remote-peer-containers.md)
+/caterva2-peer.tar

.pre-commit-config.yaml

@@ -17,11 +17,16 @@ repos:
       - id: trailing-whitespace
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.7.1
+    rev: v0.16.5
     hooks:
-      - id: ruff
+      - id: ruff-check
         args: ["--fix", "--show-fixes"]
       - id: ruff-format
+        # Python, notebooks and stubs only.  ruff-format grew Markdown support
+        # in 0.16 and the hook now claims `.md` by default, where blacken-docs
+        # below already owns those code blocks -- at black's width rather than
+        # this line-length, so the two rewrite each other on every run.
+        types_or: [python, pyi, jupyter]
         exclude: ^bench/
 
   - repo: https://github.com/adamchainz/blacken-docs

Makefile

@@ -5,6 +5,13 @@ VENV = ./venv
 #   make lite-build BIN=$(dirname $(which jupyter))
 BIN ?= $(VENV)/bin
 
+# Repo-local Jupyter search path for the caterva2-save labextension during
+# development (see lite-ext/lite-build below). Keeping it outside the active
+# environment's own share/jupyter/labextensions/ means it never collides with
+# what `pip install` puts there, so no manual cleanup is ever needed between
+# the two.
+DEV_JUPYTER_PATH = $(CURDIR)/jupyterlite-exts/.dev-jupyter-path
+
 install:
 	python3 -m venv venv
 	${BIN}/pip install -U pip
@@ -23,22 +30,26 @@ assets:
 # Build the caterva2-save JupyterLite extension (needs Node + jlpm/JupyterLab).
 # Commit the resulting jupyterlite-exts/caterva2-save/labextension/ so the
 # caterva2 wheel ships it and downstream installs need no Node toolchain.
-# `develop --overwrite` registers it for an *editable* caterva2 install (whose
-# wheel shared-data is not staged into share/jupyter/labextensions/).
+# Registered for local `jupyter lite build` runs via DEV_JUPYTER_PATH (see
+# lite-build) instead of the environment's own share/jupyter/labextensions/:
+# an *editable* caterva2 install doesn't stage shared-data there at all, and a
+# real wheel install does — either way, writing there ourselves risks
+# colliding with what `pip install` manages.
 lite-ext:
 	cd jupyterlite-exts/caterva2-save && jlpm install && jlpm build:prod
-	# Register for an *editable* caterva2 install (whose wheel shared-data is not
-	# staged into share/jupyter/labextensions/); a real wheel install does this.
-	mkdir -p ${BIN}/../share/jupyter/labextensions
-	ln -sfn $(CURDIR)/jupyterlite-exts/caterva2-save/labextension ${BIN}/../share/jupyter/labextensions/caterva2-save
+	mkdir -p $(DEV_JUPYTER_PATH)/labextensions
+	ln -sfn $(CURDIR)/jupyterlite-exts/caterva2-save/labextension $(DEV_JUPYTER_PATH)/labextensions/caterva2-save
 	git add jupyterlite-exts/caterva2-save/labextension
 
 lite-build:
 	rm -rf .jupyterlite.doit.db caterva2/services/static/jupyterlite
 	# --lite-dir points at the in-package jupyter-lite.json (contentsAllJsonFile),
 	# so the build picks it up regardless of the working directory (the deploy
 	# builds from elsewhere). See caterva2/services/lite-config/.
-	${BIN}/jupyter lite build --lite-dir caterva2/services/lite-config --output-dir caterva2/services/static/jupyterlite
+	# JUPYTER_PATH adds the dev labextension (see lite-ext) on top of whatever
+	# is really installed; harmless/no-op if lite-ext was never run.
+	JUPYTER_PATH=$(DEV_JUPYTER_PATH):$$JUPYTER_PATH \
+		${BIN}/jupyter lite build --lite-dir caterva2/services/lite-config --output-dir caterva2/services/static/jupyterlite
 
 # Installs our jupyterlite fork from a local copy, for development purposes
 # Before doing this you must run "make build" in our jupyterlite fork

README.md

@@ -83,6 +83,15 @@ Append `[extra1,extra2,...]` to any install command:
 
 **Note:** Test runs create a `_caterva2_tests` directory with state files for inspection.
 
+### Optional peer caching
+
+C2Cache is bundled as an internal Caterva2 provider; it requires no separate
+package or installation extra. It remains inactive unless the server
+configuration contains at least one `[[server.peer]]` entry. Each configured
+peer exposes that server's locally owned `@public` root under the configured
+local name and caches requested data on demand. See
+`caterva2-server.sample.toml` for the configuration fields and cache quotas.
+
 ## Quick start
 
 See [Caterva2 documentation](https://ironarray.io/caterva2-doc/index.html) for detailed tutorials.

RELEASE_NOTES.md

@@ -4,6 +4,100 @@
 
 #XXX version-specific blurb XXX#
 
+* C2Cache is bundled and registered as the internal `caterva2.c2cache`
+  provider. It is part of the Caterva2 wheel and release lifecycle, needs no
+  separate package or installed entry-point metadata, and is inert unless at
+  least one `[[server.peer]]` entry activates it.
+
+* Arrays can be filled a chunk at a time, by several writers at once. An array
+  is laid out empty first (`Client.lay_out()`, a couple of hundred bytes
+  whatever its shape) and each writer posts the chunks it owns with
+  `Client.fill_chunk()`, over the new `POST api/chunk/{path}`. Every chunk of a
+  laid-out array is a slot nothing has been written to; a writer claims one by
+  writing it, and a second write to the same slot is refused with a 409 -- so
+  two writers that both believe they own a chunk are resolved by the array
+  rather than by anything either of them holds. `Client.written_chunks()` says
+  how far a fill has got, read from the frame's own offsets, so there is no
+  bookkeeping to fall out of step with the array. A chunk is checked against the
+  whole of the array's geometry before it is stored -- its chunksize, its
+  blocksize and its typesize -- since one compressed against another typesize
+  decompresses to values in the wrong places and says nothing about it.
+
+* A filled array is published as one finished frame. Set `publish_root` in the
+  server configuration -- an fsspec URL of a directory -- and an array is copied
+  there as soon as its last chunk lands, moved into place so that what appears
+  at the destination is a whole frame or nothing. `POST api/publish/{path}`
+  (`Client.publish()`) is the primitive underneath, for finishing a publish that
+  was interrupted. The destination is the server's own configuration and never
+  something a caller names.
+
+* `api/fetch` takes an `indices` parameter: a fancy key as JSON, one entry per
+  dimension -- a list of integers for a dimension indexed by coordinates, an
+  integer for one indexed by a scalar, a string for one indexed by a slice, null
+  for one taken whole. The server gathers the points and sends those, reading
+  only the chunks they land in. The alternative is a client fetching the blocks
+  the points live in and picking them out, and a block is nearly all waste for a
+  single coordinate: nine scattered points of a 900^3 array cost 271 bytes in one
+  request this way against 237 KB in nineteen. Not combinable with `slice_`,
+  `filter` or `field`, and refused -- rather than quietly dropped -- for
+  anything that cannot gather points: a container, or a dataset mounted from a
+  peer, which a provider fetches boxes of. A key may name at most a million
+  coordinates, since it travels in a body now and is no longer bounded by what a
+  URL holds. blosc2 4.11 sends it for `C2Array[[...]]`; earlier clients never do.
+
+* `Client.get_slice()` and `ds[key]` take a fancy key -- a list or array of
+  coordinates, or a boolean mask -- and send it as `indices` for the server to
+  gather, changing verb to POST where it is too long for a URL. What comes back
+  is the points, not the chunks holding them. `slice_to_string()` now refuses an
+  index it cannot express instead of dropping it: a dropped index left an empty
+  slice string, which asks for the whole dataset and hands back all of it,
+  neither what was asked for nor smaller. An `Ellipsis` is expanded against the
+  dataset's shape rather than refused, since it names the dimensions the key
+  does not, and a bound of 0 is written out: `ds[0:0]` selects nothing, where it
+  used to be spelled `:` and select everything.
+
+* `POST api/fetch` takes the same parameters in a body that the GET takes in a
+  query, and is the same code behind them. It exists for the one thing a query
+  string cannot do: carry a key of more coordinates than a URL has room for,
+  which is a few thousand and no encoding moves by much. 200,000 coordinates go
+  through it in one request. Nothing moves off GET -- blosc2 changes verb only
+  where a query could not have been made, so every deployment goes on serving
+  what it already served, and byte ranges stay with the GET they belong to.
+
+* `api/fetch` is explicit about byte ranges, which is what lets a client read
+  single *blocks* of a dataset instead of whole chunks. A stored dataset is
+  served from its file and honours a `Range` as it always did, and now says so
+  with `Accept-Ranges: bytes`; anything built per request -- a slice, a field, a
+  lazy expression, a download -- refuses with a 416 and `Accept-Ranges: none`,
+  where before it answered 200 with the whole body, which is the download such a
+  client was trying to avoid in the first place. A leaf inside a container (a
+  `.b2z` member, or an HDF5 leaf that is a whole frame) is served by seeking to
+  its window in the file, several spans in one `multipart/byteranges` answer if
+  that is what was asked for, so it is readable block-wise like any dataset of
+  its own. `caterva2/tests/test_ranges.py` pins all of it down: a refactor
+  turning that one `FileResponse` into a `StreamingResponse` would otherwise
+  silently cost every such client its block granularity.
+
+* `api/info` reports `accept_ranges` for a dataset: `"bytes"` for one served
+  from a file, `"none"` for one mounted from a peer and re-serialized here, and
+  absent where it is not settled by the description alone (a container leaf,
+  whose answer depends on its own window, or a `.b2nd` that proxies an HDF5
+  dataset, which reads as a stored array here but is rebuilt when fetched).
+  A field an older peer does not send is simply absent, not an error. It says of `api/fetch` what that
+  endpoint would answer, a request earlier: a client reading blocks over byte
+  ranges asks `api/info` anyway, and can now tell a dataset it may range-read
+  from one it may not without spending a request to find out. A field rather
+  than an `Accept-Ranges` header, which would be claiming that `api/info` itself
+  served ranges. Clients that do not know the field are unaffected.
+
+* `api/info` and the ranged file responses carry an `ETag` of our own, built
+  from the frame's generation counter as well as its size and mtime. Starlette's
+  default is a digest of the mtime and the size, which a chunk written as a run
+  of zeros can leave untouched -- such a write stores no payload, so the frame
+  can come out of it exactly as long as it went in. A container leaf carries the
+  container's, on `api/info` and on its ranged reads alike: a leaf is a window of
+  that file, so what says the file did not change says the window did not move.
+
 ## Changes from 2025.11.17.1 to 2025.12.3
 
 * Upload in-memory objects

caterva2-server.sample.toml

@@ -11,6 +11,14 @@
 # - maxusers: if defined, it will limit the number of users (default: 0, no limit)
 # - login: if true, users will need to authenticate (default: true)
 # - register: if true, users will be able to register (default: false)
+# - publish_root: if defined, where an array filled a chunk at a time is copied
+#   once every one of its chunks has landed.  An fsspec URL of a directory
+#   ("s3://bucket/prefix", "file:///srv/published"); a dataset names the key it
+#   wants underneath it and nothing above it.  Credentials for it are the
+#   server's own.  Unset (the default) means such arrays are simply not published.
+# - peer_cache_quota: maximum total size of the optional C2Cache peer cache
+#   (default: "1G"). C2Cache ships inside Caterva2 and remains inactive unless
+#   one or more [[server.peer]] entries are configured.
 #
 [server]
 # listen = "_caterva2/state/uvicorn.socket"  # server supports listening from UNIX sockets
@@ -19,3 +27,13 @@ urlbase = "http://localhost:8000"
 quota = "10G"
 maxusers = 5
 register = true  # allow users to register
+# publish_root = "s3://a-bucket/published"
+# peer_cache_quota = "1G"
+
+# Mount a remote Caterva2 server's locally owned @public root as @labb.  This
+# activates the bundled C2Cache provider.  Repeat the table to mount more peers.
+# An optional per-peer cache_quota further bounds this peer within the pool.
+# [[server.peer]]
+# name = "labb"
+# urlbase = "http://server-b:8000"
+# cache_quota = "500M"

caterva2/__init__.py

@@ -9,16 +9,19 @@
 
 """Caterva2 - On demand access to remote Blosc2 data repositories"""
 
-from .client import BasicAuth, Client, Dataset, File, Root
+from .client import Array, BasicAuth, Client, Dataset, File, Group, Root, Table
 
 __version__ = "2025.12.4.dev0"
 """The version in use of the Caterva2 package."""
 
 __all__ = [
+    "Array",
     "BasicAuth",
     "Client",
     "Dataset",
     "File",
+    "Group",
     "Root",
+    "Table",
 ]
 """List of symbols exported by the Caterva2 package."""