Merge branch 'master' into feat/schema-namespace

Author: dimitri-yatsenko

.github/workflows/draft_release.yaml

@@ -26,6 +26,6 @@ jobs:
         with:
           config-name: release_drafter.yaml
           disable-autolabeler: true
-          name: ${{ github.event.inputs.testpypi == 'true' && 'Test $RESOLVED_VERSION' || 'Release $RESOLVED_VERSION' }}
+          name: ${{ github.event.inputs.testpypi == 'true' && 'Test DataJoint Library $RESOLVED_VERSION' || 'DataJoint Library $RESOLVED_VERSION' }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

README.md

@@ -1,6 +1,6 @@
-# DataJoint for Python
+# DataJoint Library for Python
 
-DataJoint is a framework for scientific data pipelines based on the **Relational Workflow Model** — a paradigm where your database schema is an executable specification of your workflow.
+The **DataJoint Library** is a framework for scientific data pipelines based on the **Relational Workflow Model** — a paradigm where your database schema is an executable specification of your workflow.
 
 - **Tables represent workflow steps** — Each table is a step in your pipeline
 - **Foreign keys encode dependencies** — Parent tables must be populated before child tables

pyproject.toml

@@ -32,7 +32,7 @@ maintainers = [
   {name = "DataJoint Contributors", email = "support@datajoint.com"},
 ]
 # manually sync here: https://docs.datajoint.com/core/datajoint-python/latest/#welcome-to-datajoint-for-python
-description = "DataJoint for Python is a framework for scientific workflow management based on relational principles. DataJoint is built on the foundation of the relational data model and prescribes a consistent method for organizing, populating, computing, and querying data."
+description = "The DataJoint Library for Python is a framework for scientific workflow management based on relational principles. It is built on the foundation of the relational data model and prescribes a consistent method for organizing, populating, computing, and querying data."
 readme = "README.md"
 license = {file = "LICENSE"}
 keywords = [

src/datajoint/builtin_codecs/filepath.py

@@ -164,7 +164,7 @@ def encode(
 
         return {
             "path": path,
-            "store": store_name,
+            "store": config.resolve_store_name(store_name, use_filepath_default=True),
             "size": size,
             "is_dir": is_dir,
             "timestamp": datetime.now(timezone.utc).isoformat(),

src/datajoint/builtin_codecs/npy.py

@@ -356,7 +356,7 @@ def encode(
         # Return metadata (includes numpy-specific shape/dtype)
         return {
             "path": path,
-            "store": store_name,
+            "store": config.resolve_store_name(store_name),
             "dtype": str(value.dtype),
             "shape": list(value.shape),
         }

src/datajoint/builtin_codecs/object.py

@@ -168,7 +168,7 @@ def encode(
         timestamp = datetime.now(timezone.utc)
         metadata = {
             "path": path,
-            "store": store_name,
+            "store": config.resolve_store_name(store_name),
             "size": size,
             "ext": ext,
             "is_dir": is_dir,

src/datajoint/builtin_codecs/schema.py

@@ -66,7 +66,7 @@ def encode(self, value, *, key=None, context=None, store_name=None):
                 )
                 backend = self._get_backend(store_name, config=config)
                 backend.put_buffer(serialize(value), path)
-                return {"path": path, "store": store_name, ...}
+                return {"path": path, "store": config.resolve_store_name(store_name), ...}
 
             def decode(self, stored, *, key=None, context=None):
                 config = self._codec_config(key, context)

src/datajoint/declare.py

@@ -203,6 +203,7 @@ def compile_foreign_key(
     adapter,
     fk_attribute_map: dict[str, tuple[str, str]] | None = None,
     fk_index_candidates: list[list[str]] | None = None,
+    column_comments: dict[str, str] | None = None,
 ) -> None:
     """
     Parse a foreign key line and update declaration components.
@@ -233,6 +234,11 @@ def compile_foreign_key(
         candidate supporting index; the redundancy/coverage decision is made
         post-parse (once the full primary key and index list are known).
         Updated in place.
+    column_comments : dict, optional
+        Mapping of ``column -> comment`` for backends that set column comments
+        out of line (PostgreSQL's ``COMMENT ON COLUMN``). Each copied column's
+        comment, with its core-type marker (e.g. ``:int32:``), is added so the
+        child keeps the parent's declared type. Updated in place.
 
     Raises
     ------
@@ -288,14 +294,17 @@ def compile_foreign_key(
             # Enum type names start with "enum_" (generated hash-based names)
             if sql_type.startswith("enum_") and adapter.backend == "postgresql":
                 sql_type = f"{adapter.quote_identifier(ref.database)}.{adapter.quote_identifier(sql_type)}"
+            comment = parent_attr.sql_comment
             col_def = adapter.format_column_definition(
                 name=attr,
                 sql_type=sql_type,
                 nullable=is_nullable,
                 default=None,
-                comment=parent_attr.sql_comment,
+                comment=comment,
             )
             attr_sql.append(col_def)
+            if comment and column_comments is not None:
+                column_comments[attr] = comment
 
         # Track FK attribute mapping for lineage: child_attr -> (parent_table, parent_attr)
         if fk_attribute_map is not None:
@@ -459,6 +468,7 @@ def add_attribute(line: str, in_key: bool) -> None:
                 adapter,
                 fk_attribute_map,
                 fk_index_candidates,
+                column_comments,
             )
         elif re.match(r"^(unique\s+)?index\s*\(.*\)\s*(#.*)?$", line, re.I):  # index
             compile_index(re.sub(r"\s*#.*$", "", line), index_sql, adapter)

src/datajoint/gc.py

@@ -113,8 +113,14 @@ def __init__(self, *schemas: "Schema", store: str | None = None, config=None) ->
         if not schemas:
             raise DataJointError("At least one schema must be provided")
         self.schemas = schemas
-        self.store = store
+        # An empty name (the bare ``@`` form) means the default store, as None
+        # does. Kept as "", it would match no reference in _references, and every
+        # stored file would look orphaned.
+        self.store = store or None
         self.config = config if config is not None else schemas[0].connection._config
+        # References record the resolved store name. Metadata written before
+        # that may record an empty name or none; both mean the default store.
+        self._default_store = self.config.stores.get("default")
         # Resolve the store eagerly: validates it exists and pins the backend
         # and section prefixes for this collector's lifetime (one store only).
         self.backend = get_store_backend(store, config=self.config)
@@ -174,7 +180,7 @@ def _references(self, heading_attr: str, verbose: bool) -> set[str]:
                         try:
                             for row in table.proj(attr_name).cursor(as_dict=True):
                                 for path, ref_store in attr.codec.referenced_paths(row[attr_name]):
-                                    if self.store is None or ref_store == self.store:
+                                    if self.store is None or (ref_store or self._default_store) == self.store:
                                         referenced.add(path)
                         except Exception as e:
                             logger.warning(f"Error scanning {table_name}.{attr_name}: {e}")

src/datajoint/hash_registry.py

@@ -250,7 +250,7 @@ def put_hash(
         "hash": content_hash,
         "path": path,  # Always stored for retrieval
         "schema": schema_name,
-        "store": store_name,
+        "store": config.resolve_store_name(store_name),
         "size": len(data),
     }