Merge branch 'master' into docs/remove-changelog-archive

Author: dimitri-yatsenko

src/datajoint/builtin_codecs/filepath.py

@@ -164,7 +164,7 @@ def encode(
 
         return {
             "path": path,
-            "store": store_name,
+            "store": config.resolve_store_name(store_name, use_filepath_default=True),
             "size": size,
             "is_dir": is_dir,
             "timestamp": datetime.now(timezone.utc).isoformat(),

src/datajoint/builtin_codecs/npy.py

@@ -356,7 +356,7 @@ def encode(
         # Return metadata (includes numpy-specific shape/dtype)
         return {
             "path": path,
-            "store": store_name,
+            "store": config.resolve_store_name(store_name),
             "dtype": str(value.dtype),
             "shape": list(value.shape),
         }

src/datajoint/builtin_codecs/object.py

@@ -168,7 +168,7 @@ def encode(
         timestamp = datetime.now(timezone.utc)
         metadata = {
             "path": path,
-            "store": store_name,
+            "store": config.resolve_store_name(store_name),
             "size": size,
             "ext": ext,
             "is_dir": is_dir,

src/datajoint/builtin_codecs/schema.py

@@ -66,7 +66,7 @@ def encode(self, value, *, key=None, context=None, store_name=None):
                 )
                 backend = self._get_backend(store_name, config=config)
                 backend.put_buffer(serialize(value), path)
-                return {"path": path, "store": store_name, ...}
+                return {"path": path, "store": config.resolve_store_name(store_name), ...}
 
             def decode(self, stored, *, key=None, context=None):
                 config = self._codec_config(key, context)

src/datajoint/declare.py

@@ -203,6 +203,7 @@ def compile_foreign_key(
     adapter,
     fk_attribute_map: dict[str, tuple[str, str]] | None = None,
     fk_index_candidates: list[list[str]] | None = None,
+    column_comments: dict[str, str] | None = None,
 ) -> None:
     """
     Parse a foreign key line and update declaration components.
@@ -233,6 +234,11 @@ def compile_foreign_key(
         candidate supporting index; the redundancy/coverage decision is made
         post-parse (once the full primary key and index list are known).
         Updated in place.
+    column_comments : dict, optional
+        Mapping of ``column -> comment`` for backends that set column comments
+        out of line (PostgreSQL's ``COMMENT ON COLUMN``). Each copied column's
+        comment, with its core-type marker (e.g. ``:int32:``), is added so the
+        child keeps the parent's declared type. Updated in place.
 
     Raises
     ------
@@ -288,14 +294,17 @@ def compile_foreign_key(
             # Enum type names start with "enum_" (generated hash-based names)
             if sql_type.startswith("enum_") and adapter.backend == "postgresql":
                 sql_type = f"{adapter.quote_identifier(ref.database)}.{adapter.quote_identifier(sql_type)}"
+            comment = parent_attr.sql_comment
             col_def = adapter.format_column_definition(
                 name=attr,
                 sql_type=sql_type,
                 nullable=is_nullable,
                 default=None,
-                comment=parent_attr.sql_comment,
+                comment=comment,
             )
             attr_sql.append(col_def)
+            if comment and column_comments is not None:
+                column_comments[attr] = comment
 
         # Track FK attribute mapping for lineage: child_attr -> (parent_table, parent_attr)
         if fk_attribute_map is not None:
@@ -459,6 +468,7 @@ def add_attribute(line: str, in_key: bool) -> None:
                 adapter,
                 fk_attribute_map,
                 fk_index_candidates,
+                column_comments,
             )
         elif re.match(r"^(unique\s+)?index\s*\(.*\)\s*(#.*)?$", line, re.I):  # index
             compile_index(re.sub(r"\s*#.*$", "", line), index_sql, adapter)

src/datajoint/gc.py

@@ -113,8 +113,14 @@ def __init__(self, *schemas: "Schema", store: str | None = None, config=None) ->
         if not schemas:
             raise DataJointError("At least one schema must be provided")
         self.schemas = schemas
-        self.store = store
+        # An empty name (the bare ``@`` form) means the default store, as None
+        # does. Kept as "", it would match no reference in _references, and every
+        # stored file would look orphaned.
+        self.store = store or None
         self.config = config if config is not None else schemas[0].connection._config
+        # References record the resolved store name. Metadata written before
+        # that may record an empty name or none; both mean the default store.
+        self._default_store = self.config.stores.get("default")
         # Resolve the store eagerly: validates it exists and pins the backend
         # and section prefixes for this collector's lifetime (one store only).
         self.backend = get_store_backend(store, config=self.config)
@@ -174,7 +180,7 @@ def _references(self, heading_attr: str, verbose: bool) -> set[str]:
                         try:
                             for row in table.proj(attr_name).cursor(as_dict=True):
                                 for path, ref_store in attr.codec.referenced_paths(row[attr_name]):
-                                    if self.store is None or ref_store == self.store:
+                                    if self.store is None or (ref_store or self._default_store) == self.store:
                                         referenced.add(path)
                         except Exception as e:
                             logger.warning(f"Error scanning {table_name}.{attr_name}: {e}")

src/datajoint/hash_registry.py

@@ -250,7 +250,7 @@ def put_hash(
         "hash": content_hash,
         "path": path,  # Always stored for retrieval
         "schema": schema_name,
-        "store": store_name,
+        "store": config.resolve_store_name(store_name),
         "size": len(data),
     }
 

src/datajoint/heading.py

@@ -127,13 +127,23 @@ def sql_comment(self) -> str:
         """
         Return the full SQL comment including type markers.
 
+        A core type (``int32``, ``uuid``, ``float64``, ...) is recorded in the column
+        comment as ``:type:`` so it can be reconstructed from the database; heading
+        parsing strips the marker into ``original_type``. Restoring it here keeps the
+        core type on columns copied by a foreign key.
+
         Returns
         -------
         str
-            Comment with optional ``:uuid:`` prefix.
+            Comment with the core-type prefix, e.g. ``:int32:``, when there is one.
         """
-        # UUID info is stored in the comment for reconstruction
-        return (":uuid:" if self.uuid else "") + (self.comment or "")
+        if self.original_type:
+            marker = f":{self.original_type}:"
+        elif self.uuid:
+            marker = ":uuid:"
+        else:
+            marker = ""
+        return marker + (self.comment or "")
 
     @property
     def sql(self) -> str:

src/datajoint/settings.py

@@ -447,6 +447,58 @@ def convert_path(cls, v: Any) -> Path | None:
             return None
         return Path(v) if not isinstance(v, Path) else v
 
+    def resolve_store_name(self, store: str | None = None, *, use_filepath_default: bool = False) -> str:
+        """
+        Name the configured store that a store reference resolves to.
+
+        An empty or missing name means the default store: ``stores.default``,
+        or ``stores.filepath_default`` for filepath references. Codecs record
+        the resolved name in their metadata, so a stored object stays tied to
+        the store it was written to even if the default is later re-pointed.
+
+        Parameters
+        ----------
+        store : str, optional
+            Store name as declared. ``""`` (the bare ``@`` form) and None both
+            mean the default store.
+        use_filepath_default : bool, optional
+            If True, resolve the default to stores.filepath_default instead of
+            stores.default. Default: False.
+
+        Returns
+        -------
+        str
+            Name of a key in ``stores``.
+
+        Raises
+        ------
+        DataJointError
+            If the default store is needed but not configured.
+        """
+        if store:
+            return store
+        # An empty name is the bare ``@`` form (``<npy@>``, ``<object@>``, ...),
+        # which the declaration keeps as "" so the codec can tell "store
+        # requested" from "no store"; it means the default store.
+        if use_filepath_default:
+            # Filepath references use separate default (not part of OAS)
+            if "filepath_default" not in self.stores:
+                raise DataJointError(
+                    "stores.filepath_default is not configured. "
+                    "Set stores.filepath_default or specify store explicitly with <filepath@store>"
+                )
+            store = self.stores["filepath_default"]
+        else:
+            # Integrated storage (hash, schema) uses stores.default
+            if "default" not in self.stores:
+                raise DataJointError("stores.default is not configured")
+            store = self.stores["default"]
+
+        if not isinstance(store, str):
+            default_key = "filepath_default" if use_filepath_default else "default"
+            raise DataJointError(f"stores.{default_key} must be a string")
+        return store
+
     def get_store_spec(self, store: str | None = None, *, use_filepath_default: bool = False) -> dict[str, Any]:
         """
         Get configuration for a storage store.
@@ -470,25 +522,7 @@ def get_store_spec(self, store: str | None = None, *, use_filepath_default: bool
         DataJointError
             If store is not configured or has invalid config.
         """
-        # Handle default store
-        if store is None:
-            if use_filepath_default:
-                # Filepath references use separate default (not part of OAS)
-                if "filepath_default" not in self.stores:
-                    raise DataJointError(
-                        "stores.filepath_default is not configured. "
-                        "Set stores.filepath_default or specify store explicitly with <filepath@store>"
-                    )
-                store = self.stores["filepath_default"]
-            else:
-                # Integrated storage (hash, schema) uses stores.default
-                if "default" not in self.stores:
-                    raise DataJointError("stores.default is not configured")
-                store = self.stores["default"]
-
-            if not isinstance(store, str):
-                default_key = "filepath_default" if use_filepath_default else "default"
-                raise DataJointError(f"stores.{default_key} must be a string")
+        store = self.resolve_store_name(store, use_filepath_default=use_filepath_default)
 
         # Check store exists
         if store not in self.stores:

tests/integration/test_fk_core_types.py

@@ -0,0 +1,130 @@
+"""
+A column copied by a foreign key keeps the parent's core type.
+
+Core types (``int32``, ``int16``, ``uuid``, ...) are recorded in the column comment as
+``:type:``. A foreign key copies the parent's primary-key columns into the child; the copy
+must carry the same marker, or the child's heading reports the native SQL type and every
+table derived from it (a jobs table, a further child) inherits the loss.
+"""
+
+import logging
+
+import pytest
+
+import datajoint as dj
+
+
+class _Records(logging.Handler):
+    def __init__(self):
+        super().__init__(level=logging.WARNING)
+        self.messages = []
+
+    def emit(self, record):
+        self.messages.append(record.getMessage())
+
+
+@pytest.mark.backend_agnostic
+def test_fk_columns_keep_core_types(connection_by_backend, backend, prefix):
+    schema = dj.Schema(f"{prefix}_fk_core_types_{backend}", connection=connection_by_backend)
+    try:
+
+        @schema
+        class Subject(dj.Manual):
+            definition = """
+            subject_id : int32
+            """
+
+        @schema
+        class Session(dj.Manual):
+            definition = """
+            -> Subject
+            session_idx : int16
+            """
+
+        @schema
+        class Summary(dj.Manual):
+            definition = """
+            -> Session
+            ---
+            n : int32
+            """
+
+        assert Subject.heading["subject_id"].original_type == "int32"
+        # inherited one level down
+        assert Session.heading["subject_id"].original_type == "int32"
+        assert Session.heading["session_idx"].original_type == "int16"
+        # inherited two levels down
+        assert Summary.heading["subject_id"].original_type == "int32"
+        assert Summary.heading["session_idx"].original_type == "int16"
+        # the comment text itself is unchanged
+        assert Summary.heading["subject_id"].comment == Subject.heading["subject_id"].comment
+    finally:
+        schema.drop(prompt=False)
+
+
+@pytest.mark.backend_agnostic
+def test_fk_columns_keep_uuid(connection_by_backend, backend, prefix):
+    schema = dj.Schema(f"{prefix}_fk_core_types_uuid_{backend}", connection=connection_by_backend)
+    try:
+
+        @schema
+        class Item(dj.Manual):
+            definition = """
+            item_id : uuid       # identifier of the item
+            """
+
+        @schema
+        class Tag(dj.Manual):
+            definition = """
+            -> Item
+            tag : varchar(16)
+            """
+
+        assert Tag.heading["item_id"].uuid
+        assert Tag.heading["item_id"].original_type == "uuid"
+        assert Tag.heading["item_id"].comment == "identifier of the item"
+    finally:
+        schema.drop(prompt=False)
+
+
+@pytest.mark.backend_agnostic
+def test_jobs_table_keeps_inherited_core_types(connection_by_backend, backend, prefix):
+    """A jobs table built from inherited keys declares core types and warns about nothing."""
+    schema = dj.Schema(f"{prefix}_fk_core_types_jobs_{backend}", connection=connection_by_backend)
+    records = _Records()
+    logger = logging.getLogger("datajoint")
+    try:
+
+        @schema
+        class Subject(dj.Manual):
+            definition = """
+            subject_id : int32
+            """
+
+        @schema
+        class Session(dj.Manual):
+            definition = """
+            -> Subject
+            session_idx : int16
+            """
+
+        @schema
+        class Summary(dj.Computed):
+            definition = """
+            -> Session
+            ---
+            n : int32
+            """
+
+            def make(self, key):
+                self.insert1(dict(key, n=1))
+
+        logger.addHandler(records)
+        jobs = Summary.jobs
+        assert jobs.heading["subject_id"].original_type == "int32"
+        assert jobs.heading["session_idx"].original_type == "int16"
+        native = [m for m in records.messages if m.startswith("Native type")]
+        assert native == [], native
+    finally:
+        logger.removeHandler(records)
+        schema.drop(prompt=False)