Bridge checker: read a MATLAB transpose as a transpose; bridge sqlitedb's 8 public statics

bin/check_bridge_coverage.py

@@ -156,6 +156,20 @@
 _PROPERTY = re.compile(r"^\s*([A-Za-z_]\w*)\s*(?:\(|\{|=|[A-Za-z_]|;|$)")
 
 
+def _opens_string(line: str, i: int) -> bool:
+    """Is the (unquoted) `'` at line[i] a string opener, not a transpose?
+
+    MATLAB's `'` is the transpose operator when it directly follows an
+    identifier, a closing bracket, a `.` or another transpose -- `x'`,
+    `reshape(v,[],n)'`. Reading that as a string opener swallows the rest of
+    the line, so a trailing `catch, end` goes uncounted and every method
+    after it in the block drops out of the member check (sqlitedb.m's
+    run_sql_noOpen did exactly this). Judged on the source line, not on
+    anything already blanked out.
+    """
+    return i == 0 or not (line[i - 1].isalnum() or line[i - 1] in "_)]}.'")
+
+
 def _strip_comment(line: str) -> str:
     """Drop a trailing MATLAB comment, respecting single-quoted strings."""
     out = []
@@ -169,7 +183,8 @@ def _strip_comment(line: str) -> str:
                 out.append("''")
                 i += 2
                 continue
-            in_str = not in_str
+            if in_str or _opens_string(line, i):
+                in_str = not in_str
         elif ch == "%" and not in_str:
             break
         out.append(ch)
@@ -190,7 +205,7 @@ def _outside_brackets(line: str) -> str:
     i = 0
     while i < len(line):
         ch = line[i]
-        if ch == "'" and not in_str and (not out or out[-1] in " ,(=[{"):
+        if ch == "'" and not in_str and _opens_string(line, i):
             in_str = True
         elif ch == "'" and in_str:
             if i + 1 < len(line) and line[i + 1] == "'":

src/did/did_matlab_python_bridge.yaml

@@ -939,7 +939,7 @@ classes:
   - name: document
     type: class
     matlab_path: "+did/document.m"
-    matlab_last_sync_hash: "c95b4f2"
+    matlab_last_sync_hash: "bb29861"
     status: ported
     python_path: "did/document.py"
     python_class: "Document"
@@ -1414,6 +1414,17 @@ classes:
       Both languages now accept NAME_<i> as a numbered entry only when <i>
       is a non-negative integer.
 
+      Sync hash bumped to bb29861 on 2026-10-23. That commit replaced the
+      constructor's inline char(datetime('now','TimeZone','UTCLeapSeconds'))
+      with did.fun.timestamp(), which clamps a ':60' leap second to
+      ':59.999' (DID-matlab issue #53). Ported the same way: Document.__init__
+      now stamps base.datestamp with did.fun.timestamp() (see the timestamp
+      entry in did_matlab_python_bridge_util.yaml). The module-private
+      document._utc_timestamp it replaces had its own clamp, a plain
+      ':60.' -> ':59.999' substitution that would have turned ':60.123' into
+      ':59.999123'; it is removed rather than kept beside the port.
+      Coverage: tests/test_timestamp.py.
+
   # -----------------------------------------------------------------------
   # did.query
   # -----------------------------------------------------------------------

src/did/did_matlab_python_bridge_implementations.yaml

@@ -29,7 +29,7 @@ classes:
   - name: sqlitedb
     type: class
     matlab_path: "+did/+implementations/sqlitedb.m"
-    matlab_last_sync_hash: "8bea92c"
+    matlab_last_sync_hash: "da83fa3"
     status: ported
     python_path: "did/implementations/sqlitedb.py"
     python_class: "SQLiteDB"
@@ -133,6 +133,60 @@ classes:
       new tests, including the acceptance case, the "one fetch shared
       across members" cache guarantee, and the handler-refused failure).
 
+      2026-10-23: synced through DID-matlab decd155 and da83fa3 (PRs #206,
+      #207), the slow-drive SQLITE_READONLY work. MATLAB (1) sets
+      busy_timeout=30000 in open_db, (2) retries a BUSY / READONLY failure in
+      run_sql_noOpen with backoffs 0.1, 0.2, 0.5, 1, 2, 4 s, (3) closes and
+      reopens the connection before each READONLY retry (a poisoned handle,
+      SQLITE_READONLY_DBMOVED, which waiting never clears), and (4) prints
+      filesystem diagnostics to stderr when the retries are exhausted.
+      Ported: _open_db passes timeout=30.0 to sqlite3.connect (the same
+      busy_timeout); do_run_sql_query retries on the same schedule, reopening
+      first for READONLY; _is_retryable_sql_error, _is_readonly_error and
+      _collect_readonly_diagnostics mirror the three new MATLAB helpers; the
+      debug-gated progress messages are printed when self.debug is set.
+      The three helpers are protected in MATLAB and so carry no member
+      entries here. _is_retryable_sql_error and _is_readonly_error are
+      staticmethods (MATLAB's take ~ for the object). The diagnostics run the
+      same four probes -- db file and time, -journal/-wal/-shm siblings with
+      sizes, the file's write permission (os.access for fileattrib's
+      UserWrite), and a temp-file write probe of the directory -- but print
+      the time as ISO-8601 UTC rather than MATLAB's local datestr.
+
+      DEVIATION -- where the retry may run. mksqlite autocommits every
+      statement, so in MATLAB a failed statement is the whole unit of work
+      and re-running it (even after a reopen) is safe. SQLiteDB groups each
+      add/remove/branch operation into one sqlite3 transaction. Rolling back
+      or reopening mid-transaction would silently discard the statements
+      already pending, and a retried tail would then commit a partial
+      operation. So do_run_sql_query retries only when the connection had no
+      uncommitted work before the statement; inside a transaction the error
+      is reported (with diagnostics) and raised at once, and the caller's
+      existing rollback runs.
+
+      DEVIATION -- the commit is retried too. Because Python's writes reach
+      the disk at COMMIT rather than at each statement, that is where a
+      slow-drive failure lands here. Every commit now goes through
+      _commit(), which retries a BUSY / READONLY COMMIT on the same schedule
+      while the transaction is still open. It never reopens -- that would
+      lose the transaction -- so a genuinely poisoned handle (DBMOVED) is not
+      recovered at COMMIT: the error is raised and the operation rolled back.
+      Replaying a whole operation after a reopen would cover that case, but
+      _do_add_doc also copies ingested files, so it is not obviously
+      idempotent; not attempted.
+
+      DEVIATION -- classification. Like MATLAB, BUSY vs READONLY is judged
+      from the error message (sqlite3.Error.sqlite_errorcode would be exact
+      but is 3.11+ only, and this package supports 3.10). Unlike MATLAB, only
+      sqlite3.Error instances qualify, so an unrelated Python exception whose
+      text says "read only" is never retried.
+
+      Not ported: MATLAB's run_sql_noOpen also prints every failing query
+      and its bound values to stderr. That predates these commits, and
+      Python never did it; only the new diagnostics block is printed here.
+      Coverage: tests/test_sqlitedb_transient_errors.py (real lock and
+      read-only-handle failures, with time.sleep replaced).
+
     properties:
       - name: FileDir
         python_name: ~   # not ported
@@ -174,6 +228,10 @@ classes:
           Python: do_run_sql_query(query_str, params=()).
           Python uses parameterized queries. Synchronized 2026-03-15.
 
+          Carries MATLAB run_sql_noOpen's BUSY / READONLY retry (decd155,
+          da83fa3), restricted to statements run with no transaction open;
+          see the entry-level sync_notes, 2026-10-23.
+
       - name: do_add_doc
         python_name: _do_add_doc
         out_of_sync: false
@@ -571,12 +629,115 @@ classes:
           combined "open_db / close_db" entry 2026-08-28 so each method
           resolves on its own.
 
+          Both set a 30 s busy timeout (decd155): MATLAB with pragma
+          busy_timeout=30000, Python with sqlite3.connect(timeout=30.0),
+          which is the same setting. Python's previous default was 5 s.
+
       - name: close_db
         python_name: _close_db
         decision_log: >
           See open_db. Synchronized 2026-03-15. Split out of a combined
           entry 2026-08-28.
 
+      # The eight public static helpers below were invisible to the member
+      # checks until 2026-10-23: the checker read the transpose in
+      # run_sql_noOpen's `reshape(varargin,[],numRows)'; catch, end` as the
+      # start of a string, lost the `end`, and attributed everything after it
+      # -- this whole methods (Static, Access=public) block -- to nothing.
+      # Python keeps each one private (or inlines it); none is public API here.
+      - name: dispatchCustomFileHandler
+        python_name: _dispatch_custom_file_handler
+        kind: static
+        decision_log: >
+          Same arity rule: three or more positional parameters, or *args,
+          get (dest_path, source_path, context); exactly two get the
+          two-argument call. Python counts parameters with inspect.signature
+          where MATLAB uses nargin. Two differences: a context of None is
+          always dispatched with two arguments, and a callable whose
+          signature cannot be inspected is tried with three, then two, on
+          TypeError (MATLAB's fallback is the two-argument contract).
+
+      - name: handlerTakesContext
+        python_name: ~
+        kind: static
+        decision_log: >
+          Not separated out. Its rule lives inline in
+          _dispatch_custom_file_handler. DIVERGENCE, recorded 2026-10-23:
+          MATLAB's fetchSeriesMemberBytes calls handlerTakesContext and
+          refuses to ask a two-argument handler for a series member at all,
+          since it would only see the manifest's location. Python's
+          _open_series_member has no such gate: it offers the member to
+          any handler, and a two-argument handler that fetches the manifest
+          again is caught afterwards by the same-bytes check (see
+          filesAreIdentical). The outcome is a refusal either way, but
+          Python spends one fetch to reach it, and a two-argument handler
+          that returns something else is accepted where MATLAB never asks.
+          Not changed here; that is a behaviour port, not a bridge fix.
+
+      - name: filesAreIdentical
+        python_name: ~
+        kind: static
+        decision_log: >
+          Inlined in _open_series_member, its only use (the DID-matlab#191
+          guard against a handler returning the manifest's bytes for a
+          member). Same order -- sizes first, bytes only when the sizes
+          match -- and an unreadable file counts as "not identical". Python
+          compares the two files with one read each rather than 1 MiB
+          chunks.
+
+      - name: isSafeUid
+        python_name: _is_safe_uid
+        kind: static
+        decision_log: >
+          A module-level function in did/implementations/sqlitedb.py. MATLAB
+          delegates to did.file.isSafeUid; Python's _is_safe_uid is a
+          separate copy of the same predicate as did.file.is_safe_uid (see
+          the isSafeUid entry in did_matlab_python_bridge_file.yaml). The
+          is_safe_uid docstring says the sqlitedb one delegates to it; it
+          does not, though the two bodies agree today.
+
+      - name: containsTraversal
+        python_name: ~
+        kind: static
+        decision_log: >
+          No textual '..' check in Python. MATLAB's isSafeLocalLocation
+          refuses a '..' segment, then requires an absolute path to be
+          isWithin the database directory. Python's _validated_local_path
+          resolves the location (relative ones against the database
+          directory) and requires the result to be _is_within it, which
+          refuses every escaping '..' the textual check catches. It also
+          accepts a '..' that resolves back inside the directory
+          ('sub/../x'), which MATLAB refuses. Conversely, MATLAB accepts any
+          relative path without '..' unchecked, so a relative path through a
+          symlink that points outside passes there; Python's realpath
+          comparison refuses it.
+
+      - name: isAbsolutePath
+        python_name: ~
+        kind: static
+        decision_log: >
+          os.path.isabs, called inline in _resolve_local. MATLAB treats a
+          leading '\' as absolute on every platform, and 'C:' only on
+          Windows; os.path.isabs follows the running OS's own rules.
+
+      - name: isRemoteLocation
+        python_name: _is_remote_location
+        kind: static
+        decision_log: >
+          Same rule: a file type of 'url' or 'ndicloud', or a location
+          containing '://'. Python lower-cases the type but does not strip
+          surrounding whitespace, as MATLAB's strtrim does.
+
+      - name: isWithin
+        python_name: _is_within
+        kind: static
+        decision_log: >
+          A module-level function. Both resolve symlinks before comparing
+          (os.path.realpath; java.io.File.getCanonicalPath). Python compares
+          with os.path.commonpath and returns False when the paths are on
+          different drives; MATLAB uses a prefix match with a trailing
+          separator.
+
     decision_log: >
       Core database operations are synchronized. MATLAB recently changed
       URL download mechanism (websave -> ndi.cloud.api.files.getFile) which

src/did/did_matlab_python_bridge_util.yaml

@@ -178,6 +178,41 @@ functions:
       Both provide interactive document graph visualization.
       Synchronized 2026-03-15.
 
+  # -----------------------------------------------------------------------
+  # did.fun.timestamp
+  # -----------------------------------------------------------------------
+  - name: timestamp
+    type: function
+    matlab_path: "+did/+fun/timestamp.m"
+    matlab_last_sync_hash: "bb29861"
+    status: ported
+    python_path: "did/fun.py"
+    python_name: "timestamp"
+
+    input_arguments:
+      - name: raw
+        type_matlab: "(1,:) char, default char(datetime('now','TimeZone','UTCLeapSeconds'))"
+        type_python: "str | None, default None (the current UTC time)"
+    output_arguments:
+      - name: ts
+        type_matlab: "char"
+        type_python: "str"
+
+    decision_log: >
+      Ported 2026-10-23 from DID-matlab bb29861 (issue #53). Same regex,
+      (T\d{2}:\d{2}:)60(\.\d+)? -> \g<1>59.999, so a leap second with or
+      without a fraction and with or without a trailing 'Z' clamps exactly as
+      in MATLAB, and nothing outside the seconds field is touched. The
+      default value is the current UTC time as 'yyyy-mm-ddTHH:MM:SS.fffZ',
+      the shape MATLAB's UTCLeapSeconds char renders. One difference, of
+      fact rather than of design: Python's datetime cannot represent a leap
+      second, so the default path never produces ':60' and the clamp only
+      ever acts on a supplied raw string (e.g. one written by MATLAB).
+      Python's validator (validate._parses_as_timestamp, via
+      datetime.fromisoformat) rejects ':60' as java.time.LocalDateTime.parse
+      does, so the clamp is needed on that path for the same reason.
+      Coverage: tests/test_timestamp.py, mirroring MATLAB's TestTimestamp.
+
 # =========================================================================
 # did.datastructures — Data structure utilities
 # =========================================================================

src/did/document.py

@@ -1,26 +1,9 @@
 import contextlib
 import os
-from datetime import datetime, timezone
 
 from . import ido
 from .common import PathConstants
-
-
-def _utc_timestamp():
-    """Return the current UTC time as an ISO-8601 millisecond string with 'Z'.
-
-    Ported from NDI-python (ndi.fun.timestamp) with the leap-second guard.
-    ``str(datetime.utcnow())`` previously emitted a space-separated,
-    timezone-less string ('2026-07-20 22:36:19.611068') that diverges from the
-    DID-matlab UTCLeapSeconds ISO-8601 output and from base.schema.json's own
-    default ('2018-12-05T18:36:47.241Z'); a JS ``new Date()`` parses the
-    tz-less form as LOCAL time. Emit '%Y-%m-%dT%H:%M:%S.%fZ' truncated to
-    milliseconds, clamping the (theoretical) leap-second :60 to :59.999.
-    """
-    now = datetime.now(timezone.utc)
-    ts = now.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3]
-    ts = ts.replace(":60.", ":59.999")
-    return ts + "Z"
+from .fun import timestamp
 
 
 class Document:
@@ -30,7 +13,7 @@ def __init__(self, document_type="base", **kwargs):
         else:
             self.document_properties = self.read_blank_definition(document_type)
             self.document_properties["base"]["id"] = ido.IDO.unique_id()
-            self.document_properties["base"]["datestamp"] = _utc_timestamp()
+            self.document_properties["base"]["datestamp"] = timestamp()
 
             for key, value in kwargs.items():
                 path = key.split(".")

src/did/fun.py

@@ -1,5 +1,36 @@
+import re
+from datetime import datetime, timezone
+
 import networkx as nx
 
+# A leap-second seconds field (':60' with an optional fraction), kept with its
+# 'T HH:MM:' prefix so the date portion is never touched.
+_LEAP_SECOND = re.compile(r"(T\d{2}:\d{2}:)60(\.\d+)?")
+
+
+def timestamp(raw=None):
+    """
+    A current UTC timestamp string that is safe for DID schema validation.
+
+    Returns the current time in the ISO-8601 form used throughout DID,
+    ``'2026-09-18T21:37:45.123Z'``. ``timestamp(raw)`` sanitizes the given
+    string instead of the current time, which is mainly useful for testing.
+
+    A leap second (``':60'``, with or without a fraction) is clamped back to
+    ``':59.999'``: the schema validator parses timestamps the way MATLAB's
+    ``java.time.LocalDateTime.parse`` does, which rejects any seconds value
+    above 59. Python's ``datetime`` never renders a leap second itself, so on
+    the default path the clamp is a guard rather than a fix; it matters for
+    strings that came from MATLAB's ``UTCLeapSeconds`` clock.
+
+    This function mimics the behavior of the Matlab `did.fun.timestamp`
+    function (DID-matlab issue #53).
+    """
+    if raw is None:
+        now = datetime.now(timezone.utc)
+        raw = now.strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z"
+    return _LEAP_SECOND.sub(r"\g<1>59.999", raw)
+
 
 def docs_to_graph(document_objs):
     """