From 545dc17d6f88291c97888226046d46609fbfe7f3 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 17:34:14 +0000 Subject: [PATCH 1/2] sqlitedb: announce and time the one-time search index build (port of DID-matlab f35bbb4) Opening a database without doc_data_doc_idx / doc_data_field_value builds them, silently - and for a large dataset on a slow or synced drive that takes minutes, so the open appears to hang. _ensure_indexes now prints to stderr, before each index it builds, its name, the database, the approximate number of rows (max(rowid)) and that this happens once and can take minutes; afterwards the time it took; and a build that failed. A database that already has the indexes opens without printing anything. Test: test_indexes_added_when_older_database_opened checks the messages on the first open and silence on the next. Bridge: sqlitedb -> f35bbb4. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Dv7SxVR8MCu3MKpUvwwUia --- ..._matlab_python_bridge_implementations.yaml | 6 +- src/did/implementations/sqlitedb.py | 58 +++++++++++++++---- tests/test_search_indexes.py | 17 +++++- 3 files changed, 68 insertions(+), 13 deletions(-) diff --git a/src/did/did_matlab_python_bridge_implementations.yaml b/src/did/did_matlab_python_bridge_implementations.yaml index f4617a2..c03718e 100644 --- a/src/did/did_matlab_python_bridge_implementations.yaml +++ b/src/did/did_matlab_python_bridge_implementations.yaml @@ -29,7 +29,7 @@ classes: - name: sqlitedb type: class matlab_path: "+did/+implementations/sqlitedb.m" - matlab_last_sync_hash: "83a2bda" + matlab_last_sync_hash: "f35bbb4" status: ported python_path: "did/implementations/sqlitedb.py" python_class: "SQLiteDB" @@ -55,6 +55,10 @@ classes: applies negation as a complement in Python rather than in SQL, so a negated term with candidates is the complement within the candidates rather than within the branch -- the same set once intersected. + DID-matlab f35bbb4 (2026-10-24): ensure_search_indexes announces each + index before building it (with the approximate doc_data row count), + reports the time it took, and reports a build that failed; ported to + _ensure_indexes, which prints the same lines to stderr. sync_notes: > MATLAB 14439fb (2026-06-12, audit 6.1-3) added a private escapeSqlLiteral() helper and applied it at the 14 sites where diff --git a/src/did/implementations/sqlitedb.py b/src/did/implementations/sqlitedb.py index 978bc94..d2f727d 100644 --- a/src/did/implementations/sqlitedb.py +++ b/src/did/implementations/sqlitedb.py @@ -389,6 +389,15 @@ def _ensure_indexes(self): by another writer) is left as it is, as DID-matlab leaves it: the indexes only speed searches up, and without them searches run as they did before. + + Building them is a one-time cost, but not a small one: + doc_data_field_value holds a copy of every field value, so for a large + dataset (100,000 documents can be tens of millions of doc_data rows) + on a slow or synced drive it can take minutes. Each index is announced + on stderr before it is built, with the approximate number of rows, and + timed -- otherwise the open just appears to hang. A database that + already has them opens without printing anything. Mirrors DID-matlab's + sqlitedb/ensure_search_indexes. """ self._search_indexes_ready = False try: @@ -398,18 +407,45 @@ def _ensure_indexes(self): "SELECT name FROM sqlite_master WHERE type = 'index'" ) } - missing = [idx for idx in _SEARCH_INDEXES if idx[0] not in existing] - if missing: - cursor = self.dbid.cursor() - for name, columns in missing: - cursor.execute( - f'CREATE INDEX IF NOT EXISTS "{name}" ON doc_data({columns})' - ) - self.dbid.commit() except sqlite3.Error: - with contextlib.suppress(sqlite3.Error): - self.dbid.rollback() - return + return # cannot even read the schema - search without the indexes + for name, columns in _SEARCH_INDEXES: + if name in existing: + continue + # max(rowid) is read straight from the end of the table's b-tree, + # so this estimate costs nothing (count(*) would scan) + try: + num_rows = self.dbid.execute( + "SELECT max(rowid) FROM doc_data" + ).fetchone()[0] + except sqlite3.Error: + num_rows = None + print( + f"DID: adding search index {name} to {self.connection} " + f"(about {num_rows or 0:,} rows). This happens once per database " + "and can take several minutes for a large one...", + file=sys.stderr, + ) + started = time.perf_counter() + try: + self.dbid.execute( + f'CREATE INDEX IF NOT EXISTS "{name}" ON doc_data({columns})' + ) + self.dbid.commit() + except sqlite3.Error as err: + with contextlib.suppress(sqlite3.Error): + self.dbid.rollback() + print( + f"DID: could not add search index {name} ({err}); " + "searches will run without it.", + file=sys.stderr, + ) + return + print( + f"DID: added search index {name} in " + f"{time.perf_counter() - started:.1f} s.", + file=sys.stderr, + ) self._search_indexes_ready = True def _close_db(self): diff --git a/tests/test_search_indexes.py b/tests/test_search_indexes.py index fd6bd69..eab04cd 100644 --- a/tests/test_search_indexes.py +++ b/tests/test_search_indexes.py @@ -20,6 +20,8 @@ path however quickly it happens to run on a small test database. """ +import contextlib +import io import os import sqlite3 import statistics @@ -162,17 +164,30 @@ def test_indexes_added_when_older_database_opened(self): older.dbid.commit() older._close_db() - reopened = SQLiteDB(path) + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + reopened = SQLiteDB(path) try: names = self.index_names(reopened) for name in SEARCH_INDEXES: self.assertIn(name, names, f"opening must add {name}") + # building an index on a large database can take minutes, + # so each one is announced and timed, not built silently + self.assertIn(f"adding search index {name}", stderr.getvalue()) + self.assertIn(f"added search index {name} in ", stderr.getvalue()) self.assertTrue(reopened._search_indexes_ready) q = _id_query(doc.id()) & _session_query(self.session) self.assertEqual(reopened.search(q, branch_id="a"), [doc.id()]) finally: reopened._close_db() + # ...and only once: a database that has them opens quietly + stderr = io.StringIO() + with contextlib.redirect_stderr(stderr): + again = SQLiteDB(path) + again._close_db() + self.assertNotIn("search index", stderr.getvalue()) + def test_read_only_database_without_indexes_still_searches(self): with tempfile.TemporaryDirectory() as tmp: path = os.path.join(tmp, "readonly.sqlite") From 58c4cc5bee0bc8b01ed7aa4c11743b5c35dcae9f Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 17:34:53 +0000 Subject: [PATCH 2/2] bridge: sqlitedb -> 5b7220b (DID-matlab made groupDigits private; nothing to port) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01Dv7SxVR8MCu3MKpUvwwUia --- src/did/did_matlab_python_bridge_implementations.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/did/did_matlab_python_bridge_implementations.yaml b/src/did/did_matlab_python_bridge_implementations.yaml index c03718e..11ab65c 100644 --- a/src/did/did_matlab_python_bridge_implementations.yaml +++ b/src/did/did_matlab_python_bridge_implementations.yaml @@ -29,7 +29,7 @@ classes: - name: sqlitedb type: class matlab_path: "+did/+implementations/sqlitedb.m" - matlab_last_sync_hash: "f35bbb4" + matlab_last_sync_hash: "5b7220b" status: ported python_path: "did/implementations/sqlitedb.py" python_class: "SQLiteDB" @@ -55,7 +55,7 @@ classes: applies negation as a complement in Python rather than in SQL, so a negated term with candidates is the complement within the candidates rather than within the branch -- the same set once intersected. - DID-matlab f35bbb4 (2026-10-24): ensure_search_indexes announces each + DID-matlab f35bbb4..5b7220b (2026-10-24): ensure_search_indexes announces each index before building it (with the approximate doc_data row count), reports the time it took, and reports a build that failed; ported to _ensure_indexes, which prints the same lines to stderr.