diff --git a/src/did/did_matlab_python_bridge.yaml b/src/did/did_matlab_python_bridge.yaml index 16bda24..75ddf98 100644 --- a/src/did/did_matlab_python_bridge.yaml +++ b/src/did/did_matlab_python_bridge.yaml @@ -40,12 +40,23 @@ classes: - name: database type: class matlab_path: "+did/database.m" - matlab_last_sync_hash: "bb39399" + matlab_last_sync_hash: "83a2bda" status: ported python_path: "did/database.py" python_class: "Database" inherits: "abc.ABC" out_of_sync: false + decision_log: > + DID-matlab#229 (dae960b..83a2bda, 2026-10-24) made database.m's search + fast. search_doc_ids searches the terms of an AND in rank order (base.id + lookups, other terms, base.session_id, then the terms that test every + value) and checks each later term against the documents found so far; + ORed base.id lookups become one query (search_base_ids); and 'isa' + matches its regex against the distinct superclass lists only. All of it + relies on the doc_data indexes that sqlitedb creates (hasSearchIndexes). + In Python the SQL search is SQLiteDB's, not Database's, so the port is in + did/implementations/sqlitedb.py -- see the sqlitedb entry. Nothing + changes in did/database.py. sync_notes: > Schema validation was ported on 2026-08-28, closing what had been the largest capability gap between the two languages. Python previously had diff --git a/src/did/did_matlab_python_bridge_implementations.yaml b/src/did/did_matlab_python_bridge_implementations.yaml index 5c613c6..f4617a2 100644 --- a/src/did/did_matlab_python_bridge_implementations.yaml +++ b/src/did/did_matlab_python_bridge_implementations.yaml @@ -29,13 +29,32 @@ classes: - name: sqlitedb type: class matlab_path: "+did/+implementations/sqlitedb.m" - matlab_last_sync_hash: "bb39399" + matlab_last_sync_hash: "83a2bda" status: ported python_path: "did/implementations/sqlitedb.py" python_class: "SQLiteDB" inherits_matlab: "did.database" inherits_python: "Database" out_of_sync: false + decision_log: > + DID-matlab#229 (dae960b..83a2bda, 2026-10-24): fast id and isa searches, + ported to SQLiteDB (_search_doc_ids, _search_query_head, + _search_base_ids, _query_struct_to_sql_str, _ensure_indexes). MATLAB + adopted the index names Python already created -- doc_data_doc_idx ON + doc_data(doc_idx), doc_data_field_value ON doc_data(field_idx, value) -- + so a database touched by both carries one copy of each. Both languages + now add them on open, leave a read-only file without them, and only then + (_search_indexes_ready / hasSearchIndexes) restrict terms to candidates, + collapse ORed base.id lookups, and match 'isa' against the distinct + superclass lists. Three deviations, none of which changes a result: + (1) Python also ranks the operations it brute-forces + (hasanysubfield_*, hassize, partial_struct) with regexp / + contains_string / hasmember as last-in-an-AND, since a brute-force term + loads every document it tests; (2) a brute-force term given candidates + loads only those documents (MATLAB has no brute-force path); (3) Python + applies negation as a complement in Python rather than in SQL, so a + negated term with candidates is the complement within the candidates + rather than within the branch -- the same set once intersected. sync_notes: > MATLAB 14439fb (2026-06-12, audit 6.1-3) added a private escapeSqlLiteral() helper and applied it at the 14 sites where diff --git a/src/did/implementations/sqlitedb.py b/src/did/implementations/sqlitedb.py index f4f4101..978bc94 100644 --- a/src/did/implementations/sqlitedb.py +++ b/src/did/implementations/sqlitedb.py @@ -164,6 +164,95 @@ def _sql_like_escape(value): return text +# Most document ids one search query lists when it checks a search term +# against candidate documents (see SQLiteDB._search_doc_ids); also the most +# ORed base.id values looked up in one query. DID-matlab's +# did.database.maxSearchCandidates. +_MAX_SEARCH_CANDIDATES = 1000 + +# The doc_data indexes searches rely on -- see SQLiteDB._ensure_indexes. The +# names are the same in DID-matlab (sqlitedb.searchIndexes). +_SEARCH_INDEXES = ( + ("doc_data_value", "value"), + ("doc_data_field_value", "field_idx, value"), + ("doc_data_doc_idx", "doc_idx"), +) + +# Operations that test every value they see -- in SQL or by loading every +# document (brute force) -- and so are searched last in an AND, against the +# documents the other terms left. DID-matlab ranks regexp, contains_string +# and hasmember this way; the rest are the operations only Python sends to +# brute force. +_SCAN_OPERATIONS = frozenset( + { + "regexp", + "contains_string", + "hasmember", + "hasanysubfield_exact_string", + "hasanysubfield_contains_string", + "hassize", + "partial_struct", + } +) + + +def _sql_string_list(values): + """A comma-separated list of single-quoted SQL string literals.""" + return ",".join(f"'{_sql_escape(v)}'" for v in values) + + +def _base_id_lookup_values(search_struct): + """Is a search structure a lookup of documents by base.id? + + Returns ``(True, ids)`` if *search_struct* is a ``base.id`` exact_string + match for a non-empty id, or an OR of such matches nested to any depth; + otherwise ``(False, [])``. Negated operations are not lookups. An OR's + operands are lists; a one-element list is its element, a longer one is an + AND and not a lookup. The tree is walked with a stack, not recursion, so a + query built as ``q = q | Query("base.id", ...)`` in a loop cannot run into + the recursion limit. Mirrors DID-matlab's did.database.baseIdLookupValues. + """ + found = [] + pending = [search_struct] + while pending: + item = pending.pop() + if isinstance(item, list): + if len(item) != 1: + return False, [] + item = item[0] + if not isinstance(item, dict): + return False, [] + op = str(item.get("operation") or "").strip().lower() + if op == "exact_string": + value = item.get("param1") + if ( + item.get("field") != "base.id" + or not isinstance(value, str) + or not value + ): + return False, [] + found.append(value) + elif op == "or": + pending.extend([item.get("param2"), item.get("param1")]) + else: + return False, [] + return True, found + + +def _search_rank(item): + """Where a term of an AND is searched -- see SQLiteDB._search_doc_ids.""" + if _base_id_lookup_values(item)[0]: + return 1 + if not isinstance(item, dict): + return 2 + op = str(item.get("operation") or "").lstrip("~").strip().lower() + if op in _SCAN_OPERATIONS: + return 4 + if item.get("field") == "base.session_id": + return 3 + return 2 + + def _is_safe_uid(uid): """Is ``uid`` safe to use as a filename inside FileDir? @@ -233,6 +322,8 @@ def __init__(self, filename): self._journal = None # statements of the current write operation self._pending_deletes = None # originals to delete once it commits self._db_identity = None # (st_dev, st_ino) of the file opened + # Do the _SEARCH_INDEXES exist? Set by _ensure_indexes. + self._search_indexes_ready = False self._open_db() # Create FileDir now, as DID-MATLAB's sqlitedb constructor does: @@ -287,17 +378,39 @@ def _ensure_indexes(self): Run on every open (``IF NOT EXISTS``) so databases downloaded before this fix benefit on next use. (docs.doc_id and fields.field_name are already covered by UNIQUE constraints.) + + DID-matlab creates the same indexes, under the same names + (``_SEARCH_INDEXES``), and adds them to a database it opens without + them. ``_search_doc_ids`` relies on two of them -- it names + ``doc_data_doc_idx`` with ``INDEXED BY`` -- so it uses them only once + they are known to exist (``_search_indexes_ready``). + + A database that cannot be written to (a read-only file, or one locked + by another writer) is left as it is, as DID-matlab leaves it: the + indexes only speed searches up, and without them searches run as they + did before. """ - cursor = self.dbid.cursor() - cursor.execute('CREATE INDEX IF NOT EXISTS "doc_data_value" ON doc_data(value)') - cursor.execute( - 'CREATE INDEX IF NOT EXISTS "doc_data_field_value" ' - "ON doc_data(field_idx, value)" - ) - cursor.execute( - 'CREATE INDEX IF NOT EXISTS "doc_data_doc_idx" ON doc_data(doc_idx)' - ) - self._commit() + self._search_indexes_ready = False + try: + existing = { + row[0] + for row in self.dbid.execute( + "SELECT name FROM sqlite_master WHERE type = 'index'" + ) + } + missing = [idx for idx in _SEARCH_INDEXES if idx[0] not in existing] + if missing: + cursor = self.dbid.cursor() + for name, columns in missing: + cursor.execute( + f'CREATE INDEX IF NOT EXISTS "{name}" ON doc_data({columns})' + ) + self.dbid.commit() + except sqlite3.Error: + with contextlib.suppress(sqlite3.Error): + self.dbid.rollback() + return + self._search_indexes_ready = True def _close_db(self): if self.dbid: @@ -1421,24 +1534,52 @@ def search(self, query_obj, branch_id=None): doc_ids = self._search_doc_ids(search_params, branch_id) return doc_ids - def _search_doc_ids(self, search_struct, branch_id): + def _search_doc_ids(self, search_struct, branch_id, candidates=None): """Recursively search for doc_ids matching the search structure. Matches MATLAB's search_doc_ids: struct arrays are AND'd, 'or' operations are unioned, leaf queries go through SQL. + + If *candidates* (a list of document ids) is given, only those documents + are considered, so the result is a subset of them. Once one term of an + AND has narrowed the result to at most ``_MAX_SEARCH_CANDIDATES`` + documents, the remaining terms are checked against those documents + only. The terms of an AND are searched in this order (see + ``_search_rank``): base.id lookups first, as they match at most a few + documents; then the other terms, in the order given; then + base.session_id terms, as one session usually holds most of a + database; and last the terms that test every value they see + (``_SCAN_OPERATIONS``). So ``isa daqsystem AND base.session_id = S AND + base.name regexp ...`` finds the few daqsystem documents, then checks + their session, then runs the regexp on their names only -- instead of + each term returning, or testing, every document of the branch. + + Restricting to candidates needs the search indexes; without them + (``_search_indexes_ready`` false) every term is searched over the whole + branch, as before. Mirrors DID-matlab's did.database/search_doc_ids. """ + if not self._search_indexes_ready: + candidates = None + if candidates is not None and not candidates: + return [] + if isinstance(search_struct, list): if not search_struct: return [] - # AND: intersect results from all sub-queries + # AND: intersect results from all sub-queries, the most selective + # first (sorted is stable: the given order is kept within a rank) result = None - for item in search_struct: - ids = self._search_doc_ids(item, branch_id) + for item in sorted(search_struct, key=_search_rank): + ids = self._search_doc_ids(item, branch_id, candidates) if result is None: result = set(ids) else: result &= set(ids) - return list(result) if result else [] + if not result: + return [] # no document matches every term - skip the rest + if len(result) <= _MAX_SEARCH_CANDIDATES: + candidates = list(result) + return list(result) if not isinstance(search_struct, dict): return [] @@ -1451,57 +1592,129 @@ def _search_doc_ids(self, search_struct, branch_id): op = op[1:] op_lower = op.lower() + # The documents a negated term is the complement within + if candidates is not None: + universe = candidates + else: + universe = None # every document of the branch, fetched on demand + if op_lower == "or": + is_lookup, lookup_ids = _base_id_lookup_values(search_struct) + if is_lookup and self._search_indexes_ready: + # base.id lookups ORed together: search all the ids at once + return self._search_base_ids(lookup_ids, branch_id, candidates) # OR: union results from param1 and param2 p1 = search_struct.get("param1") p2 = search_struct.get("param2") - ids1 = self._search_doc_ids(p1, branch_id) if p1 else [] - ids2 = self._search_doc_ids(p2, branch_id) if p2 else [] + ids1 = self._search_doc_ids(p1, branch_id, candidates) if p1 else [] + ids2 = self._search_doc_ids(p2, branch_id, candidates) if p2 else [] result = list(set(ids1) | set(ids2)) if negation: - all_ids = set(self._do_get_doc_ids(branch_id)) - result = list(all_ids - set(result)) + if universe is None: + universe = self._do_get_doc_ids(branch_id) + result = list(set(universe) - set(result)) return result # Leaf query: build SQL and execute try: - sql_clause = self._query_struct_to_sql_str(search_struct) + sql_clause = self._query_struct_to_sql_str( + search_struct, restricted=candidates is not None + ) except (ValueError, TypeError): # A numeric operation (exact_number/lessthan/greaterthan/...) was # given a non-numeric param1, so the float() conversion failed. # Fall back to brute force rather than aborting the whole search. - return self._brute_force_search(search_struct, branch_id) + return self._brute_force_search(search_struct, branch_id, candidates) if sql_clause is None: # Fallback to brute-force for unsupported operations - return self._brute_force_search(search_struct, branch_id) + return self._brute_force_search(search_struct, branch_id, candidates) - query = ( - "SELECT DISTINCT docs.doc_id FROM docs, branch_docs, doc_data, fields " - "WHERE docs.doc_idx = doc_data.doc_idx " - "AND docs.doc_idx = branch_docs.doc_idx " - "AND branch_docs.branch_id = ? " - "AND fields.field_idx = doc_data.field_idx " - f"AND {sql_clause}" - ) + query = self._search_query_head(candidates) + f" AND {sql_clause}" try: rows = self.do_run_sql_query(query, (branch_id,)) matched = [row["doc_id"] for row in rows] except sqlite3.OperationalError: # Fallback on SQL error - return self._brute_force_search(search_struct, branch_id) + return self._brute_force_search(search_struct, branch_id, candidates) if negation: - all_ids = set(self._do_get_doc_ids(branch_id)) - return list(all_ids - set(matched)) + if universe is None: + universe = self._do_get_doc_ids(branch_id) + return list(set(universe) - set(matched)) + + return matched + @staticmethod + def _search_query_head(candidates=None): + """The SELECT ... WHERE of a search query, before its term. + + Takes the branch id as its one bound parameter. If *candidates* (a list + of document ids) is given, only those documents are considered, and + the tables are joined with CROSS JOIN, which makes SQLite loop over + them in the order written: each candidate is found through the unique + docs.doc_id index, and its doc_data rows through doc_data_doc_idx, + named with INDEXED BY. Left to choose, the query planner (which has no + table statistics to go on) can start from every document of the + branch, or reach doc_data through an index on the searched value -- + for base.session_id, one row per document in the session, for every + candidate. Mirrors DID-matlab's did.database/get_sql_query_str. + """ + if candidates is None: + tables = "docs, branch_docs, doc_data, fields" + else: + tables = ( + "docs CROSS JOIN branch_docs CROSS JOIN fields " + "CROSS JOIN doc_data INDEXED BY doc_data_doc_idx" + ) + query = ( + f"SELECT DISTINCT docs.doc_id FROM {tables} " + "WHERE docs.doc_idx = doc_data.doc_idx " + "AND docs.doc_idx = branch_docs.doc_idx " + "AND branch_docs.branch_id = ? " + "AND fields.field_idx = doc_data.field_idx" + ) + if candidates is not None: + query += f" AND docs.doc_id IN ({_sql_string_list(candidates)})" + return query + + def _search_base_ids(self, ids, branch_id, candidates=None): + """Find the documents whose base.id is one of *ids*. + + Equivalent to ORing a base.id exact_string query for each of *ids*, in + one query per ``_MAX_SEARCH_CANDIDATES`` ids rather than one per id. A + document's doc_id is its base.id, so *ids* also serve as the candidate + documents, and are looked up through the docs.doc_id index. The + doc_data condition is kept so that the result is exactly what the ORed + exact_string queries return. Mirrors DID-matlab's + did.database/search_base_ids. + """ + ids = sorted(set(ids)) + if candidates is not None: + allowed = set(candidates) + ids = [i for i in ids if i in allowed] + matched = [] + for start in range(0, len(ids), _MAX_SEARCH_CANDIDATES): + chunk = ids[start : start + _MAX_SEARCH_CANDIDATES] + query = ( + self._search_query_head(chunk) + + " AND fields.field_name = 'base.id'" + + f" AND doc_data.value IN ({_sql_string_list(chunk)})" + ) + matched.extend( + row["doc_id"] for row in self.do_run_sql_query(query, (branch_id,)) + ) return matched - def _query_struct_to_sql_str(self, search_struct): + def _query_struct_to_sql_str(self, search_struct, restricted=False): """Convert a single query struct to a SQL WHERE clause fragment. Returns None if the operation is not supported in SQL. Matches MATLAB's query_struct_to_sql_str. + + *restricted* is true when the query is limited to a list of candidate + documents (see ``_search_query_head``); it changes how 'isa' is + matched. """ field = search_struct.get("field", "") operation = search_struct.get("operation", "") @@ -1583,10 +1796,33 @@ def _query_struct_to_sql_str(self, search_struct): classname_re = _sql_escape( _re.escape("" if param1 is None else str(param1)) ) + superclass_pattern = f"'(^|, ){classname_re}(,|$)'" + superclass_match = ( + f"regexp({superclass_pattern}, doc_data.value) IS NOT NULL" + ) + if not restricted and self._search_indexes_ready: + # Over a whole branch, running the regexp on every document's + # superclass list took seconds (~5 s for 100,000 documents in + # DID-matlab). Most documents share one of a few dozen + # superclass lists, so match the regexp against the distinct + # lists only -- read in order from the doc_data_field_value + # index -- and then find the documents holding a matching list + # through the same index. LIMIT -1 keeps SQLite from merging + # the inner DISTINCT query into the outer one, which would run + # the regexp on every row again. Negation is applied by the + # caller, so this clause is always the positive match. + superclass_match = ( + "doc_data.value IN (" + "SELECT superclasses FROM (" + "SELECT DISTINCT sc.value AS superclasses " + "FROM doc_data sc, fields scf " + "WHERE scf.field_name = 'meta.superclass' " + "AND sc.field_idx = scf.field_idx LIMIT -1) " + f"WHERE regexp({superclass_pattern}, superclasses) IS NOT NULL)" + ) return ( f"((fields.field_name = 'meta.class' AND doc_data.value = '{classname}') " - f"OR (fields.field_name = 'meta.superclass' AND " - f"regexp('(^|, ){classname_re}(,|$)', doc_data.value) IS NOT NULL))" + f"OR (fields.field_name = 'meta.superclass' AND {superclass_match}))" ) elif op_lower == "depends_on": @@ -1629,11 +1865,18 @@ def _query_struct_to_sql_str(self, search_struct): return None - def _brute_force_search(self, search_struct, branch_id): - """Fall back to brute-force field_search for unsupported SQL operations.""" + def _brute_force_search(self, search_struct, branch_id, candidates=None): + """Fall back to brute-force field_search for unsupported SQL operations. + + Loads and tests every document of the branch -- or, if *candidates* + (a list of document ids) is given, only those. + """ from ..datastructures import field_search - doc_ids = self._do_get_doc_ids(branch_id) + if candidates is not None: + doc_ids = list(candidates) + else: + doc_ids = self._do_get_doc_ids(branch_id) docs = self.get_docs(doc_ids, OnMissing="ignore") if docs is None: docs = [] diff --git a/tests/test_search_indexes.py b/tests/test_search_indexes.py new file mode 100644 index 0000000..fd6bd69 --- /dev/null +++ b/tests/test_search_indexes.py @@ -0,0 +1,369 @@ +"""The searches NDI makes most must be fast in SQLiteDB. + +NDI searches a session for documents by id and by class:: + + Query("base.id", "exact_string", ID) & Query("base.session_id", "exact_string", S) + Query("", "isa", "daqsystem") & Query("base.session_id", "exact_string", S) + +The terms of an AND used to be searched separately over the whole branch and +their results intersected, so the session term returned every document of the +session; and 'isa' ran a regexp on the superclass list of every document. Now +base.id terms are searched first, base.session_id terms after the others and +whole-value tests (regexp, brute force) last; each later term is checked +against the documents already found only; and 'isa' runs its regexp on the +distinct superclass lists only -- through the doc_data indexes doc_data_doc_idx +and doc_data_field_value. See SQLiteDB._search_doc_ids. Mirrors DID-matlab's +did.unittest.TestSearchIndexes. + +Besides the results, these tests check the number of rows each search query +returned: a lookup that returns a row per document of the session is the slow +path however quickly it happens to run on a small test database. +""" + +import os +import sqlite3 +import statistics +import tempfile +import time +import unittest +import uuid + +from did.document import Document +from did.implementations.sqlitedb import _MAX_SEARCH_CANDIDATES, SQLiteDB +from did.query import Query + +NUM_SESSION_DOCS = 1500 # documents in the main session (demoA) +NUM_OTHER_DOCS = 100 # documents in a second session (demoB) +SEARCH_INDEXES = ("doc_data_doc_idx", "doc_data_field_value") + + +class _QueryLogDB(SQLiteDB): + """A SQLiteDB that records each search query and how many rows it returned.""" + + def __init__(self, filename): + self.query_log = [] + self.loaded_doc_ids = [] + super().__init__(filename) + + def do_run_sql_query(self, query_str, params=()): + rows = super().do_run_sql_query(query_str, params) + if query_str.startswith("SELECT DISTINCT docs.doc_id"): + self.query_log.append((query_str, len(rows))) + return rows + + def get_docs(self, document_ids, *args, **kwargs): + ids = document_ids if isinstance(document_ids, list) else [document_ids] + self.loaded_doc_ids.extend(ids) + return super().get_docs(document_ids, *args, **kwargs) + + +def _doc(class_name, superclasses, session_id, value): + props = { + "base": { + "id": uuid.uuid4().hex, + "session_id": session_id, + "name": "", + "datestamp": "2026-10-24T00:00:00.000Z", + }, + "document_class": { + "class_name": class_name, + "superclasses": [{"class_name": sc} for sc in superclasses], + }, + "demoA": {"value": value}, + } + if class_name == "demoB": + props["demoB"] = {"value": value} + return Document(props) + + +def _id_query(doc_id): + return Query("base.id", "exact_string", doc_id) + + +def _session_query(session_id): + return Query("base.session_id", "exact_string", session_id) + + +def _or_of_ids(ids): + """A balanced OR of base.id queries (shallow, whatever the count).""" + if len(ids) == 1: + return _id_query(ids[0]) + half = len(ids) // 2 + return _or_of_ids(ids[:half]) | _or_of_ids(ids[half:]) + + +class TestSearchIndexes(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tmpdir = tempfile.TemporaryDirectory() + cls.db = _QueryLogDB(os.path.join(cls.tmpdir.name, "search_indexes.sqlite")) + cls.db.add_branch("a") + cls.session = uuid.uuid4().hex + cls.other_session = uuid.uuid4().hex + docs = [ + _doc("demoA", ["base"], cls.session, i) for i in range(NUM_SESSION_DOCS) + ] + other = [ + _doc("demoB", ["base", "demoA"], cls.other_session, i) + for i in range(NUM_OTHER_DOCS) + ] + cls.db.add_docs(docs + other, branch_id="a", validate=False) + cls.ids = [d.id() for d in docs] + cls.other_ids = [d.id() for d in other] + + @classmethod + def tearDownClass(cls): + cls.db._close_db() + cls.tmpdir.cleanup() + + def tearDown(self): + self.db._search_indexes_ready = True + + def search_logged(self, query): + """Search; return (ids, most rows any search query returned, query count).""" + self.db.query_log = [] + found = self.db.search(query, branch_id="a") + rows = [n for _, n in self.db.query_log] + return found, max(rows, default=0), len(rows) + + def search_both_ways(self, query): + """Search with the indexes, then as if the database had none.""" + self.db._search_indexes_ready = True + fast = self.db.search(query, branch_id="a") + self.db._search_indexes_ready = False + try: + full = self.db.search(query, branch_id="a") + finally: + self.db._search_indexes_ready = True + return sorted(fast), sorted(full) + + @staticmethod + def index_names(db): + rows = db.dbid.execute("SELECT name FROM sqlite_master WHERE type = 'index'") + return {row[0] for row in rows} + + # --- the indexes ------------------------------------------------------- + + def test_new_database_has_search_indexes(self): + names = self.index_names(self.db) + for name in SEARCH_INDEXES: + self.assertIn(name, names) + self.assertTrue(self.db._search_indexes_ready) + + def test_indexes_added_when_older_database_opened(self): + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "older.sqlite") + older = SQLiteDB(path) + older.add_branch("a") + doc = _doc("demoA", ["base"], self.session, 1) + older.add_docs([doc], branch_id="a", validate=False) + for name in SEARCH_INDEXES: + older.dbid.execute(f'DROP INDEX "{name}"') + older.dbid.commit() + older._close_db() + + reopened = SQLiteDB(path) + try: + names = self.index_names(reopened) + for name in SEARCH_INDEXES: + self.assertIn(name, names, f"opening must add {name}") + self.assertTrue(reopened._search_indexes_ready) + q = _id_query(doc.id()) & _session_query(self.session) + self.assertEqual(reopened.search(q, branch_id="a"), [doc.id()]) + finally: + reopened._close_db() + + def test_read_only_database_without_indexes_still_searches(self): + with tempfile.TemporaryDirectory() as tmp: + path = os.path.join(tmp, "readonly.sqlite") + older = SQLiteDB(path) + older.add_branch("a") + doc = _doc("demoB", ["base", "demoA"], self.session, 1) + older.add_docs([doc], branch_id="a", validate=False) + for name in SEARCH_INDEXES: + older.dbid.execute(f'DROP INDEX "{name}"') + older.dbid.commit() + older._close_db() + + db = SQLiteDB.__new__(SQLiteDB) + # Open through a read-only URI, so creating an index must fail + real_connect = sqlite3.connect + + def read_only_connect(_filename, **kwargs): + return real_connect(f"file:{path}?mode=ro", uri=True, **kwargs) + + sqlite3.connect = read_only_connect + try: + SQLiteDB.__init__(db, path) + finally: + sqlite3.connect = real_connect + try: + self.assertFalse(db._search_indexes_ready) + q = Query("", "isa", "demoA") & _session_query(self.session) + self.assertEqual(db.search(q, branch_id="a"), [doc.id()]) + finally: + db._close_db() + + # --- base.id lookups --------------------------------------------------- + + def test_id_lookup_alone(self): + doc_id = self.ids[777] + found, max_rows, _ = self.search_logged(_id_query(doc_id)) + self.assertEqual(found, [doc_id]) + self.assertLessEqual(max_rows, 1) + + def test_id_and_session_lookup_is_fast(self): + elapsed = [] + for k in (0, 1, 499, 999, NUM_SESSION_DOCS - 1): + doc_id = self.ids[k] + q = _id_query(doc_id) & _session_query(self.session) + t0 = time.perf_counter() + found, max_rows, _ = self.search_logged(q) + elapsed.append(time.perf_counter() - t0) + self.assertEqual(found, [doc_id]) + self.assertLessEqual( + max_rows, + 1, + "the session term must be checked against the one document " + "found by id, not return every document of the session", + ) + # The order of the terms must not matter + q = _session_query(self.session) & _id_query(self.ids[42]) + found, max_rows, _ = self.search_logged(q) + self.assertEqual(found, [self.ids[42]]) + self.assertLessEqual(max_rows, 1) + self.assertLess(statistics.median(elapsed), 0.25) + + def test_id_lookup_in_wrong_session_finds_nothing(self): + q = _id_query(self.ids[3]) & _session_query(self.other_session) + found, max_rows, _ = self.search_logged(q) + self.assertEqual(found, []) + self.assertLessEqual(max_rows, 1) + + def test_unknown_id_skips_the_other_terms(self): + q = _id_query("no_such_id_0123456789") & _session_query(self.session) + found, max_rows, num_queries = self.search_logged(q) + self.assertEqual(found, []) + self.assertEqual(max_rows, 0) + self.assertEqual(num_queries, 1, "the session term need not be searched") + + def test_or_of_ids_returns_all_matches_in_one_query(self): + wanted = [self.ids[k] for k in (5, 50, 500, 1234)] + [self.other_ids[7]] + q = _id_query(wanted[0]) + for doc_id in wanted[1:]: + q = q | _id_query(doc_id) # nested left to right, as a loop builds it + found, max_rows, num_queries = self.search_logged(q) + self.assertEqual(sorted(found), sorted(wanted)) + self.assertLessEqual(max_rows, len(wanted)) + self.assertEqual(num_queries, 1) + + q2 = q | _id_query("no_such_id_0123456789") + self.assertEqual(sorted(self.db.search(q2, branch_id="a")), sorted(wanted)) + + found, max_rows, _ = self.search_logged(q & _session_query(self.session)) + self.assertEqual(sorted(found), sorted(wanted[:-1])) + self.assertLessEqual(max_rows, len(wanted)) + + def test_or_of_more_ids_than_one_query_lists(self): + wanted = self.ids[:1200] + self.other_ids[:10] + self.assertGreater(len(wanted), _MAX_SEARCH_CANDIDATES) + q = _or_of_ids(wanted) + self.assertEqual(sorted(self.db.search(q, branch_id="a")), sorted(wanted)) + q = q & _session_query(self.session) + self.assertEqual( + sorted(self.db.search(q, branch_id="a")), sorted(self.ids[:1200]) + ) + + # --- isa ---------------------------------------------------------------- + + def test_isa_and_session_checks_only_the_class_documents(self): + isa = Query("", "isa", "demoB") + session = _session_query(self.other_session) + for q in (isa & session, session & isa): + found, max_rows, _ = self.search_logged(q) + self.assertEqual(sorted(found), sorted(self.other_ids)) + self.assertLessEqual( + max_rows, + NUM_OTHER_DOCS, + "the session term must be searched last, over the isa matches only", + ) + found, max_rows, _ = self.search_logged(isa & _session_query(self.session)) + self.assertEqual(found, []) + self.assertLessEqual(max_rows, NUM_OTHER_DOCS) + + def test_isa_matches_with_and_without_indexes(self): + for class_name in ("demoA", "demoB", "base", "demo", "nosuchclass"): + for op in ("isa", "~isa"): + fast, full = self.search_both_ways(Query("", op, class_name)) + self.assertEqual(fast, full, f"{op} {class_name}") + self.assertEqual( + len(self.db.search(Query("", "isa", "demoA"), branch_id="a")), + NUM_SESSION_DOCS + NUM_OTHER_DOCS, + "every demoB document is also a demoA (its superclass)", + ) + self.assertEqual(self.db.search(Query("", "isa", "demo"), branch_id="a"), []) + + # --- terms that test every value ---------------------------------------- + + def test_regexp_is_checked_last(self): + regexp = Query("base.datestamp", "regexp", "^[0-9]") + isa = Query("", "isa", "demoB") + session = _session_query(self.other_session) + for q in ( + regexp & isa & session, + session & regexp & isa, + isa & session & regexp, + ): + found, max_rows, _ = self.search_logged(q) + self.assertEqual(sorted(found), sorted(self.other_ids)) + self.assertLessEqual(max_rows, NUM_OTHER_DOCS) + + def test_brute_force_term_loads_only_the_candidates(self): + # hasmember has no SQL form, so it loads and tests documents + doc_id = self.ids[10] + q = Query("base.id", "hasmember", doc_id) & _id_query(doc_id) + self.db.loaded_doc_ids = [] + self.assertEqual( + self.db.search(q, branch_id="a"), self.db.search(q, branch_id="a") + ) + self.db.loaded_doc_ids = [] + self.db.search(q, branch_id="a") + self.assertLessEqual( + len(self.db.loaded_doc_ids), + 1, + "a brute-force term after an id lookup must load that document only", + ) + + # --- agreement ---------------------------------------------------------- + + def test_restricted_search_matches_full_search(self): + doc_id, other_id = self.ids[10], self.other_ids[10] + id_or = _id_query(doc_id) | _id_query(other_id) + terms = [ + _session_query(self.session), + Query("base.session_id", "~exact_string", self.session), + Query("", "isa", "demoB"), + Query("", "~isa", "demoB"), + Query("demoB", "hasfield"), + Query("demoA.value", "exact_number", 10), + Query("demoA.value", "greaterthan", 5), + Query("base.id", "contains_string", doc_id[:8]), + Query("base.datestamp", "regexp", "^[0-9]"), + Query("demoA.value", "exact_number", 10) | Query("", "isa", "demoB"), + ] + for k, term in enumerate(terms): + for lookup in (_id_query(doc_id), id_or): + fast, full = self.search_both_ways(lookup & term) + self.assertEqual(fast, full, f"term {k}") + + def test_searches_without_id_still_work(self): + q = _session_query(self.other_session) & Query("", "isa", "demoB") + self.assertEqual( + sorted(self.db.search(q, branch_id="a")), sorted(self.other_ids) + ) + q = _session_query(self.session) & Query("demoA.value", "lessthaneq", 2) + self.assertEqual(len(self.db.search(q, branch_id="a")), 3) + + +if __name__ == "__main__": + unittest.main()