From 68f52898273f9b7829161c34be048e74dc38be10 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 18:54:16 +0000 Subject: [PATCH 1/2] Search the operands of an OR in batches of SQL terms Port of DID-matlab ec95ef7 (DID-matlab#233). _search_doc_ids searched an OR one SQL query per operand. _search_or_operands now searches those that are single SQL terms up to 200 at a time in one query: SELECTs combined with UNION over a whole branch, and one restricted query with the conditions ORed when restricted to candidate documents. ORed base.id lookups among them go to _search_base_ids. ANDs, negated ORs, negated terms and brute-forced terms are searched one at a time as before, and a batch whose SQL fails is retried one operand at a time. tests/test_batched_or.py compares the batched search with one query per operand for every operation the SQL builder supports, in mixed ORs and inside ANDs. Closes #104 Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016Z8rXBFDpnZo3FeCE3ymyZ --- src/did/did_matlab_python_bridge.yaml | 16 +- src/did/implementations/sqlitedb.py | 113 ++++++++- tests/test_batched_or.py | 319 ++++++++++++++++++++++++++ 3 files changed, 443 insertions(+), 5 deletions(-) create mode 100644 tests/test_batched_or.py diff --git a/src/did/did_matlab_python_bridge.yaml b/src/did/did_matlab_python_bridge.yaml index 510d754..e9529b4 100644 --- a/src/did/did_matlab_python_bridge.yaml +++ b/src/did/did_matlab_python_bridge.yaml @@ -40,7 +40,7 @@ classes: - name: database type: class matlab_path: "+did/database.m" - matlab_last_sync_hash: "0de9cf2" + matlab_last_sync_hash: "ec95ef7" status: ported python_path: "did/database.py" python_class: "Database" @@ -62,6 +62,20 @@ classes: lookups, instead of recursing once per nested OR. Ported to SQLiteDB._search_doc_ids (DID-python #100 / #103); nothing changes in did/database.py. + DID-matlab ec95ef7 (2026-10-24, DID-matlab#233 / DID-python#104): + search_doc_ids hands an OR's operands to the new search_or_operands, + which searches those that are single SQL terms up to orBatchTerms + (maxOrBatchTerms = 200) at a time in one query -- UNIONed SELECTs over + a whole branch, one WHERE with the conditions ORed when restricted to + candidates -- and ORed base.id lookups among them by search_base_ids. + Ported to SQLiteDB._search_or_operands / _or_batch_terms / + _MAX_OR_BATCH_TERMS in did/implementations/sqlitedb.py; nothing changes + in did/database.py. Two deviations, neither of which changes a result: + (1) negated terms stay one query each, because Python takes a negated + term's complement in Python rather than in SQL (MATLAB batches them); + (2) terms with no SQL form are brute-forced one at a time, and a batch + whose SQL fails is retried one operand at a time so that each falls + back as it would on its own (MATLAB has no brute-force path). sync_notes: > Schema validation was ported on 2026-08-28, closing what had been the largest capability gap between the two languages. Python previously had diff --git a/src/did/implementations/sqlitedb.py b/src/did/implementations/sqlitedb.py index e0ebf75..d50d072 100644 --- a/src/did/implementations/sqlitedb.py +++ b/src/did/implementations/sqlitedb.py @@ -171,6 +171,12 @@ def _sql_like_escape(value): # did.database.maxSearchCandidates. _MAX_SEARCH_CANDIDATES = 1000 +# Most OR operands searched in one SQL query (see SQLiteDB._search_or_operands). +# SQLite allows 500 SELECTs in a compound SELECT and an expression depth of +# 1000 by default; this stays well below both. DID-matlab's +# did.database.maxOrBatchTerms. +_MAX_OR_BATCH_TERMS = 200 + # The doc_data indexes searches rely on -- see SQLiteDB._ensure_indexes. The # names are the same in DID-matlab (sqlitedb.searchIndexes). _SEARCH_INDEXES = ( @@ -1716,10 +1722,9 @@ def _search_doc_ids(self, search_struct, branch_id, candidates=None): # OR: union the results of its operands. A chain of ORs is # flattened first (or_operands), so a long one built in a loop # does not recurse once per term (issue #100). - matched = set() - for operand in or_operands(search_struct): - if operand: - matched |= set(self._search_doc_ids(operand, branch_id, candidates)) + matched = self._search_or_operands( + or_operands(search_struct), branch_id, candidates + ) result = list(matched) if negation: if universe is None: @@ -1757,6 +1762,106 @@ def _search_doc_ids(self, search_struct, branch_id, candidates=None): return matched + def _or_batch_terms(self): + """The most OR operands searched in one SQL query. + + See ``_search_or_operands``. If it is 1 or less, every operand of an + OR is searched by a query of its own. Mirrors DID-matlab's + did.database/orBatchTerms. + """ + return _MAX_OR_BATCH_TERMS + + def _search_or_operands(self, operands, branch_id, candidates=None): + """The set of documents that match any of an OR's *operands*. + + *operands* are as ``or_operands`` returns them. The result is the + union of what ``_search_doc_ids`` returns for each, but the operands + that are single SQL terms are searched up to ``_or_batch_terms()`` at + a time in one SQL query rather than one query each (issue #104). + + Over a whole branch, a batch is one SELECT per operand -- each exactly + the query the operand would run on its own -- combined with UNION. + SQLite plans each SELECT on its own, so each keeps its index ('isa' + keeps its distinct-superclass subquery). One WHERE with the operands' + conditions joined by OR returns the same documents, but the planner + then reads every row of doc_data: on 100,000 documents of 30 fields + that took ~1 s for 10 names and ~6 s for 100. + + Restricted to *candidates*, it is the other way round: each term + reads every row of every candidate (~10 ms for 1,000 candidates), + whatever it searches for. So a batch is then one restricted query + (``_search_query_head(candidates)``) with the conditions joined by + OR, which reads those rows once: 1,000 names took 0.2 s that way, + against 10 s as 1,000 queries. CROSS JOIN and INDEXED BY fix the + plan, so the OR cannot make SQLite read the whole of doc_data. + + Either way a document matches when one of its doc_data rows satisfies + one of the conditions, so the result is exactly the union of the + operands' results. + + ORed base.id lookups among the operands are searched together by + ``_search_base_ids``. The other operands are searched one at a time by + ``_search_doc_ids``, as before: ANDs, negated ORs, negated terms + (whose negation is a complement taken in Python) and terms with no + SQL form (brute-forced). Mirrors DID-matlab's + did.database/search_or_operands. + """ + batch_size = self._or_batch_terms() + restricted = candidates is not None + id_lookups = [] + terms = [] # (operand, SQL condition) of the operands searched in batches + others = [] + for operand in operands: + if not operand: + continue + item = operand + if isinstance(item, list) and len(item) == 1: + item = item[0] + if batch_size > 1 and isinstance(item, dict): + op = str(item.get("operation") or "").strip().lower() + if not op.startswith("~") and op != "or": + is_lookup, ids = _base_id_lookup_values(item) + if is_lookup and self._search_indexes_ready: + id_lookups.extend(ids) + continue + try: + condition = self._query_struct_to_sql_str( + item, restricted=restricted + ) + except (ValueError, TypeError): + condition = None # brute-forced by _search_doc_ids + if condition is not None: + terms.append((operand, condition)) + continue + others.append(operand) + + matched = set() + if id_lookups: + matched.update(self._search_base_ids(id_lookups, branch_id, candidates)) + for start in range(0, len(terms), batch_size): + batch = terms[start : start + batch_size] + if restricted: + conditions = " OR ".join(f"({condition})" for _, condition in batch) + query = self._search_query_head(candidates) + f" AND ({conditions})" + params = (branch_id,) + else: + head = self._search_query_head() + query = " UNION ".join( + f"{head} AND {condition}" for _, condition in batch + ) + params = (branch_id,) * len(batch) + try: + rows = self.do_run_sql_query(query, params) + except sqlite3.OperationalError: + # Search the operands one at a time, so that one whose SQL + # fails falls back to brute force, as on its own + others.extend(operand for operand, _ in batch) + continue + matched.update(row["doc_id"] for row in rows) + for operand in others: + matched.update(self._search_doc_ids(operand, branch_id, candidates)) + return matched + @staticmethod def _search_query_head(candidates=None): """The SELECT ... WHERE of a search query, before its term. diff --git a/tests/test_batched_or.py b/tests/test_batched_or.py new file mode 100644 index 0000000..c93e465 --- /dev/null +++ b/tests/test_batched_or.py @@ -0,0 +1,319 @@ +"""ORs are searched in batches of SQL terms, with the same results (issue #104). + +SQLiteDB._search_or_operands searches the operands of an OR that are single +SQL terms up to ``_or_batch_terms()`` at a time in one SQL query -- SELECTs +combined with UNION over a whole branch, one WHERE with the conditions ORed +together when restricted to candidate documents -- instead of one query per +operand. These tests compare every search with the same search made one query +per operand (``_or_batch_terms() == 1``, the path used before batching), for +every operation the SQL builder supports, plus the negated and brute-forced +operations that stay on that path, in mixed ORs and inside ANDs; and check that +the batched search runs fewer queries. Mirrors DID-matlab's +did.unittest.TestBatchedOr. +""" + +import os +import sqlite3 +import tempfile +import unittest +import uuid + +from did.document import Document +from did.implementations.sqlitedb import SQLiteDB +from did.query import Query + +NUM_A = 60 # demoA documents +NUM_B = 40 # demoB documents (also demoA) +NUM_C = 30 # demoC documents, depending on demoA and demoB documents + + +class _BatchDB(SQLiteDB): + """A SQLiteDB that counts search queries and can change the OR batch size.""" + + def __init__(self, filename): + self.query_log = [] + self.or_batch_size = None + super().__init__(filename) + + def do_run_sql_query(self, query_str, params=()): + rows = super().do_run_sql_query(query_str, params) + if query_str.startswith("SELECT DISTINCT docs.doc_id"): + self.query_log.append(query_str) + return rows + + def _or_batch_terms(self): + if self.or_batch_size is None: + return super()._or_batch_terms() + return self.or_batch_size + + +def _doc(class_name, superclasses, name, values, depends_on=()): + props = { + "base": { + "id": uuid.uuid4().hex, + "session_id": "s1", + "name": name, + "datestamp": "2026-10-24T00:00:00.000Z", + }, + "document_class": { + "class_name": class_name, + "superclasses": [{"class_name": sc} for sc in superclasses], + }, + "depends_on": [{"name": n, "value": v} for n, v in depends_on], + } + props.update(values) + return Document(props) + + +def _negated(query): + s = query.to_search_structure() + s = s[0] if isinstance(s, list) else s + return Query(s["field"], "~" + s["operation"], s["param1"], s["param2"]) + + +def _or_of(queries): + q = queries[0] + for other in queries[1:]: + q = q | other + return q + + +class TestBatchedOr(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tmpdir = tempfile.TemporaryDirectory() + cls.db = _BatchDB(os.path.join(cls.tmpdir.name, "batched_or.sqlite")) + cls.db.add_branch("a") + docs_a = [ + _doc("demoA", ["base"], f"n{i}", {"demoA": {"value": i}}) + for i in range(1, NUM_A + 1) + ] + docs_b = [ + _doc( + "demoB", + ["base", "demoA"], + f"b{i}", + {"demoA": {"value": i}, "demoB": {"value": 100 + i}}, + ) + for i in range(1, NUM_B + 1) + ] + cls.ids_a = [d.id() for d in docs_a] + cls.ids_b = [d.id() for d in docs_b] + docs_c = [ + _doc( + "demoC", + ["base"], + f"c{i}", + {"demoC": {"value": 200 + i}}, + depends_on=[ + ("item1", cls.ids_a[i - 1]), + ("item2", cls.ids_b[(i - 1) % NUM_B]), + ], + ) + for i in range(1, NUM_C + 1) + ] + cls.ids_c = [d.id() for d in docs_c] + cls.db.add_docs(docs_a + docs_b + docs_c, branch_id="a", validate=False) + + @classmethod + def tearDownClass(cls): + cls.db._close_db() + cls.tmpdir.cleanup() + + def tearDown(self): + self.db.or_batch_size = None + self.db._search_indexes_ready = True + + def sql_terms(self): + """A term for every operation _query_struct_to_sql_str turns into SQL.""" + return [ + Query("base.name", "exact_string", "n7"), + Query("base.name", "exact_string_anycase", "N8"), + Query("base.name", "contains_string", "b1"), + Query("base.name", "regexp", "^c2"), + Query("demoA.value", "exact_number", 12), + Query("demoA.value", "lessthan", 3), + Query("demoA.value", "lessthaneq", 3), + Query("demoB.value", "greaterthan", 138), + Query("demoB.value", "greaterthaneq", 138), + Query("demoB", "hasfield"), + Query("", "isa", "demoB"), + Query("", "isa", "demoC"), + Query("", "depends_on", "item1", self.ids_a[3]), + Query("", "depends_on", "*", self.ids_b[5]), + ] + + def per_operand_terms(self): + """Terms that stay on the one-query-per-operand path.""" + negated = [_negated(q) for q in self.sql_terms() if "hasfield" not in str(q)] + brute = [ + Query("demoC.value", "hasmember", 205), + Query("demoA.value", "exact_number", "not a number"), + Query("", "hasanysubfield_contains_string", "item1", "x"), + ] + return negated + brute + + def search(self, query, batch_size=None): + """Search with at most *batch_size* OR operands per SQL query (None = + the default); return the sorted ids and the number of search queries.""" + self.db.or_batch_size = batch_size + self.db.query_log = [] + try: + found = self.db.search(query, branch_id="a") + finally: + self.db.or_batch_size = None + return sorted(found), len(self.db.query_log) + + def assert_batched_matches_per_operand(self, query, msg): + per_operand, _ = self.search(query, 1) + batched, _ = self.search(query) + self.assertEqual(batched, per_operand, f"{msg}: batched search") + small, _ = self.search(query, 3) + self.assertEqual(small, per_operand, f"{msg}: batches of 3") + + def test_every_operation_in_an_or(self): + for term in self.sql_terms() + self.per_operand_terms(): + with self.subTest(term=str(term.to_search_structure())): + q = _or_of( + [ + term, + Query("base.name", "exact_string", "n3"), + Query("", "isa", "demoC"), + ] + ) + self.assert_batched_matches_per_operand(q, "term OR ...") + + def test_all_sql_operations_in_one_or(self): + q = _or_of(self.sql_terms()) + self.assert_batched_matches_per_operand(q, "an OR of every SQL operation") + _, num_queries = self.search(q) + self.assertEqual( + num_queries, 1, "an OR of single SQL terms must be searched in one query" + ) + + def test_every_operation_in_an_or_inside_an_and(self): + # 'isa demoB' leaves 40 candidates, so the OR is searched restricted + # to them, with the conditions ORed in one query + isa_b = Query("", "isa", "demoB") + for term in self.sql_terms() + self.per_operand_terms(): + with self.subTest(term=str(term.to_search_structure())): + or_part = _or_of( + [ + term, + Query("base.name", "exact_string", "b3"), + Query("demoA.value", "lessthan", 5), + ] + ) + q = isa_b & or_part + self.assert_batched_matches_per_operand(q, "isa demoB AND (...)") + # ...and it must match the search over the whole branch + batched, _ = self.search(q) + self.db._search_indexes_ready = False + full, _ = self.search(q, 1) + self.db._search_indexes_ready = True + self.assertEqual(batched, full, "restricted vs. whole branch") + + q = isa_b & _or_of(self.sql_terms()) + self.assert_batched_matches_per_operand(q, "isa demoB AND every SQL operation") + _, num_queries = self.search(q) + self.assertEqual( + num_queries, 2, "the isa term and the restricted OR take one query each" + ) + + def test_mixed_or(self): + # base.id lookups, an AND, a negated OR, negated and brute-forced + # terms among plain ones + name_is = lambda n: Query("base.name", "exact_string", n) + terms = [ + Query("base.id", "exact_string", self.ids_a[19]), + name_is("n5"), + Query("", "isa", "demoB") & Query("demoA.value", "lessthan", 4), + Query("", "isa", "demoC"), + Query("base.id", "exact_string", self.ids_b[19]), + Query("demoA.value", "~greaterthan", 2), + Query("base.id", "exact_string", "no_such_id"), + Query("demoC.value", "hasmember", 205), + ] + q = _or_of(terms) + not_or = { + "field": "", + "operation": "~or", + "param1": [name_is("n1").to_search_structure()[0]], + "param2": [name_is("n2").to_search_structure()[0]], + } + s = q.to_search_structure() + mixed = [{"field": "", "operation": "or", "param1": s, "param2": [not_or]}] + + class _Raw: + def __init__(self, structure): + self.structure = structure + + def to_search_structure(self): + return self.structure + + for query in (q, _Raw(mixed)): + self.assert_batched_matches_per_operand(query, "a mixed OR") + found, num_queries = self.search(query) + _, per_operand_queries = self.search(query, 1) + self.assertLess(num_queries, per_operand_queries) + self.assertTrue({self.ids_a[19], self.ids_b[19]} <= set(found)) + self.assertTrue(set(self.ids_c) <= set(found)) + + q2 = Query("base.name", "regexp", "^[nbc][0-9]$") & q + self.assert_batched_matches_per_operand(q2, "a mixed OR inside an AND") + + def test_long_or_is_searched_in_batches(self): + # 450 names: 3 queries of at most 200 terms, not 450 + names = [f"n{i}" for i in range(1, NUM_A + 1)] + [ + f"missing{i}" for i in range(390) + ] + q = _or_of([Query("base.name", "exact_string", n) for n in names]) + batched, num_queries = self.search(q) + per_operand, per_operand_queries = self.search(q, 1) + self.assertEqual(batched, per_operand) + self.assertEqual(batched, sorted(self.ids_a)) + self.assertEqual(num_queries, 3) + self.assertEqual(per_operand_queries, 450) + + # ...restricted to candidates: the isa query, then 3 batches + q2 = Query("", "isa", "demoA") & q + found, num_queries = self.search(q2) + self.assertEqual(found, sorted(self.ids_a)) + self.assertEqual(num_queries, 4) + self.assertEqual(found, self.search(q2, 1)[0]) + + def test_without_search_indexes(self): + # No candidates and no batched id lookups: the batch is a UNION over + # the whole branch + self.db._search_indexes_ready = False + q = _or_of(self.sql_terms() + [Query("base.id", "exact_string", self.ids_b[1])]) + self.assert_batched_matches_per_operand(q, "an OR without the search indexes") + q2 = Query("", "isa", "demoB") & q + self.assert_batched_matches_per_operand( + q2, "an OR in an AND without the search indexes" + ) + + def test_failing_batch_falls_back_to_per_operand(self): + # A batch whose SQL fails is searched one operand at a time + q = _or_of(self.sql_terms()) + expected, _ = self.search(q, 1) + original = SQLiteDB.do_run_sql_query + calls = [] + + def failing(db, query_str, params=()): + if " UNION " in query_str and not calls: + calls.append(query_str) + raise sqlite3.OperationalError("simulated") + return original(db, query_str, params) + + self.db.do_run_sql_query = failing.__get__(self.db) + try: + found, _ = self.search(q) + finally: + del self.db.do_run_sql_query + self.assertEqual(len(calls), 1, "the batch should have been tried") + self.assertEqual(found, expected) + + +if __name__ == "__main__": + unittest.main() From 94b9a637e8f59abdcb148e9e4db638c66dc1c397 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 19:58:08 +0000 Subject: [PATCH 2/2] bridge: record the merge of DID-matlab#235 into DID-matlab#234 (e4e02dc) Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_016Z8rXBFDpnZo3FeCE3ymyZ --- src/did/did_matlab_python_bridge.yaml | 13 +++++++++---- tests/test_batched_or.py | 3 +++ 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/src/did/did_matlab_python_bridge.yaml b/src/did/did_matlab_python_bridge.yaml index e9529b4..ab2bf1d 100644 --- a/src/did/did_matlab_python_bridge.yaml +++ b/src/did/did_matlab_python_bridge.yaml @@ -40,7 +40,7 @@ classes: - name: database type: class matlab_path: "+did/database.m" - matlab_last_sync_hash: "ec95ef7" + matlab_last_sync_hash: "e4e02dc" status: ported python_path: "did/database.py" python_class: "Database" @@ -73,9 +73,14 @@ classes: in did/database.py. Two deviations, neither of which changes a result: (1) negated terms stay one query each, because Python takes a negated term's complement in Python rather than in SQL (MATLAB batches them); - (2) terms with no SQL form are brute-forced one at a time, and a batch - whose SQL fails is retried one operand at a time so that each falls - back as it would on its own (MATLAB has no brute-force path). + (2) a batch whose SQL fails is retried one operand at a time so that + each falls back as it would on its own. + DID-matlab 4f4e9aa..e4e02dc merges DID-matlab#235 into that branch: + MATLAB now brute-forces the depends_on queries SQL cannot match exactly + (isBruteForceDependsOn) and keeps them out of its OR batches. Python + already did both -- _query_struct_to_sql_str returns None for them, so + _search_or_operands leaves them to _search_doc_ids -- so nothing more + is ported. sync_notes: > Schema validation was ported on 2026-08-28, closing what had been the largest capability gap between the two languages. Python previously had diff --git a/tests/test_batched_or.py b/tests/test_batched_or.py index c93e465..f369622 100644 --- a/tests/test_batched_or.py +++ b/tests/test_batched_or.py @@ -150,6 +150,9 @@ def per_operand_terms(self): Query("demoC.value", "hasmember", 205), Query("demoA.value", "exact_number", "not a number"), Query("", "hasanysubfield_contains_string", "item1", "x"), + # no exact SQL form (empty name): brute-forced, as in DID-matlab + Query("", "depends_on", "", self.ids_a[3]), + Query("", "~depends_on", "", self.ids_a[3]), ] return negated + brute