From 0a59bffcac961e4d3bd891f6c4eb615d58e9c0b0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 20:42:37 +0000 Subject: [PATCH 1/4] Pin hasmember quoting cases; match a scalar in field_search (DID-matlab#236) DID-matlab put a text 'hasmember' value into its SQL unquoted. Python never builds SQL for hasmember (_query_struct_to_sql_str returns None and the term is brute-forced with field_search), so its search was never affected. tests/test_hasmember_quoting.py mirrors DID-matlab's TestHasMemberQuoting: text values ('value', '1 OR 1=1', values holding " and '), numbers and ~hasmember, alone, inside an AND and inside an OR, compared with field_search and with the documents DID-matlab finds. It found that field_search never matched a number stored as a scalar (5 in 5 is a TypeError) where MATLAB's ismember(5, 5) does; field_search now compares a scalar with ==. Bridge: database matlab_last_sync_hash -> c8adf741 with a decision_log note (did/database.py does not change), and a fieldsearch note. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01C7XsYS2DKSy2ucFzEEWR3D --- src/did/datastructures.py | 16 +- src/did/did_matlab_python_bridge.yaml | 15 +- src/did/did_matlab_python_bridge_util.yaml | 8 + tests/test_hasmember_quoting.py | 182 +++++++++++++++++++++ 4 files changed, 216 insertions(+), 5 deletions(-) create mode 100644 tests/test_hasmember_quoting.py diff --git a/src/did/datastructures.py b/src/did/datastructures.py index 6d4d0b4..c16a88a 100644 --- a/src/did/datastructures.py +++ b/src/did/datastructures.py @@ -260,10 +260,18 @@ def field_search(a, search_struct): b = eq_len(np.array(value).shape, param1) elif op_lower == "hasmember": if is_there: - try: - b = param1 in value - except TypeError: - pass + if isinstance(value, (str, list, tuple, dict, set)): + try: + b = param1 in value + except TypeError: + pass + else: + # A scalar (a number read back from JSON) has one member, + # itself, as MATLAB's ismember(param1, value) has it + try: + b = bool(param1 == value) + except (TypeError, ValueError): + pass elif op_lower == "hasfield": b = is_there elif op_lower == "partial_struct": diff --git a/src/did/did_matlab_python_bridge.yaml b/src/did/did_matlab_python_bridge.yaml index 396996c..84079f1 100644 --- a/src/did/did_matlab_python_bridge.yaml +++ b/src/did/did_matlab_python_bridge.yaml @@ -40,7 +40,7 @@ classes: - name: database type: class matlab_path: "+did/database.m" - matlab_last_sync_hash: "e4e02dc" + matlab_last_sync_hash: "c8adf741" status: ported python_path: "did/database.py" python_class: "Database" @@ -985,6 +985,19 @@ classes: DID-matlab's TestDependsOnParity case for case. DID-matlab 124cf5e (same PR) only fixes how MATLAB's brute_force_search builds its did.query; nothing to port. + DID-matlab c8adf741 (DID-matlab#236, 2026-10-24): query_struct_to_sql_str + put a text 'hasmember' value into the SQL unquoted (and as a + double-quoted token, which SQLite reads as a column when it names + one), so 'value' matched every document and '1 OR 1=1' was SQL + injection; its text tests are now single-quoted literals, plus a + numeric comparison for a finite number. did/database.py and + SQLiteDB's SQL builder do not change: _query_struct_to_sql_str + returns None for hasmember, so Python brute-forces it with + field_search and never put the value into SQL. Writing the parity + test (tests/test_hasmember_quoting.py, which mirrors DID-matlab's + TestHasMemberQuoting) found that field_search did not match a number + stored as a scalar (5 in 5 is a TypeError); that is fixed in + field_search -- see the fieldsearch entry. # ----------------------------------------------------------------------- # did.document diff --git a/src/did/did_matlab_python_bridge_util.yaml b/src/did/did_matlab_python_bridge_util.yaml index 41ddc42..802139c 100644 --- a/src/did/did_matlab_python_bridge_util.yaml +++ b/src/did/did_matlab_python_bridge_util.yaml @@ -330,6 +330,14 @@ functions: holds a non-struct value (depends_on = [] read back from a database) used to return early, skipping the negation; it now matches. Python's field_search already negated it; nothing to port. + DID-matlab#236 (2026-10-24, no change to fieldsearch.m): hasmember on a + scalar value -- a number read back from JSON -- raised TypeError in + field_search (5 in 5) and so never matched, where MATLAB's + ismember(5, 5) does. field_search now compares a scalar with ==. + Remaining deviation: for a text value MATLAB's ismember tests each + character of param1 for membership in the value, Python's `in` tests + for a substring; tests/test_hasmember_quoting.py uses values on which + the two agree. - name: orOperands type: function diff --git a/tests/test_hasmember_quoting.py b/tests/test_hasmember_quoting.py new file mode 100644 index 0000000..7f0ed2f --- /dev/null +++ b/tests/test_hasmember_quoting.py @@ -0,0 +1,182 @@ +"""'hasmember' never reads its value as SQL (DID-matlab#236). + +DID-matlab's SQL search put a text 'hasmember' value into the SQL unquoted, so +'value' matched every document and '1 OR 1=1' was SQL injection. Python never +builds SQL for hasmember -- _query_struct_to_sql_str returns None, and the term +is brute-forced with field_search -- so it was never affected. These tests pin +the cases of DID-matlab's did.unittest.TestHasMemberQuoting: text values +(including 'value', '1 OR 1=1' and values holding " and '), numbers and +~hasmember, alone, inside an AND and inside an OR, compared with field_search +and with the documents DID-matlab finds. +""" + +import os +import tempfile +import unittest +import uuid + +from did.datastructures import field_search +from did.document import Document +from did.implementations.sqlitedb import SQLiteDB +from did.query import Query + +NAMES = ["value", "1 OR 1=1", 'say "hi"', "x, value", "alpha", "n7", "O'Brien"] + + +# The documents DID-matlab finds for each text value +TEXT_CASES = { + "value": ("value", "x, value"), + "1 OR 1=1": ("1 OR 1=1",), + 'say "hi"': ('say "hi"',), + "O'Brien": ("O'Brien",), + "x, value": ("x, value",), + "name17": (), + "zzz": (), +} + + +def _doc(name, value): + return Document( + { + "base": { + "id": uuid.uuid4().hex, + "session_id": "s1", + "name": name, + "datestamp": "2026-10-24T00:00:00.000Z", + }, + "document_class": { + "class_name": "demoA", + "superclasses": [{"class_name": "base"}], + }, + "depends_on": [], + "demoA": {"value": value}, + } + ) + + +def _has_member(field, value, negate=False): + return Query(field, ("~" if negate else "") + "hasmember", value) + + +def _or_of(queries): + q = queries[0] + for other in queries[1:]: + q = q | other + return q + + +class TestHasMemberQuoting(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tmpdir = tempfile.TemporaryDirectory() + cls.db = SQLiteDB(os.path.join(cls.tmpdir.name, "hasmember.sqlite")) + cls.db.add_branch("a") + cls.docs = [_doc(name, i) for i, name in enumerate(NAMES, 1)] + cls.ids = [d.id() for d in cls.docs] + cls.db.add_docs(cls.docs, branch_id="a", validate=False) + + @classmethod + def tearDownClass(cls): + cls.db._close_db() + cls.tmpdir.cleanup() + + def expected(self, query): + s = query.to_search_structure() + return sorted( + d.id() for d in self.docs if field_search(d.document_properties, s) + ) + + def verify_search(self, query, ids, msg): + found = sorted(self.db.search(query)) + self.assertEqual(found, self.expected(query), f"{msg}: vs. field_search") + self.assertEqual(found, sorted(ids), f"{msg}: vs. DID-matlab") + + def names_matching(self, *names): + return [self.ids[NAMES.index(n)] for n in names] + + def complement(self, ids): + return [i for i in self.ids if i not in ids] + + def test_text(self): + for value, names in TEXT_CASES.items(): + ids = self.names_matching(*names) + with self.subTest(value=value): + self.verify_search( + _has_member("base.name", value), ids, f"hasmember {value!r}" + ) + self.verify_search( + _has_member("base.name", value, True), + self.complement(ids), + f"~hasmember {value!r}", + ) + + def test_number(self): + for value in (3, 5, 99): + ids = [self.ids[value - 1]] if value <= len(NAMES) else [] + with self.subTest(value=value): + self.verify_search( + _has_member("demoA.value", value), ids, f"hasmember {value}" + ) + self.verify_search( + _has_member("demoA.value", value, True), + self.complement(ids), + f"~hasmember {value}", + ) + + def test_in_an_and(self): + every_doc = Query("demoA.value", "lessthan", 100) + for value, names in TEXT_CASES.items(): + ids = self.names_matching(*names) + with self.subTest(value=value): + self.verify_search( + every_doc & _has_member("base.name", value), + ids, + f"... AND hasmember {value!r}", + ) + self.verify_search( + every_doc & _has_member("base.name", value, True), + self.complement(ids), + f"... AND ~hasmember {value!r}", + ) + self.verify_search( + every_doc & _has_member("demoA.value", 5), [self.ids[4]], "... AND 5" + ) + + def test_in_an_or(self): + terms = [ + ( + _has_member("base.name", "value"), + self.names_matching("value", "x, value"), + ), + (_has_member("base.name", "1 OR 1=1"), self.names_matching("1 OR 1=1")), + (_has_member("base.name", 'say "hi"'), self.names_matching('say "hi"')), + ( + _has_member("base.name", "O'Brien", True), + self.complement(self.names_matching("O'Brien")), + ), + (_has_member("demoA.value", 6), [self.ids[5]]), + (_has_member("demoA.value", 2, True), self.complement([self.ids[1]])), + ] + alpha = Query("base.name", "exact_string", "alpha") + for k, (q, ids) in enumerate(terms): + other, other_ids = terms[(k + 1) % len(terms)] + with self.subTest(term=k): + self.verify_search( + q | alpha, + sorted(set(ids) | set(self.names_matching("alpha"))), + "... OR alpha", + ) + self.verify_search( + q | other, sorted(set(ids) | set(other_ids)), "... OR hasmember" + ) + everything = sorted(set().union(*(ids for _, ids in terms))) + self.verify_search(_or_of([q for q, _ in terms]), everything, "an OR of all") + self.verify_search( + Query("demoA.value", "lessthan", 100) & _or_of([q for q, _ in terms]), + everything, + "... AND an OR of all", + ) + + +if __name__ == "__main__": + unittest.main() From fdfc261455af78ab62d6976c283f24cfb6829ad0 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 21:09:24 +0000 Subject: [PATCH 2/4] One 'hasmember' rule in field_search, as in DID-matlab (#236) DID-matlab f4ed409e gives fieldsearch and its SQL search one hasmember rule; field_search now matches it too (_has_member): text is a member of text that is the whole value or an item of its comma-separated list, anything of a list holding an equal element, a number of an equal number; text never of a number, nor a number of text; a dict has no members. Before, `in` tested text for a substring and dict keys. test_hasmember_quoting.py: test_rule runs the table of DID-matlab's TestHasMemberQuoting/testRule; new cases for text vs. numbers and a struct field. Bridge: database and fieldsearch matlab_last_sync_hash -> f4ed409e, with their decision_log notes rewritten for the shared rule. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01C7XsYS2DKSy2ucFzEEWR3D --- src/did/datastructures.py | 45 ++++++++++---- src/did/did_matlab_python_bridge.yaml | 33 +++++----- src/did/did_matlab_python_bridge_util.yaml | 24 +++++--- tests/test_hasmember_quoting.py | 72 +++++++++++++++++++--- 4 files changed, 128 insertions(+), 46 deletions(-) diff --git a/src/did/datastructures.py b/src/did/datastructures.py index c16a88a..248c481 100644 --- a/src/did/datastructures.py +++ b/src/did/datastructures.py @@ -1,4 +1,5 @@ import json +import numbers import re import numpy as np @@ -190,6 +191,37 @@ def or_operands(search_struct): return operands +def _has_member(value, member): + """Is *member* a member of *value*, for field_search's 'hasmember'? + + Text is a member of text that is the whole value or an item of its + comma-separated list (white space after a comma, and before one, is + ignored); anything is a member of a list that holds an equal element; a + number is a member of an equal number. Text is never a member of a + number, nor a number of text. This is what DID-matlab's fieldsearch and + its SQL search match (DID-matlab#236). + """ + + def is_number(x): + return isinstance(x, numbers.Real) and not isinstance(x, str) + + if isinstance(value, np.ndarray): + value = value.tolist() + if isinstance(member, str): + if isinstance(value, str): + pattern = r"(^|,\s*)" + re.escape(member) + r"\s*(,|$)" + return re.search(pattern, value) is not None + if isinstance(value, (list, tuple)): + return any(isinstance(v, str) and v == member for v in value) + return False + if is_number(member): + if is_number(value): + return value == member + if isinstance(value, (list, tuple)): + return any(is_number(v) and v == member for v in value) + return False + + def field_search(a, search_struct): """ Searches a dictionary to determine if it matches a search structure. @@ -260,18 +292,7 @@ def field_search(a, search_struct): b = eq_len(np.array(value).shape, param1) elif op_lower == "hasmember": if is_there: - if isinstance(value, (str, list, tuple, dict, set)): - try: - b = param1 in value - except TypeError: - pass - else: - # A scalar (a number read back from JSON) has one member, - # itself, as MATLAB's ismember(param1, value) has it - try: - b = bool(param1 == value) - except (TypeError, ValueError): - pass + b = _has_member(value, param1) elif op_lower == "hasfield": b = is_there elif op_lower == "partial_struct": diff --git a/src/did/did_matlab_python_bridge.yaml b/src/did/did_matlab_python_bridge.yaml index 2a2e2a3..1aa81da 100644 --- a/src/did/did_matlab_python_bridge.yaml +++ b/src/did/did_matlab_python_bridge.yaml @@ -40,7 +40,7 @@ classes: - name: database type: class matlab_path: "+did/database.m" - matlab_last_sync_hash: "2c08472c" + matlab_last_sync_hash: "f4ed409e" status: ported python_path: "did/database.py" python_class: "Database" @@ -125,23 +125,20 @@ classes: tests/test_negated_or.py pins the behaviour with the same cases as MATLAB's did.unittest.TestNegatedOr. - DID-matlab c8adf741 (DID-matlab#236, 2026-10-24): query_struct_to_sql_str - put a text 'hasmember' value into the SQL unquoted (and as a - double-quoted token, which SQLite reads as a column when it names - one), so 'value' matched every document and '1 OR 1=1' was SQL - injection; its text tests are now single-quoted literals, plus a - numeric comparison for a finite number. did/database.py and - SQLiteDB's SQL builder do not change: _query_struct_to_sql_str - returns None for hasmember, so Python brute-forces it with - field_search and never put the value into SQL. Writing the parity - test (tests/test_hasmember_quoting.py, which mirrors DID-matlab's - TestHasMemberQuoting) found that field_search did not match a number - stored as a scalar (5 in 5 is a TypeError); that is fixed in - field_search -- see the fieldsearch entry. - - DID-matlab 2c08472c merges main (2c9d8fa7, #237) into the #236 branch - (c8adf741); the merged database.m is exactly those two changes, both - covered above. + DID-matlab c8adf741..f4ed409e (DID-matlab#236, 2026-10-24): + query_struct_to_sql_str put a text 'hasmember' value into the SQL + unquoted (and as a double-quoted token, which SQLite reads as a column + when it names one), so 'value' matched every document and '1 OR 1=1' + was SQL injection. It now matches the rule fieldsearch and field_search + share (see the fieldsearch entry): a text member as a single-quoted + regex literal against text values only, a numeric member as a number + against numbers only, and on the field itself, no longer on its + subfields. 2c08472c merges DID-matlab main (2c9d8fa7, #237, above) into + that branch. did/database.py and SQLiteDB's SQL builder do not change: + _query_struct_to_sql_str returns None for hasmember, so Python + brute-forces it with field_search and never put the value into SQL. + tests/test_hasmember_quoting.py mirrors DID-matlab's + TestHasMemberQuoting. sync_notes: > Schema validation was ported on 2026-08-28, closing what had been the largest capability gap between the two languages. Python previously had diff --git a/src/did/did_matlab_python_bridge_util.yaml b/src/did/did_matlab_python_bridge_util.yaml index 802139c..2bfd15d 100644 --- a/src/did/did_matlab_python_bridge_util.yaml +++ b/src/did/did_matlab_python_bridge_util.yaml @@ -317,7 +317,7 @@ functions: - name: fieldsearch type: function matlab_path: "+did/+datastructures/fieldsearch.m" - matlab_last_sync_hash: "051f535" + matlab_last_sync_hash: "f4ed409e" status: ported python_path: "did/datastructures.py" python_name: "field_search" @@ -330,14 +330,20 @@ functions: holds a non-struct value (depends_on = [] read back from a database) used to return early, skipping the negation; it now matches. Python's field_search already negated it; nothing to port. - DID-matlab#236 (2026-10-24, no change to fieldsearch.m): hasmember on a - scalar value -- a number read back from JSON -- raised TypeError in - field_search (5 in 5) and so never matched, where MATLAB's - ismember(5, 5) does. field_search now compares a scalar with ==. - Remaining deviation: for a text value MATLAB's ismember tests each - character of param1 for membership in the value, Python's `in` tests - for a substring; tests/test_hasmember_quoting.py uses values on which - the two agree. + DID-matlab f4ed409e (DID-matlab#236, 2026-10-24): 'hasmember' had a + different rule in each place. MATLAB's fieldsearch used + ismember(param1, value), which compares two texts character by + character; its SQL search matched an item of a comma-separated list; + field_search used `in` (a substring test, and a TypeError -- no match + -- on a scalar number). All three now match one rule, MATLAB's in the + local function fieldsearch/hasMember and Python's in _has_member: text + is a member of text that is the whole value or an item of its + comma-separated list (pattern (^|,\s*)M\s*(,|$)); anything is a + member of a list (MATLAB cell or numeric array) that holds an equal + element, and a number of an equal number; text is never a member of a + number, nor a number of text; a dict (MATLAB struct) has no members. + tests/test_hasmember_quoting.py test_rule runs the same table of + cases as DID-matlab's TestHasMemberQuoting/testRule. - name: orOperands type: function diff --git a/tests/test_hasmember_quoting.py b/tests/test_hasmember_quoting.py index 7f0ed2f..f700fe4 100644 --- a/tests/test_hasmember_quoting.py +++ b/tests/test_hasmember_quoting.py @@ -1,13 +1,24 @@ -"""'hasmember' never reads its value as SQL (DID-matlab#236). +"""'hasmember': one rule in field_search and in SQL (DID-matlab#236). DID-matlab's SQL search put a text 'hasmember' value into the SQL unquoted, so 'value' matched every document and '1 OR 1=1' was SQL injection. Python never builds SQL for hasmember -- _query_struct_to_sql_str returns None, and the term -is brute-forced with field_search -- so it was never affected. These tests pin -the cases of DID-matlab's did.unittest.TestHasMemberQuoting: text values -(including 'value', '1 OR 1=1' and values holding " and '), numbers and -~hasmember, alone, inside an AND and inside an OR, compared with field_search -and with the documents DID-matlab finds. +is brute-forced with field_search -- so it was never affected. But DID-matlab's +fieldsearch (ismember, which compares two texts character by character), its +SQL search and field_search (``in``: a substring test) each matched another +rule. All three now match one: + +- text is a member of text that is the whole value or an item of its + comma-separated list; +- anything is a member of a list that holds an equal element, and a number + of an equal number; +- text is never a member of a number, nor a number of text. + +test_rule checks field_search against the table of DID-matlab's +did.unittest.TestHasMemberQuoting/testRule. The other tests pin the cases of +that class: text values (including 'value', '1 OR 1=1' and values holding " +and '), numbers and ~hasmember, alone, inside an AND and inside an OR, +compared with field_search and with the documents DID-matlab finds. """ import os @@ -20,7 +31,7 @@ from did.implementations.sqlitedb import SQLiteDB from did.query import Query -NAMES = ["value", "1 OR 1=1", 'say "hi"', "x, value", "alpha", "n7", "O'Brien"] +NAMES = ["value", "1 OR 1=1", 'say "hi"', "x, value", "alpha", "n7", "O'Brien", "5"] # The documents DID-matlab finds for each text value @@ -32,8 +43,33 @@ "x, value": ("x, value",), "name17": (), "zzz": (), + "5": ("5",), + "": (), } +# (value, member, is a member): DID-matlab's TestHasMemberQuoting/testRule +RULE_CASES = [ + ("value", "value", True), + ("x, value", "value", True), + ("x,value ,y", "value", True), + ("valuex", "value", False), + ("value", "x, value", False), + ("eulav", "value", False), # MATLAB's ismember matched this + ("a.b", ".", False), # the member is not a pattern + ("", "", True), + (5, 5, True), + ([1, 2, 3], 2, True), + ([1, 2, 3], 4, False), + (True, 1, True), + (["a", "b"], "b", True), + (["a", "b"], "a,b", False), + ([1, "a"], 1, True), + ("5", 5, False), + (5, "5", False), + ("a", 97, False), # MATLAB's ismember matched this + ({"a": 1}, "a", False), +] + def _doc(name, value): return Document( @@ -97,6 +133,28 @@ def names_matching(self, *names): def complement(self, ids): return [i for i in self.ids if i not in ids] + def test_rule(self): + for k, (value, member, is_member) in enumerate(RULE_CASES, 1): + with self.subTest(case=k): + a = {"f": value} + s = {"field": "f", "operation": "hasmember", "param1": member} + self.assertEqual(field_search(a, s), is_member) + s["operation"] = "~hasmember" + self.assertEqual(field_search(a, s), not is_member) + + def test_text_and_numbers_do_not_mix(self): + # base.name "5" is text; demoA.value 3 is a number + self.verify_search(_has_member("base.name", 5), [], "hasmember 5 in '5'") + self.verify_search( + _has_member("base.name", "5"), self.names_matching("5"), "'5' in '5'" + ) + self.verify_search(_has_member("demoA.value", "3"), [], "'3' in 3") + self.verify_search(_has_member("demoA.value", 3), [self.ids[2]], "3 in 3") + + def test_struct_field(self): + # A struct (base) has no members: its fields are not searched + self.verify_search(_has_member("base", "value"), [], "'value' in base") + def test_text(self): for value, names in TEXT_CASES.items(): ids = self.names_matching(*names) From 47db1bcdda7b8ecd88ee215e08e31ef8ad07f311 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 21:16:02 +0000 Subject: [PATCH 3/4] test_hasmember_quoting: Inf and NaN members, as DID-matlab Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01C7XsYS2DKSy2ucFzEEWR3D --- tests/test_hasmember_quoting.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/test_hasmember_quoting.py b/tests/test_hasmember_quoting.py index f700fe4..bcd1e97 100644 --- a/tests/test_hasmember_quoting.py +++ b/tests/test_hasmember_quoting.py @@ -151,6 +151,15 @@ def test_text_and_numbers_do_not_mix(self): self.verify_search(_has_member("demoA.value", "3"), [], "'3' in 3") self.verify_search(_has_member("demoA.value", 3), [self.ids[2]], "3 in 3") + def test_numbers_no_value_holds(self): + # DID-matlab writes Inf as 9e999 and makes NaN a member of nothing + for member in (float("inf"), float("-inf"), float("nan")): + with self.subTest(member=member): + q = _has_member("demoA.value", member) + self.verify_search(q, [], f"hasmember {member}") + q = _has_member("demoA.value", member, True) + self.verify_search(q, self.ids, f"~hasmember {member}") + def test_struct_field(self): # A struct (base) has no members: its fields are not searched self.verify_search(_has_member("base", "value"), [], "'value' in base") From 780a3f7478925145aa60d8543e9446ae566acd55 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 21:28:53 +0000 Subject: [PATCH 4/4] Bridge: DID-matlab now brute-forces hasmember too (#236) DID-matlab 74ae15f1 searches hasmember by brute force with fieldsearch, as SQLiteDB always has, so neither did/database.py nor sqlitedb.py changes. database matlab_last_sync_hash -> 74ae15f1, its decision_log note rewritten. test_hasmember_quoting.py adds the cases brute force now covers in both languages: a missing field and list-valued fields. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01C7XsYS2DKSy2ucFzEEWR3D --- src/did/did_matlab_python_bridge.yaml | 26 ++++++++------ tests/test_hasmember_quoting.py | 49 +++++++++++++++++++++++++-- 2 files changed, 62 insertions(+), 13 deletions(-) diff --git a/src/did/did_matlab_python_bridge.yaml b/src/did/did_matlab_python_bridge.yaml index 1aa81da..e8c147a 100644 --- a/src/did/did_matlab_python_bridge.yaml +++ b/src/did/did_matlab_python_bridge.yaml @@ -40,7 +40,7 @@ classes: - name: database type: class matlab_path: "+did/database.m" - matlab_last_sync_hash: "f4ed409e" + matlab_last_sync_hash: "74ae15f1" status: ported python_path: "did/database.py" python_class: "Database" @@ -125,19 +125,23 @@ classes: tests/test_negated_or.py pins the behaviour with the same cases as MATLAB's did.unittest.TestNegatedOr. - DID-matlab c8adf741..f4ed409e (DID-matlab#236, 2026-10-24): + DID-matlab c8adf741..74ae15f1 (DID-matlab#236, 2026-10-24): query_struct_to_sql_str put a text 'hasmember' value into the SQL unquoted (and as a double-quoted token, which SQLite reads as a column when it names one), so 'value' matched every document and '1 OR 1=1' - was SQL injection. It now matches the rule fieldsearch and field_search - share (see the fieldsearch entry): a text member as a single-quoted - regex literal against text values only, a numeric member as a number - against numbers only, and on the field itself, no longer on its - subfields. 2c08472c merges DID-matlab main (2c9d8fa7, #237, above) into - that branch. did/database.py and SQLiteDB's SQL builder do not change: - _query_struct_to_sql_str returns None for hasmember, so Python - brute-forces it with field_search and never put the value into SQL. - tests/test_hasmember_quoting.py mirrors DID-matlab's + was SQL injection; and the SQL could not see the members of an array + or cell, which doc2sql stores only as '[1 3] double'. MATLAB now + searches hasmember by brute force with fieldsearch, as Python always + has: the new protected static isBruteForceTerm (hasmember, and the + depends_on queries of isBruteForceDependsOn) sends a term to + brute_force_search, searched last in an AND and kept out of the OR + batches, and query_struct_to_sql_str's 'hasmember' case is an error. + That is what SQLiteDB already does (_query_struct_to_sql_str returns + None for hasmember; _SCAN_OPERATIONS ranks it last), so did/database.py + and did/implementations/sqlitedb.py do not change; the rule itself + changed in fieldsearch and field_search -- see the fieldsearch entry. + 2c08472c merges DID-matlab main (2c9d8fa7, #237, above) into that + branch. tests/test_hasmember_quoting.py mirrors DID-matlab's TestHasMemberQuoting. sync_notes: > Schema validation was ported on 2026-08-28, closing what had been the diff --git a/tests/test_hasmember_quoting.py b/tests/test_hasmember_quoting.py index bcd1e97..7a3eb96 100644 --- a/tests/test_hasmember_quoting.py +++ b/tests/test_hasmember_quoting.py @@ -1,4 +1,4 @@ -"""'hasmember': one rule in field_search and in SQL (DID-matlab#236). +"""'hasmember': one rule, searched by brute force (DID-matlab#236). DID-matlab's SQL search put a text 'hasmember' value into the SQL unquoted, so 'value' matched every document and '1 OR 1=1' was SQL injection. Python never @@ -6,7 +6,8 @@ is brute-forced with field_search -- so it was never affected. But DID-matlab's fieldsearch (ismember, which compares two texts character by character), its SQL search and field_search (``in``: a substring test) each matched another -rule. All three now match one: +rule. All three now match one, and DID-matlab searches hasmember by brute +force with fieldsearch, as Python always has: - text is a member of text that is the whole value or an item of its comma-separated list; @@ -160,6 +161,50 @@ def test_numbers_no_value_holds(self): q = _has_member("demoA.value", member, True) self.verify_search(q, self.ids, f"~hasmember {member}") + def test_missing_field(self): + # No document has demoB.value: ~hasmember matches every one + self.verify_search(_has_member("demoB.value", 1), [], "missing field") + self.verify_search(_has_member("demoB.value", 1, True), self.ids, "~missing") + + def test_list_fields(self): + # DID-matlab's doc2sql stores these as '[1 3] double' / '[1 2] cell', + # which its SQL could not search; brute force can, in both languages + with tempfile.TemporaryDirectory() as tmp: + db = SQLiteDB(os.path.join(tmp, "list_fields.sqlite")) + db.add_branch("a") + docs = [ + _doc("numbers", [1, 2, 3]), + _doc("other", [4, 5]), + _doc("names", ["alpha", "beta"]), + ] + ids = [d.id() for d in docs] + db.add_docs(docs, branch_id="a", validate=False) + cases = [ + ("demoA.value", 2, [0]), + ("demoA.value", 5, [1]), + ("demoA.value", 6, []), + ("demoA.value", "beta", [2]), + ("demoA.value", "gamma", []), + ("base.name", "other", [1]), + ] + try: + for field, member, matching in cases: + for negate in (False, True): + with self.subTest(field=field, member=member, negate=negate): + q = _has_member(field, member, negate) + rows = [i for i in range(3) if (i in matching) != negate] + expected = sorted(ids[i] for i in rows) + self.assertEqual(sorted(db.search(q)), expected) + s = q.to_search_structure() + by_field_search = sorted( + d.id() + for d in docs + if field_search(d.document_properties, s) + ) + self.assertEqual(by_field_search, expected) + finally: + db._close_db() + def test_struct_field(self): # A struct (base) has no members: its fields are not searched self.verify_search(_has_member("base", "value"), [], "'value' in base")