diff --git a/src/did/datastructures.py b/src/did/datastructures.py index 6d4d0b4..248c481 100644 --- a/src/did/datastructures.py +++ b/src/did/datastructures.py @@ -1,4 +1,5 @@ import json +import numbers import re import numpy as np @@ -190,6 +191,37 @@ def or_operands(search_struct): return operands +def _has_member(value, member): + """Is *member* a member of *value*, for field_search's 'hasmember'? + + Text is a member of text that is the whole value or an item of its + comma-separated list (white space after a comma, and before one, is + ignored); anything is a member of a list that holds an equal element; a + number is a member of an equal number. Text is never a member of a + number, nor a number of text. This is what DID-matlab's fieldsearch and + its SQL search match (DID-matlab#236). + """ + + def is_number(x): + return isinstance(x, numbers.Real) and not isinstance(x, str) + + if isinstance(value, np.ndarray): + value = value.tolist() + if isinstance(member, str): + if isinstance(value, str): + pattern = r"(^|,\s*)" + re.escape(member) + r"\s*(,|$)" + return re.search(pattern, value) is not None + if isinstance(value, (list, tuple)): + return any(isinstance(v, str) and v == member for v in value) + return False + if is_number(member): + if is_number(value): + return value == member + if isinstance(value, (list, tuple)): + return any(is_number(v) and v == member for v in value) + return False + + def field_search(a, search_struct): """ Searches a dictionary to determine if it matches a search structure. @@ -260,10 +292,7 @@ def field_search(a, search_struct): b = eq_len(np.array(value).shape, param1) elif op_lower == "hasmember": if is_there: - try: - b = param1 in value - except TypeError: - pass + b = _has_member(value, param1) elif op_lower == "hasfield": b = is_there elif op_lower == "partial_struct": diff --git a/src/did/did_matlab_python_bridge.yaml b/src/did/did_matlab_python_bridge.yaml index 580c623..e8c147a 100644 --- a/src/did/did_matlab_python_bridge.yaml +++ b/src/did/did_matlab_python_bridge.yaml @@ -40,7 +40,7 @@ classes: - name: database type: class matlab_path: "+did/database.m" - matlab_last_sync_hash: "2c9d8fa7" + matlab_last_sync_hash: "74ae15f1" status: ported python_path: "did/database.py" python_class: "Database" @@ -124,6 +124,25 @@ classes: did/database.py nor did/implementations/sqlitedb.py changes. tests/test_negated_or.py pins the behaviour with the same cases as MATLAB's did.unittest.TestNegatedOr. + + DID-matlab c8adf741..74ae15f1 (DID-matlab#236, 2026-10-24): + query_struct_to_sql_str put a text 'hasmember' value into the SQL + unquoted (and as a double-quoted token, which SQLite reads as a column + when it names one), so 'value' matched every document and '1 OR 1=1' + was SQL injection; and the SQL could not see the members of an array + or cell, which doc2sql stores only as '[1 3] double'. MATLAB now + searches hasmember by brute force with fieldsearch, as Python always + has: the new protected static isBruteForceTerm (hasmember, and the + depends_on queries of isBruteForceDependsOn) sends a term to + brute_force_search, searched last in an AND and kept out of the OR + batches, and query_struct_to_sql_str's 'hasmember' case is an error. + That is what SQLiteDB already does (_query_struct_to_sql_str returns + None for hasmember; _SCAN_OPERATIONS ranks it last), so did/database.py + and did/implementations/sqlitedb.py do not change; the rule itself + changed in fieldsearch and field_search -- see the fieldsearch entry. + 2c08472c merges DID-matlab main (2c9d8fa7, #237, above) into that + branch. tests/test_hasmember_quoting.py mirrors DID-matlab's + TestHasMemberQuoting. sync_notes: > Schema validation was ported on 2026-08-28, closing what had been the largest capability gap between the two languages. Python previously had diff --git a/src/did/did_matlab_python_bridge_util.yaml b/src/did/did_matlab_python_bridge_util.yaml index 41ddc42..2bfd15d 100644 --- a/src/did/did_matlab_python_bridge_util.yaml +++ b/src/did/did_matlab_python_bridge_util.yaml @@ -317,7 +317,7 @@ functions: - name: fieldsearch type: function matlab_path: "+did/+datastructures/fieldsearch.m" - matlab_last_sync_hash: "051f535" + matlab_last_sync_hash: "f4ed409e" status: ported python_path: "did/datastructures.py" python_name: "field_search" @@ -330,6 +330,20 @@ functions: holds a non-struct value (depends_on = [] read back from a database) used to return early, skipping the negation; it now matches. Python's field_search already negated it; nothing to port. + DID-matlab f4ed409e (DID-matlab#236, 2026-10-24): 'hasmember' had a + different rule in each place. MATLAB's fieldsearch used + ismember(param1, value), which compares two texts character by + character; its SQL search matched an item of a comma-separated list; + field_search used `in` (a substring test, and a TypeError -- no match + -- on a scalar number). All three now match one rule, MATLAB's in the + local function fieldsearch/hasMember and Python's in _has_member: text + is a member of text that is the whole value or an item of its + comma-separated list (pattern (^|,\s*)M\s*(,|$)); anything is a + member of a list (MATLAB cell or numeric array) that holds an equal + element, and a number of an equal number; text is never a member of a + number, nor a number of text; a dict (MATLAB struct) has no members. + tests/test_hasmember_quoting.py test_rule runs the same table of + cases as DID-matlab's TestHasMemberQuoting/testRule. - name: orOperands type: function diff --git a/tests/test_hasmember_quoting.py b/tests/test_hasmember_quoting.py new file mode 100644 index 0000000..7a3eb96 --- /dev/null +++ b/tests/test_hasmember_quoting.py @@ -0,0 +1,294 @@ +"""'hasmember': one rule, searched by brute force (DID-matlab#236). + +DID-matlab's SQL search put a text 'hasmember' value into the SQL unquoted, so +'value' matched every document and '1 OR 1=1' was SQL injection. Python never +builds SQL for hasmember -- _query_struct_to_sql_str returns None, and the term +is brute-forced with field_search -- so it was never affected. But DID-matlab's +fieldsearch (ismember, which compares two texts character by character), its +SQL search and field_search (``in``: a substring test) each matched another +rule. All three now match one, and DID-matlab searches hasmember by brute +force with fieldsearch, as Python always has: + +- text is a member of text that is the whole value or an item of its + comma-separated list; +- anything is a member of a list that holds an equal element, and a number + of an equal number; +- text is never a member of a number, nor a number of text. + +test_rule checks field_search against the table of DID-matlab's +did.unittest.TestHasMemberQuoting/testRule. The other tests pin the cases of +that class: text values (including 'value', '1 OR 1=1' and values holding " +and '), numbers and ~hasmember, alone, inside an AND and inside an OR, +compared with field_search and with the documents DID-matlab finds. +""" + +import os +import tempfile +import unittest +import uuid + +from did.datastructures import field_search +from did.document import Document +from did.implementations.sqlitedb import SQLiteDB +from did.query import Query + +NAMES = ["value", "1 OR 1=1", 'say "hi"', "x, value", "alpha", "n7", "O'Brien", "5"] + + +# The documents DID-matlab finds for each text value +TEXT_CASES = { + "value": ("value", "x, value"), + "1 OR 1=1": ("1 OR 1=1",), + 'say "hi"': ('say "hi"',), + "O'Brien": ("O'Brien",), + "x, value": ("x, value",), + "name17": (), + "zzz": (), + "5": ("5",), + "": (), +} + +# (value, member, is a member): DID-matlab's TestHasMemberQuoting/testRule +RULE_CASES = [ + ("value", "value", True), + ("x, value", "value", True), + ("x,value ,y", "value", True), + ("valuex", "value", False), + ("value", "x, value", False), + ("eulav", "value", False), # MATLAB's ismember matched this + ("a.b", ".", False), # the member is not a pattern + ("", "", True), + (5, 5, True), + ([1, 2, 3], 2, True), + ([1, 2, 3], 4, False), + (True, 1, True), + (["a", "b"], "b", True), + (["a", "b"], "a,b", False), + ([1, "a"], 1, True), + ("5", 5, False), + (5, "5", False), + ("a", 97, False), # MATLAB's ismember matched this + ({"a": 1}, "a", False), +] + + +def _doc(name, value): + return Document( + { + "base": { + "id": uuid.uuid4().hex, + "session_id": "s1", + "name": name, + "datestamp": "2026-10-24T00:00:00.000Z", + }, + "document_class": { + "class_name": "demoA", + "superclasses": [{"class_name": "base"}], + }, + "depends_on": [], + "demoA": {"value": value}, + } + ) + + +def _has_member(field, value, negate=False): + return Query(field, ("~" if negate else "") + "hasmember", value) + + +def _or_of(queries): + q = queries[0] + for other in queries[1:]: + q = q | other + return q + + +class TestHasMemberQuoting(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tmpdir = tempfile.TemporaryDirectory() + cls.db = SQLiteDB(os.path.join(cls.tmpdir.name, "hasmember.sqlite")) + cls.db.add_branch("a") + cls.docs = [_doc(name, i) for i, name in enumerate(NAMES, 1)] + cls.ids = [d.id() for d in cls.docs] + cls.db.add_docs(cls.docs, branch_id="a", validate=False) + + @classmethod + def tearDownClass(cls): + cls.db._close_db() + cls.tmpdir.cleanup() + + def expected(self, query): + s = query.to_search_structure() + return sorted( + d.id() for d in self.docs if field_search(d.document_properties, s) + ) + + def verify_search(self, query, ids, msg): + found = sorted(self.db.search(query)) + self.assertEqual(found, self.expected(query), f"{msg}: vs. field_search") + self.assertEqual(found, sorted(ids), f"{msg}: vs. DID-matlab") + + def names_matching(self, *names): + return [self.ids[NAMES.index(n)] for n in names] + + def complement(self, ids): + return [i for i in self.ids if i not in ids] + + def test_rule(self): + for k, (value, member, is_member) in enumerate(RULE_CASES, 1): + with self.subTest(case=k): + a = {"f": value} + s = {"field": "f", "operation": "hasmember", "param1": member} + self.assertEqual(field_search(a, s), is_member) + s["operation"] = "~hasmember" + self.assertEqual(field_search(a, s), not is_member) + + def test_text_and_numbers_do_not_mix(self): + # base.name "5" is text; demoA.value 3 is a number + self.verify_search(_has_member("base.name", 5), [], "hasmember 5 in '5'") + self.verify_search( + _has_member("base.name", "5"), self.names_matching("5"), "'5' in '5'" + ) + self.verify_search(_has_member("demoA.value", "3"), [], "'3' in 3") + self.verify_search(_has_member("demoA.value", 3), [self.ids[2]], "3 in 3") + + def test_numbers_no_value_holds(self): + # DID-matlab writes Inf as 9e999 and makes NaN a member of nothing + for member in (float("inf"), float("-inf"), float("nan")): + with self.subTest(member=member): + q = _has_member("demoA.value", member) + self.verify_search(q, [], f"hasmember {member}") + q = _has_member("demoA.value", member, True) + self.verify_search(q, self.ids, f"~hasmember {member}") + + def test_missing_field(self): + # No document has demoB.value: ~hasmember matches every one + self.verify_search(_has_member("demoB.value", 1), [], "missing field") + self.verify_search(_has_member("demoB.value", 1, True), self.ids, "~missing") + + def test_list_fields(self): + # DID-matlab's doc2sql stores these as '[1 3] double' / '[1 2] cell', + # which its SQL could not search; brute force can, in both languages + with tempfile.TemporaryDirectory() as tmp: + db = SQLiteDB(os.path.join(tmp, "list_fields.sqlite")) + db.add_branch("a") + docs = [ + _doc("numbers", [1, 2, 3]), + _doc("other", [4, 5]), + _doc("names", ["alpha", "beta"]), + ] + ids = [d.id() for d in docs] + db.add_docs(docs, branch_id="a", validate=False) + cases = [ + ("demoA.value", 2, [0]), + ("demoA.value", 5, [1]), + ("demoA.value", 6, []), + ("demoA.value", "beta", [2]), + ("demoA.value", "gamma", []), + ("base.name", "other", [1]), + ] + try: + for field, member, matching in cases: + for negate in (False, True): + with self.subTest(field=field, member=member, negate=negate): + q = _has_member(field, member, negate) + rows = [i for i in range(3) if (i in matching) != negate] + expected = sorted(ids[i] for i in rows) + self.assertEqual(sorted(db.search(q)), expected) + s = q.to_search_structure() + by_field_search = sorted( + d.id() + for d in docs + if field_search(d.document_properties, s) + ) + self.assertEqual(by_field_search, expected) + finally: + db._close_db() + + def test_struct_field(self): + # A struct (base) has no members: its fields are not searched + self.verify_search(_has_member("base", "value"), [], "'value' in base") + + def test_text(self): + for value, names in TEXT_CASES.items(): + ids = self.names_matching(*names) + with self.subTest(value=value): + self.verify_search( + _has_member("base.name", value), ids, f"hasmember {value!r}" + ) + self.verify_search( + _has_member("base.name", value, True), + self.complement(ids), + f"~hasmember {value!r}", + ) + + def test_number(self): + for value in (3, 5, 99): + ids = [self.ids[value - 1]] if value <= len(NAMES) else [] + with self.subTest(value=value): + self.verify_search( + _has_member("demoA.value", value), ids, f"hasmember {value}" + ) + self.verify_search( + _has_member("demoA.value", value, True), + self.complement(ids), + f"~hasmember {value}", + ) + + def test_in_an_and(self): + every_doc = Query("demoA.value", "lessthan", 100) + for value, names in TEXT_CASES.items(): + ids = self.names_matching(*names) + with self.subTest(value=value): + self.verify_search( + every_doc & _has_member("base.name", value), + ids, + f"... AND hasmember {value!r}", + ) + self.verify_search( + every_doc & _has_member("base.name", value, True), + self.complement(ids), + f"... AND ~hasmember {value!r}", + ) + self.verify_search( + every_doc & _has_member("demoA.value", 5), [self.ids[4]], "... AND 5" + ) + + def test_in_an_or(self): + terms = [ + ( + _has_member("base.name", "value"), + self.names_matching("value", "x, value"), + ), + (_has_member("base.name", "1 OR 1=1"), self.names_matching("1 OR 1=1")), + (_has_member("base.name", 'say "hi"'), self.names_matching('say "hi"')), + ( + _has_member("base.name", "O'Brien", True), + self.complement(self.names_matching("O'Brien")), + ), + (_has_member("demoA.value", 6), [self.ids[5]]), + (_has_member("demoA.value", 2, True), self.complement([self.ids[1]])), + ] + alpha = Query("base.name", "exact_string", "alpha") + for k, (q, ids) in enumerate(terms): + other, other_ids = terms[(k + 1) % len(terms)] + with self.subTest(term=k): + self.verify_search( + q | alpha, + sorted(set(ids) | set(self.names_matching("alpha"))), + "... OR alpha", + ) + self.verify_search( + q | other, sorted(set(ids) | set(other_ids)), "... OR hasmember" + ) + everything = sorted(set().union(*(ids for _, ids in terms))) + self.verify_search(_or_of([q for q, _ in terms]), everything, "an OR of all") + self.verify_search( + Query("demoA.value", "lessthan", 100) & _or_of([q for q, _ in terms]), + everything, + "... AND an OR of all", + ) + + +if __name__ == "__main__": + unittest.main()