From c8adf741aa238d6e1c24ba94382e5c9b9501c7f6 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 20:38:16 +0000 Subject: [PATCH 1/5] Quote the text tests of 'hasmember' in the SQL search (#236) query_struct_to_sql_str put a hasmember value into the SQL unquoted, and also compared it as a double-quoted token. A text value was then read as SQL: 'name17' as a column (no such column), 'value' as doc_data.value (every document matched) and '1 OR 1=1' as an expression (SQL injection). The double-quoted test had the same problem for "value". The list-member regex and the whole-value test now use single-quoted literals (sqlStringList). doc2sql stores a number as a number, which never equals a text literal, so a finite numeric value is also compared as a number, written by sprintf('%.17g') so it can only be a numeric literal. TestHasMemberQuoting compares db.search with fieldsearch for text values (including 'value', '1 OR 1=1' and values holding " and '), numbers and ~hasmember, alone, inside an AND (restricted to candidates) and inside an OR (searched in a batch). Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01C7XsYS2DKSy2ucFzEEWR3D --- src/did/+did/database.m | 21 ++- tests/+did/+unittest/TestHasMemberQuoting.m | 173 ++++++++++++++++++++ 2 files changed, 189 insertions(+), 5 deletions(-) create mode 100644 tests/+did/+unittest/TestHasMemberQuoting.m diff --git a/src/did/+did/database.m b/src/did/+did/database.m index 32db0c2c..7196fdfd 100644 --- a/src/did/+did/database.m +++ b/src/did/+did/database.m @@ -1095,11 +1095,22 @@ function close_doc(database_obj, file_obj) case 'hasmember' fieldNameLike = regexprep(field,{'\\','\*','_'},{'\\\\','%','\\_'}); field_check = ['(' field_check ' OR fields.field_name like "' fieldNameLike '.%" ESCAPE "\")']; - %value_check= ['(doc_data.value like "%' param1 ',%"' ... - value_check = ['(regex(doc_data.value,"^(.*,\s*)*' did.database.sqlEscapeLiteral(param1RegexEsc) '\s*(,.*)*$") NOT NULL' ... - ' OR doc_data.value=' param1StrEsc ... - ' OR doc_data.value="' param1StrEsc '")']; - sql_str = [field_check ' AND ' notStr value_check]; + % PARAM1 is a member of a comma-separated list, or the + % whole value. The text tests use single-quoted literals + % (see sqlStringList): a double-quoted "value" is read as + % the column doc_data.value, and an unquoted value as SQL + % (issue #236). doc2sql stores a number as a number, which + % never equals a text literal, so a finite numeric PARAM1 + % is also compared as a number, written by sprintf so it + % can only be a numeric literal. + memberRegex = did.database.sqlStringList({['^(.*,\s*)*' param1RegexEsc '\s*(,.*)*$']}); + value_check = ['regex(doc_data.value,' memberRegex ') NOT NULL' ... + ' OR doc_data.value=' did.database.sqlStringList({param1Str})]; + if (isnumeric(param1) || islogical(param1)) && isscalar(param1) ... + && isreal(param1) && isfinite(param1) + value_check = [value_check ' OR doc_data.value=' param1Val]; + end + sql_str = [field_check ' AND ' notStr '(' value_check ')']; case 'hasfield' fieldNameLike = regexprep(field,{'\\','\*','_'},{'\\\\','%','\\_'}); sql_str = [field_check ' OR fields.field_name like "' fieldNameLike '.%" ESCAPE "\"']; diff --git a/tests/+did/+unittest/TestHasMemberQuoting.m b/tests/+did/+unittest/TestHasMemberQuoting.m new file mode 100644 index 00000000..00e7c956 --- /dev/null +++ b/tests/+did/+unittest/TestHasMemberQuoting.m @@ -0,0 +1,173 @@ +classdef TestHasMemberQuoting < matlab.unittest.TestCase + % Test that 'hasmember' never reads its value as SQL + % + % did.database/query_struct_to_sql_str put a text 'hasmember' value + % into the SQL unquoted, so 'name17' was read as a column (an error), + % 'value' as the column doc_data.value (every document matched) and + % '1 OR 1=1' as SQL (SQL injection). These tests compare db.search with + % did.datastructures.fieldsearch for text values, numbers and + % '~hasmember', on their own, inside an AND (searched restricted to + % candidate documents) and inside an OR (searched in a batch by + % search_or_operands). See issue #236. + % + % fieldsearch tests hasmember of a text value character by character + % (ismember), so the names below are chosen so that no name holds + % all the characters of another's except where SQL matches too. + + properties (Constant) + db_filename = 'test_db_hasmember.sqlite' + names = {'value', '1 OR 1=1', 'say "hi"', 'x, value', 'alpha', 'n7', 'O''Brien'} + end + + properties + db % did.test.helper.QueryLogSqlitedb + docs % the did.documents added, in order + end + + properties (TestParameter) + textValue = {'value', '1 OR 1=1', 'say "hi"', 'O''Brien', 'x, value', 'name17', 'zzz'} + numberValue = {3, 5, 99} + end + + methods (TestClassSetup) + function setupClass(testCase) + testCase.applyFixture(matlab.unittest.fixtures.WorkingFolderFixture); + testCase.applyFixture(did.test.fixture.PathConstantFixture); + + testCase.db = did.test.helper.QueryLogSqlitedb(testCase.db_filename); + testCase.db.add_branch('a'); + testCase.docs = cell(1, numel(testCase.names)); + for i = 1:numel(testCase.names) + testCase.docs{i} = did.document('demoA', 'base.name', testCase.names{i}, ... + 'demoA.value', i); + end + testCase.db.add_docs(testCase.docs, 'a', 'Validate', false); + end + end + + methods + function ids = expectedIds(testCase, s) + % The ids of the documents that fieldsearch matches with S + ids = {}; + for k = 1:numel(testCase.docs) + if did.datastructures.fieldsearch(testCase.docs{k}.document_properties, s) + ids{end+1} = testCase.docs{k}.id(); %#ok + end + end + ids = sort(ids(:)); + end + + function verifySearch(testCase, s, msg) + found = testCase.db.search(s); + testCase.verifyClass(found, 'cell', msg); + testCase.verifyEqual(sort(found(:)), testCase.expectedIds(s), msg); + end + + function s = hasMember(~, field, value, negate) + s = did.query(field, 'hasmember', value).searchstructure; + if negate + s.operation = '~hasmember'; + end + end + + function s = orOf(~, terms) + % Left-nested OR of search structures, as q = q | ... builds it + s = terms{1}; + for k = 2:numel(terms) + s = struct('field', '', 'operation', 'or', 'param1', s, 'param2', terms{k}); + end + end + + function s = allDocs(~) + % A term that matches every document, and that search_doc_ids + % searches before hasmember (fieldsearch has no 'isa') + s = did.query('demoA.value', 'lessthan', 100).searchstructure; + end + + function numQueries = countDocQueries(testCase) + entries = testCase.db.queryLog; + numQueries = nnz(startsWith({entries.query}, 'SELECT DISTINCT docs.doc_id')); + end + end + + methods (Test) + function testText(testCase, textValue) + for negate = [false true] + s = testCase.hasMember('base.name', textValue, negate); + testCase.verifySearch(s, sprintf('%s ''%s''', s.operation, textValue)); + end + end + + function testNumber(testCase, numberValue) + % doc2sql stores demoA.value as a number + for negate = [false true] + s = testCase.hasMember('demoA.value', numberValue, negate); + testCase.verifySearch(s, sprintf('%s %g', s.operation, numberValue)); + end + s = testCase.hasMember('demoA.value', numberValue, false); + testCase.verifyNumElements(testCase.db.search(s), ... + double(numberValue <= numel(testCase.names)), ... + sprintf('hasmember %g must match the one document holding it', numberValue)); + end + + function testInjectionMatchesOnlyItsOwnName(testCase) + s = testCase.hasMember('base.name', '1 OR 1=1', false); + found = testCase.db.search(s); + testCase.verifyEqual(found(:), {testCase.docs{2}.id()}, ... + 'hasmember ''1 OR 1=1'' must match only the document of that name'); + s = testCase.hasMember('base.name', 'value', false); + found = testCase.db.search(s); + testCase.verifyEqual(sort(found(:)), sort({testCase.docs{1}.id(); testCase.docs{4}.id()}), ... + 'hasmember ''value'' must match the name and the list holding it'); + end + + function testInAnAnd(testCase, textValue) + % 'demoA.value < 100' is searched first and leaves every + % document as a candidate, so hasmember is searched restricted + % to them + for negate = [false true] + s = [testCase.allDocs(), ... + testCase.hasMember('base.name', textValue, negate)]; + testCase.db.clearQueryLog(); + testCase.verifySearch(s, sprintf('demoA.value < 100 AND %s ''%s''', s(2).operation, textValue)); + entries = testCase.db.queryLog; + testCase.verifyTrue(any(contains({entries.query}, 'INDEXED BY')), ... + 'hasmember in an AND must be searched restricted to candidates'); + end + for negate = [false true] + s = [testCase.allDocs(), ... + testCase.hasMember('demoA.value', 5, negate)]; + testCase.verifySearch(s, sprintf('demoA.value < 100 AND %s 5', s(2).operation)); + end + end + + function testInAnOr(testCase) + terms = {testCase.hasMember('base.name', 'value', false), ... + testCase.hasMember('base.name', '1 OR 1=1', false), ... + testCase.hasMember('base.name', 'say "hi"', false), ... + testCase.hasMember('base.name', 'O''Brien', true), ... + testCase.hasMember('demoA.value', 6, false), ... + testCase.hasMember('demoA.value', 2, true)}; + for k = 1:numel(terms) + others = terms([1:k-1 k+1:end]); + s = testCase.orOf([terms(k), {did.query('base.name', 'exact_string', 'alpha').searchstructure}]); + testCase.verifySearch(s, sprintf('%s ''%s'' OR ...', terms{k}.operation, ... + num2str(terms{k}.param1))); + s = testCase.orOf([terms(k) others(1)]); + testCase.verifySearch(s, sprintf('%s ''%s'' OR another hasmember', ... + terms{k}.operation, num2str(terms{k}.param1))); + end + + % every operand is a single SQL term, so the OR is one batch + s = testCase.orOf(terms(1:3)); + testCase.db.clearQueryLog(); + testCase.verifySearch(s, 'an OR of text hasmembers'); + testCase.verifyEqual(testCase.countDocQueries(), 1, ... + 'an OR of hasmembers must be searched in one batched query'); + + % ...and restricted to candidates, inside an AND + s = [testCase.allDocs(), testCase.orOf(terms)]; + testCase.verifySearch(s, 'demoA.value < 100 AND an OR of hasmembers'); + end + end +end From c92931786ccb6b04afe223f64e6bb502f912fa62 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 20:49:38 +0000 Subject: [PATCH 2/5] TestHasMemberQuoting: test text hasmember against the list-member rule CI showed that fieldsearch is no oracle for a text hasmember: ismember(param1, value) compares two char arrays character by character, and the logical array it returns is negated and ANDed element by element, so '~hasmember value' matched only names sharing no character with 'value' and an AND matched any name sharing one. The SQL results in those runs were the intended ones. The expected documents now come from matches/memberOf: hasmember holds when the value is the whole field or an item of its comma-separated list, and every other operation is still tested by fieldsearch. Number searches are also compared with fieldsearch directly, which agrees for them. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01C7XsYS2DKSy2ucFzEEWR3D --- tests/+did/+unittest/TestHasMemberQuoting.m | 79 +++++++++++++++++---- 1 file changed, 67 insertions(+), 12 deletions(-) diff --git a/tests/+did/+unittest/TestHasMemberQuoting.m b/tests/+did/+unittest/TestHasMemberQuoting.m index 00e7c956..9377d342 100644 --- a/tests/+did/+unittest/TestHasMemberQuoting.m +++ b/tests/+did/+unittest/TestHasMemberQuoting.m @@ -4,15 +4,21 @@ % did.database/query_struct_to_sql_str put a text 'hasmember' value % into the SQL unquoted, so 'name17' was read as a column (an error), % 'value' as the column doc_data.value (every document matched) and - % '1 OR 1=1' as SQL (SQL injection). These tests compare db.search with - % did.datastructures.fieldsearch for text values, numbers and - % '~hasmember', on their own, inside an AND (searched restricted to - % candidate documents) and inside an OR (searched in a batch by - % search_or_operands). See issue #236. + % '1 OR 1=1' as SQL (SQL injection). These tests check db.search for + % text values, numbers and '~hasmember', on their own, inside an AND + % (searched restricted to candidate documents) and inside an OR + % (searched in a batch by search_or_operands). See issue #236. % - % fieldsearch tests hasmember of a text value character by character - % (ismember), so the names below are chosen so that no name holds - % all the characters of another's except where SQL matches too. + % The expected documents are found by matches, which tests hasmember + % by memberOf - the value is the whole field or an item of its + % comma-separated list, as the SQL search has it - and every other + % operation by did.datastructures.fieldsearch. fieldsearch itself is + % no guide for a text hasmember: ismember(param1, value) compares + % them character by character, and the logical array it returns is + % then negated and ANDed element by element, so '~hasmember value' + % matches only the names that share no character with 'value'. + % For numbers, fieldsearch agrees, and the search is compared with + % it too. properties (Constant) db_filename = 'test_db_hasmember.sqlite' @@ -46,17 +52,62 @@ function setupClass(testCase) end methods - function ids = expectedIds(testCase, s) - % The ids of the documents that fieldsearch matches with S + function ids = expectedIds(testCase, s, useFieldsearch) + % The ids of the documents that match S (see matches), or, if + % USEFIELDSEARCH is true, that fieldsearch matches with S + if nargin < 3, useFieldsearch = false; end ids = {}; for k = 1:numel(testCase.docs) - if did.datastructures.fieldsearch(testCase.docs{k}.document_properties, s) + props = testCase.docs{k}.document_properties; + if useFieldsearch + isMatch = did.datastructures.fieldsearch(props, s); + else + isMatch = testCase.matches(props, s); + end + if isMatch ids{end+1} = testCase.docs{k}.id(); %#ok end end ids = sort(ids(:)); end + function tf = matches(testCase, props, s) + % Does a document with properties PROPS match the search + % structure S? hasmember is tested by memberOf, the rest by + % fieldsearch + if numel(s) > 1 % AND + tf = true; + for k = 1:numel(s) + tf = tf && testCase.matches(props, s(k)); + end + return + end + op = lower(s.operation); + switch op + case 'or' + tf = testCase.matches(props, s.param1) || testCase.matches(props, s.param2); + case {'hasmember', '~hasmember'} + [isthere, value] = did.datastructures.isfullfield(props, s.field); + tf = isthere && testCase.memberOf(s.param1, value); + if op(1) == '~' + tf = ~tf; % as fieldsearch negates + end + otherwise + tf = logical(did.datastructures.fieldsearch(props, s)); + end + end + + function tf = memberOf(~, member, value) + % Is MEMBER the whole of VALUE, or an item of the + % comma-separated list in VALUE? + if ischar(member) + tf = ischar(value) && (strcmp(value, member) || ... + any(strcmp(strtrim(strsplit(value, ',')), member))); + else + tf = isnumeric(value) && isscalar(member) && any(value(:) == member); + end + end + function verifySearch(testCase, s, msg) found = testCase.db.search(s); testCase.verifyClass(found, 'cell', msg); @@ -102,7 +153,11 @@ function testNumber(testCase, numberValue) % doc2sql stores demoA.value as a number for negate = [false true] s = testCase.hasMember('demoA.value', numberValue, negate); - testCase.verifySearch(s, sprintf('%s %g', s.operation, numberValue)); + msg = sprintf('%s %g', s.operation, numberValue); + testCase.verifySearch(s, msg); + found = testCase.db.search(s); + testCase.verifyEqual(sort(found(:)), testCase.expectedIds(s, true), ... + [msg ': vs. fieldsearch']); end s = testCase.hasMember('demoA.value', numberValue, false); testCase.verifyNumElements(testCase.db.search(s), ... From f4ed409e4cfb9126e365f45aff2a17f4cd9d569b Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 21:08:08 +0000 Subject: [PATCH 3/5] One 'hasmember' rule in fieldsearch and the SQL search (#236) fieldsearch tested hasmember with ismember(param1, value), which compares two texts character by character and returns a logical array, so it matched another rule than the SQL search did (and DID-python's field_search, a substring test, a third). All three now match one rule, in a new local function fieldsearch/hasMember: - text is a member of text that is the whole value or an item of its comma-separated list (white space around the comma ignored); - anything is a member of a cell array that holds an equal element, and a number of a numeric or logical array that holds it; - text is never a member of a number, nor a number of text. query_struct_to_sql_str matches the same rule: a text member is tested only against text values (typeof), with the pattern (^|,\s*)M\s*(,|$), and a numeric member only against numbers. It no longer extends the field to its subfields (fields.field_name like 'field.%'), so a struct has no members, as in fieldsearch. TestHasMemberQuoting now compares db.search with fieldsearch again, and testRule pins fieldsearch to a table of cases shared with DID-python. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01C7XsYS2DKSy2ucFzEEWR3D --- src/did/+did/+datastructures/fieldsearch.m | 39 ++++- src/did/+did/database.m | 46 ++++-- tests/+did/+unittest/TestHasMemberQuoting.m | 171 +++++++++++--------- 3 files changed, 158 insertions(+), 98 deletions(-) diff --git a/src/did/+did/+datastructures/fieldsearch.m b/src/did/+did/+datastructures/fieldsearch.m index 270467eb..f29779b6 100644 --- a/src/did/+did/+datastructures/fieldsearch.m +++ b/src/did/+did/+datastructures/fieldsearch.m @@ -25,6 +25,10 @@ % | 'greaterthaneq' - is the field value greater than or equal to 'param1' (and comparable size) % | 'hassize' - does the field value have the size as indicated in 'param1' [x y z ...] % | 'hasmember' - does the field value have the member indicated in 'param1' ? + % | Text is a member of text that is the whole value or an item of its + % | comma-separated list; anything is a member of a cell array that holds + % | an equal element; a number is a member of a numeric array that holds + % | it. Text is never a member of a number, nor a number of text. % | 'hasfield' - is the field present? (no role for 'param1' or 'param2') % | 'partial_struct' - is the field value a structure that has all the fields of 'param1' with the same values? % | (note that it may have additional fields not found in the structure in param1) @@ -148,9 +152,7 @@ end case 'hasmember' if isthere - try - b = ismember(searchstruct.param1,value); - end + b = hasMember(value, searchstruct.param1); end case 'hasfield' b = isthere; @@ -218,3 +220,34 @@ if negation b = ~b; end + +function tf = hasMember(value, member) + % HASMEMBER - is MEMBER a member of VALUE, for 'hasmember'? + % + % Text is a member of text that is the whole value or an item of its + % comma-separated list (white space after a comma, and before one, is + % ignored); anything is a member of a cell array that holds an equal + % element; a number is a member of a numeric or logical array that + % holds it. Text is never a member of a number, nor a number of text. + % This is what did.database's SQL search matches, and DID-python's + % field_search. (ismember compared text character by character.) + tf = false; + if isstring(member) && isscalar(member), member = char(member); end + if isstring(value) && isscalar(value), value = char(value); end + isNumber = @(x) (isnumeric(x) || islogical(x)) && isreal(x); + if ischar(member) + if ischar(value) && isempty(value) + tf = isempty(member); + elseif ischar(value) + pattern = ['(^|,\s*)' regexptranslate('escape', member) '\s*(,|$)']; + tf = ~isempty(regexp(value, pattern, 'once', 'start')); + elseif iscell(value) + tf = any(cellfun(@(v) ischar(v) && strcmp(v, member), value(:))); + end + elseif isNumber(member) && isscalar(member) + if isNumber(value) + tf = any(value(:) == member); + elseif iscell(value) + tf = any(cellfun(@(v) isNumber(v) && isscalar(v) && v == member, value(:))); + end + end diff --git a/src/did/+did/database.m b/src/did/+did/database.m index 7065937c..1ae30d8f 100644 --- a/src/did/+did/database.m +++ b/src/did/+did/database.m @@ -1093,22 +1093,36 @@ function close_doc(database_obj, file_obj) case 'hassize' %TODO error('DID:Database:SQL','Query operation "%s" is not yet implemented',op); case 'hasmember' - fieldNameLike = regexprep(field,{'\\','\*','_'},{'\\\\','%','\\_'}); - field_check = ['(' field_check ' OR fields.field_name like "' fieldNameLike '.%" ESCAPE "\")']; - % PARAM1 is a member of a comma-separated list, or the - % whole value. The text tests use single-quoted literals - % (see sqlStringList): a double-quoted "value" is read as - % the column doc_data.value, and an unquoted value as SQL - % (issue #236). doc2sql stores a number as a number, which - % never equals a text literal, so a finite numeric PARAM1 - % is also compared as a number, written by sprintf so it - % can only be a numeric literal. - memberRegex = did.database.sqlStringList({['^(.*,\s*)*' param1RegexEsc '\s*(,.*)*$']}); - value_check = ['regex(doc_data.value,' memberRegex ') NOT NULL' ... - ' OR doc_data.value=' did.database.sqlStringList({param1Str})]; - if (isnumeric(param1) || islogical(param1)) && isscalar(param1) ... - && isreal(param1) && isfinite(param1) - value_check = [value_check ' OR doc_data.value=' param1Val]; + % Matched as did.datastructures.fieldsearch matches it + % (and DID-python's field_search): text is a member of + % text that is the whole value or an item of its + % comma-separated list, and a number of an equal number; + % text is never a member of a number, nor a number of + % text. The pattern is a single-quoted literal (see + % sqlStringList): an unquoted value was read as SQL, and + % a double-quoted "value" as the column doc_data.value + % (issue #236). doc2sql stores a number as a number, so a + % numeric PARAM1 is compared as a number, written by + % sprintf so that it can only be a numeric literal. + % Values doc2sql stores as a summary of their size and + % class (arrays and cells of more than one element) are + % not matched. + if ischar(param1) || (isstring(param1) && isscalar(param1)) + memberRegex = did.database.sqlStringList({['(^|,\s*)' ... + regexptranslate('escape', char(param1)) '\s*(,|$)']}); + value_check = ['typeof(doc_data.value)=''text'' AND ' ... + 'regex(doc_data.value,' memberRegex ') NOT NULL']; + elseif (isnumeric(param1) || islogical(param1)) && isscalar(param1) ... + && isreal(param1) && ~isnan(param1) + if isinf(param1) + numberLiteral = sprintf('%s9e999', repmat('-', 1, param1 < 0)); + else + numberLiteral = sprintf('%.17g', double(param1)); + end + value_check = ['typeof(doc_data.value) IN (''integer'',''real'') AND ' ... + 'doc_data.value=' numberLiteral]; + else + value_check = '0'; % a member no value can have end sql_str = [field_check ' AND ' notStr '(' value_check ')']; case 'hasfield' diff --git a/tests/+did/+unittest/TestHasMemberQuoting.m b/tests/+did/+unittest/TestHasMemberQuoting.m index 9377d342..03d41c6a 100644 --- a/tests/+did/+unittest/TestHasMemberQuoting.m +++ b/tests/+did/+unittest/TestHasMemberQuoting.m @@ -1,28 +1,29 @@ classdef TestHasMemberQuoting < matlab.unittest.TestCase - % Test that 'hasmember' never reads its value as SQL + % Test 'hasmember': one rule in fieldsearch and SQL, never read as SQL % % did.database/query_struct_to_sql_str put a text 'hasmember' value % into the SQL unquoted, so 'name17' was read as a column (an error), % 'value' as the column doc_data.value (every document matched) and - % '1 OR 1=1' as SQL (SQL injection). These tests check db.search for - % text values, numbers and '~hasmember', on their own, inside an AND - % (searched restricted to candidate documents) and inside an OR - % (searched in a batch by search_or_operands). See issue #236. + % '1 OR 1=1' as SQL (SQL injection). And did.datastructures.fieldsearch + % used ismember, which compares two texts character by character, so + % it matched another rule than the SQL did. Both now match one rule + % (see fieldsearch), as DID-python's field_search does: + % - text is a member of text that is the whole value or an item of + % its comma-separated list; + % - anything is a member of a cell array that holds an equal element, + % and a number of a numeric array that holds it; + % - text is never a member of a number, nor a number of text. % - % The expected documents are found by matches, which tests hasmember - % by memberOf - the value is the whole field or an item of its - % comma-separated list, as the SQL search has it - and every other - % operation by did.datastructures.fieldsearch. fieldsearch itself is - % no guide for a text hasmember: ismember(param1, value) compares - % them character by character, and the logical array it returns is - % then negated and ANDed element by element, so '~hasmember value' - % matches only the names that share no character with 'value'. - % For numbers, fieldsearch agrees, and the search is compared with - % it too. + % testRule checks fieldsearch against a table of cases (the same table + % as DID-python's tests/test_hasmember_quoting.py). The other tests + % compare db.search with fieldsearch for text values, numbers and + % '~hasmember', on their own, inside an AND (searched restricted to + % candidate documents) and inside an OR (searched in a batch by + % search_or_operands). See issue #236. properties (Constant) db_filename = 'test_db_hasmember.sqlite' - names = {'value', '1 OR 1=1', 'say "hi"', 'x, value', 'alpha', 'n7', 'O''Brien'} + names = {'value', '1 OR 1=1', 'say "hi"', 'x, value', 'alpha', 'n7', 'O''Brien', '5'} end properties @@ -31,7 +32,7 @@ end properties (TestParameter) - textValue = {'value', '1 OR 1=1', 'say "hi"', 'O''Brien', 'x, value', 'name17', 'zzz'} + textValue = {'value', '1 OR 1=1', 'say "hi"', 'O''Brien', 'x, value', 'name17', 'zzz', '5', ''} numberValue = {3, 5, 99} end @@ -52,68 +53,31 @@ function setupClass(testCase) end methods - function ids = expectedIds(testCase, s, useFieldsearch) - % The ids of the documents that match S (see matches), or, if - % USEFIELDSEARCH is true, that fieldsearch matches with S - if nargin < 3, useFieldsearch = false; end + function ids = expectedIds(testCase, s) + % The ids of the documents that fieldsearch matches with S ids = {}; for k = 1:numel(testCase.docs) - props = testCase.docs{k}.document_properties; - if useFieldsearch - isMatch = did.datastructures.fieldsearch(props, s); - else - isMatch = testCase.matches(props, s); - end - if isMatch + if did.datastructures.fieldsearch(testCase.docs{k}.document_properties, s) ids{end+1} = testCase.docs{k}.id(); %#ok end end ids = sort(ids(:)); end - function tf = matches(testCase, props, s) - % Does a document with properties PROPS match the search - % structure S? hasmember is tested by memberOf, the rest by - % fieldsearch - if numel(s) > 1 % AND - tf = true; - for k = 1:numel(s) - tf = tf && testCase.matches(props, s(k)); - end - return - end - op = lower(s.operation); - switch op - case 'or' - tf = testCase.matches(props, s.param1) || testCase.matches(props, s.param2); - case {'hasmember', '~hasmember'} - [isthere, value] = did.datastructures.isfullfield(props, s.field); - tf = isthere && testCase.memberOf(s.param1, value); - if op(1) == '~' - tf = ~tf; % as fieldsearch negates - end - otherwise - tf = logical(did.datastructures.fieldsearch(props, s)); - end - end - - function tf = memberOf(~, member, value) - % Is MEMBER the whole of VALUE, or an item of the - % comma-separated list in VALUE? - if ischar(member) - tf = ischar(value) && (strcmp(value, member) || ... - any(strcmp(strtrim(strsplit(value, ',')), member))); - else - tf = isnumeric(value) && isscalar(member) && any(value(:) == member); - end - end - function verifySearch(testCase, s, msg) found = testCase.db.search(s); testCase.verifyClass(found, 'cell', msg); testCase.verifyEqual(sort(found(:)), testCase.expectedIds(s), msg); end + function verifyIds(testCase, s, docIndexes, msg) + % The search and fieldsearch must both find the documents + % DOCINDEXES, no more + expected = cellfun(@(d) d.id(), testCase.docs(docIndexes), 'UniformOutput', false); + testCase.verifyEqual(testCase.expectedIds(s), sort(expected(:)), [msg ': fieldsearch']); + testCase.verifySearch(s, msg); + end + function s = hasMember(~, field, value, negate) s = did.query(field, 'hasmember', value).searchstructure; if negate @@ -142,6 +106,41 @@ function verifySearch(testCase, s, msg) end methods (Test) + function testRule(testCase) + % {value, member, is a member}; the same table as DID-python's + cases = { + 'value', 'value', true + 'x, value', 'value', true + 'x,value ,y', 'value', true + 'valuex', 'value', false + 'value', 'x, value', false + 'eulav', 'value', false % ismember matched this + 'a.b', '.', false % the member is not a pattern + '', '', true + 5, 5, true + [1 2 3], 2, true + [1 2 3], 4, false + true, 1, true + {'a', 'b'}, 'b', true + {'a', 'b'}, 'a,b', false + {1, 'a'}, 1, true + '5', 5, false + 5, '5', false + 'a', 97, false % ismember matched this + struct('a',1), 'a', false + }; + for k = 1:size(cases, 1) + [value, member, isMember] = cases{k, :}; + A = struct('f', {value}); + s = struct('field', 'f', 'operation', 'hasmember', 'param1', member, 'param2', ''); + testCase.verifyEqual(logical(did.datastructures.fieldsearch(A, s)), isMember, ... + sprintf('case %d: hasmember', k)); + s.operation = '~hasmember'; + testCase.verifyEqual(logical(did.datastructures.fieldsearch(A, s)), ~isMember, ... + sprintf('case %d: ~hasmember', k)); + end + end + function testText(testCase, textValue) for negate = [false true] s = testCase.hasMember('base.name', textValue, negate); @@ -153,11 +152,7 @@ function testNumber(testCase, numberValue) % doc2sql stores demoA.value as a number for negate = [false true] s = testCase.hasMember('demoA.value', numberValue, negate); - msg = sprintf('%s %g', s.operation, numberValue); - testCase.verifySearch(s, msg); - found = testCase.db.search(s); - testCase.verifyEqual(sort(found(:)), testCase.expectedIds(s, true), ... - [msg ': vs. fieldsearch']); + testCase.verifySearch(s, sprintf('%s %g', s.operation, numberValue)); end s = testCase.hasMember('demoA.value', numberValue, false); testCase.verifyNumElements(testCase.db.search(s), ... @@ -165,15 +160,33 @@ function testNumber(testCase, numberValue) sprintf('hasmember %g must match the one document holding it', numberValue)); end - function testInjectionMatchesOnlyItsOwnName(testCase) - s = testCase.hasMember('base.name', '1 OR 1=1', false); - found = testCase.db.search(s); - testCase.verifyEqual(found(:), {testCase.docs{2}.id()}, ... - 'hasmember ''1 OR 1=1'' must match only the document of that name'); - s = testCase.hasMember('base.name', 'value', false); - found = testCase.db.search(s); - testCase.verifyEqual(sort(found(:)), sort({testCase.docs{1}.id(); testCase.docs{4}.id()}), ... - 'hasmember ''value'' must match the name and the list holding it'); + function testExpectedDocuments(testCase) + testCase.verifyIds(testCase.hasMember('base.name', '1 OR 1=1', false), 2, ... + 'hasmember ''1 OR 1=1'''); + testCase.verifyIds(testCase.hasMember('base.name', 'value', false), [1 4], ... + 'hasmember ''value'' (the name and the list holding it)'); + testCase.verifyIds(testCase.hasMember('base.name', 'value', true), [2 3 5 6 7 8], ... + '~hasmember ''value'''); + testCase.verifyIds(testCase.hasMember('base.name', 'O''Brien', false), 7, ... + 'hasmember ''O''''Brien'''); + end + + function testTextAndNumbersDoNotMix(testCase) + % base.name '5' is text; demoA.value 3 is a number + testCase.verifyIds(testCase.hasMember('base.name', 5, false), [], ... + 'hasmember 5 in the text ''5'''); + testCase.verifyIds(testCase.hasMember('base.name', '5', false), 8, ... + 'hasmember ''5'' in the text ''5'''); + testCase.verifyIds(testCase.hasMember('demoA.value', '3', false), [], ... + 'hasmember ''3'' in the number 3'); + testCase.verifyIds(testCase.hasMember('demoA.value', 3, false), 3, ... + 'hasmember 3 in the number 3'); + end + + function testStructField(testCase) + % A struct (base) has no members: its fields are not searched + testCase.verifyIds(testCase.hasMember('base', 'value', false), [], ... + 'hasmember ''value'' in the struct base'); end function testInAnAnd(testCase, textValue) From 0171a4faee2063d177b2b9d520b98a521e11e157 Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 21:16:00 +0000 Subject: [PATCH 4/5] TestHasMemberQuoting: cover Inf and NaN members hasmember Inf / -Inf (written as 9e999 in the SQL) and NaN (a member of nothing) match no document, and ~hasmember matches every one; this covers the two SQL branches the other cases did not reach. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01C7XsYS2DKSy2ucFzEEWR3D --- tests/+did/+unittest/TestHasMemberQuoting.m | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/+did/+unittest/TestHasMemberQuoting.m b/tests/+did/+unittest/TestHasMemberQuoting.m index 03d41c6a..498ef5d8 100644 --- a/tests/+did/+unittest/TestHasMemberQuoting.m +++ b/tests/+did/+unittest/TestHasMemberQuoting.m @@ -183,6 +183,17 @@ function testTextAndNumbersDoNotMix(testCase) 'hasmember 3 in the number 3'); end + function testNumbersNoValueHolds(testCase) + % Inf is written as 9e999 and NaN is a member of nothing; no + % document holds either, so ~hasmember matches every one + for member = {Inf, -Inf, NaN} + msg = sprintf('hasmember %g', member{1}); + testCase.verifyIds(testCase.hasMember('demoA.value', member{1}, false), [], msg); + testCase.verifyIds(testCase.hasMember('demoA.value', member{1}, true), ... + 1:numel(testCase.names), ['~' msg]); + end + end + function testStructField(testCase) % A struct (base) has no members: its fields are not searched testCase.verifyIds(testCase.hasMember('base', 'value', false), [], ... From 74ae15f174427b8e3bfbebeaf0f11807aac9b86a Mon Sep 17 00:00:00 2001 From: Claude Date: Sat, 24 Oct 2026 21:27:35 +0000 Subject: [PATCH 5/5] Search 'hasmember' by brute force, as DID-python does (#236) doc2sql stores an array or cell of more than one element only as a summary of its size and class ('[1 3] double'), so the SQL search could not see the members of such fields: hasmember missed them and ~hasmember matched them. search_doc_ids now tests every hasmember and ~hasmember term with fieldsearch (brute_force_search), as DID-python's SQLiteDB does, so both languages apply the same rule the same way and no hasmember value goes into SQL. The new isBruteForceTerm names the terms searched this way (hasmember, and the depends_on queries isBruteForceDependsOn already sent there); search_doc_ids still searches them last in an AND, against the candidates the other terms leave, and search_or_operands keeps them out of its batches. query_struct_to_sql_str's 'hasmember' case is now an error, as no search reaches it. Tests: TestHasMemberQuoting checks that an AND loads only its candidates, that an OR of hasmembers runs no SQL search, and covers array and cell fields and a missing field; TestBatchedOr moves hasmember from its SQL terms to a brute-force-in-an-OR test. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01C7XsYS2DKSy2ucFzEEWR3D --- src/did/+did/database.m | 74 ++++++------- tests/+did/+unittest/TestBatchedOr.m | 19 +++- tests/+did/+unittest/TestHasMemberQuoting.m | 116 +++++++++++++++----- 3 files changed, 142 insertions(+), 67 deletions(-) diff --git a/src/did/+did/database.m b/src/did/+did/database.m index 1ae30d8f..7e193350 100644 --- a/src/did/+did/database.m +++ b/src/did/+did/database.m @@ -1093,38 +1093,10 @@ function close_doc(database_obj, file_obj) case 'hassize' %TODO error('DID:Database:SQL','Query operation "%s" is not yet implemented',op); case 'hasmember' - % Matched as did.datastructures.fieldsearch matches it - % (and DID-python's field_search): text is a member of - % text that is the whole value or an item of its - % comma-separated list, and a number of an equal number; - % text is never a member of a number, nor a number of - % text. The pattern is a single-quoted literal (see - % sqlStringList): an unquoted value was read as SQL, and - % a double-quoted "value" as the column doc_data.value - % (issue #236). doc2sql stores a number as a number, so a - % numeric PARAM1 is compared as a number, written by - % sprintf so that it can only be a numeric literal. - % Values doc2sql stores as a summary of their size and - % class (arrays and cells of more than one element) are - % not matched. - if ischar(param1) || (isstring(param1) && isscalar(param1)) - memberRegex = did.database.sqlStringList({['(^|,\s*)' ... - regexptranslate('escape', char(param1)) '\s*(,|$)']}); - value_check = ['typeof(doc_data.value)=''text'' AND ' ... - 'regex(doc_data.value,' memberRegex ') NOT NULL']; - elseif (isnumeric(param1) || islogical(param1)) && isscalar(param1) ... - && isreal(param1) && ~isnan(param1) - if isinf(param1) - numberLiteral = sprintf('%s9e999', repmat('-', 1, param1 < 0)); - else - numberLiteral = sprintf('%.17g', double(param1)); - end - value_check = ['typeof(doc_data.value) IN (''integer'',''real'') AND ' ... - 'doc_data.value=' numberLiteral]; - else - value_check = '0'; % a member no value can have - end - sql_str = [field_check ' AND ' notStr '(' value_check ')']; + % Searched by brute force in search_doc_ids (see + % isBruteForceTerm), so never gets here + error('DID:Database:SQL', ['hasmember is searched by brute force, ' ... + 'not in SQL']); case 'hasfield' fieldNameLike = regexprep(field,{'\\','\*','_'},{'\\\\','%','\\_'}); sql_str = [field_check ' OR fields.field_name like "' fieldNameLike '.%" ESCAPE "\"']; @@ -1274,8 +1246,8 @@ function close_doc(database_obj, file_obj) termOp(termOp=='~') = ''; if did.database.baseIdLookupValues(query_struct(i)) rank(i) = 1; - elseif any(strcmp(termOp, {'regexp','contains_string','hasmember'})) ... - || did.database.isBruteForceDependsOn(query_struct(i)) + elseif any(strcmp(termOp, {'regexp','contains_string'})) ... + || did.database.isBruteForceTerm(query_struct(i)) rank(i) = 4; elseif strcmp(char(query_struct(i).field), 'base.session_id') rank(i) = 3; @@ -1341,7 +1313,7 @@ function close_doc(database_obj, file_obj) if size(doc_ids,1)==1 && size(doc_ids,2)>1 doc_ids = doc_ids'; % ensure column vector end - elseif did.database.isBruteForceDependsOn(query_struct) + elseif did.database.isBruteForceTerm(query_struct) doc_ids = brute_force_search(sqlitedb_obj, query_struct, branch_id, candidate_ids); else % leaf scalar query query_str = get_sql_query_str(sqlitedb_obj, query_struct, branch_id, candidate_ids); @@ -1393,9 +1365,9 @@ function close_doc(database_obj, file_obj) % ORed base.id lookups among the operands are searched together % by search_base_ids, as for an OR of nothing but base.id % lookups. Operands that are not single SQL terms - ANDs (struct - % arrays), negated ORs and the depends_on queries that are - % brute-forced (see isBruteForceDependsOn) - are searched by - % search_doc_ids, one at a time, as before. + % arrays), negated ORs and the terms that are brute-forced + % (see isBruteForceTerm) - are searched by search_doc_ids, one + % at a time, as before. batchSize = sqlitedb_obj.orBatchTerms(); numOperands = numel(operands); @@ -1414,7 +1386,7 @@ function close_doc(database_obj, file_obj) if strcmp(op, 'or') continue % a negated OR (a plain one was flattened) end - if did.database.isBruteForceDependsOn(operand) + if did.database.isBruteForceTerm(operand) continue % no SQL form: brute_force_search end [isId, id] = did.database.baseIdLookupValues(operand); @@ -1470,7 +1442,7 @@ function close_doc(database_obj, file_obj) % the ones that did.datastructures.fieldsearch matches against the % search structure of QUERY_STRUCT (see did.query/to_searchstructure). % Used for the queries that SQL cannot match exactly (see - % isBruteForceDependsOn), as DID-python does. + % isBruteForceTerm), as DID-python does. if nargin > 3 && iscell(candidate_ids) ids = candidate_ids; else @@ -2399,6 +2371,28 @@ function validate_field_type_and_value(database_obj, doc_name, field_name, value isempty(did.database.dependsOnSqlMatch(query_struct.param1, query_struct.param2)); end % isBruteForceDependsOn() + function tf = isBruteForceTerm(query_struct) + % isBruteForceTerm - is a search term tested by brute force? + % + % TF = isBruteForceTerm(QUERY_STRUCT) + % + % True if QUERY_STRUCT (a single search structure) is searched + % by brute_force_search - its documents tested one by one with + % did.datastructures.fieldsearch - rather than in SQL, as + % DID-python searches it: + % - hasmember and ~hasmember, always. Their members are often + % those of an array or cell, which doc2sql stores only as a + % summary of its size and class ('[1 3] double'), so SQL + % cannot see them (issue #236). + % - the depends_on queries SQL cannot match exactly (see + % isBruteForceDependsOn). + % search_doc_ids searches these last in an AND, so they test + % only the documents the other terms leave. + op = strtrim(lower(char(query_struct.operation))); + op(op=='~') = ''; + tf = strcmp(op, 'hasmember') || did.database.isBruteForceDependsOn(query_struct); + end % isBruteForceTerm() + function [tf, ids] = baseIdLookupValues(query_struct) % baseIdLookupValues - is a search structure a lookup by base.id? % diff --git a/tests/+did/+unittest/TestBatchedOr.m b/tests/+did/+unittest/TestBatchedOr.m index 3a58636b..14b0d46c 100644 --- a/tests/+did/+unittest/TestBatchedOr.m +++ b/tests/+did/+unittest/TestBatchedOr.m @@ -78,14 +78,13 @@ function restoreDefaults(testCase) did.query('demoA.value', 'lessthaneq', 3) did.query('demoB.value', 'greaterthan', 138) did.query('demoB.value', 'greaterthaneq', 138) - did.query('demoC.value', 'hasmember', 205) did.query('demoB', 'hasfield') did.query('', 'depends_on', 'item1', testCase.idsA{4}) did.query('base.name', 'regexp', '^c2') did.query('', 'isa', 'demoB') did.query('', 'isa', 'demoC') }; - negatable = terms([1:9 11:14]); + negatable = terms([1:8 10:13]); for k = 1:numel(negatable) s = negatable{k}.searchstructure; s.operation = ['~' s.operation]; @@ -255,6 +254,22 @@ function testBruteForceDependsOnInAnOr(testCase) end end + function testBruteForceHasMemberInAnOr(testCase) + % hasmember is always searched by brute_force_search (see + % did.database/isBruteForceTerm), on its own, not in a batch + member = did.query('demoC.value', 'hasmember', 205); + notMember = member.searchstructure; + notMember.operation = '~hasmember'; + for term = {member, notMember} + s = testCase.toStruct(term{1}); + q = testCase.orOf({term{1}, did.query('base.name', 'exact_string', 'n3'), ... + did.query('', 'isa', 'demoB')}); + testCase.verifyBatchedMatchesPerOperand(q, [s.operation ' in an OR']); + q2 = [testCase.toStruct(did.query('', 'isa', 'demoC')), q]; + testCase.verifyBatchedMatchesPerOperand(q2, [s.operation ' in an OR inside an AND']); + end + end + function testWithoutSearchIndexes(testCase) % Without the indexes there are no candidates and no batched % id lookups; the batch is a UNION over the whole branch diff --git a/tests/+did/+unittest/TestHasMemberQuoting.m b/tests/+did/+unittest/TestHasMemberQuoting.m index 498ef5d8..8f0ddefb 100644 --- a/tests/+did/+unittest/TestHasMemberQuoting.m +++ b/tests/+did/+unittest/TestHasMemberQuoting.m @@ -1,25 +1,29 @@ classdef TestHasMemberQuoting < matlab.unittest.TestCase - % Test 'hasmember': one rule in fieldsearch and SQL, never read as SQL + % Test 'hasmember': one rule, searched by brute force, never as SQL % % did.database/query_struct_to_sql_str put a text 'hasmember' value % into the SQL unquoted, so 'name17' was read as a column (an error), % 'value' as the column doc_data.value (every document matched) and - % '1 OR 1=1' as SQL (SQL injection). And did.datastructures.fieldsearch - % used ismember, which compares two texts character by character, so - % it matched another rule than the SQL did. Both now match one rule - % (see fieldsearch), as DID-python's field_search does: + % '1 OR 1=1' as SQL (SQL injection). And each implementation had its + % own rule: did.datastructures.fieldsearch used ismember, which + % compares two texts character by character; the SQL matched an item + % of a comma-separated list, and could not see the members of arrays + % and cells at all (doc2sql stores them as '[1 3] double'); DID-python + % tested for a substring. Now hasmember has one rule (see fieldsearch): % - text is a member of text that is the whole value or an item of % its comma-separated list; % - anything is a member of a cell array that holds an equal element, % and a number of a numeric array that holds it; - % - text is never a member of a number, nor a number of text. + % - text is never a member of a number, nor a number of text; + % and the SQLite search tests it by brute force with fieldsearch (see + % did.database/isBruteForceTerm), as DID-python does, so no hasmember + % value goes into SQL. % % testRule checks fieldsearch against a table of cases (the same table % as DID-python's tests/test_hasmember_quoting.py). The other tests - % compare db.search with fieldsearch for text values, numbers and - % '~hasmember', on their own, inside an AND (searched restricted to - % candidate documents) and inside an OR (searched in a batch by - % search_or_operands). See issue #236. + % compare db.search with fieldsearch for text values, numbers, array + % and cell fields and '~hasmember', on their own, inside an AND (only + % the candidates are tested) and inside an OR. See issue #236. properties (Constant) db_filename = 'test_db_hasmember.sqlite' @@ -100,9 +104,21 @@ function verifyIds(testCase, s, docIndexes, msg) end function numQueries = countDocQueries(testCase) + % The SQL searches run since the query log was cleared entries = testCase.db.queryLog; numQueries = nnz(startsWith({entries.query}, 'SELECT DISTINCT docs.doc_id')); end + + function verifyLoadedOnly(testCase, numDocs, msg) + % Since the query log was cleared, NUMDOCS documents were + % loaded, and the documents of the branch were not listed + entries = testCase.db.queryLog; + queries = {entries.query}; + testCase.verifyFalse(any(startsWith(queries, 'SELECT docs.doc_id FROM docs')), ... + [msg ': the branch must not be listed']); + testCase.verifyEqual(nnz(startsWith(queries, 'SELECT json_code FROM docs')), numDocs, ... + [msg ': documents loaded']); + end end methods (Test) @@ -194,6 +210,59 @@ function testNumbersNoValueHolds(testCase) end end + function testMissingField(testCase) + % No document has demoB.value: none holds a member of it, so + % ~hasmember matches every one (as in fieldsearch) + testCase.verifyIds(testCase.hasMember('demoB.value', 1, false), [], ... + 'hasmember in a missing field'); + testCase.verifyIds(testCase.hasMember('demoB.value', 1, true), ... + 1:numel(testCase.names), '~hasmember in a missing field'); + end + + function testArrayAndCellFields(testCase) + % doc2sql stores these as a summary ('[1 3] double', '[1 2] + % cell'), which SQL could not search; brute force can + testCase.applyFixture(matlab.unittest.fixtures.WorkingFolderFixture); + db = did.implementations.sqlitedb('list_fields.sqlite'); + db.add_branch('a'); + docs = {did.document('demoA', 'base.name', 'numbers', 'demoA.value', [1 2 3]), ... + did.document('demoA', 'base.name', 'other', 'demoA.value', [4 5]), ... + did.document('demoA', 'base.name', 'names', 'demoA.value', {'alpha', 'beta'})}; + db.add_docs(docs, 'a', 'Validate', false); + ids = cellfun(@(d) d.id(), docs, 'UniformOutput', false); + stored = db.get_docs(ids); % the documents as the search sees them + cases = { + 'demoA.value', 2, 1 + 'demoA.value', 5, 2 + 'demoA.value', 6, [] + 'demoA.value', 'beta', 3 + 'demoA.value', 'gamma', [] + 'base.name', 'other', 2 + }; + for k = 1:size(cases, 1) + [field, member, matching] = cases{k, :}; + for negate = [false true] + s = testCase.hasMember(field, member, negate); + if negate + expected = ids(setdiff(1:3, matching)); + else + expected = ids(matching); + end + expected = sort(expected(:)); + msg = sprintf('%s %s in %s', s.operation, num2str(member), field); + found = db.search(s); + testCase.verifyEqual(sort(found(:)), expected, msg); + fieldsearchIds = cell(0, 1); + for j = 1:numel(stored) + if did.datastructures.fieldsearch(stored(j).document_properties, s) + fieldsearchIds{end+1, 1} = stored(j).id(); %#ok + end + end + testCase.verifyEqual(sort(fieldsearchIds), expected, [msg ': fieldsearch']); + end + end + end + function testStructField(testCase) % A struct (base) has no members: its fields are not searched testCase.verifyIds(testCase.hasMember('base', 'value', false), [], ... @@ -201,22 +270,19 @@ function testStructField(testCase) end function testInAnAnd(testCase, textValue) - % 'demoA.value < 100' is searched first and leaves every - % document as a candidate, so hasmember is searched restricted - % to them + % 'demoA.value < 4' is searched first and leaves 3 candidates, + % so hasmember loads and tests those 3 documents only + firstThree = did.query('demoA.value', 'lessthan', 4).searchstructure; for negate = [false true] - s = [testCase.allDocs(), ... - testCase.hasMember('base.name', textValue, negate)]; + s = [firstThree, testCase.hasMember('base.name', textValue, negate)]; + msg = sprintf('demoA.value < 4 AND %s ''%s''', s(2).operation, textValue); testCase.db.clearQueryLog(); - testCase.verifySearch(s, sprintf('demoA.value < 100 AND %s ''%s''', s(2).operation, textValue)); - entries = testCase.db.queryLog; - testCase.verifyTrue(any(contains({entries.query}, 'INDEXED BY')), ... - 'hasmember in an AND must be searched restricted to candidates'); + testCase.verifySearch(s, msg); + testCase.verifyLoadedOnly(3, msg); end for negate = [false true] - s = [testCase.allDocs(), ... - testCase.hasMember('demoA.value', 5, negate)]; - testCase.verifySearch(s, sprintf('demoA.value < 100 AND %s 5', s(2).operation)); + s = [firstThree, testCase.hasMember('demoA.value', 2, negate)]; + testCase.verifySearch(s, sprintf('demoA.value < 4 AND %s 2', s(2).operation)); end end @@ -237,12 +303,12 @@ function testInAnOr(testCase) terms{k}.operation, num2str(terms{k}.param1))); end - % every operand is a single SQL term, so the OR is one batch + % every operand is brute-forced, so no SQL search is run s = testCase.orOf(terms(1:3)); testCase.db.clearQueryLog(); testCase.verifySearch(s, 'an OR of text hasmembers'); - testCase.verifyEqual(testCase.countDocQueries(), 1, ... - 'an OR of hasmembers must be searched in one batched query'); + testCase.verifyEqual(testCase.countDocQueries(), 0, ... + 'hasmember must not be searched in SQL'); % ...and restricted to candidates, inside an AND s = [testCase.allDocs(), testCase.orOf(terms)];