Skip to content

hasmember: one rule, searched by brute force as in DID-python (#236) #239

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
stevevanhooser merged 6 commits into from
Oct 24, 2026
Merged

hasmember: one rule, searched by brute force as in DID-python (#236) #239

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
c8adf74
Quote the text tests of 'hasmember' in the SQL search (#236)
claude Oct 24, 2026
c929317
TestHasMemberQuoting: test text hasmember against the list-member rule
claude Oct 24, 2026
2c08472
Merge remote-tracking branch 'origin/main' into claude/did-hasmember-…
claude Oct 24, 2026
f4ed409
One 'hasmember' rule in fieldsearch and the SQL search (#236)
claude Oct 24, 2026
0171a4f
TestHasMemberQuoting: cover Inf and NaN members
claude Oct 24, 2026
74ae15f
Search 'hasmember' by brute force, as DID-python does (#236)
claude Oct 24, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 36 additions & 3 deletions src/did/+did/+datastructures/fieldsearch.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,10 @@
% | 'greaterthaneq' - is the field value greater than or equal to 'param1' (and comparable size)
% | 'hassize' - does the field value have the size as indicated in 'param1' [x y z ...]
% | 'hasmember' - does the field value have the member indicated in 'param1' ?
% | Text is a member of text that is the whole value or an item of its
% | comma-separated list; anything is a member of a cell array that holds
% | an equal element; a number is a member of a numeric array that holds
% | it. Text is never a member of a number, nor a number of text.
% | 'hasfield' - is the field present? (no role for 'param1' or 'param2')
% | 'partial_struct' - is the field value a structure that has all the fields of 'param1' with the same values?
% | (note that it may have additional fields not found in the structure in param1)
Expand Down Expand Up @@ -148,9 +152,7 @@
end
case 'hasmember'
if isthere
try
b = ismember(searchstruct.param1,value);
end
b = hasMember(value, searchstruct.param1);
end
case 'hasfield'
b = isthere;
Expand Down Expand Up @@ -218,3 +220,34 @@
if negation
b = ~b;
end

function tf = hasMember(value, member)
% HASMEMBER - is MEMBER a member of VALUE, for 'hasmember'?
%
% Text is a member of text that is the whole value or an item of its
% comma-separated list (white space after a comma, and before one, is
% ignored); anything is a member of a cell array that holds an equal
% element; a number is a member of a numeric or logical array that
% holds it. Text is never a member of a number, nor a number of text.
% This is what did.database's SQL search matches, and DID-python's
% field_search. (ismember compared text character by character.)
tf = false;
if isstring(member) && isscalar(member), member = char(member); end
if isstring(value) && isscalar(value), value = char(value); end
isNumber = @(x) (isnumeric(x) || islogical(x)) && isreal(x);
if ischar(member)
if ischar(value) && isempty(value)
tf = isempty(member);
elseif ischar(value)
pattern = ['(^|,\s*)' regexptranslate('escape', member) '\s*(,|$)'];
tf = ~isempty(regexp(value, pattern, 'once', 'start'));
elseif iscell(value)
tf = any(cellfun(@(v) ischar(v) && strcmp(v, member), value(:)));
end
elseif isNumber(member) && isscalar(member)
if isNumber(value)
tf = any(value(:) == member);
elseif iscell(value)
tf = any(cellfun(@(v) isNumber(v) && isscalar(v) && v == member, value(:)));
end
end
49 changes: 34 additions & 15 deletions src/did/+did/database.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1093,13 +1093,10 @@ function close_doc(database_obj, file_obj)
case 'hassize' %TODO
error('DID:Database:SQL','Query operation "%s" is not yet implemented',op);
case 'hasmember'
fieldNameLike = regexprep(field,{'\\','\*','_'},{'\\\\','%','\\_'});
field_check = ['(' field_check ' OR fields.field_name like "' fieldNameLike '.%" ESCAPE "\")'];
%value_check= ['(doc_data.value like "%' param1 ',%"' ...
value_check = ['(regex(doc_data.value,"^(.*,\s*)*' did.database.sqlEscapeLiteral(param1RegexEsc) '\s*(,.*)*$") NOT NULL' ...
' OR doc_data.value=' param1StrEsc ...
' OR doc_data.value="' param1StrEsc '")'];
sql_str = [field_check ' AND ' notStr value_check];
% Searched by brute force in search_doc_ids (see
% isBruteForceTerm), so never gets here
error('DID:Database:SQL', ['hasmember is searched by brute force, ' ...
'not in SQL']);
case 'hasfield'
fieldNameLike = regexprep(field,{'\\','\*','_'},{'\\\\','%','\\_'});
sql_str = [field_check ' OR fields.field_name like "' fieldNameLike '.%" ESCAPE "\"'];
Expand Down Expand Up @@ -1249,8 +1246,8 @@ function close_doc(database_obj, file_obj)
termOp(termOp=='~') = '';
if did.database.baseIdLookupValues(query_struct(i))
rank(i) = 1;
elseif any(strcmp(termOp, {'regexp','contains_string','hasmember'})) ...
|| did.database.isBruteForceDependsOn(query_struct(i))
elseif any(strcmp(termOp, {'regexp','contains_string'})) ...
|| did.database.isBruteForceTerm(query_struct(i))
rank(i) = 4;
elseif strcmp(char(query_struct(i).field), 'base.session_id')
rank(i) = 3;
Expand Down Expand Up @@ -1316,7 +1313,7 @@ function close_doc(database_obj, file_obj)
if size(doc_ids,1)==1 && size(doc_ids,2)>1
doc_ids = doc_ids'; % ensure column vector
end
elseif did.database.isBruteForceDependsOn(query_struct)
elseif did.database.isBruteForceTerm(query_struct)
doc_ids = brute_force_search(sqlitedb_obj, query_struct, branch_id, candidate_ids);
else % leaf scalar query
query_str = get_sql_query_str(sqlitedb_obj, query_struct, branch_id, candidate_ids);
Expand Down Expand Up @@ -1368,9 +1365,9 @@ function close_doc(database_obj, file_obj)
% ORed base.id lookups among the operands are searched together
% by search_base_ids, as for an OR of nothing but base.id
% lookups. Operands that are not single SQL terms - ANDs (struct
% arrays), negated ORs and the depends_on queries that are
% brute-forced (see isBruteForceDependsOn) - are searched by
% search_doc_ids, one at a time, as before.
% arrays), negated ORs and the terms that are brute-forced
% (see isBruteForceTerm) - are searched by search_doc_ids, one
% at a time, as before.

batchSize = sqlitedb_obj.orBatchTerms();
numOperands = numel(operands);
Expand All @@ -1389,7 +1386,7 @@ function close_doc(database_obj, file_obj)
if strcmp(op, 'or')
continue % a negated OR (a plain one was flattened)
end
if did.database.isBruteForceDependsOn(operand)
if did.database.isBruteForceTerm(operand)
continue % no SQL form: brute_force_search
end
[isId, id] = did.database.baseIdLookupValues(operand);
Expand Down Expand Up @@ -1445,7 +1442,7 @@ function close_doc(database_obj, file_obj)
% the ones that did.datastructures.fieldsearch matches against the
% search structure of QUERY_STRUCT (see did.query/to_searchstructure).
% Used for the queries that SQL cannot match exactly (see
% isBruteForceDependsOn), as DID-python does.
% isBruteForceTerm), as DID-python does.
if nargin > 3 && iscell(candidate_ids)
ids = candidate_ids;
else
Expand Down Expand Up @@ -2374,6 +2371,28 @@ function validate_field_type_and_value(database_obj, doc_name, field_name, value
isempty(did.database.dependsOnSqlMatch(query_struct.param1, query_struct.param2));
end % isBruteForceDependsOn()

function tf = isBruteForceTerm(query_struct)
% isBruteForceTerm - is a search term tested by brute force?
%
% TF = isBruteForceTerm(QUERY_STRUCT)
%
% True if QUERY_STRUCT (a single search structure) is searched
% by brute_force_search - its documents tested one by one with
% did.datastructures.fieldsearch - rather than in SQL, as
% DID-python searches it:
% - hasmember and ~hasmember, always. Their members are often
% those of an array or cell, which doc2sql stores only as a
% summary of its size and class ('[1 3] double'), so SQL
% cannot see them (issue #236).
% - the depends_on queries SQL cannot match exactly (see
% isBruteForceDependsOn).
% search_doc_ids searches these last in an AND, so they test
% only the documents the other terms leave.
op = strtrim(lower(char(query_struct.operation)));
op(op=='~') = '';
tf = strcmp(op, 'hasmember') || did.database.isBruteForceDependsOn(query_struct);
end % isBruteForceTerm()

function [tf, ids] = baseIdLookupValues(query_struct)
% baseIdLookupValues - is a search structure a lookup by base.id?
%
Expand Down
19 changes: 17 additions & 2 deletions tests/+did/+unittest/TestBatchedOr.m
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,14 +78,13 @@ function restoreDefaults(testCase)
did.query('demoA.value', 'lessthaneq', 3)
did.query('demoB.value', 'greaterthan', 138)
did.query('demoB.value', 'greaterthaneq', 138)
did.query('demoC.value', 'hasmember', 205)
did.query('demoB', 'hasfield')
did.query('', 'depends_on', 'item1', testCase.idsA{4})
did.query('base.name', 'regexp', '^c2')
did.query('', 'isa', 'demoB')
did.query('', 'isa', 'demoC')
};
negatable = terms([1:9 11:14]);
negatable = terms([1:8 10:13]);
for k = 1:numel(negatable)
s = negatable{k}.searchstructure;
s.operation = ['~' s.operation];
Expand Down Expand Up @@ -255,6 +254,22 @@ function testBruteForceDependsOnInAnOr(testCase)
end
end

function testBruteForceHasMemberInAnOr(testCase)
% hasmember is always searched by brute_force_search (see
% did.database/isBruteForceTerm), on its own, not in a batch
member = did.query('demoC.value', 'hasmember', 205);
notMember = member.searchstructure;
notMember.operation = '~hasmember';
for term = {member, notMember}
s = testCase.toStruct(term{1});
q = testCase.orOf({term{1}, did.query('base.name', 'exact_string', 'n3'), ...
did.query('', 'isa', 'demoB')});
testCase.verifyBatchedMatchesPerOperand(q, [s.operation ' in an OR']);
q2 = [testCase.toStruct(did.query('', 'isa', 'demoC')), q];
testCase.verifyBatchedMatchesPerOperand(q2, [s.operation ' in an OR inside an AND']);
end
end

function testWithoutSearchIndexes(testCase)
% Without the indexes there are no candidates and no batched
% id lookups; the batch is a UNION over the whole branch
Expand Down
Loading
Loading