From bd7fce18d58934584a90a03346769d9997631143 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josh=20Goldberg=20=E2=9C=A8?= Date: Tue, 8 Sep 2026 12:06:54 -0400 Subject: [PATCH 01/10] feat(logs): add regex search support to the EAP query syntax Adds a `Matches` operator to the search grammar, using the same reserved-unicode marker encoding as the existing Contains/StartsWith/EndsWith wildcard operators, and resolves it to a `ComparisonFilter` with `OP_REGEXP` for EAP datasets. Unlike the wildcard operators, the pattern is carried verbatim rather than being rewritten: `SearchValue.is_regex` suppresses wildcard translation and escape sequence handling, so `\*` stays a literal asterisk and `a*b` stays a quantifier. Patterns are validated at parse time, rejecting empty patterns and the backreference and lookaround syntax that RE2 does not support. Regex is rejected on virtual column contexts and on keys backed by a filter alias, since those converters resolve values against Sentry models and would match the pattern as a literal. There is no `OP_NOT_REGEXP`, so `!=` and `NOT IN` wrap the match in a `NotFilter`. `OP_REGEXP` is referenced by its wire value until sentry-protos publishes a release containing it (getsentry/sentry-protos#420). Snuba's translation to ClickHouse `match()` is likewise not yet released (getsentry/snuba#8437), so this path is not end-to-end testable yet. Fixes LOGS-958 --- src/sentry/api/event_search.py | 75 ++++++++++-- src/sentry/search/eap/constants.py | 18 +++ src/sentry/search/eap/resolver.py | 49 ++++++++ src/sentry/search/events/constants.py | 7 ++ tests/sentry/api/test_event_search.py | 91 +++++++++++++++ tests/sentry/search/eap/test_ourlogs.py | 146 ++++++++++++++++++++++++ 6 files changed, 378 insertions(+), 8 deletions(-) diff --git a/src/sentry/api/event_search.py b/src/sentry/api/event_search.py index 1a1d7f403cf6..e638e8402929 100644 --- a/src/sentry/api/event_search.py +++ b/src/sentry/api/event_search.py @@ -17,12 +17,14 @@ DURATION_UNITS, NOT_HAS_FILTER_ERROR_MESSAGE, OPERATOR_NEGATION_MAP, + REGEX_OPERATOR, SEARCH_MAP, SEMVER_ALIAS, SEMVER_BUILD_ALIAS, SIZE_UNITS, TAG_KEY_RE, TEAM_KEY_TRANSACTION_ALIAS, + UNSUPPORTED_REGEX_SYNTAX, WILDCARD_OPERATOR_MAP, ) from sentry.search.events.fields import FIELD_ALIASES, FUNCTIONS @@ -188,7 +190,7 @@ # NOTE: These wildcard operators are internal implementation details and # should not be included in product docs. Users should use `*` instead. -wildcard_op = wildcard_unicode (contains / starts_with / ends_with) wildcard_unicode +wildcard_op = wildcard_unicode (contains / starts_with / ends_with / matches) wildcard_unicode # See: https://stackoverflow.com/a/39617181/790169 in_value_termination = in_value_char (!in_value_end in_value_char)* in_value_end @@ -232,6 +234,7 @@ contains = "Contains" starts_with = "StartsWith" ends_with = "EndsWith" +matches = "Matches" comma = "," spaces = " "* @@ -417,6 +420,34 @@ def get_wildcard_op(node: Node | Sequence[Node]) -> str: return "" +def has_regex_op(node: Node | Sequence[Node]) -> bool: + return get_wildcard_op(node) == REGEX_OPERATOR + + +def quote_regex_pattern(pattern: str) -> str: + """Regex patterns are always quoted when serialized back to a query string, because the + unquoted value grammar rejects the parentheses and spaces that patterns routinely contain.""" + escaped = pattern.replace('"', '\\"') + return f'"{escaped}"' + + +def validate_regex_pattern(key: str, pattern: str) -> None: + if not pattern: + raise InvalidSearchQuery(f"{key}: Empty regex pattern") + + unsupported = UNSUPPORTED_REGEX_SYNTAX.search(pattern) + if unsupported is not None: + raise InvalidSearchQuery( + f"{key}: Invalid regex: `{unsupported.group()}` is not supported. " + "Backreferences and lookaround are unavailable." + ) + + try: + re.compile(pattern) + except re.error as exc: + raise InvalidSearchQuery(f"{key}: Invalid regex: {exc.msg}") + + def add_leading_wildcard(value: str) -> str: if value.startswith('"') and value.endswith('"'): return f"*{value[1:-1]}" @@ -525,11 +556,16 @@ class SearchValue(NamedTuple): raw_value: str | float | datetime | Sequence[float] | Sequence[str] # Used for top events where we don't want to modify the raw value at all use_raw_value: bool = False + is_regex: bool = False @property def value(self) -> Any: if self.use_raw_value: return self.raw_value + elif self.is_regex: + # Escape sequences are meaningful to the regex engine, so the pattern passes through + # untouched. `\*` is a literal asterisk here, not an escaped wildcard. + return self.raw_value elif self.is_wildcard() and isinstance(self.raw_value, str): return translate_wildcard(self.raw_value) elif self.is_wildcard() and isinstance(self.raw_value, (list, tuple)): @@ -548,11 +584,18 @@ def to_query_string(self) -> str: # we do that because a simple str() would not be usable for strings # str(["a","b"]) == "['a', 'b']" but we would like "[a,b]" if isinstance(self.raw_value, (list, tuple)): - ret_val = ", ".join(str(x) for x in self.raw_value) + values = ( + [quote_regex_pattern(str(x)) for x in self.raw_value] + if self.is_regex + else [str(x) for x in self.raw_value] + ) + ret_val = ", ".join(values) ret_val = f"[{ret_val}]" return ret_val elif isinstance(self.raw_value, datetime): return self.raw_value.isoformat() + elif self.is_regex: + return quote_regex_pattern(str(self.value)) else: return str(self.value) @@ -560,6 +603,9 @@ def is_wildcard(self) -> bool: # If we're using the raw value only it'll never be a wildcard if self.use_raw_value: return False + # A `*` in a regex is a quantifier, not a wildcard + if self.is_regex: + return False if self.is_str_sequence(): return isinstance(self.raw_value, list) and any( _is_wildcard(value) for value in self.raw_value @@ -664,12 +710,14 @@ def __str__(self) -> str: return f"{self.key.name}{self.operator}{self.value.raw_value}" def to_query_string(self) -> str: + # The marker sits between the `:` and the operator, matching the grammar's ordering + marker = REGEX_OPERATOR if self.value.is_regex else "" if self.operator == "IN": - return f"{self.key.name}:{self.value.to_query_string()}" + return f"{self.key.name}:{marker}{self.value.to_query_string()}" elif self.operator == "NOT IN": - return f"!{self.key.name}:{self.value.to_query_string()}" + return f"!{self.key.name}:{marker}{self.value.to_query_string()}" else: - return f"{self.key.name}:{self.operator}{self.value.to_query_string()}" + return f"{self.key.name}:{marker}{self.operator}{self.value.to_query_string()}" @property def is_negation(self) -> bool: @@ -1442,7 +1490,12 @@ def visit_text_in_filter( operator = handle_negation(negation, operator) - if has_wildcard_op(wildcard_op) and isinstance(search_value.raw_value, list): + if has_regex_op(wildcard_op) and isinstance(search_value.raw_value, list): + for value in search_value.raw_value: + if isinstance(value, str): + validate_regex_pattern(search_key.name, value) + search_value = search_value._replace(is_regex=True) + elif has_wildcard_op(wildcard_op) and isinstance(search_value.raw_value, list): wildcarded_values = [] found_wildcard_op = get_wildcard_op(wildcard_op) for value in search_value.raw_value: @@ -1481,7 +1534,10 @@ def visit_text_filter( operator_s = handle_negation(negation, operator_s) - if has_wildcard_op(wildcard_op) and isinstance(search_value.raw_value, str): + if has_regex_op(wildcard_op) and isinstance(search_value.raw_value, str): + validate_regex_pattern(search_key.name, search_value.raw_value) + search_value = search_value._replace(is_regex=True) + elif has_wildcard_op(wildcard_op) and isinstance(search_value.raw_value, str): wildcarded_value = gen_wildcard_value( search_value.raw_value, get_wildcard_op(wildcard_op) ) @@ -1940,7 +1996,10 @@ def visit_array_includes_filter( raise InvalidSearchQuery("In Array Queries, only EQUAL/NOT_EQUAL operators are allowed") operator_s = handle_negation(negation, operator_s) - if has_wildcard_op(wildcard_op) and isinstance(search_value.raw_value, str): + if has_regex_op(wildcard_op) and isinstance(search_value.raw_value, str): + validate_regex_pattern(search_key.name, search_value.raw_value) + search_value = search_value._replace(is_regex=True) + elif has_wildcard_op(wildcard_op) and isinstance(search_value.raw_value, str): wildcard_value = gen_wildcard_value( search_value.raw_value, get_wildcard_op(wildcard_op) ) diff --git a/src/sentry/search/eap/constants.py b/src/sentry/search/eap/constants.py index 36dddf44b2b2..01567fdf06b4 100644 --- a/src/sentry/search/eap/constants.py +++ b/src/sentry/search/eap/constants.py @@ -55,6 +55,24 @@ } IN_OPERATORS = ["IN", "NOT IN"] +# ComparisonFilter.OP_REGEXP is not in a published sentry-protos release yet, so it is referenced +# by its wire value. Op is an open proto3 enum, so this serializes identically to the generated +# member. Replace with ComparisonFilter.OP_REGEXP once the dependency is bumped. +# https://github.com/getsentry/sentry-protos/pull/420 +OP_REGEXP: ComparisonFilter.Op.ValueType = getattr( + ComparisonFilter, "OP_REGEXP", ComparisonFilter.Op.ValueType(13) +) + +# Snuba applies the same type rules to OP_REGEXP as to OP_LIKE: the pattern matches string +# values, or the string elements of a string array. +REGEXP_ATTRIBUTE_TYPES = frozenset( + { + AttributeKey.TYPE_STRING, + AttributeKey.TYPE_ARRAY, + AttributeKey.TYPE_ARRAY_STRING, + } +) + AGGREGATION_OPERATOR_MAP = { "=": AggregationComparisonFilter.OP_EQUALS, "!=": AggregationComparisonFilter.OP_NOT_EQUALS, diff --git a/src/sentry/search/eap/resolver.py b/src/sentry/search/eap/resolver.py index 57972fc795a4..8288a0645138 100644 --- a/src/sentry/search/eap/resolver.py +++ b/src/sentry/search/eap/resolver.py @@ -541,6 +541,10 @@ def convert_term(self, term: event_search.SearchFilter) -> list[event_search.Sea converter = self.definitions.filter_aliases.get(name) if converter is not None: + if term.value.is_regex: + # The converters resolve values against Sentry models, so they would treat the + # pattern as a literal rather than matching against it + raise InvalidSearchQuery(f"Cannot use regular expressions with {name}") return converter(self.params, term, self) return [term] @@ -581,6 +585,14 @@ def _resolve_term( if term.value.is_wildcard(): # Avoiding this for now, but we could theoretically do a wildcard search on the resolved contexts raise InvalidSearchQuery(f"Cannot use wildcards with {term.key.name}") + if term.value.is_regex: + raise InvalidSearchQuery(f"Cannot use regular expressions with {term.key.name}") + + if term.value.is_regex: + return ( + self._resolve_regex_term(term, resolved_column), + context_definition, + ) if term.value.is_wildcard(): is_list = False @@ -886,6 +898,43 @@ def resolve_aggregate_term( context, ) + def _resolve_regex_term( + self, + term: event_search.SearchFilter, + resolved_column: ResolvedAttribute, + ) -> TraceItemFilter: + if resolved_column.proto_definition.type not in constants.REGEXP_ATTRIBUTE_TYPES: + raise InvalidSearchQuery( + f"Cannot use regular expressions with {term.key.name}, it is not a string attribute" + ) + + patterns = to_list(term.value.raw_value) + matches = [ + TraceItemFilter( + comparison_filter=ComparisonFilter( + key=resolved_column.proto_definition, + op=constants.OP_REGEXP, + value=AttributeValue(val_str=str(pattern)), + ignore_case=self.params.case_insensitive, + ) + ) + for pattern in patterns + ] + + matches_any = ( + matches[0] + if len(matches) == 1 + else TraceItemFilter(or_filter=OrFilter(filters=matches)) + ) + + if term.operator in ("=", "IN"): + return matches_any + elif term.operator in ("!=", "NOT IN"): + # There is no OP_NOT_REGEXP, so negation is expressed by wrapping the match + return TraceItemFilter(not_filter=NotFilter(filters=[matches_any])) + + raise InvalidSearchQuery(f"Cannot use operator: {term.operator} with regular expressions") + def _resolve_search_value( self, column: ResolvedAttribute, diff --git a/src/sentry/search/events/constants.py b/src/sentry/search/events/constants.py index 4beb624f906b..ab0c4fa5ce64 100644 --- a/src/sentry/search/events/constants.py +++ b/src/sentry/search/events/constants.py @@ -327,6 +327,13 @@ class ThresholdDict(TypedDict): "ends_with": f"{WILDCARD_UNICODE}EndsWith{WILDCARD_UNICODE}", } +# Deliberately kept out of WILDCARD_OPERATOR_MAP: it shares the marker encoding, but a regex +# pattern must reach the backend verbatim rather than being rewritten into a wildcard pattern. +REGEX_OPERATOR = f"{WILDCARD_UNICODE}Matches{WILDCARD_UNICODE}" + +# RE2, which backs the ClickHouse `match` this compiles to, has no backreferences or lookarounds. +UNSUPPORTED_REGEX_SYNTAX = re.compile(r"\\[1-9]|\(\?[=!<]") + MAX_SEARCH_RELEASES = 1000 SEMVER_EMPTY_RELEASE = "____SENTRY_EMPTY_RELEASE____" SEMVER_WILDCARDS = frozenset(["X", "*"]) diff --git a/tests/sentry/api/test_event_search.py b/tests/sentry/api/test_event_search.py index afe449755524..cc4d9135d36e 100644 --- a/tests/sentry/api/test_event_search.py +++ b/tests/sentry/api/test_event_search.py @@ -27,6 +27,7 @@ from sentry.constants import MODULE_ROOT from sentry.exceptions import IncompatibleMetricsQuery, InvalidSearchQuery from sentry.search.events.constants import ( + REGEX_OPERATOR, TEAM_KEY_TRANSACTION_ALIAS, WILDCARD_OPERATOR_MAP, WILDCARD_UNICODE, @@ -1477,6 +1478,96 @@ def test_handles_starts_with_wildcard_op_translations(query, expected) -> None: assert actual == expected +@pytest.mark.parametrize( + ["query", "expected_operator", "expected_value"], + [ + pytest.param(f"span.op:{REGEX_OPERATOR}^test$", "=", "^test$", id="anchored"), + pytest.param(f"!span.op:{REGEX_OPERATOR}^test$", "!=", "^test$", id="negated"), + pytest.param(f"span.op:{REGEX_OPERATOR}a*b", "=", "a*b", id="quantifier"), + pytest.param(f"span.op:{REGEX_OPERATOR}a\\*b", "=", "a\\*b", id="escaped asterisk"), + pytest.param(f"span.op:{REGEX_OPERATOR}a\\d+", "=", "a\\d+", id="character class"), + pytest.param(f'span.op:{REGEX_OPERATOR}"a b|c"', "=", "a b|c", id="quoted"), + pytest.param(f"span.op:{REGEX_OPERATOR}[^foo, bar$]", "IN", ["^foo", "bar$"], id="in list"), + pytest.param( + f"!span.op:{REGEX_OPERATOR}[^foo, bar$]", "NOT IN", ["^foo", "bar$"], id="not in list" + ), + ], +) +def test_parses_regex_op_without_rewriting_the_pattern( + query, expected_operator, expected_value +) -> None: + filters = parse_search_query(query) + assert len(filters) == 1 + assert isinstance(filters[0], SearchFilter) + assert filters[0].operator == expected_operator + assert filters[0].value.is_regex is True + assert filters[0].value.is_wildcard() is False + assert filters[0].value.value == expected_value + + +@pytest.mark.parametrize( + "query", + [ + pytest.param(f"span.op:{REGEX_OPERATOR}^test$", id="scalar"), + pytest.param(f"!span.op:{REGEX_OPERATOR}^test$", id="negated"), + pytest.param(f"span.op:{REGEX_OPERATOR}[^foo, bar$]", id="in list"), + pytest.param(f"!span.op:{REGEX_OPERATOR}[^foo, bar$]", id="not in list"), + pytest.param(f'span.op:{REGEX_OPERATOR}"^(foo|bar) baz$"', id="parens and spaces"), + pytest.param(f'span.op:{REGEX_OPERATOR}"\\"quoted\\""', id="embedded quotes"), + pytest.param(f'span.op:{REGEX_OPERATOR}["^(a|b)", "(c|d)$"]', id="parens in list"), + ], +) +def test_round_trips_a_regex_op_through_to_query_string(query) -> None: + filters = parse_search_query(query) + assert len(filters) == 1 + assert isinstance(filters[0], SearchFilter) + assert parse_search_query(filters[0].to_query_string()) == filters + + +@pytest.mark.parametrize( + ["query", "expected_message"], + [ + pytest.param( + f"span.op:{REGEX_OPERATOR}[a-", + "span.op: Invalid regex: unterminated character set", + id="unterminated character set", + ), + pytest.param( + f'span.op:{REGEX_OPERATOR}"(foo"', + "span.op: Invalid regex: missing ), unterminated subpattern", + id="unterminated group", + ), + pytest.param( + f'span.op:{REGEX_OPERATOR}"(foo)\\1"', + "span.op: Invalid regex: `\\1` is not supported. " + "Backreferences and lookaround are unavailable.", + id="backreference", + ), + pytest.param( + f'span.op:{REGEX_OPERATOR}"foo(?=bar)"', + "span.op: Invalid regex: `(?=` is not supported. " + "Backreferences and lookaround are unavailable.", + id="lookahead", + ), + pytest.param( + f'span.op:{REGEX_OPERATOR}"foo(? None: + with pytest.raises(InvalidSearchQuery) as err: + parse_search_query(query) + assert str(err.value) == expected_message + + @pytest.mark.parametrize( ["query", "expected"], [ diff --git a/tests/sentry/search/eap/test_ourlogs.py b/tests/sentry/search/eap/test_ourlogs.py index 3586a8438442..bb331534dd8f 100644 --- a/tests/sentry/search/eap/test_ourlogs.py +++ b/tests/sentry/search/eap/test_ourlogs.py @@ -13,13 +13,17 @@ from sentry_protos.snuba.v1.trace_item_filter_pb2 import ( AndFilter, ComparisonFilter, + NotFilter, OrFilter, TraceItemFilter, ) +from sentry.exceptions import InvalidSearchQuery +from sentry.search.eap import constants from sentry.search.eap.ourlogs.definitions import OURLOG_DEFINITIONS from sentry.search.eap.resolver import SearchResolver from sentry.search.eap.types import SearchResolverConfig +from sentry.search.events.constants import REGEX_OPERATOR from sentry.search.events.types import SnubaParams @@ -305,6 +309,148 @@ def test_internal_name_resolves_with_normalizer(self) -> None: ) assert having is None + def test_regex_query(self) -> None: + where, having, _ = self.resolver.resolve_query(f"message:{REGEX_OPERATOR}^ERROR") + assert where == TraceItemFilter( + comparison_filter=ComparisonFilter( + key=AttributeKey(name="sentry.body", type=AttributeKey.Type.TYPE_STRING), + op=constants.OP_REGEXP, + value=AttributeValue(val_str="^ERROR"), + ) + ) + assert having is None + + def test_regex_query_negated(self) -> None: + where, having, _ = self.resolver.resolve_query(f"!message:{REGEX_OPERATOR}^ERROR") + assert where == TraceItemFilter( + not_filter=NotFilter( + filters=[ + TraceItemFilter( + comparison_filter=ComparisonFilter( + key=AttributeKey( + name="sentry.body", type=AttributeKey.Type.TYPE_STRING + ), + op=constants.OP_REGEXP, + value=AttributeValue(val_str="^ERROR"), + ) + ) + ] + ) + ) + assert having is None + + def test_regex_query_on_an_attribute(self) -> None: + where, having, _ = self.resolver.resolve_query(f"foo:{REGEX_OPERATOR}ba[rz]") + assert where == TraceItemFilter( + comparison_filter=ComparisonFilter( + key=AttributeKey(name="foo", type=AttributeKey.Type.TYPE_STRING), + op=constants.OP_REGEXP, + value=AttributeValue(val_str="ba[rz]"), + ) + ) + assert having is None + + def test_regex_query_keeps_the_pattern_verbatim(self) -> None: + """Regex metacharacters must not be rewritten the way wildcard patterns are.""" + where, _, _ = self.resolver.resolve_query(f"message:{REGEX_OPERATOR}a*b%c_d\\*e") + assert where == TraceItemFilter( + comparison_filter=ComparisonFilter( + key=AttributeKey(name="sentry.body", type=AttributeKey.Type.TYPE_STRING), + op=constants.OP_REGEXP, + value=AttributeValue(val_str="a*b%c_d\\*e"), + ) + ) + + def test_regex_query_is_case_insensitive_when_requested(self) -> None: + resolver = SearchResolver( + params=SnubaParams(case_insensitive=True), + config=SearchResolverConfig(), + definitions=OURLOG_DEFINITIONS, + ) + where, _, _ = resolver.resolve_query(f"message:{REGEX_OPERATOR}^error") + assert where == TraceItemFilter( + comparison_filter=ComparisonFilter( + key=AttributeKey(name="sentry.body", type=AttributeKey.Type.TYPE_STRING), + op=constants.OP_REGEXP, + value=AttributeValue(val_str="^error"), + ignore_case=True, + ) + ) + + def test_regex_in_filter(self) -> None: + where, having, _ = self.resolver.resolve_query(f"message:{REGEX_OPERATOR}[^ERROR, ^WARN]") + assert where == TraceItemFilter( + or_filter=OrFilter( + filters=[ + TraceItemFilter( + comparison_filter=ComparisonFilter( + key=AttributeKey( + name="sentry.body", type=AttributeKey.Type.TYPE_STRING + ), + op=constants.OP_REGEXP, + value=AttributeValue(val_str="^ERROR"), + ) + ), + TraceItemFilter( + comparison_filter=ComparisonFilter( + key=AttributeKey( + name="sentry.body", type=AttributeKey.Type.TYPE_STRING + ), + op=constants.OP_REGEXP, + value=AttributeValue(val_str="^WARN"), + ) + ), + ] + ) + ) + assert having is None + + def test_regex_not_in_filter(self) -> None: + where, having, _ = self.resolver.resolve_query(f"!message:{REGEX_OPERATOR}[^ERROR, ^WARN]") + assert where == TraceItemFilter( + not_filter=NotFilter( + filters=[ + TraceItemFilter( + or_filter=OrFilter( + filters=[ + TraceItemFilter( + comparison_filter=ComparisonFilter( + key=AttributeKey( + name="sentry.body", + type=AttributeKey.Type.TYPE_STRING, + ), + op=constants.OP_REGEXP, + value=AttributeValue(val_str="^ERROR"), + ) + ), + TraceItemFilter( + comparison_filter=ComparisonFilter( + key=AttributeKey( + name="sentry.body", + type=AttributeKey.Type.TYPE_STRING, + ), + op=constants.OP_REGEXP, + value=AttributeValue(val_str="^WARN"), + ) + ), + ] + ) + ) + ] + ) + ) + assert having is None + + def test_regex_query_raises_when_the_key_is_backed_by_a_filter_alias(self) -> None: + with pytest.raises(InvalidSearchQuery) as err: + self.resolver.resolve_query(f"release:{REGEX_OPERATOR}^1\\.2") + assert str(err.value) == "Cannot use regular expressions with release" + + def test_regex_query_raises_when_the_attribute_is_not_a_string(self) -> None: + with pytest.raises(InvalidSearchQuery) as err: + self.resolver.resolve_query(f"tags[foo,boolean]:{REGEX_OPERATOR}tru.") + assert "not a string attribute" in str(err.value) + def test_internal_trace_id_resolves_with_normalizer(self) -> None: """Using the internal name 'sentry.trace_id' resolves with normalizer.""" where, having, _ = self.resolver.resolve_query( From 09b6f0e92affd89a07f4f9247254a037fa44f9a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josh=20Goldberg=20=E2=9C=A8?= Date: Thu, 10 Sep 2026 10:57:52 -0400 Subject: [PATCH 02/10] remove workarounds for sentry-protos#420 --- pyproject.toml | 2 +- src/sentry/search/eap/constants.py | 8 -------- src/sentry/search/eap/resolver.py | 2 +- tests/sentry/search/eap/test_ourlogs.py | 19 +++++++++---------- uv.lock | 6 +++--- 5 files changed, 14 insertions(+), 23 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 42cdb2eeadd7..de64598dcc7e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -92,7 +92,7 @@ dependencies = [ "sentry-ophio>=1.1.3", # sentry-options is only used in getsentry for now "sentry-options>=1.2.8", - "sentry-protos>=0.67.0", + "sentry-protos>=0.68.0", "sentry-redis-tools>=0.5.0", "sentry-relay>=0.9.28", "sentry-scm==1.5.0", diff --git a/src/sentry/search/eap/constants.py b/src/sentry/search/eap/constants.py index 01567fdf06b4..c9ae55d977e2 100644 --- a/src/sentry/search/eap/constants.py +++ b/src/sentry/search/eap/constants.py @@ -55,14 +55,6 @@ } IN_OPERATORS = ["IN", "NOT IN"] -# ComparisonFilter.OP_REGEXP is not in a published sentry-protos release yet, so it is referenced -# by its wire value. Op is an open proto3 enum, so this serializes identically to the generated -# member. Replace with ComparisonFilter.OP_REGEXP once the dependency is bumped. -# https://github.com/getsentry/sentry-protos/pull/420 -OP_REGEXP: ComparisonFilter.Op.ValueType = getattr( - ComparisonFilter, "OP_REGEXP", ComparisonFilter.Op.ValueType(13) -) - # Snuba applies the same type rules to OP_REGEXP as to OP_LIKE: the pattern matches string # values, or the string elements of a string array. REGEXP_ATTRIBUTE_TYPES = frozenset( diff --git a/src/sentry/search/eap/resolver.py b/src/sentry/search/eap/resolver.py index 8288a0645138..4aa9d15f3406 100644 --- a/src/sentry/search/eap/resolver.py +++ b/src/sentry/search/eap/resolver.py @@ -913,7 +913,7 @@ def _resolve_regex_term( TraceItemFilter( comparison_filter=ComparisonFilter( key=resolved_column.proto_definition, - op=constants.OP_REGEXP, + op=ComparisonFilter.OP_REGEXP, value=AttributeValue(val_str=str(pattern)), ignore_case=self.params.case_insensitive, ) diff --git a/tests/sentry/search/eap/test_ourlogs.py b/tests/sentry/search/eap/test_ourlogs.py index bb331534dd8f..5f30cac91068 100644 --- a/tests/sentry/search/eap/test_ourlogs.py +++ b/tests/sentry/search/eap/test_ourlogs.py @@ -19,7 +19,6 @@ ) from sentry.exceptions import InvalidSearchQuery -from sentry.search.eap import constants from sentry.search.eap.ourlogs.definitions import OURLOG_DEFINITIONS from sentry.search.eap.resolver import SearchResolver from sentry.search.eap.types import SearchResolverConfig @@ -314,7 +313,7 @@ def test_regex_query(self) -> None: assert where == TraceItemFilter( comparison_filter=ComparisonFilter( key=AttributeKey(name="sentry.body", type=AttributeKey.Type.TYPE_STRING), - op=constants.OP_REGEXP, + op=ComparisonFilter.OP_REGEXP, value=AttributeValue(val_str="^ERROR"), ) ) @@ -330,7 +329,7 @@ def test_regex_query_negated(self) -> None: key=AttributeKey( name="sentry.body", type=AttributeKey.Type.TYPE_STRING ), - op=constants.OP_REGEXP, + op=ComparisonFilter.OP_REGEXP, value=AttributeValue(val_str="^ERROR"), ) ) @@ -344,7 +343,7 @@ def test_regex_query_on_an_attribute(self) -> None: assert where == TraceItemFilter( comparison_filter=ComparisonFilter( key=AttributeKey(name="foo", type=AttributeKey.Type.TYPE_STRING), - op=constants.OP_REGEXP, + op=ComparisonFilter.OP_REGEXP, value=AttributeValue(val_str="ba[rz]"), ) ) @@ -356,7 +355,7 @@ def test_regex_query_keeps_the_pattern_verbatim(self) -> None: assert where == TraceItemFilter( comparison_filter=ComparisonFilter( key=AttributeKey(name="sentry.body", type=AttributeKey.Type.TYPE_STRING), - op=constants.OP_REGEXP, + op=ComparisonFilter.OP_REGEXP, value=AttributeValue(val_str="a*b%c_d\\*e"), ) ) @@ -371,7 +370,7 @@ def test_regex_query_is_case_insensitive_when_requested(self) -> None: assert where == TraceItemFilter( comparison_filter=ComparisonFilter( key=AttributeKey(name="sentry.body", type=AttributeKey.Type.TYPE_STRING), - op=constants.OP_REGEXP, + op=ComparisonFilter.OP_REGEXP, value=AttributeValue(val_str="^error"), ignore_case=True, ) @@ -387,7 +386,7 @@ def test_regex_in_filter(self) -> None: key=AttributeKey( name="sentry.body", type=AttributeKey.Type.TYPE_STRING ), - op=constants.OP_REGEXP, + op=ComparisonFilter.OP_REGEXP, value=AttributeValue(val_str="^ERROR"), ) ), @@ -396,7 +395,7 @@ def test_regex_in_filter(self) -> None: key=AttributeKey( name="sentry.body", type=AttributeKey.Type.TYPE_STRING ), - op=constants.OP_REGEXP, + op=ComparisonFilter.OP_REGEXP, value=AttributeValue(val_str="^WARN"), ) ), @@ -419,7 +418,7 @@ def test_regex_not_in_filter(self) -> None: name="sentry.body", type=AttributeKey.Type.TYPE_STRING, ), - op=constants.OP_REGEXP, + op=ComparisonFilter.OP_REGEXP, value=AttributeValue(val_str="^ERROR"), ) ), @@ -429,7 +428,7 @@ def test_regex_not_in_filter(self) -> None: name="sentry.body", type=AttributeKey.Type.TYPE_STRING, ), - op=constants.OP_REGEXP, + op=ComparisonFilter.OP_REGEXP, value=AttributeValue(val_str="^WARN"), ) ), diff --git a/uv.lock b/uv.lock index 210934d8b7ee..2dc52534c11c 100644 --- a/uv.lock +++ b/uv.lock @@ -2385,7 +2385,7 @@ requires-dist = [ { name = "sentry-kafka-schemas", specifier = ">=2.2.0" }, { name = "sentry-ophio", specifier = ">=1.1.3" }, { name = "sentry-options", specifier = ">=1.2.8" }, - { name = "sentry-protos", specifier = ">=0.67.0" }, + { name = "sentry-protos", specifier = ">=0.68.0" }, { name = "sentry-redis-tools", specifier = ">=0.5.0" }, { name = "sentry-relay", specifier = ">=0.9.28" }, { name = "sentry-scm", specifier = "==1.5.0" }, @@ -2565,7 +2565,7 @@ wheels = [ [[package]] name = "sentry-protos" -version = "0.67.0" +version = "0.69.0" source = { registry = "https://pypi.devinfra.sentry.io/simple" } dependencies = [ { name = "grpc-stubs", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, @@ -2573,7 +2573,7 @@ dependencies = [ { name = "protobuf", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, ] wheels = [ - { url = "https://pypi.devinfra.sentry.io/wheels/sentry_protos-0.67.0-py3-none-any.whl", hash = "sha256:e09a083a630e7e58f5599b9c18ac777305077e066ffb4245721ea603eaed616f" }, + { url = "https://pypi.devinfra.sentry.io/wheels/sentry_protos-0.69.0-py3-none-any.whl", hash = "sha256:5ae4d39464c75ea3382c76c186fa4da952fad927a9f41230a579e91e809010c7" }, ] [[package]] From 53bf745d554e7dc29072b32ba9388a5780b746dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josh=20Goldberg=20=E2=9C=A8?= Date: Thu, 10 Sep 2026 12:04:44 -0400 Subject: [PATCH 03/10] fix(logs): reject regex searches on virtual columns in timeseries requests The guard sat inside `if context_definition:`, but the timeseries branch above clears context_definition after remapping, so it never fired. The pattern was then looked up in the context's value map as a literal, surfacing as "Unknown value ^sen" instead of the intended error. --- src/sentry/search/eap/resolver.py | 5 +++-- tests/sentry/search/eap/test_ourlogs.py | 15 +++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/src/sentry/search/eap/resolver.py b/src/sentry/search/eap/resolver.py index 4aa9d15f3406..b1bd8b883d39 100644 --- a/src/sentry/search/eap/resolver.py +++ b/src/sentry/search/eap/resolver.py @@ -570,6 +570,9 @@ def _resolve_term( resolved_column, context_definition = self.resolve_column(term.key.name) self._raise_if_hidden_api_attribute(term.key.name, resolved_column) + if context_definition is not None and term.value.is_regex: + raise InvalidSearchQuery(f"Cannot use regular expressions with {term.key.name}") + value = term.value.value if self.params.is_timeseries_request and context_definition is not None: resolved_column, value = self.map_search_term_context_to_original_column( @@ -585,8 +588,6 @@ def _resolve_term( if term.value.is_wildcard(): # Avoiding this for now, but we could theoretically do a wildcard search on the resolved contexts raise InvalidSearchQuery(f"Cannot use wildcards with {term.key.name}") - if term.value.is_regex: - raise InvalidSearchQuery(f"Cannot use regular expressions with {term.key.name}") if term.value.is_regex: return ( diff --git a/tests/sentry/search/eap/test_ourlogs.py b/tests/sentry/search/eap/test_ourlogs.py index 5f30cac91068..02ea92fd7a16 100644 --- a/tests/sentry/search/eap/test_ourlogs.py +++ b/tests/sentry/search/eap/test_ourlogs.py @@ -445,6 +445,21 @@ def test_regex_query_raises_when_the_key_is_backed_by_a_filter_alias(self) -> No self.resolver.resolve_query(f"release:{REGEX_OPERATOR}^1\\.2") assert str(err.value) == "Cannot use regular expressions with release" + def test_regex_query_raises_when_the_key_is_backed_by_a_virtual_column(self) -> None: + with pytest.raises(InvalidSearchQuery) as err: + self.resolver.resolve_query(f"project:{REGEX_OPERATOR}^sen") + assert str(err.value) == "Cannot use regular expressions with project" + + def test_regex_query_raises_on_a_virtual_column_in_a_timeseries_request(self) -> None: + resolver = SearchResolver( + params=SnubaParams(granularity_secs=60), + config=SearchResolverConfig(), + definitions=OURLOG_DEFINITIONS, + ) + with pytest.raises(InvalidSearchQuery) as err: + resolver.resolve_query(f"project:{REGEX_OPERATOR}^sen") + assert str(err.value) == "Cannot use regular expressions with project" + def test_regex_query_raises_when_the_attribute_is_not_a_string(self) -> None: with pytest.raises(InvalidSearchQuery) as err: self.resolver.resolve_query(f"tags[foo,boolean]:{REGEX_OPERATOR}tru.") From 81665daebe89f61f02b01ab698f8c876b87b28fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josh=20Goldberg=20=E2=9C=A8?= Date: Thu, 10 Sep 2026 12:04:54 -0400 Subject: [PATCH 04/10] ref(logs): dedupe regex handling in the search visitor Three visitor sites repeated the same validate-then-mark block; SearchValue branched on is_regex in three places where one condition covers it. --- src/sentry/api/event_search.py | 54 +++++++++++++++------------------- 1 file changed, 24 insertions(+), 30 deletions(-) diff --git a/src/sentry/api/event_search.py b/src/sentry/api/event_search.py index fcf9f9eaa92a..3866aba65c96 100644 --- a/src/sentry/api/event_search.py +++ b/src/sentry/api/event_search.py @@ -449,6 +449,14 @@ def validate_regex_pattern(key: str, pattern: str) -> None: raise InvalidSearchQuery(f"{key}: Invalid regex: {exc.msg}") +def as_regex_value(key: str, value: SearchValue) -> SearchValue: + patterns = value.raw_value if isinstance(value.raw_value, (list, tuple)) else [value.raw_value] + for pattern in patterns: + if isinstance(pattern, str): + validate_regex_pattern(key, pattern) + return value._replace(is_regex=True) + + def add_leading_wildcard(value: str) -> str: if value.startswith('"') and value.endswith('"'): return f"*{value[1:-1]}" @@ -561,11 +569,9 @@ class SearchValue(NamedTuple): @property def value(self) -> Any: - if self.use_raw_value: - return self.raw_value - elif self.is_regex: - # Escape sequences are meaningful to the regex engine, so the pattern passes through - # untouched. `\*` is a literal asterisk here, not an escaped wildcard. + # Escape sequences are meaningful to the regex engine, so a pattern passes through + # untouched. `\*` is a literal asterisk there, not an escaped wildcard. + if self.use_raw_value or self.is_regex: return self.raw_value elif self.is_wildcard() and isinstance(self.raw_value, str): return translate_wildcard(self.raw_value) @@ -585,27 +591,20 @@ def to_query_string(self) -> str: # we do that because a simple str() would not be usable for strings # str(["a","b"]) == "['a', 'b']" but we would like "[a,b]" if isinstance(self.raw_value, (list, tuple)): - values = ( - [quote_regex_pattern(str(x)) for x in self.raw_value] - if self.is_regex - else [str(x) for x in self.raw_value] - ) - ret_val = ", ".join(values) + ret_val = ", ".join(self._serialize(x) for x in self.raw_value) ret_val = f"[{ret_val}]" return ret_val elif isinstance(self.raw_value, datetime): return self.raw_value.isoformat() - elif self.is_regex: - return quote_regex_pattern(str(self.value)) else: - return str(self.value) + return self._serialize(self.value) + + def _serialize(self, value: Any) -> str: + return quote_regex_pattern(str(value)) if self.is_regex else str(value) def is_wildcard(self) -> bool: - # If we're using the raw value only it'll never be a wildcard - if self.use_raw_value: - return False - # A `*` in a regex is a quantifier, not a wildcard - if self.is_regex: + # The raw value is never a wildcard, and a `*` in a regex is a quantifier + if self.use_raw_value or self.is_regex: return False if self.is_str_sequence(): return isinstance(self.raw_value, list) and any( @@ -1491,11 +1490,8 @@ def visit_text_in_filter( operator = handle_negation(negation, operator) - if has_regex_op(wildcard_op) and isinstance(search_value.raw_value, list): - for value in search_value.raw_value: - if isinstance(value, str): - validate_regex_pattern(search_key.name, value) - search_value = search_value._replace(is_regex=True) + if has_regex_op(wildcard_op): + search_value = as_regex_value(search_key.name, search_value) elif has_wildcard_op(wildcard_op) and isinstance(search_value.raw_value, list): wildcarded_values = [] found_wildcard_op = get_wildcard_op(wildcard_op) @@ -1535,9 +1531,8 @@ def visit_text_filter( operator_s = handle_negation(negation, operator_s) - if has_regex_op(wildcard_op) and isinstance(search_value.raw_value, str): - validate_regex_pattern(search_key.name, search_value.raw_value) - search_value = search_value._replace(is_regex=True) + if has_regex_op(wildcard_op): + search_value = as_regex_value(search_key.name, search_value) elif has_wildcard_op(wildcard_op) and isinstance(search_value.raw_value, str): wildcarded_value = gen_wildcard_value( search_value.raw_value, get_wildcard_op(wildcard_op) @@ -2000,9 +1995,8 @@ def visit_array_includes_filter( raise InvalidSearchQuery("In Array Queries, only EQUAL/NOT_EQUAL operators are allowed") operator_s = handle_negation(negation, operator_s) - if has_regex_op(wildcard_op) and isinstance(search_value.raw_value, str): - validate_regex_pattern(search_key.name, search_value.raw_value) - search_value = search_value._replace(is_regex=True) + if has_regex_op(wildcard_op): + search_value = as_regex_value(search_key.name, search_value) elif has_wildcard_op(wildcard_op) and isinstance(search_value.raw_value, str): wildcard_value = gen_wildcard_value( search_value.raw_value, get_wildcard_op(wildcard_op) From 4ff7e940fc3f77cd907f3780bab5ca285ce95c07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josh=20Goldberg=20=E2=9C=A8?= Date: Thu, 10 Sep 2026 12:04:56 -0400 Subject: [PATCH 05/10] fix(tools): skip validated_data stub tests when sentry is not installed The mypy config points django_settings_module at sentry.conf.server_mypy, so mypy cannot construct the django plugin and exits before checking anything. make test-tools runs under uv sync --only-dev in the dev env workflow, where the four tests asserting diagnostics have been failing since they were added in #123923 - that PR touched no path that triggers the workflow, so it never ran there. --- .../mypy_helpers/test_typed_validated_data.py | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/tests/tools/mypy_helpers/test_typed_validated_data.py b/tests/tools/mypy_helpers/test_typed_validated_data.py index d5ff640a5483..295b6273866d 100644 --- a/tests/tools/mypy_helpers/test_typed_validated_data.py +++ b/tests/tools/mypy_helpers/test_typed_validated_data.py @@ -1,20 +1,30 @@ """The serializer stub gives `validated_data` a declared shape. -Exercised through mypy under the repo's own config, because the mechanism is a -stub with nothing to import and CI resolves it the same way. If the stub ever -stops being found, `validated_data` falls back to `Any` and these cases would -pass without checking anything, so `test_stub_is_in_effect` guards that. +Exercised through mypy under the repo's own config. If the stub ever stops being +found, `validated_data` falls back to `Any` and these cases would pass without +checking anything, so `test_stub_is_in_effect` guards that. """ from __future__ import annotations +import importlib.util import os.path import subprocess import sys import tempfile +import pytest + REPO = os.path.join(os.path.dirname(__file__), "..", "..", "..") +# The mypy config points django_settings_module at `sentry.conf.server_mypy`, so the django +# plugin cannot be constructed without the project installed and mypy exits before checking +# anything. `make test-tools` runs under a `uv sync --only-dev` env in CI, which is that case. +pytestmark = pytest.mark.skipif( + importlib.util.find_spec("sentry") is None, + reason="mypy's django plugin needs the sentry package for django_settings_module", +) + PRELUDE = """\ from typing import Any, NotRequired, TypedDict From 2d15b11c5df090c73e53ec5ac214bb2c8918b01f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josh=20Goldberg=20=E2=9C=A8?= Date: Thu, 10 Sep 2026 13:44:46 -0400 Subject: [PATCH 06/10] Revert "fix(tools): skip validated_data stub tests when sentry is not installed" This reverts commit 4ff7e940fc3f77cd907f3780bab5ca285ce95c07. #124057 fixed the same devenv/test-tools failure at the root: the checks now run under an isolated mypy config with the vendored stubs on MYPYPATH, so the django plugin is never constructed and sentry does not need to be installed. The skip is no longer needed, and would drop the coverage that fix restores. --- .../mypy_helpers/test_typed_validated_data.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/tests/tools/mypy_helpers/test_typed_validated_data.py b/tests/tools/mypy_helpers/test_typed_validated_data.py index 295b6273866d..d5ff640a5483 100644 --- a/tests/tools/mypy_helpers/test_typed_validated_data.py +++ b/tests/tools/mypy_helpers/test_typed_validated_data.py @@ -1,30 +1,20 @@ """The serializer stub gives `validated_data` a declared shape. -Exercised through mypy under the repo's own config. If the stub ever stops being -found, `validated_data` falls back to `Any` and these cases would pass without -checking anything, so `test_stub_is_in_effect` guards that. +Exercised through mypy under the repo's own config, because the mechanism is a +stub with nothing to import and CI resolves it the same way. If the stub ever +stops being found, `validated_data` falls back to `Any` and these cases would +pass without checking anything, so `test_stub_is_in_effect` guards that. """ from __future__ import annotations -import importlib.util import os.path import subprocess import sys import tempfile -import pytest - REPO = os.path.join(os.path.dirname(__file__), "..", "..", "..") -# The mypy config points django_settings_module at `sentry.conf.server_mypy`, so the django -# plugin cannot be constructed without the project installed and mypy exits before checking -# anything. `make test-tools` runs under a `uv sync --only-dev` env in CI, which is that case. -pytestmark = pytest.mark.skipif( - importlib.util.find_spec("sentry") is None, - reason="mypy's django plugin needs the sentry package for django_settings_module", -) - PRELUDE = """\ from typing import Any, NotRequired, TypedDict From a6060ffc2c6108deaf9b1ef3d03e34f6d28bdb1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josh=20Goldberg=20=E2=9C=A8?= Date: Thu, 10 Sep 2026 16:16:33 -0400 Subject: [PATCH 07/10] fix(logs): Express case-insensitive regex with the RE2 inline flag Snuba lowercases the pattern along with the value when a comparison filter sets ignore_case, which rewrites classes like [A-Z] and inverts escapes like \D. Prefixing the pattern with (?i) instead keeps it intact. --- src/sentry/search/eap/resolver.py | 6 ++++-- tests/sentry/search/eap/test_ourlogs.py | 5 ++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/sentry/search/eap/resolver.py b/src/sentry/search/eap/resolver.py index b1bd8b883d39..bfba1907e724 100644 --- a/src/sentry/search/eap/resolver.py +++ b/src/sentry/search/eap/resolver.py @@ -910,13 +910,15 @@ def _resolve_regex_term( ) patterns = to_list(term.value.raw_value) + # Snuba's `ignore_case` lowercases the pattern along with the value, rewriting `[A-Z]` + # and inverting escapes like `\D`. RE2's inline flag leaves the pattern intact. + prefix = "(?i)" if self.params.case_insensitive else "" matches = [ TraceItemFilter( comparison_filter=ComparisonFilter( key=resolved_column.proto_definition, op=ComparisonFilter.OP_REGEXP, - value=AttributeValue(val_str=str(pattern)), - ignore_case=self.params.case_insensitive, + value=AttributeValue(val_str=f"{prefix}{pattern}"), ) ) for pattern in patterns diff --git a/tests/sentry/search/eap/test_ourlogs.py b/tests/sentry/search/eap/test_ourlogs.py index 02ea92fd7a16..a0f17af44a25 100644 --- a/tests/sentry/search/eap/test_ourlogs.py +++ b/tests/sentry/search/eap/test_ourlogs.py @@ -366,13 +366,12 @@ def test_regex_query_is_case_insensitive_when_requested(self) -> None: config=SearchResolverConfig(), definitions=OURLOG_DEFINITIONS, ) - where, _, _ = resolver.resolve_query(f"message:{REGEX_OPERATOR}^error") + where, _, _ = resolver.resolve_query(f"message:{REGEX_OPERATOR}^[A-Z]rror") assert where == TraceItemFilter( comparison_filter=ComparisonFilter( key=AttributeKey(name="sentry.body", type=AttributeKey.Type.TYPE_STRING), op=ComparisonFilter.OP_REGEXP, - value=AttributeValue(val_str="^error"), - ignore_case=True, + value=AttributeValue(val_str="(?i)^[A-Z]rror"), ) ) From 7e23b821e41d7a2cd8d264bff8a36dbaba035afd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josh=20Goldberg=20=E2=9C=A8?= Date: Thu, 10 Sep 2026 16:16:40 -0400 Subject: [PATCH 08/10] fix(logs): Reject the regex syntax RE2 cannot compile ClickHouse only reports an uncompilable pattern once the query runs, and Snuba surfaces that as a 500, so \Z, atomic groups, conditionals, inline comments and named backreferences are now caught alongside backreferences and lookaround. --- src/sentry/api/event_search.py | 3 ++- src/sentry/search/events/constants.py | 6 +++-- tests/sentry/api/test_event_search.py | 35 ++++++++++++++++++++++----- 3 files changed, 35 insertions(+), 9 deletions(-) diff --git a/src/sentry/api/event_search.py b/src/sentry/api/event_search.py index 3866aba65c96..ea21467da5c1 100644 --- a/src/sentry/api/event_search.py +++ b/src/sentry/api/event_search.py @@ -440,7 +440,8 @@ def validate_regex_pattern(key: str, pattern: str) -> None: if unsupported is not None: raise InvalidSearchQuery( f"{key}: Invalid regex: `{unsupported.group()}` is not supported. " - "Backreferences and lookaround are unavailable." + "Patterns are matched with RE2, which has no backreferences, lookaround, " + "or other PCRE extensions." ) try: diff --git a/src/sentry/search/events/constants.py b/src/sentry/search/events/constants.py index ab0c4fa5ce64..f8acc6eff484 100644 --- a/src/sentry/search/events/constants.py +++ b/src/sentry/search/events/constants.py @@ -331,8 +331,10 @@ class ThresholdDict(TypedDict): # pattern must reach the backend verbatim rather than being rewritten into a wildcard pattern. REGEX_OPERATOR = f"{WILDCARD_UNICODE}Matches{WILDCARD_UNICODE}" -# RE2, which backs the ClickHouse `match` this compiles to, has no backreferences or lookarounds. -UNSUPPORTED_REGEX_SYNTAX = re.compile(r"\\[1-9]|\(\?[=!<]") +# RE2, which backs the ClickHouse `match` this compiles to, rejects the PCRE extensions that +# Python's `re` accepts, and ClickHouse only reports that as a query failure once the pattern +# has already reached it. +UNSUPPORTED_REGEX_SYNTAX = re.compile(r"\\[1-9]|\\Z|\(\?(?:[=!>#(]|<[=!]|P=)") MAX_SEARCH_RELEASES = 1000 SEMVER_EMPTY_RELEASE = "____SENTRY_EMPTY_RELEASE____" diff --git a/tests/sentry/api/test_event_search.py b/tests/sentry/api/test_event_search.py index 4c898cf01517..40fde3e2c399 100644 --- a/tests/sentry/api/test_event_search.py +++ b/tests/sentry/api/test_event_search.py @@ -1548,6 +1548,12 @@ def test_round_trips_a_regex_op_through_to_query_string(query) -> None: assert parse_search_query(filters[0].to_query_string()) == filters +UNSUPPORTED_REGEX_MESSAGE = ( + "Patterns are matched with RE2, which has no backreferences, lookaround, " + "or other PCRE extensions." +) + + @pytest.mark.parametrize( ["query", "expected_message"], [ @@ -1563,22 +1569,39 @@ def test_round_trips_a_regex_op_through_to_query_string(query) -> None: ), pytest.param( f'span.op:{REGEX_OPERATOR}"(foo)\\1"', - "span.op: Invalid regex: `\\1` is not supported. " - "Backreferences and lookaround are unavailable.", + "span.op: Invalid regex: `\\1` is not supported. " + UNSUPPORTED_REGEX_MESSAGE, id="backreference", ), pytest.param( f'span.op:{REGEX_OPERATOR}"foo(?=bar)"', - "span.op: Invalid regex: `(?=` is not supported. " - "Backreferences and lookaround are unavailable.", + "span.op: Invalid regex: `(?=` is not supported. " + UNSUPPORTED_REGEX_MESSAGE, id="lookahead", ), pytest.param( f'span.op:{REGEX_OPERATOR}"foo(?foo)"', + "span.op: Invalid regex: `(?>` is not supported. " + UNSUPPORTED_REGEX_MESSAGE, + id="atomic group", + ), + pytest.param( + f'span.op:{REGEX_OPERATOR}"(foo)(?(1)bar|baz)"', + "span.op: Invalid regex: `(?(` is not supported. " + UNSUPPORTED_REGEX_MESSAGE, + id="conditional", + ), + pytest.param( + f'span.op:{REGEX_OPERATOR}"(?Pfoo)(?P=name)"', + "span.op: Invalid regex: `(?P=` is not supported. " + UNSUPPORTED_REGEX_MESSAGE, + id="named backreference", + ), pytest.param( f'span.op:{REGEX_OPERATOR}""', "span.op: Empty regex pattern", From a7d246e1dba6632fbe30512268a2e7f7b470baaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josh=20Goldberg=20=E2=9C=A8?= Date: Thu, 10 Sep 2026 16:16:46 -0400 Subject: [PATCH 09/10] test(logs): Cover regex log search end to end Exercises matching, negation, IN lists, attribute keys and case insensitivity against Snuba now that OP_REGEXP has landed there. --- .../test_organization_events_ourlogs.py | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) diff --git a/tests/snuba/api/endpoints/test_organization_events_ourlogs.py b/tests/snuba/api/endpoints/test_organization_events_ourlogs.py index 3c43c20d0b6a..27eee02cb1e9 100644 --- a/tests/snuba/api/endpoints/test_organization_events_ourlogs.py +++ b/tests/snuba/api/endpoints/test_organization_events_ourlogs.py @@ -8,6 +8,7 @@ from sentry.conf.types.sentry_config import SentryMode from sentry.constants import DataCategory from sentry.search.eap import constants +from sentry.search.events.constants import REGEX_OPERATOR from sentry.testutils.cases import OutcomesSnubaTest from sentry.testutils.helpers import parse_link_header from sentry.testutils.helpers.datetime import before_now @@ -140,6 +141,150 @@ def test_free_text_wildcard_filter(self) -> None: assert meta["dataset"] == self.dataset + def test_regex_filter(self) -> None: + logs = [ + self.create_ourlog( + {"body": "ERROR [42] disk full"}, + timestamp=self.ten_mins_ago, + ), + self.create_ourlog( + {"body": "WARN [7] disk filling up"}, + timestamp=self.nine_mins_ago, + ), + ] + self.store_eap_items(logs) + response = self.do_request( + { + "field": ["log.body"], + "query": f'message:{REGEX_OPERATOR}"^ERROR \\[\\d+\\]"', + "project": self.project.id, + "dataset": self.dataset, + } + ) + + assert response.status_code == 200, response.content + assert [log["log.body"] for log in response.data["data"]] == ["ERROR [42] disk full"] + + def test_regex_filter_negated(self) -> None: + logs = [ + self.create_ourlog( + {"body": "ERROR [42] disk full"}, + timestamp=self.ten_mins_ago, + ), + self.create_ourlog( + {"body": "WARN [7] disk filling up"}, + timestamp=self.nine_mins_ago, + ), + ] + self.store_eap_items(logs) + response = self.do_request( + { + "field": ["log.body"], + "query": f'!message:{REGEX_OPERATOR}"^ERROR \\[\\d+\\]"', + "project": self.project.id, + "dataset": self.dataset, + } + ) + + assert response.status_code == 200, response.content + assert [log["log.body"] for log in response.data["data"]] == ["WARN [7] disk filling up"] + + def test_regex_filter_in_list(self) -> None: + logs = [ + self.create_ourlog( + {"body": "ERROR [42] disk full"}, + timestamp=self.ten_mins_ago, + ), + self.create_ourlog( + {"body": "WARN [7] disk filling up"}, + timestamp=self.nine_mins_ago, + ), + self.create_ourlog( + {"body": "INFO [1] all good"}, + timestamp=self.nine_mins_ago, + ), + ] + self.store_eap_items(logs) + response = self.do_request( + { + "field": ["log.body"], + "query": f'message:{REGEX_OPERATOR}["^ERROR", "^WARN"]', + "orderby": "log.body", + "project": self.project.id, + "dataset": self.dataset, + } + ) + + assert response.status_code == 200, response.content + assert [log["log.body"] for log in response.data["data"]] == [ + "ERROR [42] disk full", + "WARN [7] disk filling up", + ] + + def test_regex_filter_on_an_attribute(self) -> None: + logs = [ + self.create_ourlog( + {"body": "first"}, + attributes={"release": "1.2.3"}, + timestamp=self.ten_mins_ago, + ), + self.create_ourlog( + {"body": "second"}, + attributes={"release": "nightly"}, + timestamp=self.nine_mins_ago, + ), + ] + self.store_eap_items(logs) + response = self.do_request( + { + "field": ["log.body"], + "query": f'tags[release,string]:{REGEX_OPERATOR}"^\\d+\\.\\d+"', + "project": self.project.id, + "dataset": self.dataset, + } + ) + + assert response.status_code == 200, response.content + assert [log["log.body"] for log in response.data["data"]] == ["first"] + + def test_regex_filter_case_insensitive(self) -> None: + logs = [ + self.create_ourlog( + {"body": "Error: disk full"}, + timestamp=self.ten_mins_ago, + ), + self.create_ourlog( + {"body": "0 problems"}, + timestamp=self.nine_mins_ago, + ), + ] + self.store_eap_items(logs) + response = self.do_request( + { + "field": ["log.body"], + "query": f'message:{REGEX_OPERATOR}"^[A-Z]RROR\\D+"', + "caseInsensitive": "1", + "project": self.project.id, + "dataset": self.dataset, + } + ) + + assert response.status_code == 200, response.content + assert [log["log.body"] for log in response.data["data"]] == ["Error: disk full"] + + def test_regex_filter_rejects_an_invalid_pattern(self) -> None: + response = self.do_request( + { + "field": ["log.body"], + "query": f"message:{REGEX_OPERATOR}[a-", + "project": self.project.id, + "dataset": self.dataset, + } + ) + + assert response.status_code == 400, response.content + assert "Invalid regex" in response.data["detail"] + def test_pagination(self) -> None: logs = [ self.create_ourlog( From 8570ae640cfd8b7673088cf1566b0f7752f227b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Josh=20Goldberg=20=E2=9C=A8?= Date: Thu, 10 Sep 2026 16:42:26 -0400 Subject: [PATCH 10/10] fix(logs): Read an escaped backslash before a digit as a literal The unsupported-syntax scan treated any backslash-digit pair as a backreference, so a pattern matching a literal backslash followed by a digit was rejected before it reached Snuba. --- src/sentry/api/event_search.py | 15 ++++++++------- src/sentry/search/events/constants.py | 5 +++-- tests/sentry/api/test_event_search.py | 6 ++++++ 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/sentry/api/event_search.py b/src/sentry/api/event_search.py index ea21467da5c1..b7cd92fc4728 100644 --- a/src/sentry/api/event_search.py +++ b/src/sentry/api/event_search.py @@ -436,13 +436,14 @@ def validate_regex_pattern(key: str, pattern: str) -> None: if not pattern: raise InvalidSearchQuery(f"{key}: Empty regex pattern") - unsupported = UNSUPPORTED_REGEX_SYNTAX.search(pattern) - if unsupported is not None: - raise InvalidSearchQuery( - f"{key}: Invalid regex: `{unsupported.group()}` is not supported. " - "Patterns are matched with RE2, which has no backreferences, lookaround, " - "or other PCRE extensions." - ) + for match in UNSUPPORTED_REGEX_SYNTAX.finditer(pattern): + unsupported = match.group("unsupported") + if unsupported is not None: + raise InvalidSearchQuery( + f"{key}: Invalid regex: `{unsupported}` is not supported. " + "Patterns are matched with RE2, which has no backreferences, lookaround, " + "or other PCRE extensions." + ) try: re.compile(pattern) diff --git a/src/sentry/search/events/constants.py b/src/sentry/search/events/constants.py index f8acc6eff484..73487abd32d5 100644 --- a/src/sentry/search/events/constants.py +++ b/src/sentry/search/events/constants.py @@ -333,8 +333,9 @@ class ThresholdDict(TypedDict): # RE2, which backs the ClickHouse `match` this compiles to, rejects the PCRE extensions that # Python's `re` accepts, and ClickHouse only reports that as a query failure once the pattern -# has already reached it. -UNSUPPORTED_REGEX_SYNTAX = re.compile(r"\\[1-9]|\\Z|\(\?(?:[=!>#(]|<[=!]|P=)") +# has already reached it. The first branch consumes escaped backslashes, so that a pattern +# like `\\1` reads as a literal backslash followed by a digit. +UNSUPPORTED_REGEX_SYNTAX = re.compile(r"\\\\|(?P\\[1-9]|\\Z|\(\?(?:[=!>#(]|<[=!]|P=))") MAX_SEARCH_RELEASES = 1000 SEMVER_EMPTY_RELEASE = "____SENTRY_EMPTY_RELEASE____" diff --git a/tests/sentry/api/test_event_search.py b/tests/sentry/api/test_event_search.py index 40fde3e2c399..69a6c36cab08 100644 --- a/tests/sentry/api/test_event_search.py +++ b/tests/sentry/api/test_event_search.py @@ -1510,6 +1510,12 @@ def test_handles_starts_with_wildcard_op_translations(query, expected) -> None: pytest.param(f"span.op:{REGEX_OPERATOR}a*b", "=", "a*b", id="quantifier"), pytest.param(f"span.op:{REGEX_OPERATOR}a\\*b", "=", "a\\*b", id="escaped asterisk"), pytest.param(f"span.op:{REGEX_OPERATOR}a\\d+", "=", "a\\d+", id="character class"), + pytest.param( + f"span.op:{REGEX_OPERATOR}a\\\\1", "=", "a\\\\1", id="escaped backslash before digit" + ), + pytest.param( + f"span.op:{REGEX_OPERATOR}a\\\\Z", "=", "a\\\\Z", id="escaped backslash before Z" + ), pytest.param(f'span.op:{REGEX_OPERATOR}"a b|c"', "=", "a b|c", id="quoted"), pytest.param(f"span.op:{REGEX_OPERATOR}[^foo, bar$]", "IN", ["^foo", "bar$"], id="in list"), pytest.param(