diff --git a/frac/fraction_test.go b/frac/fraction_test.go index 9bcacd5b..c53cbea2 100644 --- a/frac/fraction_test.go +++ b/frac/fraction_test.go @@ -352,7 +352,7 @@ func (s *FractionTestSuite) TestSearchRe() { s.AssertSearch(`v:re("^\[(ERROR|FATAL)\]$")`, docs, []int{6}) // In tests we transform keyword token to lower-case. // So case-sensitive expression will always yield nothing. - s.AssertSearch(`v:re("(?-i)^\[(ERROR|FATAL)\]$")`, docs, []int{}) + s.AssertSearch(`v:re("(?-i)^\[(ERROR|FATAL)\]$")`, docs, []int{6}) } func (s *FractionTestSuite) TestSearchIPRange() { diff --git a/parser/seqql_filter_test.go b/parser/seqql_filter_test.go index 64902186..c7da5367 100644 --- a/parser/seqql_filter_test.go +++ b/parser/seqql_filter_test.go @@ -395,8 +395,8 @@ func TestParseSeqQLError(t *testing.T) { test(`keyword:re()`, "parsing `re` filter: invalid syntax") test(`keyword:re(")`, "parsing `re` filter: invalid syntax") test(`keyword:re(""invalid)`, "parsing `re` filter: expected ')', got \"invalid\"") - test(`keyword:re("[invalid")`, "parsing `re` filter: invalid expression for `re` filter: error parsing regexp: missing closing ]: `[invalid)$`") - test(`keyword:re("invalid)")`, "parsing `re` filter: invalid expression for `re` filter: error parsing regexp: unexpected ): `^(?i:invalid))$`") + test(`keyword:re("[invalid")`, "parsing `re` filter: invalid expression for `re` filter: error parsing regexp: missing closing ]: `[invalid`") + test(`keyword:re("invalid)")`, "parsing `re` filter: invalid expression for `re` filter: error parsing regexp: unexpected ): `invalid)`") test(`keyword:re("[z-a]")`, "parsing `re` filter: invalid expression for `re` filter: error parsing regexp: invalid character class range: `z-a`") test(`keyword:re("*invalid")`, "parsing `re` filter: invalid expression for `re` filter: error parsing regexp: missing argument to repetition operator: `*`") diff --git a/parser/token_re.go b/parser/token_re.go index 879fd31c..0b9567f7 100644 --- a/parser/token_re.go +++ b/parser/token_re.go @@ -3,13 +3,28 @@ package parser import ( "fmt" "regexp" + "regexp/syntax" + "slices" "strings" + + "github.com/ozontech/seq-db/config" + "github.com/ozontech/seq-db/util" ) +type ReLiteral struct { + Value []byte + Foldable bool +} + type Re struct { - Field string + Field string + Expression Term CompiledExpression *regexp.Regexp + + Prefix ReLiteral + Middle []ReLiteral + Suffix ReLiteral } func (r *Re) DumpSeqQL(b *strings.Builder) { @@ -43,6 +58,12 @@ func parseReFilter(lex *lexer, fieldName string) (*Re, error) { // This behaviour has negative impact on language extendability. expr := lex.Token + lex.Next() + if !lex.IsKeyword(")") { + return nil, fmt.Errorf("expected ')', got %q", lex.Token) + } + lex.Next() + // Here are two important things to keep in mind: // - We perform case-insensitive search by default; // - We force anchoring for the expression; @@ -52,22 +73,161 @@ func parseReFilter(lex *lexer, fieldName string) (*Re, error) { // // See Prometheus TSDB `FastRegexMatcher` for a similar approach: // https://github.com/prometheus/prometheus/blob/19fd0b0b1dbfe01a5e49f5d04544a7c5853c12bb/model/labels/regexp.go#L70 - expr = "^(?i:" + expr + ")$" - compiled, err := regexp.Compile(expr) + re, err := syntax.Parse(expr, syntax.Perl) if err != nil { return nil, fmt.Errorf("invalid expression for `re` filter: %s", err) } - lex.Next() - if !lex.IsKeyword(")") { - return nil, fmt.Errorf("expected ')', got %q", lex.Token) + // NOTE(dkharms): We do not allow overriding of case-sensitivity + // in case if search is case-insensitive. + // + // This way `re` filter works consistently with how `keyword` + // and `text` search behave. + overridable := config.CaseSensitive + if !overridable && hasCaseSensitivityOverride(re) { + return nil, fmt.Errorf( + "store is configured for case-insensitive search: " + + "you cannot override this option", + ) } - lex.Next() + // NOTE(dkharms): Again, if store works in case-insensitive mode + // we simulate behaviour of `keyword` and `text` indexes. + // Should we really do this? + if !overridable { + expr = strings.ToLower(expr) + } + + // NOTE(dkharms): We force anchoring for the expression. + // Anchoring is necessary for prefix and suffix search optimization. + expr = "^" + expr + "$" + exp, err := regexp.Compile(expr) + if err != nil { + return nil, fmt.Errorf( + "it's likely you've encountered a bug: " + + "please contact seq-db team", + ) + } + + re = optimizeRe(re) + return &Re{ - Field: fieldName, + Field: fieldName, + Expression: newTextTerm(expr), - CompiledExpression: compiled, + CompiledExpression: exp, + + Prefix: prefix(re), + Middle: middle(re), + Suffix: suffix(re), }, nil } + +func hasCaseSensitivityOverride(re *syntax.Regexp) bool { + switch re.Op { + case syntax.OpLiteral, syntax.OpCharClass: + return (re.Flags & syntax.FoldCase) == syntax.FoldCase + default: + return slices.ContainsFunc(re.Sub, hasCaseSensitivityOverride) + } +} + +// eliminateCapture transforms regular expression into +// semantically equivalent one but without capturing groups. +func eliminateCapture(re *syntax.Regexp) { + if re.Op == syntax.OpCapture { + *re = *re.Sub[0] + } + + for _, s := range re.Sub { + eliminateCapture(s) + } +} + +func optimizeRe(re *syntax.Regexp) *syntax.Regexp { + // Eliminate captures in-place. + // + // They do not change semantics of regular expression + // but make simplification and extraction of literals more difficult. + eliminateCapture(re) + + // Well, some simplification are not reflected at already parsed tree. + // So we do it again :) + re, err := syntax.Parse(re.Simplify().String(), syntax.Perl) + if err != nil { + panic(fmt.Sprintf("BUG: cannot parse re after optimization pass: %s", err)) + } + + return re +} + +func prefix(re *syntax.Regexp) ReLiteral { + // For example, we work with regular expression `seqdb-(stg|prod)-[1-9]+`. + subs := []*syntax.Regexp{re} + + // Yep, this is concatention. We are interesed in its subexpressions. + if subs[0].Op == syntax.OpConcat { + subs = subs[0].Sub + } + + // Skip symbols like `^`. + if subs[0].Op == syntax.OpBeginText || subs[0].Op == syntax.OpBeginLine { + subs = subs[1:] + } + + // Well, not today. + if len(subs) == 0 || subs[0].Op != syntax.OpLiteral { + return ReLiteral{} + } + + return ReLiteral{ + // TODO(dkharms): Check whether it is safe. + Value: util.StringToByteUnsafe(string(subs[0].Rune)), + Foldable: (subs[0].Flags & syntax.FoldCase) == syntax.FoldCase, + } +} + +func middle(re *syntax.Regexp) []ReLiteral { + var m []ReLiteral + + subs := []*syntax.Regexp{re} + if subs[0].Op == syntax.OpConcat { + subs = subs[0].Sub + } + + for len(subs) > 0 { + if subs[0].Op == syntax.OpLiteral { + m = append(m, ReLiteral{ + // TODO(dkharms): Check whether it is safe. + Value: util.StringToByteUnsafe(string(subs[0].Rune)), + Foldable: (subs[0].Flags & syntax.FoldCase) == syntax.FoldCase, + }) + } + subs = subs[1:] + } + + return m +} + +func suffix(re *syntax.Regexp) ReLiteral { + subs := []*syntax.Regexp{re} + + if subs[0].Op == syntax.OpConcat { + subs = subs[0].Sub + } + + if subs[len(subs)-1].Op == syntax.OpBeginText || subs[len(subs)-1].Op == syntax.OpBeginLine { + subs = subs[:len(subs)-1] + } + + if len(subs) == 0 || subs[len(subs)-1].Op != syntax.OpLiteral { + return ReLiteral{} + } + + return ReLiteral{ + // TODO(dkharms): Check whether it is safe. + Value: util.StringToByteUnsafe(string(subs[len(subs)-1].Rune)), + Foldable: (subs[len(subs)-1].Flags & syntax.FoldCase) == syntax.FoldCase, + } +} diff --git a/parser/token_re_test.go b/parser/token_re_test.go new file mode 100644 index 00000000..fb990a08 --- /dev/null +++ b/parser/token_re_test.go @@ -0,0 +1,144 @@ +package parser + +import ( + "regexp/syntax" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestOptimizationPass(t *testing.T) { + parse := func(s string) *syntax.Regexp { + t.Helper() + + re, err := syntax.Parse(s, syntax.Perl) + require.NoError(t, err) + + return re + } + + cases := []struct { + re string + expected string + }{ + { + // Remove capturing groups. + re: "(a(b(c(d))))", + expected: "abcd", + }, + { + re: "(seqdb-prod)?", + // This is non-capturing group. + expected: "(?:seqdb-prod)?", + }, + { + re: "[1][2][3]", + // This is non-capturing group. + expected: "123", + }, + } + + for _, cc := range cases { + assert.Equal(t, cc.expected, optimizeRe(parse(cc.re)).String()) + } +} + +func TestLiteralExtraction(t *testing.T) { + parse := func(s string) *syntax.Regexp { + t.Helper() + + re, err := syntax.Parse(s, syntax.Perl) + require.NoError(t, err) + + return optimizeRe(re) + } + + prefix := func(re *syntax.Regexp) []ReLiteral { + if r := prefix(re); len(r.Value) > 0 { + return []ReLiteral{r} + } + return []ReLiteral{} + } + + suffix := func(re *syntax.Regexp) []ReLiteral { + if r := suffix(re); len(r.Value) > 0 { + return []ReLiteral{r} + } + return []ReLiteral{} + } + + cases := []struct { + re *syntax.Regexp + expected []ReLiteral + fn func(*syntax.Regexp) []ReLiteral + }{ + { + re: parse("simple"), + expected: []ReLiteral{{Value: []byte("simple"), Foldable: false}}, + fn: prefix, + }, + { + re: parse("easy-prefix-[a-zA-Z]"), + expected: []ReLiteral{{Value: []byte("easy-prefix-"), Foldable: false}}, + fn: prefix, + }, + { + re: parse("(video|vodka)-prefix-[a-zA-Z]"), + expected: []ReLiteral{{Value: []byte("v"), Foldable: false}}, + fn: prefix, + }, + { + re: parse("(?i)prefix-[a-zA-Z]"), + expected: []ReLiteral{{Value: []byte("PREFIX-"), Foldable: true}}, + fn: prefix, + }, + { + re: parse("(no|prefix)-suffix"), + expected: []ReLiteral{}, + fn: prefix, + }, + + { + re: parse("simple"), + expected: []ReLiteral{{Value: []byte("simple"), Foldable: false}}, + fn: suffix, + }, + { + re: parse("((a)b)"), + expected: []ReLiteral{{Value: []byte("ab"), Foldable: false}}, + fn: suffix, + }, + { + re: parse("prefix-[a-zA-Z]-suffix"), + expected: []ReLiteral{{Value: []byte("-suffix"), Foldable: false}}, + fn: suffix, + }, + { + re: parse("suffix-(no|literal)"), + expected: []ReLiteral{}, + fn: suffix, + }, + + { + re: parse("(something|nothing)-in-(a|the)-way-(song)?"), + expected: []ReLiteral{ + {Value: []byte("-in-"), Foldable: false}, + {Value: []byte("-way-"), Foldable: false}, + }, + fn: middle, + }, + { + re: parse("([a-z]+-one-([0-9]-two))-three"), + expected: []ReLiteral{ + {Value: []byte("-one-"), Foldable: false}, + {Value: []byte("-two-three"), Foldable: false}, + }, + fn: middle, + }, + } + + for _, cc := range cases { + assert.Equal(t, cc.expected, cc.fn(cc.re)) + } +} diff --git a/pattern/pattern.go b/pattern/pattern.go index 504d3929..b5cdb76e 100644 --- a/pattern/pattern.go +++ b/pattern/pattern.go @@ -131,10 +131,12 @@ func cut(b []byte, l int) []byte { func (s *wildcardSearch) Narrow(tp tokenProvider) { s.narrowed = true l := len(s.prefix) + s.first = util.BinSearchInRange(s.first, s.last, func(tid int) bool { tokenPrefix := cut(tp.GetToken(uint32(tid)), l) return bytes.Compare(tokenPrefix, s.prefix) >= 0 }) + s.last = util.BinSearchInRange(s.first, s.last, func(tid int) bool { tokenPrefix := cut(tp.GetToken(uint32(tid)), l) return bytes.Compare(tokenPrefix, s.prefix) > 0 @@ -334,26 +336,137 @@ func (s *rangeIpSearch) Check(rawVal []byte) (bool, error) { type reSearch struct { baseSearch - r *regexp.Regexp - checked int + r *regexp.Regexp + + prefix parser.ReLiteral + middle []parser.ReLiteral + suffix parser.ReLiteral + + letters util.LettersBitset + narrowed bool + checked int } func newReSearch(base baseSearch, token *parser.Re) *reSearch { if token.Expression.Kind != parser.TermText { panic("BUG: wrong term kind in re") } - return &reSearch{baseSearch: base, r: token.CompiledExpression} + + var b util.LetterBitsetBuilder + b.Add(token.Prefix.Value) + b.Add(token.Suffix.Value) + + for i := range token.Middle { + b.Add(token.Middle[i].Value) + } + + return &reSearch{ + baseSearch: base, + r: token.CompiledExpression, + + prefix: token.Prefix, + middle: token.Middle, + suffix: token.Suffix, + + letters: b.Build(), + } } -func (s *reSearch) Check(rawVal []byte) (bool, error) { +func (s *reSearch) Narrow(tp tokenProvider) { + // TODO(dkharms): Handle case-insensitive search. + if s.prefix.Foldable { + return + } + + s.narrowed = true + l := len(s.prefix.Value) + s.first = util.BinSearchInRange(s.first, s.last, func(tid int) bool { + tokenPrefix := cut(tp.GetToken(uint32(tid)), l) + return bytes.Compare(tokenPrefix, s.prefix.Value) >= 0 + }) + + s.last = util.BinSearchInRange(s.first, s.last, func(tid int) bool { + tokenPrefix := cut(tp.GetToken(uint32(tid)), l) + return bytes.Compare(tokenPrefix, s.prefix.Value) > 0 + }) - 1 +} + +func (s *reSearch) CheckEntry(letters util.LettersBitset) bool { + return letters.IsNil() || letters.ContainsAll(s.letters) +} + +func (s *reSearch) Check(val []byte) (bool, error) { + if max(len(s.prefix.Value), len(s.suffix.Value)) > len(val) { + return false, nil + } + + if !s.checkPrefix(val) || !s.checkSuffix(val) || !s.checkMiddle(val) { + return false, nil + } + if config.MaxRegexTokensCheck > 0 && s.checked >= config.MaxRegexTokensCheck { return false, errors.New( "'re' filter exceeded token limit: " + "consider using regular filters", ) } + s.checked++ - return s.r.Match(rawVal), nil + return s.r.Match(val), nil +} + +func (s *reSearch) checkPrefix(val []byte) bool { + prefix := s.prefix.Value + + if s.narrowed || len(prefix) == 0 { + return true + } + + if s.prefix.Foldable { + return bytes.EqualFold(prefix, val[:len(prefix)]) + } + + return bytes.Equal(prefix, val[:len(prefix)]) +} + +func (s *reSearch) checkMiddle(val []byte) bool { + if len(s.middle) == 0 { + return true + } + + for i := range s.middle { + lit := s.middle[i] + + // We have to perform case-insensitive substring search, + // so at this point it's just easier to give up and check token + // via compiled regular expression. + if lit.Foldable { + return true + } + + start := bytes.Index(val, lit.Value) + if start == -1 { + return false + } + + val = val[start+len(lit.Value):] + } + + return true +} + +func (s *reSearch) checkSuffix(val []byte) bool { + suffix := s.suffix.Value + + if len(suffix) == 0 { + return true + } + + if s.suffix.Foldable { + return bytes.EqualFold(suffix, val[len(val)-len(suffix):]) + } + + return bytes.Equal(suffix, val[len(val)-len(suffix):]) } type Searcher interface { @@ -403,7 +516,11 @@ func newSearcher(token parser.Token, tp tokenProvider) Searcher { // // There are other techniques which are more complicated so it's // worth studying Apache Lucene, TSDB (Prometheus) etc. - return newReSearch(base, t) + s := newReSearch(base, t) + if tp.Ordered() { + s.Narrow(tp) + } + return s } panic(fmt.Sprintf("unknown token type: %T", token)) } diff --git a/pattern/pattern_test.go b/pattern/pattern_test.go index f89766e9..ed596b27 100644 --- a/pattern/pattern_test.go +++ b/pattern/pattern_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/ozontech/seq-db/config" "github.com/ozontech/seq-db/parser" ) @@ -496,6 +497,24 @@ func TestPatternSymbols(t *testing.T) { } func TestPatternRe(t *testing.T) { + t.Run("match-simple", func(t *testing.T) { + needles := []string{"simple"} + + data := append( + []string{"not-uuid", "not uuid as well"}, + needles..., + ) + + tp := newTestTokenProvider(data) + + testAll(t, tp, []testCase{ + { + `re("simple")`, + needles, + }, + }) + }) + t.Run("match-uuid", func(t *testing.T) { needles := []string{ "7313c25b-2eae-4839-b773-91dff1f24f1f", @@ -564,12 +583,12 @@ func TestPatternRe(t *testing.T) { t.Run("match-level", func(t *testing.T) { needles := []string{ - "[ERROR] connection refused", - "[WARN] timeout exceeded", + "[error] connection refused", + "[warn] timeout exceeded", } data := append( - []string{"[INFO] all good", "[DEBUG] trace value"}, + []string{"[info] all good", "[debug] trace value"}, needles..., ) @@ -577,7 +596,7 @@ func TestPatternRe(t *testing.T) { testAll(t, tp, []testCase{ { - `re("\[(ERROR|WARN)\].*")`, + `re("\[(error|warn)\].*")`, needles, }, }) @@ -649,12 +668,12 @@ func TestPatternRe(t *testing.T) { t.Run("match-anchored", func(t *testing.T) { needles := []string{ - "ERROR: disk empty", - "ERROR: disk full", + "error: disk empty", + "error: disk full", } data := append( - []string{"some ERROR in the middle", "info: no error"}, + []string{"some error in the middle", "info: no error"}, needles..., ) @@ -662,7 +681,7 @@ func TestPatternRe(t *testing.T) { testAll(t, tp, []testCase{ { - `re("^ERROR: disk (empty|full)$")`, + `re("^error: disk (empty|full)$")`, needles, }, }) @@ -716,7 +735,10 @@ func TestPatternRe(t *testing.T) { }) }) - t.Run("match-case-insensitive", func(t *testing.T) { + t.Run("match-case-sensitive", func(t *testing.T) { + defer func(v bool) { config.CaseSensitive = v }(config.CaseSensitive) + config.CaseSensitive = true + needles := []string{ "Error occurred", "ERROR occurred", @@ -732,10 +754,24 @@ func TestPatternRe(t *testing.T) { testAll(t, tp, []testCase{ { - `re("(?i)error.*")`, + `re("(?i)error.*red")`, needles, }, }) + + testAll(t, tp, []testCase{ + { + `re("error.*")`, + []string{needles[2]}, + }, + }) + + testAll(t, tp, []testCase{ + { + `re("error.*RED")`, + []string{}, + }, + }) }) t.Run("match-range", func(t *testing.T) { diff --git a/util/letters_bitset.go b/util/letters_bitset.go index 3202cd06..7f91c7d2 100644 --- a/util/letters_bitset.go +++ b/util/letters_bitset.go @@ -54,9 +54,11 @@ func NewLettersBitsetFromArray(letters [30]bool) LettersBitset { return s } -func NewLettersBitset(data []byte) LettersBitset { +func NewLettersBitset(data ...[]byte) LettersBitset { var builder LetterBitsetBuilder - builder.Add(data) + for i := range data { + builder.Add(data[i]) + } return builder.Build() }