diff --git a/parser.go b/parser.go index 56efab8..72df5f0 100644 --- a/parser.go +++ b/parser.go @@ -46,12 +46,11 @@ func newInfParser(t *Tokenizer, reader io.Reader, bufferSize uint) *parsing { if bufferSize == 0 { bufferSize = DefaultChunkSize } - buffer := make([]byte, bufferSize) tok := t.allocToken() tok.line = 1 return &parsing{ t: t, - str: buffer, + str: make([]byte, 0, bufferSize), // the first chunk is read by the first parse() call reader: reader, line: 1, chunkSize: int(bufferSize), @@ -59,15 +58,14 @@ func newInfParser(t *Tokenizer, reader io.Reader, bufferSize uint) *parsing { } } +// ensureBytes makes sure that the byte at p.pos+n is loaded. +// A reader may return less data than requested, so chunks are loaded until there is enough data or the stream ends. func (p *parsing) ensureBytes(n int) bool { - if p.pos+n >= len(p.str) { - if p.reader != nil { - p.loadChunk() - if p.pos+n < len(p.str) { - return true - } + for p.pos+n >= len(p.str) { + if p.reader == nil { + return false } - return false + p.loadChunk() } return true } @@ -90,48 +88,38 @@ func (p *parsing) nextByte() byte { return 0 } -func (p *parsing) slice(from, to int) []byte { - if to < len(p.str) { - return p.str[from:to] - } - return p.str[from:] -} - -func (p *parsing) preload() { - n, err := p.reader.Read(p.str) - if n < p.chunkSize { - p.str = p.str[:n] - p.reader = nil - } - if err != nil { - p.reader = nil - if err != io.EOF { - p.err = err - } - } -} +// maxEmptyReads is the number of consecutive empty reads (0, nil) after which the reader is treated as broken, like in bufio. +const maxEmptyReads = 100 +// loadChunk reads the next portion of data from the reader and appends it to p.str. +// A short read is not the end of the data: the stream ends only when the reader returns an error (io.EOF included). +// Returns the number of loaded bytes; zero means the end of the stream. func (p *parsing) loadChunk() int { - // chunk size = new chunk size + size of tail of prev chunk - chunk := make([]byte, len(p.str)+p.chunkSize) - copy(chunk, p.str) - n, err := p.reader.Read(chunk[len(p.str):]) - - if n < p.chunkSize { - p.str = chunk[:len(p.str)+n] - p.reader = nil - } else { + if len(p.str) == cap(p.str) { + // chunk size = new chunk size + size of tail of prev chunk. + // The old buffer is not reused because the parsed tokens point into it. + chunk := make([]byte, len(p.str), len(p.str)+p.chunkSize) + copy(chunk, p.str) p.str = chunk } - - if err != nil { - p.reader = nil - if err != io.EOF { - p.err = err + p.resume = false + for i := 0; i < maxEmptyReads; i++ { + n, err := p.reader.Read(p.str[len(p.str):cap(p.str)]) + p.str = p.str[:len(p.str)+n] + if err != nil { + p.reader = nil + if err != io.EOF { + p.err = err + } + return n + } + if n > 0 { + return n } } - p.resume = false - return n + p.reader = nil + p.err = io.ErrNoProgress + return 0 } // checkPoint reset internal values for next chunk of data @@ -157,7 +145,8 @@ func (p *parsing) parse() { } p.curr = p.str[p.pos] p.resume = true - for p.checkPoint() { + // a loaded chunk stops parsing after the current token (lazy parsing), but an injection (p.stopKeys) is parsed up to its end token. + for p.checkPoint() || p.stopKeys != nil { if p.stopKeys != nil { for _, t := range p.stopKeys { if p.ptr.key == t.Key { @@ -248,8 +237,10 @@ func (p *parsing) parseKeyword() bool { } var r rune var size int - p.ensureBytes(4) - r, size = utf8.DecodeRune(p.slice(p.pos, p.pos+4)) + for p.reader != nil && !utf8.FullRune(p.str[p.pos:]) { // the rune may be split between chunks + p.loadChunk() + } + r, size = utf8.DecodeRune(p.str[p.pos:]) if unicode.IsLetter(r) || runeExists(p.t.kwMajorSymbols, r) || (start != -1 && runeExists(p.t.kwMinorSymbols, r)) { if start == -1 { start = p.pos @@ -328,7 +319,10 @@ func (p *parsing) parseNumber() bool { return false } end = end + 1 - p.pos = end + if p.pos != end { // roll back the bytes that are not a part of the number, e.g. "e+" in "1e+" + p.pos = end + p.curr = p.str[end] + } p.token.value = p.str[start:end] if floatTraitPos == -1 || floatTraitPos > end-1 { p.token.key = TokenInteger @@ -392,6 +386,7 @@ func (p *parsing) parseQuote() bool { } } escapes := false +chars: for p.curr != 0 { if escapes { escapes = false @@ -400,7 +395,6 @@ func (p *parsing) parseQuote() bool { } else if p.match(quote.EndToken, true) { break } else if quote.Injects != nil { - loop := true for i, inject := range quote.Injects { for _, token := range injectTokens[i] { if p.match(token.Token, true) { @@ -419,13 +413,10 @@ func (p *parsing) parseQuote() bool { p.token.offset = p.offset + p.pos p.token.string = quote start = p.pos - loop = false - break + // p.curr is the first byte after the injection (or 0 at the end of data): check it, don't skip it + continue chars } } - if !loop { - break - } } } if p.curr == newLine { diff --git a/readme.md b/readme.md index db39943..1843d9b 100644 --- a/readme.md +++ b/readme.md @@ -396,7 +396,7 @@ The `*Stream` is a bidirectional iterator over the parsed tokens. Common methods | `IsValid() bool` | `true` while the pointer is on a real token. | | `GoNext() *Stream` | Move to the next token (loads the next chunk in stream mode). | | `GoPrev() *Stream` | Move to the previous token (bounded by `SetHistorySize`). | -| `GoTo(id int) *Stream` | Move to a token by its id. | +| `GoTo(id int) *Stream` | Move to a token by its id (the pointer is not moved if there is no such token). | | `CurrentToken() *Token` | The current token (a `TokenUndef` token when out of range). | | `NextToken() / PrevToken() *Token` | Peek without moving the pointer. | | `GoNextIfNextIs(key, ...) bool` | Advance only if the next token matches. | diff --git a/stream.go b/stream.go index b279282..07da107 100644 --- a/stream.go +++ b/stream.go @@ -101,7 +101,14 @@ func (s *Stream) GetParsedLength() int { // ensureParseNext lazily parses the next chunk of data when the token list ends mid-stream. func (s *Stream) ensureParseNext() { - if s.current.next == nil && s.current != undefToken && s.p != nil { + if s.current != undefToken { + s.ensureParsedAfter(s.current) + } +} + +// ensureParsedAfter lazily parses the next chunk of data when the token list ends at the token t mid-stream. +func (s *Stream) ensureParsedAfter(t *Token) { + if t.next == nil && s.p != nil { n := s.p.n s.p.parse() s.len += s.p.n - n @@ -111,6 +118,7 @@ func (s *Stream) ensureParseNext() { // GoNext moves the stream pointer to the next token. // If there is no token, it initiates the parsing of the next chunk of data. // If there is no data, the pointer will point to the TokenUndef token. +// If the pointer is before the beginning of the stream (see GoPrev), it moves to the first token. func (s *Stream) GoNext() *Stream { s.ensureParseNext() if s.current.next != nil { @@ -122,55 +130,75 @@ func (s *Stream) GoNext() *Stream { s.len-- } } else if s.current == undefToken { - s.current = s.prev - s.prev = nil + if s.next != nil { // before the beginning of the stream: step back to the first token + s.current = s.next + s.next = nil + } + // beyond the end of the stream: stay there } else { s.prev = s.current + s.next = nil s.current = undefToken } return s } -// GoPrev moves the pointer of stream to the next token. +// GoPrev moves the pointer of stream to the previous token. // The number of possible calls is limited if you specified SetHistorySize. // If the beginning of the stream or the end of the history is reached, the pointer will point to the TokenUndef token. +// If the pointer is beyond the end of the stream (see GoNext), it moves to the last token. func (s *Stream) GoPrev() *Stream { if s.current.prev != nil { s.current = s.current.prev } else if s.current == undefToken { - s.current = s.next - s.prev = nil + if s.prev != nil { // beyond the end of the stream: step back to the last token + s.current = s.prev + s.prev = nil + } + // before the beginning of the stream: stay there } else { s.next = s.current + s.prev = nil s.current = undefToken } return s } -// GoTo moves the pointer of stream to specific token. +// GoTo moves the pointer of stream to the token with the specific id. +// If there is no such token (it is beyond the end of the stream or before the beginning of the history), the pointer is not moved. func (s *Stream) GoTo(id int) *Stream { - if s.current == undefToken { - if s.prev != nil && id <= s.prev.id { // we at the end of the stream - s.GoPrev() // now current is available - for s.current != nil && id != s.current.id { - s.GoPrev() - } - } else if s.next != nil && id >= s.prev.id { // we at the beginning of the stream - s.GoNext() // now current is available - for s.current != nil && id != s.current.id { - s.GoNext() - } + // search without moving the pointer: GoNext frees the history behind the pointer, so the old position could not be restored + ptr := s.current + if ptr == undefToken { + if s.prev != nil { // beyond the end of the stream + ptr = s.prev + } else if s.next != nil { // before the beginning of the stream + ptr = s.next + } else { // empty stream + return s } - return s } - if id > s.current.id { - for s.current != nil && id != s.current.id { - s.GoNext() - } - } else if id < s.current.id { - for s.current != nil && id != s.current.id { - s.GoPrev() + for id < ptr.id && ptr.prev != nil { + ptr = ptr.prev + } + for id > ptr.id { + s.ensureParsedAfter(ptr) + if ptr.next == nil { + break } + ptr = ptr.next + } + if ptr.id != id { + return s + } + s.current = ptr + s.prev = nil + s.next = nil + for s.historySize != 0 && s.current.id-s.head.id > s.historySize { + t := s.head + s.head = s.head.unlink() + s.t.freeToken(t) + s.len-- } return s } @@ -192,14 +220,14 @@ func (s *Stream) IsNextSequence(keys ...TokenKey) bool { s.historySize = len(keys) } - start := s.current + start, prev, next := s.current, s.prev, s.next for _, key := range keys { if !s.GoNext().CurrentToken().Is(key) { result = false break } } - s.current = start + s.current, s.prev, s.next = start, prev, next if hSize != 0 { s.SetHistorySize(hSize) @@ -218,7 +246,7 @@ func (s *Stream) IsAnyNextSequence(keys ...[]TokenKey) bool { s.historySize = len(keys) } - start := s.current + start, prev, next := s.current, s.prev, s.next for _, key := range keys { found := false for _, k := range key { @@ -232,7 +260,7 @@ func (s *Stream) IsAnyNextSequence(keys ...[]TokenKey) bool { break } } - s.current = start + s.current, s.prev, s.next = start, prev, next if hSize != 0 { s.SetHistorySize(hSize) diff --git a/stream_test.go b/stream_test.go index 2ef3b32..b7536db 100644 --- a/stream_test.go +++ b/stream_test.go @@ -2,8 +2,12 @@ package tokenizer import ( "bytes" + "errors" "fmt" + "io" + "strings" "testing" + "testing/iotest" "time" "github.com/stretchr/testify/require" @@ -360,6 +364,282 @@ func TestStreamOverflow(t *testing.T) { require.False(t, stream.IsNextSequence(TokenKeyword, TokenKeyword, TokenKeyword, TokenKeyword)) } +// streamToken is a comparable copy of a token without the links of the token list. +type streamToken struct { + id int + key TokenKey + value string + line int + offset int + indent string + string *StringSettings +} + +func collectTokens(stream *Stream) []streamToken { + var tokens []streamToken + for stream.IsValid() { + c := stream.CurrentToken() + tokens = append(tokens, streamToken{c.id, c.key, string(c.value), c.line, c.offset, string(c.indent), c.string}) + stream.GoNext() + } + return tokens +} + +// TestStreamLargerThanChunk parses a stream that is many times larger than the parser's buffer. +func TestStreamLargerThanChunk(t *testing.T) { + const commaKey = TokenKey(1) + parser := New() + parser.DefineTokens(commaKey, []string{","}) + + for _, repeat := range []int{5, 5000} { + for _, size := range []uint{4, 16, 64, 4096} { + stream := parser.ParseStream(strings.NewReader(strings.Repeat("a, ", repeat)), size) + n := 0 + for ; stream.IsValid(); n++ { + if n%2 == 0 { + require.Truef(t, stream.CurrentToken().IsKeyword(), "repeat %d, buffer size %d, token %d", repeat, size, n) + require.Equal(t, "a", stream.CurrentToken().ValueString()) + } else { + require.Truef(t, stream.CurrentToken().Is(commaKey), "repeat %d, buffer size %d, token %d", repeat, size, n) + } + stream.GoNext() + } + require.Equalf(t, 2*repeat, n, "repeat %d, buffer size %d", repeat, size) + stream.Close() + } + } +} + +// TestStreamChunkBoundaries checks that a stream produces exactly the same tokens as the in-memory parser when tokens, runes, numbers, strings with injections and whitespaces cross chunk boundaries. +// Readers that return less data than requested (like pipes and sockets) are checked too. +func TestStreamChunkBoundaries(t *testing.T) { + parser := newBenchTokenizer() + parser.DefineTokens(100, []string{"или"}) + parser.DefineStringToken(101, ``) + + inputs := map[string]string{ + "tokens": strings.Repeat("a, ", 50), + "mixed": string(benchSource(3)), + "injections": strings.Repeat(`x = "value2 {{ value3 }} value4"; "{{ a }}{{ b }}"; `, 10) + `"tail {{ c`, + "unicode": strings.Repeat("привет или мир, функция\n", 10), + "numbers": strings.Repeat("1.5e10 2_000 .5 3E-2 1e+ 7. ", 10), + "operators": strings.Repeat("a <= b >= c == d != e ", 10), + "whitespace": " \n\t trailing ws \n\n ", + } + readers := map[string]func(string) io.Reader{ + "full": func(s string) io.Reader { return strings.NewReader(s) }, + "half": func(s string) io.Reader { return iotest.HalfReader(strings.NewReader(s)) }, + "onebyte": func(s string) io.Reader { return iotest.OneByteReader(strings.NewReader(s)) }, + } + sizes := []uint{4096} + for size := uint(1); size <= 64; size++ { + sizes = append(sizes, size) + } + + for name, input := range inputs { + expected := parser.ParseString(input) + expectedTokens := collectTokens(expected) + for readerName, reader := range readers { + for _, size := range sizes { + stream := parser.ParseStream(reader(input), size) + require.Equalf(t, expectedTokens, collectTokens(stream), "input %s, reader %s, buffer size %d", name, readerName, size) + require.Equalf(t, expected.GetParsedLength(), stream.GetParsedLength(), "input %s, reader %s, buffer size %d", name, readerName, size) + stream.Close() + } + } + expected.Close() + } +} + +// stutterReader returns (0, nil) several times before each byte of data. +type stutterReader struct { + data []byte + empty int + misses int +} + +func (r *stutterReader) Read(p []byte) (int, error) { + if len(r.data) == 0 { + return 0, io.EOF + } + if r.misses < r.empty || len(p) == 0 { + r.misses++ + return 0, nil + } + r.misses = 0 + p[0] = r.data[0] + r.data = r.data[1:] + return 1, nil +} + +type noProgressReader struct{} + +func (noProgressReader) Read([]byte) (int, error) { + return 0, nil +} + +// TestStreamReaders checks readers that return no data or an error. +func TestStreamReaders(t *testing.T) { + parser := New() + + stream := parser.ParseStream(&stutterReader{data: []byte("one two three"), empty: 3}, 4) + require.Equal(t, []string{"one", "two", "three"}, streamValues(stream)) + stream.Close() + + // a reader that never returns data must not hang the stream + stream = parser.ParseStream(noProgressReader{}, 4) + require.False(t, stream.IsValid()) + stream.Close() + + // the stream ends on a read error with the tokens parsed before it + stream = parser.ParseStream(iotest.TimeoutReader(strings.NewReader("one two three")), 8) + require.Equal(t, []string{"one", "two"}, streamValues(stream)) + stream.Close() + + stream = parser.ParseStream(iotest.DataErrReader(strings.NewReader("one two three")), 4) + require.Equal(t, []string{"one", "two", "three"}, streamValues(stream)) + stream.Close() + + stream = parser.ParseStream(iotest.ErrReader(errors.New("broken")), 4) + require.False(t, stream.IsValid()) + stream.Close() +} + +func streamValues(stream *Stream) []string { + var values []string + for stream.IsValid() { + values = append(values, stream.CurrentToken().ValueString()) + stream.GoNext() + } + return values +} + +// goTo calls stream.GoTo and fails the test if it does not return. +func goTo(t *testing.T, stream *Stream, id int) *Stream { + t.Helper() + done := make(chan struct{}) + go func() { + stream.GoTo(id) + close(done) + }() + select { + case <-done: + case <-time.After(time.Second): + t.Fatalf("GoTo(%d) did not return", id) + } + return stream +} + +// TestStreamGoTo GoTo moves the pointer to the token with the given id and doesn't move it if there is no such token. +func TestStreamGoTo(t *testing.T) { + parser := New() + stream := parser.ParseString("a b c") // ids 0..2 + defer stream.Close() + + require.Equal(t, 2, goTo(t, stream, 2).CurrentToken().ID()) + require.Equal(t, 0, goTo(t, stream, 0).CurrentToken().ID()) + require.Equal(t, 1, goTo(t, stream, 1).CurrentToken().ID()) + for _, id := range []int{3, 100, -1, -5} { + require.Equalf(t, 1, goTo(t, stream, id).CurrentToken().ID(), "GoTo(%d)", id) + } + + // beyond the end of the stream + stream.GoNext().GoNext() + require.False(t, stream.IsValid()) + for _, id := range []int{3, 100, -1, -5} { + require.Falsef(t, goTo(t, stream, id).IsValid(), "GoTo(%d)", id) + } + require.Equal(t, 2, stream.GoPrev().CurrentToken().ID()) + stream.GoNext() + require.Equal(t, 1, goTo(t, stream, 1).CurrentToken().ID()) + + // before the beginning of the stream + stream.GoPrev().GoPrev() + require.False(t, stream.IsValid()) + for _, id := range []int{3, 100, -1, -5} { + require.Falsef(t, goTo(t, stream, id).IsValid(), "GoTo(%d)", id) + } + require.Equal(t, 0, stream.GoNext().CurrentToken().ID()) + stream.GoPrev() + require.Equal(t, 2, goTo(t, stream, 2).CurrentToken().ID()) + + empty := parser.ParseString("") + defer empty.Close() + require.False(t, goTo(t, empty, 0).IsValid()) +} + +// TestStreamGoToInfinite GoTo parses the next chunks lazily and keeps all tokens when there is no such token. +func TestStreamGoToInfinite(t *testing.T) { + parser := New() + stream := parser.ParseStream(strings.NewReader(strings.Repeat("a ", 100)), 4) // ids 0..99 + defer stream.Close() + + require.Equal(t, 0, goTo(t, stream, 100).CurrentToken().ID()) + n := 0 + for ; stream.IsValid(); n++ { + require.Equal(t, n, stream.CurrentToken().ID()) + stream.GoNext() + } + require.Equal(t, 100, n) + require.Equal(t, 0, goTo(t, stream, 0).CurrentToken().ID()) + require.Equal(t, 99, goTo(t, stream, 99).CurrentToken().ID()) + require.Equal(t, 99, goTo(t, stream, 100).CurrentToken().ID()) +} + +// TestStreamGoToHistory GoTo frees the history behind the new position only, and stays if the token is before the beginning of the history. +func TestStreamGoToHistory(t *testing.T) { + parser := New() + stream := parser.ParseString("0 1 2 3 4 5 6 7 8 9").SetHistorySize(3) + defer stream.Close() + + stream.GoNext() + require.Equal(t, 1, goTo(t, stream, 100).CurrentToken().ID()) + require.Equal(t, 0, stream.HeadToken().ID()) + require.Equal(t, int64(0), stream.GoPrev().CurrentToken().ValueInt64()) + + require.Equal(t, 9, goTo(t, stream, 9).CurrentToken().ID()) + require.Equal(t, 6, stream.HeadToken().ID()) + require.Equal(t, 4, stream.len) + require.Equal(t, 9, goTo(t, stream, 0).CurrentToken().ID()) + require.Equal(t, 6, goTo(t, stream, 6).CurrentToken().ID()) + + stream = parser.ParseStream(strings.NewReader(strings.Repeat("a ", 100)), 4).SetHistorySize(5) + defer stream.Close() + require.Equal(t, 50, goTo(t, stream, 50).CurrentToken().ID()) + require.Equal(t, 45, stream.HeadToken().ID()) + require.Equal(t, 50, goTo(t, stream, 44).CurrentToken().ID()) +} + +// TestStreamBounds GoNext and GoPrev stay out of the stream once they cross its bounds and step back with the opposite move. +func TestStreamBounds(t *testing.T) { + parser := New() + stream := parser.ParseString("a b c") + defer stream.Close() + + stream.GoNext().GoNext().GoNext() + require.False(t, stream.IsValid()) + require.False(t, stream.GoNext().IsValid()) + require.Equal(t, 2, stream.GoPrev().CurrentToken().ID()) + + stream.GoTo(0).GoPrev() + require.False(t, stream.IsValid()) + require.False(t, stream.GoPrev().IsValid()) + require.Equal(t, 0, stream.GoNext().CurrentToken().ID()) + + // a look-ahead before the beginning of the stream keeps the position + stream.GoPrev() + require.True(t, stream.IsNextSequence(TokenKeyword, TokenKeyword, TokenKeyword)) + require.False(t, stream.IsAnyNextSequence([]TokenKey{TokenKeyword}, []TokenKey{TokenInteger})) + require.False(t, stream.IsValid()) + require.Equal(t, 0, stream.GoNext().CurrentToken().ID()) + + empty := parser.ParseString("") + defer empty.Close() + require.False(t, goTo(t, empty, 0).IsValid()) + require.False(t, empty.GoNext().IsValid()) + require.False(t, empty.GoPrev().IsValid()) +} + func TestIssue7(t *testing.T) { p := New() buf := bytes.NewBuffer(nil) diff --git a/tokenizer.go b/tokenizer.go index e02a40d..b7d558a 100644 --- a/tokenizer.go +++ b/tokenizer.go @@ -290,7 +290,6 @@ func (t *Tokenizer) ParseBytes(str []byte) *Stream { // ParseStream parse and convert infinite stream of bytes into infinite stream of tokens. func (t *Tokenizer) ParseStream(r io.Reader, bufferSize uint) *Stream { p := newInfParser(t, r, bufferSize) - p.preload() p.parse() return NewInfStream(p) } diff --git a/tokenizer_test.go b/tokenizer_test.go index e302b16..6413e99 100644 --- a/tokenizer_test.go +++ b/tokenizer_test.go @@ -193,6 +193,68 @@ func TestTokenizeEdgeCases(t *testing.T) { }) } +// TestNumberRollback the bytes after a number that are not a part of it (like "E+" in "0E+") are parsed as the next tokens. +func TestNumberRollback(t *testing.T) { + plusKey := TokenKey(1) + parser := New() + parser.DefineTokens(plusKey, []string{"+"}) + + stream := parser.ParseString("0E+") + defer stream.Close() + require.Equal(t, []Token{ + {key: TokenInteger, value: s2b("0"), offset: 0, line: 1, id: 0}, + {key: TokenKeyword, value: s2b("E"), offset: 1, line: 1, id: 1}, + {key: plusKey, value: s2b("+"), offset: 2, line: 1, id: 2}, + }, stream.GetSnippet(10, 10)) +} + +// TestInjectionBounds the byte right after an injection is parsed as a part of the string, and a string that ends inside or right after an injection doesn't panic. +func TestInjectionBounds(t *testing.T) { + const ( + openKey TokenKey = iota + 1 + closeKey + quoteKey + ) + parser := New() + parser.DefineTokens(openKey, []string{"{{"}) + parser.DefineTokens(closeKey, []string{"}}"}) + parser.DefineStringToken(quoteKey, `"`, `"`).SetEscapeSymbol('\\').AddInjection(openKey, closeKey) + + type tok struct { + key TokenKey + value string + } + data := map[string][]tok{ + `"a {{ b }}" c`: { + {TokenStringFragment, `"a `}, {openKey, "{{"}, {TokenKeyword, "b"}, {closeKey, "}}"}, {TokenStringFragment, `"`}, + {TokenKeyword, "c"}, + }, + `"a {{ b }}{{ c }}" d`: { + {TokenStringFragment, `"a `}, {openKey, "{{"}, {TokenKeyword, "b"}, {closeKey, "}}"}, {TokenStringFragment, ``}, + {openKey, "{{"}, {TokenKeyword, "c"}, {closeKey, "}}"}, {TokenStringFragment, `"`}, + {TokenKeyword, "d"}, + }, + `"a {{`: { + {TokenStringFragment, `"a `}, {openKey, "{{"}, {TokenStringFragment, ``}, + }, + `"a {{ b`: { + {TokenStringFragment, `"a `}, {openKey, "{{"}, {TokenKeyword, "b"}, {TokenStringFragment, ``}, + }, + `"a {{ b }}`: { + {TokenStringFragment, `"a `}, {openKey, "{{"}, {TokenKeyword, "b"}, {closeKey, "}}"}, {TokenStringFragment, ``}, + }, + } + for str, expected := range data { + stream := parser.ParseString(str) + var actual []tok + for ; stream.IsValid(); stream.GoNext() { + actual = append(actual, tok{stream.CurrentToken().Key(), stream.CurrentToken().ValueString()}) + } + require.Equalf(t, expected, actual, "parse %s", str) + stream.Close() + } +} + func TestTokenizeComplex(t *testing.T) { tokenizer := New() compareTokenKey := TokenKey(10)