Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
101 changes: 46 additions & 55 deletions parser.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,28 +46,26 @@ func newInfParser(t *Tokenizer, reader io.Reader, bufferSize uint) *parsing {
if bufferSize == 0 {
bufferSize = DefaultChunkSize
}
buffer := make([]byte, bufferSize)
tok := t.allocToken()
tok.line = 1
return &parsing{
t: t,
str: buffer,
str: make([]byte, 0, bufferSize), // the first chunk is read by the first parse() call
reader: reader,
line: 1,
chunkSize: int(bufferSize),
token: tok,
}
}

// ensureBytes makes sure that the byte at p.pos+n is loaded.
// A reader may return less data than requested, so chunks are loaded until there is enough data or the stream ends.
func (p *parsing) ensureBytes(n int) bool {
if p.pos+n >= len(p.str) {
if p.reader != nil {
p.loadChunk()
if p.pos+n < len(p.str) {
return true
}
for p.pos+n >= len(p.str) {
if p.reader == nil {
return false
}
return false
p.loadChunk()
}
return true
}
Expand All @@ -90,48 +88,38 @@ func (p *parsing) nextByte() byte {
return 0
}

func (p *parsing) slice(from, to int) []byte {
if to < len(p.str) {
return p.str[from:to]
}
return p.str[from:]
}

func (p *parsing) preload() {
n, err := p.reader.Read(p.str)
if n < p.chunkSize {
p.str = p.str[:n]
p.reader = nil
}
if err != nil {
p.reader = nil
if err != io.EOF {
p.err = err
}
}
}
// maxEmptyReads is the number of consecutive empty reads (0, nil) after which the reader is treated as broken, like in bufio.
const maxEmptyReads = 100

// loadChunk reads the next portion of data from the reader and appends it to p.str.
// A short read is not the end of the data: the stream ends only when the reader returns an error (io.EOF included).
// Returns the number of loaded bytes; zero means the end of the stream.
func (p *parsing) loadChunk() int {
// chunk size = new chunk size + size of tail of prev chunk
chunk := make([]byte, len(p.str)+p.chunkSize)
copy(chunk, p.str)
n, err := p.reader.Read(chunk[len(p.str):])

if n < p.chunkSize {
p.str = chunk[:len(p.str)+n]
p.reader = nil
} else {
if len(p.str) == cap(p.str) {
// chunk size = new chunk size + size of tail of prev chunk.
// The old buffer is not reused because the parsed tokens point into it.
chunk := make([]byte, len(p.str), len(p.str)+p.chunkSize)
copy(chunk, p.str)
p.str = chunk
}

if err != nil {
p.reader = nil
if err != io.EOF {
p.err = err
p.resume = false
for i := 0; i < maxEmptyReads; i++ {
n, err := p.reader.Read(p.str[len(p.str):cap(p.str)])
p.str = p.str[:len(p.str)+n]
if err != nil {
p.reader = nil
if err != io.EOF {
p.err = err
}
return n
}
if n > 0 {
return n
}
}
p.resume = false
return n
p.reader = nil
p.err = io.ErrNoProgress
return 0
}

// checkPoint reset internal values for next chunk of data
Expand All @@ -157,7 +145,8 @@ func (p *parsing) parse() {
}
p.curr = p.str[p.pos]
p.resume = true
for p.checkPoint() {
// a loaded chunk stops parsing after the current token (lazy parsing), but an injection (p.stopKeys) is parsed up to its end token.
for p.checkPoint() || p.stopKeys != nil {
if p.stopKeys != nil {
for _, t := range p.stopKeys {
if p.ptr.key == t.Key {
Expand Down Expand Up @@ -248,8 +237,10 @@ func (p *parsing) parseKeyword() bool {
}
var r rune
var size int
p.ensureBytes(4)
r, size = utf8.DecodeRune(p.slice(p.pos, p.pos+4))
for p.reader != nil && !utf8.FullRune(p.str[p.pos:]) { // the rune may be split between chunks
p.loadChunk()
}
r, size = utf8.DecodeRune(p.str[p.pos:])
if unicode.IsLetter(r) || runeExists(p.t.kwMajorSymbols, r) || (start != -1 && runeExists(p.t.kwMinorSymbols, r)) {
if start == -1 {
start = p.pos
Expand Down Expand Up @@ -328,7 +319,10 @@ func (p *parsing) parseNumber() bool {
return false
}
end = end + 1
p.pos = end
if p.pos != end { // roll back the bytes that are not a part of the number, e.g. "e+" in "1e+"
p.pos = end
p.curr = p.str[end]
}
p.token.value = p.str[start:end]
if floatTraitPos == -1 || floatTraitPos > end-1 {
p.token.key = TokenInteger
Expand Down Expand Up @@ -392,6 +386,7 @@ func (p *parsing) parseQuote() bool {
}
}
escapes := false
chars:
for p.curr != 0 {
if escapes {
escapes = false
Expand All @@ -400,7 +395,6 @@ func (p *parsing) parseQuote() bool {
} else if p.match(quote.EndToken, true) {
break
} else if quote.Injects != nil {
loop := true
for i, inject := range quote.Injects {
for _, token := range injectTokens[i] {
if p.match(token.Token, true) {
Expand All @@ -419,13 +413,10 @@ func (p *parsing) parseQuote() bool {
p.token.offset = p.offset + p.pos
p.token.string = quote
start = p.pos
loop = false
break
// p.curr is the first byte after the injection (or 0 at the end of data): check it, don't skip it
continue chars
}
}
if !loop {
break
}
}
}
if p.curr == newLine {
Expand Down
2 changes: 1 addition & 1 deletion readme.md
Original file line number Diff line number Diff line change
Expand Up @@ -396,7 +396,7 @@ The `*Stream` is a bidirectional iterator over the parsed tokens. Common methods
| `IsValid() bool` | `true` while the pointer is on a real token. |
| `GoNext() *Stream` | Move to the next token (loads the next chunk in stream mode). |
| `GoPrev() *Stream` | Move to the previous token (bounded by `SetHistorySize`). |
| `GoTo(id int) *Stream` | Move to a token by its id. |
| `GoTo(id int) *Stream` | Move to a token by its id (the pointer is not moved if there is no such token). |
| `CurrentToken() *Token` | The current token (a `TokenUndef` token when out of range). |
| `NextToken() / PrevToken() *Token` | Peek without moving the pointer. |
| `GoNextIfNextIs(key, ...) bool` | Advance only if the next token matches. |
Expand Down
88 changes: 58 additions & 30 deletions stream.go
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,14 @@ func (s *Stream) GetParsedLength() int {

// ensureParseNext lazily parses the next chunk of data when the token list ends mid-stream.
func (s *Stream) ensureParseNext() {
if s.current.next == nil && s.current != undefToken && s.p != nil {
if s.current != undefToken {
s.ensureParsedAfter(s.current)
}
}

// ensureParsedAfter lazily parses the next chunk of data when the token list ends at the token t mid-stream.
func (s *Stream) ensureParsedAfter(t *Token) {
if t.next == nil && s.p != nil {
n := s.p.n
s.p.parse()
s.len += s.p.n - n
Expand All @@ -111,6 +118,7 @@ func (s *Stream) ensureParseNext() {
// GoNext moves the stream pointer to the next token.
// If there is no token, it initiates the parsing of the next chunk of data.
// If there is no data, the pointer will point to the TokenUndef token.
// If the pointer is before the beginning of the stream (see GoPrev), it moves to the first token.
func (s *Stream) GoNext() *Stream {
s.ensureParseNext()
if s.current.next != nil {
Expand All @@ -122,55 +130,75 @@ func (s *Stream) GoNext() *Stream {
s.len--
}
} else if s.current == undefToken {
s.current = s.prev
s.prev = nil
if s.next != nil { // before the beginning of the stream: step back to the first token
s.current = s.next
s.next = nil
}
// beyond the end of the stream: stay there
} else {
s.prev = s.current
s.next = nil
s.current = undefToken
}
return s
}

// GoPrev moves the pointer of stream to the next token.
// GoPrev moves the pointer of stream to the previous token.
// The number of possible calls is limited if you specified SetHistorySize.
// If the beginning of the stream or the end of the history is reached, the pointer will point to the TokenUndef token.
// If the pointer is beyond the end of the stream (see GoNext), it moves to the last token.
func (s *Stream) GoPrev() *Stream {
if s.current.prev != nil {
s.current = s.current.prev
} else if s.current == undefToken {
s.current = s.next
s.prev = nil
if s.prev != nil { // beyond the end of the stream: step back to the last token
s.current = s.prev
s.prev = nil
}
// before the beginning of the stream: stay there
} else {
s.next = s.current
s.prev = nil
s.current = undefToken
}
return s
}

// GoTo moves the pointer of stream to specific token.
// GoTo moves the pointer of stream to the token with the specific id.
// If there is no such token (it is beyond the end of the stream or before the beginning of the history), the pointer is not moved.
func (s *Stream) GoTo(id int) *Stream {
if s.current == undefToken {
if s.prev != nil && id <= s.prev.id { // we at the end of the stream
s.GoPrev() // now current is available
for s.current != nil && id != s.current.id {
s.GoPrev()
}
} else if s.next != nil && id >= s.prev.id { // we at the beginning of the stream
s.GoNext() // now current is available
for s.current != nil && id != s.current.id {
s.GoNext()
}
// search without moving the pointer: GoNext frees the history behind the pointer, so the old position could not be restored
ptr := s.current
if ptr == undefToken {
if s.prev != nil { // beyond the end of the stream
ptr = s.prev
} else if s.next != nil { // before the beginning of the stream
ptr = s.next
} else { // empty stream
return s
}
return s
}
if id > s.current.id {
for s.current != nil && id != s.current.id {
s.GoNext()
}
} else if id < s.current.id {
for s.current != nil && id != s.current.id {
s.GoPrev()
for id < ptr.id && ptr.prev != nil {
ptr = ptr.prev
}
for id > ptr.id {
s.ensureParsedAfter(ptr)
if ptr.next == nil {
break
}
ptr = ptr.next
}
if ptr.id != id {
return s
}
s.current = ptr
s.prev = nil
s.next = nil
for s.historySize != 0 && s.current.id-s.head.id > s.historySize {
t := s.head
s.head = s.head.unlink()
s.t.freeToken(t)
s.len--
}
return s
}
Expand All @@ -192,14 +220,14 @@ func (s *Stream) IsNextSequence(keys ...TokenKey) bool {
s.historySize = len(keys)
}

start := s.current
start, prev, next := s.current, s.prev, s.next
for _, key := range keys {
if !s.GoNext().CurrentToken().Is(key) {
result = false
break
}
}
s.current = start
s.current, s.prev, s.next = start, prev, next

if hSize != 0 {
s.SetHistorySize(hSize)
Expand All @@ -218,7 +246,7 @@ func (s *Stream) IsAnyNextSequence(keys ...[]TokenKey) bool {
s.historySize = len(keys)
}

start := s.current
start, prev, next := s.current, s.prev, s.next
for _, key := range keys {
found := false
for _, k := range key {
Expand All @@ -232,7 +260,7 @@ func (s *Stream) IsAnyNextSequence(keys ...[]TokenKey) bool {
break
}
}
s.current = start
s.current, s.prev, s.next = start, prev, next

if hSize != 0 {
s.SetHistorySize(hSize)
Expand Down
Loading
Loading