diff options
| -rw-r--r-- | ccl.go | 6 | ||||
| -rw-r--r-- | ccl_test.go | 2 | ||||
| -rw-r--r-- | lexer.go | 29 |
3 files changed, 16 insertions, 21 deletions
@@ -258,13 +258,13 @@ func (p *parser) peek() ([]byte, error) { if p.err != nil || p.tok != nil { return p.tok, p.err } - tok, err := p.lexer.next() + i, tok, err := p.lexer.next() if err != nil { p.err = err return nil, p.err } - p.tok = tok.b - p.i = tok.i + p.i = i + p.tok = tok return p.tok, nil } diff --git a/ccl_test.go b/ccl_test.go index e7405f8..8aaca98 100644 --- a/ccl_test.go +++ b/ccl_test.go @@ -718,7 +718,7 @@ func BenchmarkLex(b *testing.B) { for b.Loop() { l := &lexer{data: msg} for { - _, err := l.next() + _, _, err := l.next() if err != nil { if err == errEOF { break @@ -6,11 +6,6 @@ import ( "unicode/utf8" ) -type token struct { - i int - b []byte -} - type lexer struct { data []byte i int @@ -20,10 +15,10 @@ func (l *lexer) error(reason string, args ...any) error { return newSyntaxError(l.data, l.i, reason, args...) } -func (l *lexer) yield(n int) token { - t := token{l.i, l.data[l.i : l.i+n]} +func (l *lexer) yield(n int) (int, []byte, error) { + i := l.i l.i += n - return t + return i, l.data[i : i+n], nil } func (l *lexer) skipSpace() error { @@ -76,12 +71,12 @@ func fieldTailByte(b byte) bool { '0' <= b && b <= '9' } -func (l *lexer) next() (token, error) { +func (l *lexer) next() (int, []byte, error) { if err := l.skipSpace(); err != nil { - return token{}, err + return 0, nil, err } if l.i == len(l.data) { - return token{}, errEOF + return 0, nil, errEOF } switch l.data[l.i] { case @@ -92,7 +87,7 @@ func (l *lexer) next() (token, error) { ':', ',': - return l.yield(1), nil + return l.yield(1) case '\'', '"': q := l.data[l.i] i := l.i + 1 @@ -102,21 +97,21 @@ func (l *lexer) next() (token, error) { } } if i >= len(l.data) { - return token{}, l.error("unterminated string") + return 0, nil, l.error("unterminated string") } - return l.yield(i + 1 - l.i), nil + return l.yield(i + 1 - l.i) } switch b := l.data[l.i]; { case numFirstByte(b): i := l.i + 1 for ; i < len(l.data) && numTailByte(l.data[i]); i++ { } - return l.yield(i - l.i), nil + return l.yield(i - l.i) case fieldFirstByte(b): i := l.i + 1 for ; i < len(l.data) && fieldTailByte(l.data[i]); i++ { } - return l.yield(i - l.i), nil + return l.yield(i - l.i) } - return token{}, l.error("invalid lexeme") + return 0, nil, l.error("invalid lexeme") } |