diff options
| -rw-r--r-- | ccl.go | 13 | ||||
| -rw-r--r-- | ccl_test.go | 7 | ||||
| -rw-r--r-- | lexer.go | 112 |
3 files changed, 54 insertions, 78 deletions
@@ -164,7 +164,6 @@ import ( "encoding/base64" "errors" "fmt" - "iter" "math" "reflect" "strconv" @@ -243,7 +242,7 @@ func fieldMap(out map[structField]int, types map[reflect.Type]bool, s reflect.Ty } type parser struct { - nextTok func() (token, error, bool) + nextTok func() (token, error) tok []byte err error data []byte @@ -261,11 +260,7 @@ func (p *parser) peek() ([]byte, error) { if p.err != nil || p.tok != nil { return p.tok, p.err } - tok, err, ok := p.nextTok() - if !ok { - p.err = errEOF - return nil, p.err - } + tok, err := p.nextTok() if err != nil { p.err = err return nil, p.err @@ -747,7 +742,5 @@ func Unmarshal(data []byte, v any) error { if err := fieldMap(fields, make(map[reflect.Type]bool), val.Type().Elem()); err != nil { return err } - nextToken, stop := iter.Pull2(tokens(data)) - defer stop() - return (&parser{nextTok: nextToken, data: data, fieldMap: fields}).parse(val.Elem()) + return (&parser{nextTok: newLexer(data).next, data: data, fieldMap: fields}).parse(val.Elem()) } diff --git a/ccl_test.go b/ccl_test.go index ab4833f..53b2a24 100644 --- a/ccl_test.go +++ b/ccl_test.go @@ -684,8 +684,13 @@ func BenchmarkLex(b *testing.B) { repeated: [5, 6] `) for b.Loop() { - for _, err := range tokens(msg) { + l := newLexer(msg) + for { + _, err := l.next() if err != nil { + if err == errEOF { + break + } b.Fatal(err) } } @@ -2,7 +2,6 @@ package ccl import ( "bytes" - "iter" "unicode" "unicode/utf8" ) @@ -13,21 +12,18 @@ type token struct { } type lexer struct { - data []byte - i int - yieldTok func(token, error) bool + data []byte + i int } -func (l *lexer) error(reason string, args ...any) { - l.yieldTok(token{}, newSyntaxError(l.data, l.i, reason, args...)) +func (l *lexer) error(reason string, args ...any) error { + return newSyntaxError(l.data, l.i, reason, args...) } -func (l *lexer) yield(n int) bool { - if !l.yieldTok(token{l.i, l.data[l.i : l.i+n]}, nil) { - return false - } +func (l *lexer) yield(n int) token { + t := token{l.i, l.data[l.i : l.i+n]} l.i += n - return true + return t } func (l *lexer) skipSpace() { @@ -75,67 +71,49 @@ func fieldTailByte(b byte) bool { '0' <= b && b <= '9' } -func (l *lexer) tokens() { - for l.i = 0; ; { - l.skipSpace() - if l.i == len(l.data) { - break - } - switch l.data[l.i] { - case - '{', - '}', - '[', - ']', - ':', - ',': +func (l *lexer) next() (token, error) { + l.skipSpace() + if l.i == len(l.data) { + return token{}, errEOF + } + switch l.data[l.i] { + case + '{', + '}', + '[', + ']', + ':', + ',': - if !l.yield(1) { - return - } - continue - case '\'', '"': - q := l.data[l.i] - i := l.i + 1 - for ; i < len(l.data) && l.data[i] != q; i++ { - if l.data[i] == '\\' { - i++ - } + return l.yield(1), nil + case '\'', '"': + q := l.data[l.i] + i := l.i + 1 + for ; i < len(l.data) && l.data[i] != q; i++ { + if l.data[i] == '\\' { + i++ } - if i >= len(l.data) { - l.error("unterminated string") - return - } - if !l.yield(i + 1 - l.i) { - return - } - continue } - switch b := l.data[l.i]; { - case numFirstByte(b): - i := l.i + 1 - for ; i < len(l.data) && numTailByte(l.data[i]); i++ { - } - if !l.yield(i - l.i) { - return - } - continue - case fieldFirstByte(b): - i := l.i + 1 - for ; i < len(l.data) && fieldTailByte(l.data[i]); i++ { - } - if !l.yield(i - l.i) { - return - } - continue + if i >= len(l.data) { + return token{}, l.error("unterminated string") } - l.error("invalid lexeme") - return + return l.yield(i + 1 - l.i), nil } + switch b := l.data[l.i]; { + case numFirstByte(b): + i := l.i + 1 + for ; i < len(l.data) && numTailByte(l.data[i]); i++ { + } + return l.yield(i - l.i), nil + case fieldFirstByte(b): + i := l.i + 1 + for ; i < len(l.data) && fieldTailByte(l.data[i]); i++ { + } + return l.yield(i - l.i), nil + } + return token{}, l.error("invalid lexeme") } -func tokens(data []byte) iter.Seq2[token, error] { - return func(yield func(token, error) bool) { - (&lexer{data: data, yieldTok: yield}).tokens() - } +func newLexer(data []byte) *lexer { + return &lexer{data: data} } |