Separate the lexer

3 files changed, 228 insertions, 117 deletions

diff --git a/asspb.go b/asspb.go
index 8665d08..c649edd 100644
--- a/asspb.go
+++ b/asspb.go
@@ -150,13 +150,37 @@ package asspb
 import (
 	"bytes"
 	"encoding/json"
+	"errors"
 	"fmt"
+	"iter"
 	"regexp"
 	"strconv"
 	"strings"
 	"unicode/utf8"
 )
 
+type syntaxError struct {
+	line, col int
+	reason    string
+}
+
+func newSyntaxError(data []byte, idx int, reason string, args ...any) error {
+	line, col := 1, 1
+	for _, b := range data[:idx] {
+		if b == '\n' {
+			line++
+			col = 1
+		} else {
+			col++
+		}
+	}
+	return &syntaxError{line, col, fmt.Sprintf(reason, args...)}
+}
+
+func (e *syntaxError) Error() string {
+	return fmt.Sprintf("%d:%d syntax error: %s", e.line, e.col, e.reason)
+}
+
 func appendAnyRepeated(prev any, new ...any) []any {
 	if prev == nil {
 		return new
@@ -175,76 +199,70 @@ func appendAny(prev any, new any) any {
 }
 
 type parser struct {
-	data []byte
-	i    int
-}
-
-type syntaxError struct {
-	line, col int
-	reason    string
-}
-
-func (e *syntaxError) Error() string {
-	return fmt.Sprintf("%d:%d syntax error: %s", e.line, e.col, e.reason)
+	nextTok func() (token, error, bool)
+	tok     []byte
+	err     error
+	data    []byte
+	i       int
 }
 
 func (p *parser) error(reason string, args ...any) error {
-	line, col := 1, 1
-	for _, b := range p.data[:p.i] {
-		if b == '\n' {
-			line++
-			col = 1
-		} else {
-			col++
-		}
-	}
-	return &syntaxError{line, col, fmt.Sprintf(reason, args...)}
+	return newSyntaxError(p.data, p.i, reason, args...)
 }
 
-var spaceRE = regexp.MustCompile(`^([[:space:]\p{Zs}]|(#|//)[^\n]*|/\*([^*]|\*[^/])*\*?\*/)*`)
+var errEOF = errors.New("premature EOF")
 
-func (p *parser) skipSpace() {
-	p.i += len(spaceRE.Find(p.data[p.i:]))
+func (p *parser) peek() ([]byte, error) {
+	if p.err != nil || p.tok != nil {
+		return p.tok, p.err
+	}
+	tok, err, ok := p.nextTok()
+	if !ok {
+		p.err = errEOF
+		return nil, p.err
+	}
+	if err != nil {
+		p.err = err
+		return nil, p.err
+	}
+	p.tok = tok.b
+	p.i = tok.i
+	return p.tok, nil
 }
 
-func (p *parser) parseLit(s string) bool {
-	p.skipSpace()
-	if len(p.data[p.i:]) >= len(s) && string(p.data[p.i:p.i+len(s)]) == s {
-		p.i += len(s)
-		return true
+func (p *parser) next() ([]byte, error) {
+	tok, err := p.peek()
+	if err != nil {
+		return nil, err
 	}
-	return false
+	p.tok = nil
+	return tok, nil
 }
 
-var numRE = regexp.MustCompile(`^[-+]?(0[xX][0-9a-fA-F]+|((0|[1-9][0-9]*)(\.[0-9]*)?|\.[0-9]+)([eE][-+]?[0-9]+)?)`)
+var numRE = regexp.MustCompile(`^[-+]?(0[xX][0-9a-fA-F]+|((0|[1-9][0-9]*)(\.[0-9]*)?|\.[0-9]+)([eE][-+]?[0-9]+)?)$`)
 
-func (p *parser) parseNum() (any, bool) {
-	p.skipSpace()
-	numBytes := numRE.Find(p.data[p.i:])
-	if numBytes == nil {
+func parseNum(numBytes []byte) (any, bool) {
+	if !numRE.Match(numBytes) {
 		return nil, false
 	}
-	if bytes.ContainsAny(numBytes, ".eE") {
-		n, err := strconv.ParseFloat(string(numBytes), 64)
+	if bytes.HasPrefix(numBytes, []byte("0x")) || bytes.HasPrefix(numBytes, []byte("0X")) {
+		n, err := strconv.ParseInt(string(numBytes[2:]), 16, 64)
 		if err != nil {
 			return nil, false
 		}
-		p.i += len(numBytes)
 		return n, true
 	}
-	if bytes.HasPrefix(numBytes, []byte("0x")) || bytes.HasPrefix(numBytes, []byte("0X")) {
-		n, err := strconv.ParseInt(string(numBytes[2:]), 16, 64)
+	if bytes.ContainsAny(numBytes, ".eE") {
+		n, err := strconv.ParseFloat(string(numBytes), 64)
 		if err != nil {
 			return nil, false
 		}
-		p.i += len(numBytes)
 		return n, true
 	}
 	n, err := strconv.ParseInt(string(numBytes), 10, 64)
 	if err != nil {
 		return nil, false
 	}
-	p.i += len(numBytes)
 	return n, true
 }
 
@@ -324,88 +342,60 @@ func (p *parser) unescape(rawStr []byte) ([]byte, error) {
 	return escaped, nil
 }
 
-var (
-	stringRE       = regexp.MustCompile(`(?s)^(([^'\\]|\\.)*)'`)
-	doubleStringRE = regexp.MustCompile(`(?s)^(([^"\\]|\\.)*)"`)
-)
-
-func (p *parser) parseString(double bool) (string, error) {
-	re := stringRE
-	if double {
-		re = doubleStringRE
-	}
+func (p *parser) parseString(tok []byte) (string, error) {
 	s := new(strings.Builder)
 	for {
-		p.skipSpace()
-		rawStr := re.FindSubmatch(p.data[p.i:])
-		if rawStr == nil {
-			return "", p.error("invalid string")
-		}
-		ss, err := p.unescape(rawStr[1])
+		ss, err := p.unescape(tok[1 : len(tok)-1])
 		if err != nil {
 			return "", err
 		}
 		s.Write(ss)
-		p.i += len(rawStr[0])
-		switch {
-		case p.parseLit("'"):
-			re = stringRE
-			continue
-		case p.parseLit(`"`):
-			re = doubleStringRE
-			continue
+		nextTok, err := p.peek()
+		if err != nil || nextTok[0] != '\'' && nextTok[0] != '"' {
+			return s.String(), nil
 		}
-		return s.String(), nil
+		p.next()
+		tok = nextTok
 	}
 }
 
-var fieldRE = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z_0-9]*`)
-
-func (p *parser) parseField() ([]byte, error) {
-	p.skipSpace()
-	fieldName := fieldRE.Find(p.data[p.i:])
-	if fieldName == nil {
-		return nil, p.error("expecting field name")
-	}
-	p.i += len(fieldName)
-	return fieldName, nil
-}
-
 func (p *parser) parseMessage() (map[string]any, error) {
 	m := make(map[string]any)
 	for {
-		if p.parseLit("}") {
-			return m, nil
+		tok, err := p.next()
+		if err != nil || tok[0] == '}' {
+			return m, err
 		}
-		if err := p.parseFieldVal(m); err != nil {
+		if err := p.parseFieldVal(m, tok); err != nil {
 			return nil, err
 		}
 	}
 }
 
-func (p *parser) parseVal() (any, error) {
+func (p *parser) parseVal(tok []byte) (any, error) {
 	var (
 		val any
 		err error
 	)
-	switch {
-	case p.parseLit("{"):
+	switch tok[0] {
+	case '{':
 		val, err = p.parseMessage()
-	case p.parseLit("["):
+	case '[':
 		val, err = p.parseList()
-	case p.parseLit("'"):
-		val, err = p.parseString(false)
-	case p.parseLit(`"`):
-		val, err = p.parseString(true)
-	case p.parseLit("true"), p.parseLit("yes"), p.parseLit("on"):
-		val = true
-	case p.parseLit("false"), p.parseLit("no"), p.parseLit("off"):
-		val = false
+	case '\'', '"':
+		val, err = p.parseString(tok)
 	default:
-		var ok bool
-		val, ok = p.parseNum()
-		if !ok {
-			return nil, p.error("expecting field value")
+		switch string(tok) {
+		case "true", "yes", "on":
+			val = true
+		case "false", "no", "off":
+			val = false
+		default:
+			var ok bool
+			val, ok = parseNum(tok)
+			if !ok {
+				return nil, p.error("expecting field value")
+			}
 		}
 	}
 	return val, err
@@ -414,18 +404,20 @@ func (p *parser) parseVal() (any, error) {
 func (p *parser) parseList() ([]any, error) {
 	l := []any{}
 	for i := 0; ; i++ {
-		if p.parseLit("]") {
-			return l, nil
+		tok, err := p.next()
+		if err != nil || tok[0] == ']' {
+			return l, err
 		}
 		if i > 0 {
-			if !p.parseLit(",") {
+			if tok[0] != ',' {
 				return nil, p.error("expecting comma")
 			}
-			if p.parseLit("]") { // allow trailing comma
-				return l, nil
+			tok, err = p.next()
+			if err != nil || tok[0] == ']' { // allow trailing comma
+				return l, err
 			}
 		}
-		val, err := p.parseVal()
+		val, err := p.parseVal(tok)
 		if err != nil {
 			return nil, err
 		}
@@ -433,26 +425,33 @@ func (p *parser) parseList() ([]any, error) {
 	}
 }
 
-func (p *parser) parseFieldVal(out map[string]any) error {
-	field, err := p.parseField()
+func (p *parser) parseFieldVal(out map[string]any, field []byte) error {
+	if b := field[0]; !(b == '_' || 'a' <= b && b <= 'z' || 'A' <= b && b <= 'Z') {
+		return p.error("expecting field")
+	}
+	tok, err := p.next()
 	if err != nil {
 		return err
 	}
 	var val any
-	switch {
-	case p.parseLit("{"):
+	switch tok[0] {
+	case '{':
 		if val, err = p.parseMessage(); err != nil {
 			return err
 		}
-	case p.parseLit("["):
+	case '[':
 		val, err := p.parseList()
 		if err != nil {
 			return err
 		}
 		out[string(field)] = appendAnyRepeated(out[string(field)], val...)
 		return nil
-	case p.parseLit(":"):
-		if val, err = p.parseVal(); err != nil {
+	case ':':
+		tok, err := p.next()
+		if err != nil {
+			return err
+		}
+		if val, err = p.parseVal(tok); err != nil {
 			return err
 		}
 		if l, ok := val.([]any); ok {
@@ -469,11 +468,14 @@ func (p *parser) parseFieldVal(out map[string]any) error {
 func (p *parser) parse() (map[string]any, error) {
 	m := make(map[string]any)
 	for {
-		p.skipSpace()
-		if p.i == len(p.data) {
-			return m, nil
+		tok, err := p.next()
+		if err != nil {
+			if err == errEOF {
+				return m, nil
+			}
+			return nil, err
 		}
-		if err := p.parseFieldVal(m); err != nil {
+		if err := p.parseFieldVal(m, tok); err != nil {
 			return nil, err
 		}
 	}
@@ -490,8 +492,9 @@ func (p *parser) parse() (map[string]any, error) {
 //	key1: "val1"
 //	key2: "val2"
 func Unmarshal(data []byte, v any) error {
-	p := &parser{data: data}
-	m, err := p.parse()
+	nextToken, stop := iter.Pull2(tokens(data))
+	defer stop()
+	m, err := (&parser{nextTok: nextToken, data: data}).parse()
 	if err != nil {
 		return err
 	}
diff --git a/asspb_test.go b/asspb_test.go
index 4280dae..63361ef 100644
--- a/asspb_test.go
+++ b/asspb_test.go
@@ -275,6 +275,9 @@ func TestUnmarshal_Invalid(t *testing.T) {
 	}, {
 		desc: "InvalidUTF8",
 		msg:  `field: "\x80"`,
+	}, {
+		desc: "FieldMissingVal",
+		msg:  `field`,
 	}} {
 		t.Run(tc.desc, func(t *testing.T) {
 			t.Parallel()
diff --git a/lexer.go b/lexer.go
new file mode 100644
index 0000000..ee72b33
--- /dev/null
+++ b/lexer.go
@@ -0,0 +1,105 @@
+package asspb
+
+import (
+	"iter"
+	"regexp"
+)
+
+type token struct {
+	i int
+	b []byte
+}
+
+type lexer struct {
+	data     []byte
+	i        int
+	yieldTok func(token, error) bool
+}
+
+func (l *lexer) error(reason string, args ...any) {
+	l.yieldTok(token{}, newSyntaxError(l.data, l.i, reason, args...))
+}
+
+func (l *lexer) yield(n int) bool {
+	if !l.yieldTok(token{l.i, l.data[l.i : l.i+n]}, nil) {
+		return false
+	}
+	l.i += n
+	return true
+}
+
+var spaceRE = regexp.MustCompile(`^([[:space:]\p{Zs}]|(#|//)[^\n]*|/\*([^*]|\*[^/])*\*?\*/)*`)
+
+func (l *lexer) skipSpace() {
+	l.i += len(spaceRE.Find(l.data[l.i:]))
+}
+
+var (
+	stringRE       = regexp.MustCompile(`(?s)^(([^'\\]|\\.)*)'`)
+	doubleStringRE = regexp.MustCompile(`(?s)^(([^"\\]|\\.)*)"`)
+	lexNumRE       = regexp.MustCompile(`^[-+.0-9][-+.0-9a-zA-Z]*`)
+	fieldRE        = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z_0-9]*`)
+)
+
+func (l *lexer) tokens() {
+	for l.i = 0; ; {
+		l.skipSpace()
+		if l.i == len(l.data) {
+			break
+		}
+		switch l.data[l.i] {
+		case
+			'{',
+			'}',
+			'[',
+			']',
+			':',
+			',':
+
+			if !l.yield(1) {
+				return
+			}
+			continue
+		case '\'':
+			str := stringRE.Find(l.data[l.i+1:])
+			if str == nil {
+				l.error("invaild string")
+				return
+			}
+			if !l.yield(1 + len(str)) {
+				return
+			}
+			continue
+		case '"':
+			str := doubleStringRE.Find(l.data[l.i+1:])
+			if str == nil {
+				l.error("invalid string")
+				return
+			}
+			if !l.yield(1 + len(str)) {
+				return
+			}
+			continue
+		}
+		if n := lexNumRE.Find(l.data[l.i:]); n != nil {
+			if !l.yield(len(n)) {
+				return
+			}
+			continue
+		}
+		if n := fieldRE.Find(l.data[l.i:]); n != nil {
+			if !l.yield(len(n)) {
+				return
+			}
+			continue
+		}
+		l.error("invalid lexeme")
+		return
+	}
+}
+
+func tokens(data []byte) iter.Seq2[token, error] {
+	return func(yield func(token, error) bool) {
+		(&lexer{data: data, yieldTok: yield}).tokens()
+	}
+}