From 5ecab26da844e843ad30776d97d8da3fe28098a6 Mon Sep 17 00:00:00 2001
From: Rose Hogenson <rosehogenson@posteo.net>
Date: Sun, 26 Oct 2025 18:36:01 -0700
Subject: Separate the lexer

---
 lexer.go | 105 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 105 insertions(+)
 create mode 100644 lexer.go

(limited to 'lexer.go')

diff --git a/lexer.go b/lexer.go
new file mode 100644
index 0000000..ee72b33
--- /dev/null
+++ b/lexer.go
@@ -0,0 +1,105 @@
+package asspb
+
+import (
+	"iter"
+	"regexp"
+)
+
+type token struct {
+	i int
+	b []byte
+}
+
+type lexer struct {
+	data     []byte
+	i        int
+	yieldTok func(token, error) bool
+}
+
+func (l *lexer) error(reason string, args ...any) {
+	l.yieldTok(token{}, newSyntaxError(l.data, l.i, reason, args...))
+}
+
+func (l *lexer) yield(n int) bool {
+	if !l.yieldTok(token{l.i, l.data[l.i : l.i+n]}, nil) {
+		return false
+	}
+	l.i += n
+	return true
+}
+
+var spaceRE = regexp.MustCompile(`^([[:space:]\p{Zs}]|(#|//)[^\n]*|/\*([^*]|\*[^/])*\*?\*/)*`)
+
+func (l *lexer) skipSpace() {
+	l.i += len(spaceRE.Find(l.data[l.i:]))
+}
+
+var (
+	stringRE       = regexp.MustCompile(`(?s)^(([^'\\]|\\.)*)'`)
+	doubleStringRE = regexp.MustCompile(`(?s)^(([^"\\]|\\.)*)"`)
+	lexNumRE       = regexp.MustCompile(`^[-+.0-9][-+.0-9a-zA-Z]*`)
+	fieldRE        = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z_0-9]*`)
+)
+
+func (l *lexer) tokens() {
+	for l.i = 0; ; {
+		l.skipSpace()
+		if l.i == len(l.data) {
+			break
+		}
+		switch l.data[l.i] {
+		case
+			'{',
+			'}',
+			'[',
+			']',
+			':',
+			',':
+
+			if !l.yield(1) {
+				return
+			}
+			continue
+		case '\'':
+			str := stringRE.Find(l.data[l.i+1:])
+			if str == nil {
+				l.error("invaild string")
+				return
+			}
+			if !l.yield(1 + len(str)) {
+				return
+			}
+			continue
+		case '"':
+			str := doubleStringRE.Find(l.data[l.i+1:])
+			if str == nil {
+				l.error("invalid string")
+				return
+			}
+			if !l.yield(1 + len(str)) {
+				return
+			}
+			continue
+		}
+		if n := lexNumRE.Find(l.data[l.i:]); n != nil {
+			if !l.yield(len(n)) {
+				return
+			}
+			continue
+		}
+		if n := fieldRE.Find(l.data[l.i:]); n != nil {
+			if !l.yield(len(n)) {
+				return
+			}
+			continue
+		}
+		l.error("invalid lexeme")
+		return
+	}
+}
+
+func tokens(data []byte) iter.Seq2[token, error] {
+	return func(yield func(token, error) bool) {
+		(&lexer{data: data, yieldTok: yield}).tokens()
+	}
+}
-- 
cgit v1.3.1