1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
|
package asspb
import (
"iter"
"regexp"
)
type token struct {
i int
b []byte
}
type lexer struct {
data []byte
i int
yieldTok func(token, error) bool
}
func (l *lexer) error(reason string, args ...any) {
l.yieldTok(token{}, newSyntaxError(l.data, l.i, reason, args...))
}
func (l *lexer) yield(n int) bool {
if !l.yieldTok(token{l.i, l.data[l.i : l.i+n]}, nil) {
return false
}
l.i += n
return true
}
var spaceRE = regexp.MustCompile(`^([[:space:]\p{Zs}]|(#|//)[^\n]*|/\*([^*]|\*[^/])*\*?\*/)*`)
func (l *lexer) skipSpace() {
l.i += len(spaceRE.Find(l.data[l.i:]))
}
var (
stringRE = regexp.MustCompile(`(?s)^(([^'\\]|\\.)*)'`)
doubleStringRE = regexp.MustCompile(`(?s)^(([^"\\]|\\.)*)"`)
lexNumRE = regexp.MustCompile(`^[-+.0-9][-+.0-9a-zA-Z]*`)
fieldRE = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z_0-9]*`)
)
func (l *lexer) tokens() {
for l.i = 0; ; {
l.skipSpace()
if l.i == len(l.data) {
break
}
switch l.data[l.i] {
case
'{',
'}',
'[',
']',
':',
',':
if !l.yield(1) {
return
}
continue
case '\'':
str := stringRE.Find(l.data[l.i+1:])
if str == nil {
l.error("invalid string")
return
}
if !l.yield(1 + len(str)) {
return
}
continue
case '"':
str := doubleStringRE.Find(l.data[l.i+1:])
if str == nil {
l.error("invalid string")
return
}
if !l.yield(1 + len(str)) {
return
}
continue
}
if n := lexNumRE.Find(l.data[l.i:]); n != nil {
if !l.yield(len(n)) {
return
}
continue
}
if n := fieldRE.Find(l.data[l.i:]); n != nil {
if !l.yield(len(n)) {
return
}
continue
}
l.error("invalid lexeme")
return
}
}
func tokens(data []byte) iter.Seq2[token, error] {
return func(yield func(token, error) bool) {
(&lexer{data: data, yieldTok: yield}).tokens()
}
}
|
