Remove the regexes

Wow it doesn't look good for regex

goos: linux
goarch: amd64
pkg: roseh.moe/pkg/ccl
cpu: AMD Ryzen 9 5900X 12-Core Processor
         │ baseline.txt │            regexn't.txt             │
         │    sec/op    │   sec/op     vs base                │
Lex-24      8.625µ ± 1%   1.429µ ± 0%  -83.43% (p=0.000 n=20)
Parse-24   18.305µ ± 0%   8.856µ ± 0%  -51.62% (p=0.000 n=20)
geomean     12.57µ        3.557µ       -71.69%

3 files changed, 236 insertions, 155 deletions

diff --git a/ccl.go b/ccl.go
index c4c660e..521fbba 100644
--- a/ccl.go
+++ b/ccl.go
@@ -167,7 +167,6 @@ import (
 	"iter"
 	"math"
 	"reflect"
-	"regexp"
 	"strconv"
 	"strings"
 	"unicode/utf8"
@@ -284,121 +283,159 @@ func (p *parser) next() ([]byte, error) {
 	return tok, nil
 }
 
-var (
-	numRE = regexp.MustCompile(`^[-+]?(0[xX][0-9a-fA-F]+|((0|[1-9][0-9]*)(\.[0-9]*)?|\.[0-9]+)([eE][-+]?[0-9]+)?)$`)
-	hexRE = regexp.MustCompile(`^([-+]?)0[xX]`)
-)
+func checkNum(b []byte) bool {
+	if bytes.Equal(b, []byte("0")) {
+		return true
+	}
+	if len(b) == 0 || !(b[0] == '.' || '1' <= b[0] && b[0] <= '9') {
+		return false
+	}
+	haveDigits := false
+	for ; len(b) > 0 && '0' <= b[0] && b[0] <= '9'; b = b[1:] {
+		haveDigits = true
+	}
+	if len(b) > 0 && b[0] == '.' {
+		b = b[1:]
+		for ; len(b) > 0 && '0' <= b[0] && b[0] <= '9'; b = b[1:] {
+			haveDigits = true
+		}
+	}
+	if !haveDigits {
+		return false
+	}
+	if len(b) == 0 || !(b[0] == 'e' || b[0] == 'E') {
+		return true
+	}
+	b = b[1:]
+	if len(b) > 0 && b[0] == '-' || b[0] == '+' {
+		b = b[1:]
+	}
+	if len(b) == 0 {
+		return false
+	}
+	for ; len(b) > 0 && '0' <= b[0] && b[0] <= '9'; b = b[1:] {
+	}
+	return len(b) == 0
+}
+
+type integer struct {
+	n   uint64
+	sgn int8
+}
 
 func (p *parser) parseNum(numBytes []byte) (any, error) {
-	if !numRE.Match(numBytes) {
-		return nil, p.error("invalid number")
+	n := numBytes
+	var sgn int8 = 1
+	switch numBytes[0] {
+	case '-':
+		sgn = -1
+		n = numBytes[1:]
+	case '+':
+		n = numBytes[1:]
 	}
-	if hex := hexRE.FindSubmatch(numBytes); hex != nil {
-		if string(hex[1]) == "-" {
-			n, err := strconv.ParseInt(string(numBytes[len(hex[0]):]), 16, 64)
-			if err != nil {
-				return nil, p.error("invalid number")
-			}
-			return -n, nil
-		} else {
-			n, err := strconv.ParseUint(string(numBytes[len(hex[0]):]), 16, 64)
-			if err != nil {
-				return nil, p.error("invalid number")
-			}
-			return n, nil
+	if len(n) > 2 && n[0] == '0' && (n[1] == 'x' || n[1] == 'X') {
+		n, err := strconv.ParseUint(string(n[2:]), 16, 64)
+		if err != nil {
+			return nil, p.error("invalid hex number: %s", err)
 		}
+		return &integer{n, sgn}, nil
+	}
+	if !checkNum(n) {
+		return nil, p.error("invalid number")
 	}
-	if bytes.ContainsAny(numBytes, ".eE") {
+	if bytes.ContainsAny(n, ".eE") {
 		n, err := strconv.ParseFloat(string(numBytes), 64)
 		if err != nil {
-			return nil, p.error("invalid number")
+			return nil, p.error("invalid number (unreachable)")
 		}
 		return n, nil
 	}
-	if bytes.HasPrefix(numBytes, []byte("-")) {
-		n, err := strconv.ParseInt(string(numBytes), 10, 64)
-		if err != nil {
-			return nil, p.error("invalid number")
-		}
-		return n, nil
-	} else {
-		n, err := strconv.ParseUint(string(bytes.TrimPrefix(numBytes, []byte("+"))), 10, 64)
-		if err != nil {
-			return nil, p.error("invalid number")
-		}
-		return n, nil
+	un, err := strconv.ParseUint(string(n), 10, 64)
+	if err != nil {
+		return nil, p.error("invalid number (unreachable)")
 	}
-}
-
-var escapesRE = regexp.MustCompile(`(?s)\\(.|\r\n|[0-7]{3}|x[0-9a-fA-F]{2}|u[0-9a-fA-F]{4}|U[0-9a-fA-F]{8})`)
-
-func init() {
-	escapesRE.Longest()
+	return &integer{un, sgn}, nil
 }
 
 func (p *parser) unescape(rawStr []byte) ([]byte, error) {
-	var err error
-	escaped := escapesRE.ReplaceAllFunc(rawStr, func(escape []byte) []byte {
-		switch string(escape) {
-		case `\'`:
-			return []byte("'")
-		case `\"`:
-			return []byte(`"`)
-		case `\?`:
-			return []byte("?")
-		case `\\`:
-			return []byte(`\`)
-		case `\a`:
-			return []byte("\a")
-		case `\b`:
-			return []byte("\b")
-		case `\f`:
-			return []byte("\f")
-		case `\n`:
-			return []byte("\n")
-		case `\r`:
-			return []byte("\r")
-		case `\t`:
-			return []byte("\t")
-		case `\v`:
-			return []byte("\v")
-		case "\\\n", "\\\r\n":
-			return nil
+	var escaped []byte
+	for i := 0; i < len(rawStr); i++ {
+		if rawStr[i] != '\\' {
+			escaped = append(escaped, rawStr[i])
+			continue
 		}
-		switch {
-		case bytes.HasPrefix(escape, []byte(`\x`)):
-			var n uint64
-			if n, err = strconv.ParseUint(string(escape[2:]), 16, 8); err != nil {
-				err = p.error("invalid hex escape %q: %s", escape, err)
-				return nil
+		i++
+		var b []byte
+		switch rawStr[i] {
+		case '\'':
+			b = []byte("'")
+		case '"':
+			b = []byte(`"`)
+		case '?':
+			b = []byte("?")
+		case '\\':
+			b = []byte(`\`)
+		case 'a':
+			b = []byte("\a")
+		case 'b':
+			b = []byte("\b")
+		case 'f':
+			b = []byte("\f")
+		case 'n':
+			b = []byte("\n")
+		case 'r':
+			b = []byte("\r")
+		case 't':
+			b = []byte("\t")
+		case 'v':
+			b = []byte("\v")
+		case '\n':
+			b = nil
+		case '\r':
+			i++
+			if i < len(rawStr) && rawStr[i] == '\n' {
+				b = nil
+			} else {
+				return nil, fmt.Errorf("invalid escape sequence %q", rawStr[i-2:min(i+1, len(rawStr))])
 			}
-			return []byte{byte(n)}
-		case bytes.HasPrefix(escape, []byte(`\u`)), bytes.HasPrefix(escape, []byte(`\U`)):
-			var n int64
-			if n, err = strconv.ParseInt(string(escape[2:]), 16, 32); err != nil {
-				err = p.error("invalid unicode escape %q: %s", escape, err)
-				return nil
+		case 'x':
+			i++
+			if i+2 > len(rawStr) {
+				return nil, fmt.Errorf("invalid hex escape %q", rawStr[i-2:min(i+2, len(rawStr))])
 			}
-			return utf8.AppendRune(nil, rune(n))
-		default:
-			if len(escape) != 4 {
-				err = p.error("invalid string escape %q", escape)
-				return nil
+			n, err := strconv.ParseUint(string(rawStr[i:i+2]), 16, 8)
+			if err != nil {
+				return nil, fmt.Errorf("invalid hex escape %q: %s", rawStr[i-2:i+2], err)
 			}
-			var n int64
-			if n, err = strconv.ParseInt(string(escape[1:]), 8, 32); err != nil {
-				err = p.error("invalid string escape %q", escape)
-				return nil
+			i++
+			b = []byte{byte(n)}
+		case 'u', 'U':
+			nBytes := 4
+			if rawStr[i] == 'U' {
+				nBytes = 8
 			}
-			if n > 255 {
-				err = p.error("invalid octal escape %q %d > 255", escape, n)
-				return nil
+			i++
+			if i+nBytes > len(rawStr) {
+				return nil, fmt.Errorf("invalid unicode escape %q", rawStr[i-2:min(i+nBytes, len(rawStr))])
 			}
-			return []byte{byte(n)}
+			n, err := strconv.ParseUint(string(rawStr[i:i+nBytes]), 16, 31)
+			if err != nil {
+				return nil, fmt.Errorf("invalid hex escape %q: %s", rawStr[i-2:i+2], err)
+			}
+			i += nBytes - 1
+			b = utf8.AppendRune(nil, rune(n))
+		default:
+			if i+3 > len(rawStr) {
+				return nil, fmt.Errorf("invalid string escape %q", rawStr[i-1:i+1])
+			}
+			n, err := strconv.ParseUint(string(rawStr[i:i+3]), 8, 8)
+			if err != nil {
+				return nil, fmt.Errorf("invalid octal escape %q: %s", rawStr[i:i+3], err)
+			}
+			i += 2
+			b = []byte{byte(n)}
 		}
-	})
-	if err != nil {
-		return nil, err
+		escaped = append(escaped, b...)
 	}
 	if !utf8.Valid(escaped) {
 		return nil, p.error("syntax error: string %q is not UTF-8 encoded", escaped)
@@ -499,7 +536,7 @@ func appendAny(prevVal any, newVal any) any {
 	var l []any
 	if ll, ok := prevVal.([]any); ok {
 		l = ll
-	} else if prevVal != nil {
+	} else {
 		l = []any{prevVal}
 	}
 	if ll, ok := newVal.([]any); ok {
@@ -555,18 +592,18 @@ func (p *parser) parse() (map[string]any, error) {
 	}
 }
 
-func intLimits(kind reflect.Kind) (min int64, max uint64, ok bool) {
+func intLimits(kind reflect.Kind) (min, max uint64, ok bool) {
 	switch kind {
 	case reflect.Int:
-		return math.MinInt, math.MaxInt, true
+		return -math.MinInt, math.MaxInt, true
 	case reflect.Int8:
-		return math.MinInt8, math.MaxInt8, true
+		return -math.MinInt8, math.MaxInt8, true
 	case reflect.Int16:
-		return math.MinInt16, math.MaxInt16, true
+		return -math.MinInt16, math.MaxInt16, true
 	case reflect.Int32:
-		return math.MinInt32, math.MaxInt32, true
+		return -math.MinInt32, math.MaxInt32, true
 	case reflect.Int64:
-		return math.MinInt64, math.MaxInt64, true
+		return -math.MinInt64, math.MaxInt64, true
 	case reflect.Uint:
 		return 0, math.MaxUint, true
 	case reflect.Uint8:
@@ -603,41 +640,23 @@ func unpackVal(fieldVal reflect.Value, fieldMap map[structField]int, val any, fi
 		default:
 			return fmt.Errorf("field %q should have type bool", field)
 		}
-	case uint64:
-		switch fieldVal.Kind() {
-		case reflect.Float32, reflect.Float64:
-			fieldVal.SetFloat(float64(val))
-			return nil
-		}
-		min, max, ok := intLimits(fieldVal.Kind())
-		if !ok {
-			return fmt.Errorf("field %q should have type int", field)
-		}
-		if val > max {
-			return fmt.Errorf("number %d is out of range for %s", val, fieldVal.Kind())
-		}
-		if min == 0 { // unsigned
-			fieldVal.SetUint(val)
-		} else {
-			fieldVal.SetInt(int64(val))
-		}
-	case int64:
+	case *integer:
 		switch fieldVal.Kind() {
 		case reflect.Float32, reflect.Float64:
-			fieldVal.SetFloat(float64(val))
+			fieldVal.SetFloat(float64(val.sgn) * float64(val.n))
 			return nil
 		}
 		min, max, ok := intLimits(fieldVal.Kind())
 		if !ok {
 			return fmt.Errorf("field %q should have type int", field)
 		}
-		if val < min || val > 0 && uint64(val) > max {
+		if val.sgn < 0 && val.n > min || val.sgn > 0 && val.n > max {
 			return fmt.Errorf("number %d is out of range for %s", val, fieldVal.Kind())
 		}
 		if min == 0 { // unsigned
-			fieldVal.SetUint(uint64(val))
+			fieldVal.SetUint(val.n)
 		} else {
-			fieldVal.SetInt(val)
+			fieldVal.SetInt(int64(val.sgn) * int64(val.n))
 		}
 	case float64:
 		switch fieldVal.Kind() {
@@ -684,7 +703,7 @@ func unpackVal(fieldVal reflect.Value, fieldMap map[structField]int, val any, fi
 	case []any:
 		return fmt.Errorf("invalid repeated field")
 	default:
-		return fmt.Errorf("unexpected AST node (internal failure)")
+		return fmt.Errorf("unexpected AST node (unreachable)")
 	}
 	return nil
 }
diff --git a/ccl_test.go b/ccl_test.go
index 628009d..938af09 100644
--- a/ccl_test.go
+++ b/ccl_test.go
@@ -260,6 +260,10 @@ can just span multiple lines"`,
 		msg:  `string: '\u2014'`,
 		want: message{String: "—"},
 	}, {
+		desc: "StringBigUnicode",
+		msg:  `string: '\U0001f600'`,
+		want: message{String: "😀"},
+	}, {
 		desc: "StringOctal",
 		msg:  `string: '\033'`,
 		want: message{String: "\033"},
@@ -357,6 +361,7 @@ func TestUnmarshal_Invalid(t *testing.T) {
 	type message struct {
 		Int            int64             `ccl:"int"`
 		Int8           int8              `ccl:"int8"`
+		Float          float64           `ccl:"float"`
 		String         string            `ccl:"string"`
 		Msg            nestedMessage     `ccl:"msg"`
 		Repeated       []int64           `ccl:"repeated"`
@@ -372,12 +377,33 @@ func TestUnmarshal_Invalid(t *testing.T) {
 		desc: "BadNum",
 		msg:  `int: .`,
 	}, {
+		desc: "WeirdNum",
+		msg:  `float:1e+`,
+	}, {
+		desc: "BadHex",
+		msg:  `int:0xgg`,
+	}, {
 		desc: "BadStringEscape",
 		msg:  `string: '\g'`,
 	}, {
 		desc: "BadDoubleStringEscape",
 		msg:  `string: "\g"`,
 	}, {
+		desc: "StringBadReturnEscape",
+		msg:  "string:'\\\r'",
+	}, {
+		desc: "StringShortHex",
+		msg:  `string:"\x1"`,
+	}, {
+		desc: "StringBadHex",
+		msg:  `string:"\xgg"`,
+	}, {
+		desc: "StringShortUnicode",
+		msg:  `string:"\u001"`,
+	}, {
+		desc: "StringBadUnicode",
+		msg:  `string:"\ugggg"`,
+	}, {
 		desc: "UnterminatedString",
 		msg:  `string: '`,
 	}, {
diff --git a/lexer.go b/lexer.go
index edc2444..c7c0631 100644
--- a/lexer.go
+++ b/lexer.go
@@ -1,8 +1,10 @@
 package ccl
 
 import (
+	"bytes"
 	"iter"
-	"regexp"
+	"unicode"
+	"unicode/utf8"
 )
 
 type token struct {
@@ -28,18 +30,50 @@ func (l *lexer) yield(n int) bool {
 	return true
 }
 
-var spaceRE = regexp.MustCompile(`^([[:space:]\p{Zs}]|(#|//)[^\n]*|/\*([^*]|\*[^/])*\*?\*/)*`)
-
 func (l *lexer) skipSpace() {
-	l.i += len(spaceRE.Find(l.data[l.i:]))
+	for l.i < len(l.data) {
+		if bytes.HasPrefix(l.data[l.i:], []byte("#")) || bytes.HasPrefix(l.data[l.i:], []byte("//")) {
+			for ; l.i < len(l.data) && l.data[l.i] != '\n'; l.i++ {
+			}
+			continue
+		}
+		if bytes.HasPrefix(l.data[l.i:], []byte("/*")) {
+			for ; l.i < len(l.data) && !bytes.HasPrefix(l.data[l.i:], []byte("*/")); l.i++ {
+			}
+			l.i += 2
+			continue
+		}
+		if r, n := utf8.DecodeRune(l.data[l.i:]); unicode.IsSpace(r) {
+			l.i += n
+			continue
+		}
+		break
+	}
 }
 
-var (
-	stringRE       = regexp.MustCompile(`(?s)^(([^'\\]|\\.)*)'`)
-	doubleStringRE = regexp.MustCompile(`(?s)^(([^"\\]|\\.)*)"`)
-	lexNumRE       = regexp.MustCompile(`^[-+.0-9][-+.0-9a-zA-Z]*`)
-	fieldRE        = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z_0-9]*`)
-)
+func numFirstByte(b byte) bool {
+	return b == '-' ||
+		b == '+' ||
+		b == '.' ||
+		'0' <= b && b <= '9'
+}
+
+func numTailByte(b byte) bool {
+	return numFirstByte(b) ||
+		'a' <= b && b <= 'z' ||
+		'A' <= b && b <= 'Z'
+}
+
+func fieldFirstByte(b byte) bool {
+	return b == '_' ||
+		'a' <= b && b <= 'z' ||
+		'A' <= b && b <= 'Z'
+}
+
+func fieldTailByte(b byte) bool {
+	return fieldFirstByte(b) ||
+		'0' <= b && b <= '9'
+}
 
 func (l *lexer) tokens() {
 	for l.i = 0; ; {
@@ -60,35 +94,37 @@ func (l *lexer) tokens() {
 				return
 			}
 			continue
-		case '\'':
-			str := stringRE.Find(l.data[l.i+1:])
-			if str == nil {
-				l.error("invalid string")
-				return
+		case '\'', '"':
+			q := l.data[l.i]
+			i := l.i + 1
+			for ; i < len(l.data) && l.data[i] != q; i++ {
+				if l.data[i] == '\\' {
+					i++
+				}
 			}
-			if !l.yield(1 + len(str)) {
+			if i >= len(l.data) {
+				l.error("unterminated string")
 				return
 			}
-			continue
-		case '"':
-			str := doubleStringRE.Find(l.data[l.i+1:])
-			if str == nil {
-				l.error("invalid string")
-				return
-			}
-			if !l.yield(1 + len(str)) {
+			if !l.yield(i + 1 - l.i) {
 				return
 			}
 			continue
 		}
-		if n := lexNumRE.Find(l.data[l.i:]); n != nil {
-			if !l.yield(len(n)) {
+		switch b := l.data[l.i]; {
+		case numFirstByte(b):
+			i := l.i + 1
+			for ; i < len(l.data) && numTailByte(l.data[i]); i++ {
+			}
+			if !l.yield(i - l.i) {
 				return
 			}
 			continue
-		}
-		if n := fieldRE.Find(l.data[l.i:]); n != nil {
-			if !l.yield(len(n)) {
+		case fieldFirstByte(b):
+			i := l.i + 1
+			for ; i < len(l.data) && fieldTailByte(l.data[i]); i++ {
+			}
+			if !l.yield(i - l.i) {
 				return
 			}
 			continue