From b1e37119da439b52a2c1f1aca4df2fdc80462f97 Mon Sep 17 00:00:00 2001
From: Rose Hogenson <rosehogenson@posteo.net>
Date: Thu, 30 Oct 2025 11:04:23 -0700
Subject: Remove the regexes
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Wow it doesn't look good for regex

goos: linux
goarch: amd64
pkg: roseh.moe/pkg/ccl
cpu: AMD Ryzen 9 5900X 12-Core Processor
         │ baseline.txt │            regexn't.txt             │
         │    sec/op    │   sec/op     vs base                │
Lex-24      8.625µ ± 1%   1.429µ ± 0%  -83.43% (p=0.000 n=20)
Parse-24   18.305µ ± 0%   8.856µ ± 0%  -51.62% (p=0.000 n=20)
geomean     12.57µ        3.557µ       -71.69%
---
 lexer.go | 94 ++++++++++++++++++++++++++++++++++++++++++++--------------------
 1 file changed, 65 insertions(+), 29 deletions(-)

(limited to 'lexer.go')

diff --git a/lexer.go b/lexer.go
index edc2444..c7c0631 100644
--- a/lexer.go
+++ b/lexer.go
@@ -1,8 +1,10 @@
 package ccl
 
 import (
+	"bytes"
 	"iter"
-	"regexp"
+	"unicode"
+	"unicode/utf8"
 )
 
 type token struct {
@@ -28,18 +30,50 @@ func (l *lexer) yield(n int) bool {
 	return true
 }
 
-var spaceRE = regexp.MustCompile(`^([[:space:]\p{Zs}]|(#|//)[^\n]*|/\*([^*]|\*[^/])*\*?\*/)*`)
-
 func (l *lexer) skipSpace() {
-	l.i += len(spaceRE.Find(l.data[l.i:]))
+	for l.i < len(l.data) {
+		if bytes.HasPrefix(l.data[l.i:], []byte("#")) || bytes.HasPrefix(l.data[l.i:], []byte("//")) {
+			for ; l.i < len(l.data) && l.data[l.i] != '\n'; l.i++ {
+			}
+			continue
+		}
+		if bytes.HasPrefix(l.data[l.i:], []byte("/*")) {
+			for ; l.i < len(l.data) && !bytes.HasPrefix(l.data[l.i:], []byte("*/")); l.i++ {
+			}
+			l.i += 2
+			continue
+		}
+		if r, n := utf8.DecodeRune(l.data[l.i:]); unicode.IsSpace(r) {
+			l.i += n
+			continue
+		}
+		break
+	}
 }
 
-var (
-	stringRE       = regexp.MustCompile(`(?s)^(([^'\\]|\\.)*)'`)
-	doubleStringRE = regexp.MustCompile(`(?s)^(([^"\\]|\\.)*)"`)
-	lexNumRE       = regexp.MustCompile(`^[-+.0-9][-+.0-9a-zA-Z]*`)
-	fieldRE        = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z_0-9]*`)
-)
+func numFirstByte(b byte) bool {
+	return b == '-' ||
+		b == '+' ||
+		b == '.' ||
+		'0' <= b && b <= '9'
+}
+
+func numTailByte(b byte) bool {
+	return numFirstByte(b) ||
+		'a' <= b && b <= 'z' ||
+		'A' <= b && b <= 'Z'
+}
+
+func fieldFirstByte(b byte) bool {
+	return b == '_' ||
+		'a' <= b && b <= 'z' ||
+		'A' <= b && b <= 'Z'
+}
+
+func fieldTailByte(b byte) bool {
+	return fieldFirstByte(b) ||
+		'0' <= b && b <= '9'
+}
 
 func (l *lexer) tokens() {
 	for l.i = 0; ; {
@@ -60,35 +94,37 @@ func (l *lexer) tokens() {
 				return
 			}
 			continue
-		case '\'':
-			str := stringRE.Find(l.data[l.i+1:])
-			if str == nil {
-				l.error("invalid string")
-				return
+		case '\'', '"':
+			q := l.data[l.i]
+			i := l.i + 1
+			for ; i < len(l.data) && l.data[i] != q; i++ {
+				if l.data[i] == '\\' {
+					i++
+				}
 			}
-			if !l.yield(1 + len(str)) {
+			if i >= len(l.data) {
+				l.error("unterminated string")
 				return
 			}
-			continue
-		case '"':
-			str := doubleStringRE.Find(l.data[l.i+1:])
-			if str == nil {
-				l.error("invalid string")
-				return
-			}
-			if !l.yield(1 + len(str)) {
+			if !l.yield(i + 1 - l.i) {
 				return
 			}
 			continue
 		}
-		if n := lexNumRE.Find(l.data[l.i:]); n != nil {
-			if !l.yield(len(n)) {
+		switch b := l.data[l.i]; {
+		case numFirstByte(b):
+			i := l.i + 1
+			for ; i < len(l.data) && numTailByte(l.data[i]); i++ {
+			}
+			if !l.yield(i - l.i) {
 				return
 			}
 			continue
-		}
-		if n := fieldRE.Find(l.data[l.i:]); n != nil {
-			if !l.yield(len(n)) {
+		case fieldFirstByte(b):
+			i := l.i + 1
+			for ; i < len(l.data) && fieldTailByte(l.data[i]); i++ {
+			}
+			if !l.yield(i - l.i) {
 				return
 			}
 			continue
-- 
cgit v1.3.1