diff options
| -rw-r--r-- | ccl.go | 271 | ||||
| -rw-r--r-- | ccl_test.go | 26 | ||||
| -rw-r--r-- | lexer.go | 94 |
3 files changed, 236 insertions, 155 deletions
@@ -167,7 +167,6 @@ import ( "iter" "math" "reflect" - "regexp" "strconv" "strings" "unicode/utf8" @@ -284,121 +283,159 @@ func (p *parser) next() ([]byte, error) { return tok, nil } -var ( - numRE = regexp.MustCompile(`^[-+]?(0[xX][0-9a-fA-F]+|((0|[1-9][0-9]*)(\.[0-9]*)?|\.[0-9]+)([eE][-+]?[0-9]+)?)$`) - hexRE = regexp.MustCompile(`^([-+]?)0[xX]`) -) +func checkNum(b []byte) bool { + if bytes.Equal(b, []byte("0")) { + return true + } + if len(b) == 0 || !(b[0] == '.' || '1' <= b[0] && b[0] <= '9') { + return false + } + haveDigits := false + for ; len(b) > 0 && '0' <= b[0] && b[0] <= '9'; b = b[1:] { + haveDigits = true + } + if len(b) > 0 && b[0] == '.' { + b = b[1:] + for ; len(b) > 0 && '0' <= b[0] && b[0] <= '9'; b = b[1:] { + haveDigits = true + } + } + if !haveDigits { + return false + } + if len(b) == 0 || !(b[0] == 'e' || b[0] == 'E') { + return true + } + b = b[1:] + if len(b) > 0 && b[0] == '-' || b[0] == '+' { + b = b[1:] + } + if len(b) == 0 { + return false + } + for ; len(b) > 0 && '0' <= b[0] && b[0] <= '9'; b = b[1:] { + } + return len(b) == 0 +} + +type integer struct { + n uint64 + sgn int8 +} func (p *parser) parseNum(numBytes []byte) (any, error) { - if !numRE.Match(numBytes) { - return nil, p.error("invalid number") + n := numBytes + var sgn int8 = 1 + switch numBytes[0] { + case '-': + sgn = -1 + n = numBytes[1:] + case '+': + n = numBytes[1:] } - if hex := hexRE.FindSubmatch(numBytes); hex != nil { - if string(hex[1]) == "-" { - n, err := strconv.ParseInt(string(numBytes[len(hex[0]):]), 16, 64) - if err != nil { - return nil, p.error("invalid number") - } - return -n, nil - } else { - n, err := strconv.ParseUint(string(numBytes[len(hex[0]):]), 16, 64) - if err != nil { - return nil, p.error("invalid number") - } - return n, nil + if len(n) > 2 && n[0] == '0' && (n[1] == 'x' || n[1] == 'X') { + n, err := strconv.ParseUint(string(n[2:]), 16, 64) + if err != nil { + return nil, p.error("invalid hex number: %s", err) } + return &integer{n, sgn}, nil + } + if !checkNum(n) { + return nil, p.error("invalid number") } - if bytes.ContainsAny(numBytes, ".eE") { + if bytes.ContainsAny(n, ".eE") { n, err := strconv.ParseFloat(string(numBytes), 64) if err != nil { - return nil, p.error("invalid number") + return nil, p.error("invalid number (unreachable)") } return n, nil } - if bytes.HasPrefix(numBytes, []byte("-")) { - n, err := strconv.ParseInt(string(numBytes), 10, 64) - if err != nil { - return nil, p.error("invalid number") - } - return n, nil - } else { - n, err := strconv.ParseUint(string(bytes.TrimPrefix(numBytes, []byte("+"))), 10, 64) - if err != nil { - return nil, p.error("invalid number") - } - return n, nil + un, err := strconv.ParseUint(string(n), 10, 64) + if err != nil { + return nil, p.error("invalid number (unreachable)") } -} - -var escapesRE = regexp.MustCompile(`(?s)\\(.|\r\n|[0-7]{3}|x[0-9a-fA-F]{2}|u[0-9a-fA-F]{4}|U[0-9a-fA-F]{8})`) - -func init() { - escapesRE.Longest() + return &integer{un, sgn}, nil } func (p *parser) unescape(rawStr []byte) ([]byte, error) { - var err error - escaped := escapesRE.ReplaceAllFunc(rawStr, func(escape []byte) []byte { - switch string(escape) { - case `\'`: - return []byte("'") - case `\"`: - return []byte(`"`) - case `\?`: - return []byte("?") - case `\\`: - return []byte(`\`) - case `\a`: - return []byte("\a") - case `\b`: - return []byte("\b") - case `\f`: - return []byte("\f") - case `\n`: - return []byte("\n") - case `\r`: - return []byte("\r") - case `\t`: - return []byte("\t") - case `\v`: - return []byte("\v") - case "\\\n", "\\\r\n": - return nil + var escaped []byte + for i := 0; i < len(rawStr); i++ { + if rawStr[i] != '\\' { + escaped = append(escaped, rawStr[i]) + continue } - switch { - case bytes.HasPrefix(escape, []byte(`\x`)): - var n uint64 - if n, err = strconv.ParseUint(string(escape[2:]), 16, 8); err != nil { - err = p.error("invalid hex escape %q: %s", escape, err) - return nil + i++ + var b []byte + switch rawStr[i] { + case '\'': + b = []byte("'") + case '"': + b = []byte(`"`) + case '?': + b = []byte("?") + case '\\': + b = []byte(`\`) + case 'a': + b = []byte("\a") + case 'b': + b = []byte("\b") + case 'f': + b = []byte("\f") + case 'n': + b = []byte("\n") + case 'r': + b = []byte("\r") + case 't': + b = []byte("\t") + case 'v': + b = []byte("\v") + case '\n': + b = nil + case '\r': + i++ + if i < len(rawStr) && rawStr[i] == '\n' { + b = nil + } else { + return nil, fmt.Errorf("invalid escape sequence %q", rawStr[i-2:min(i+1, len(rawStr))]) } - return []byte{byte(n)} - case bytes.HasPrefix(escape, []byte(`\u`)), bytes.HasPrefix(escape, []byte(`\U`)): - var n int64 - if n, err = strconv.ParseInt(string(escape[2:]), 16, 32); err != nil { - err = p.error("invalid unicode escape %q: %s", escape, err) - return nil + case 'x': + i++ + if i+2 > len(rawStr) { + return nil, fmt.Errorf("invalid hex escape %q", rawStr[i-2:min(i+2, len(rawStr))]) } - return utf8.AppendRune(nil, rune(n)) - default: - if len(escape) != 4 { - err = p.error("invalid string escape %q", escape) - return nil + n, err := strconv.ParseUint(string(rawStr[i:i+2]), 16, 8) + if err != nil { + return nil, fmt.Errorf("invalid hex escape %q: %s", rawStr[i-2:i+2], err) } - var n int64 - if n, err = strconv.ParseInt(string(escape[1:]), 8, 32); err != nil { - err = p.error("invalid string escape %q", escape) - return nil + i++ + b = []byte{byte(n)} + case 'u', 'U': + nBytes := 4 + if rawStr[i] == 'U' { + nBytes = 8 } - if n > 255 { - err = p.error("invalid octal escape %q %d > 255", escape, n) - return nil + i++ + if i+nBytes > len(rawStr) { + return nil, fmt.Errorf("invalid unicode escape %q", rawStr[i-2:min(i+nBytes, len(rawStr))]) } - return []byte{byte(n)} + n, err := strconv.ParseUint(string(rawStr[i:i+nBytes]), 16, 31) + if err != nil { + return nil, fmt.Errorf("invalid hex escape %q: %s", rawStr[i-2:i+2], err) + } + i += nBytes - 1 + b = utf8.AppendRune(nil, rune(n)) + default: + if i+3 > len(rawStr) { + return nil, fmt.Errorf("invalid string escape %q", rawStr[i-1:i+1]) + } + n, err := strconv.ParseUint(string(rawStr[i:i+3]), 8, 8) + if err != nil { + return nil, fmt.Errorf("invalid octal escape %q: %s", rawStr[i:i+3], err) + } + i += 2 + b = []byte{byte(n)} } - }) - if err != nil { - return nil, err + escaped = append(escaped, b...) } if !utf8.Valid(escaped) { return nil, p.error("syntax error: string %q is not UTF-8 encoded", escaped) @@ -499,7 +536,7 @@ func appendAny(prevVal any, newVal any) any { var l []any if ll, ok := prevVal.([]any); ok { l = ll - } else if prevVal != nil { + } else { l = []any{prevVal} } if ll, ok := newVal.([]any); ok { @@ -555,18 +592,18 @@ func (p *parser) parse() (map[string]any, error) { } } -func intLimits(kind reflect.Kind) (min int64, max uint64, ok bool) { +func intLimits(kind reflect.Kind) (min, max uint64, ok bool) { switch kind { case reflect.Int: - return math.MinInt, math.MaxInt, true + return -math.MinInt, math.MaxInt, true case reflect.Int8: - return math.MinInt8, math.MaxInt8, true + return -math.MinInt8, math.MaxInt8, true case reflect.Int16: - return math.MinInt16, math.MaxInt16, true + return -math.MinInt16, math.MaxInt16, true case reflect.Int32: - return math.MinInt32, math.MaxInt32, true + return -math.MinInt32, math.MaxInt32, true case reflect.Int64: - return math.MinInt64, math.MaxInt64, true + return -math.MinInt64, math.MaxInt64, true case reflect.Uint: return 0, math.MaxUint, true case reflect.Uint8: @@ -603,41 +640,23 @@ func unpackVal(fieldVal reflect.Value, fieldMap map[structField]int, val any, fi default: return fmt.Errorf("field %q should have type bool", field) } - case uint64: - switch fieldVal.Kind() { - case reflect.Float32, reflect.Float64: - fieldVal.SetFloat(float64(val)) - return nil - } - min, max, ok := intLimits(fieldVal.Kind()) - if !ok { - return fmt.Errorf("field %q should have type int", field) - } - if val > max { - return fmt.Errorf("number %d is out of range for %s", val, fieldVal.Kind()) - } - if min == 0 { // unsigned - fieldVal.SetUint(val) - } else { - fieldVal.SetInt(int64(val)) - } - case int64: + case *integer: switch fieldVal.Kind() { case reflect.Float32, reflect.Float64: - fieldVal.SetFloat(float64(val)) + fieldVal.SetFloat(float64(val.sgn) * float64(val.n)) return nil } min, max, ok := intLimits(fieldVal.Kind()) if !ok { return fmt.Errorf("field %q should have type int", field) } - if val < min || val > 0 && uint64(val) > max { + if val.sgn < 0 && val.n > min || val.sgn > 0 && val.n > max { return fmt.Errorf("number %d is out of range for %s", val, fieldVal.Kind()) } if min == 0 { // unsigned - fieldVal.SetUint(uint64(val)) + fieldVal.SetUint(val.n) } else { - fieldVal.SetInt(val) + fieldVal.SetInt(int64(val.sgn) * int64(val.n)) } case float64: switch fieldVal.Kind() { @@ -684,7 +703,7 @@ func unpackVal(fieldVal reflect.Value, fieldMap map[structField]int, val any, fi case []any: return fmt.Errorf("invalid repeated field") default: - return fmt.Errorf("unexpected AST node (internal failure)") + return fmt.Errorf("unexpected AST node (unreachable)") } return nil } diff --git a/ccl_test.go b/ccl_test.go index 628009d..938af09 100644 --- a/ccl_test.go +++ b/ccl_test.go @@ -260,6 +260,10 @@ can just span multiple lines"`, msg: `string: '\u2014'`, want: message{String: "—"}, }, { + desc: "StringBigUnicode", + msg: `string: '\U0001f600'`, + want: message{String: "😀"}, + }, { desc: "StringOctal", msg: `string: '\033'`, want: message{String: "\033"}, @@ -357,6 +361,7 @@ func TestUnmarshal_Invalid(t *testing.T) { type message struct { Int int64 `ccl:"int"` Int8 int8 `ccl:"int8"` + Float float64 `ccl:"float"` String string `ccl:"string"` Msg nestedMessage `ccl:"msg"` Repeated []int64 `ccl:"repeated"` @@ -372,12 +377,33 @@ func TestUnmarshal_Invalid(t *testing.T) { desc: "BadNum", msg: `int: .`, }, { + desc: "WeirdNum", + msg: `float:1e+`, + }, { + desc: "BadHex", + msg: `int:0xgg`, + }, { desc: "BadStringEscape", msg: `string: '\g'`, }, { desc: "BadDoubleStringEscape", msg: `string: "\g"`, }, { + desc: "StringBadReturnEscape", + msg: "string:'\\\r'", + }, { + desc: "StringShortHex", + msg: `string:"\x1"`, + }, { + desc: "StringBadHex", + msg: `string:"\xgg"`, + }, { + desc: "StringShortUnicode", + msg: `string:"\u001"`, + }, { + desc: "StringBadUnicode", + msg: `string:"\ugggg"`, + }, { desc: "UnterminatedString", msg: `string: '`, }, { @@ -1,8 +1,10 @@ package ccl import ( + "bytes" "iter" - "regexp" + "unicode" + "unicode/utf8" ) type token struct { @@ -28,18 +30,50 @@ func (l *lexer) yield(n int) bool { return true } -var spaceRE = regexp.MustCompile(`^([[:space:]\p{Zs}]|(#|//)[^\n]*|/\*([^*]|\*[^/])*\*?\*/)*`) - func (l *lexer) skipSpace() { - l.i += len(spaceRE.Find(l.data[l.i:])) + for l.i < len(l.data) { + if bytes.HasPrefix(l.data[l.i:], []byte("#")) || bytes.HasPrefix(l.data[l.i:], []byte("//")) { + for ; l.i < len(l.data) && l.data[l.i] != '\n'; l.i++ { + } + continue + } + if bytes.HasPrefix(l.data[l.i:], []byte("/*")) { + for ; l.i < len(l.data) && !bytes.HasPrefix(l.data[l.i:], []byte("*/")); l.i++ { + } + l.i += 2 + continue + } + if r, n := utf8.DecodeRune(l.data[l.i:]); unicode.IsSpace(r) { + l.i += n + continue + } + break + } } -var ( - stringRE = regexp.MustCompile(`(?s)^(([^'\\]|\\.)*)'`) - doubleStringRE = regexp.MustCompile(`(?s)^(([^"\\]|\\.)*)"`) - lexNumRE = regexp.MustCompile(`^[-+.0-9][-+.0-9a-zA-Z]*`) - fieldRE = regexp.MustCompile(`^[a-zA-Z_][a-zA-Z_0-9]*`) -) +func numFirstByte(b byte) bool { + return b == '-' || + b == '+' || + b == '.' || + '0' <= b && b <= '9' +} + +func numTailByte(b byte) bool { + return numFirstByte(b) || + 'a' <= b && b <= 'z' || + 'A' <= b && b <= 'Z' +} + +func fieldFirstByte(b byte) bool { + return b == '_' || + 'a' <= b && b <= 'z' || + 'A' <= b && b <= 'Z' +} + +func fieldTailByte(b byte) bool { + return fieldFirstByte(b) || + '0' <= b && b <= '9' +} func (l *lexer) tokens() { for l.i = 0; ; { @@ -60,35 +94,37 @@ func (l *lexer) tokens() { return } continue - case '\'': - str := stringRE.Find(l.data[l.i+1:]) - if str == nil { - l.error("invalid string") - return + case '\'', '"': + q := l.data[l.i] + i := l.i + 1 + for ; i < len(l.data) && l.data[i] != q; i++ { + if l.data[i] == '\\' { + i++ + } } - if !l.yield(1 + len(str)) { + if i >= len(l.data) { + l.error("unterminated string") return } - continue - case '"': - str := doubleStringRE.Find(l.data[l.i+1:]) - if str == nil { - l.error("invalid string") - return - } - if !l.yield(1 + len(str)) { + if !l.yield(i + 1 - l.i) { return } continue } - if n := lexNumRE.Find(l.data[l.i:]); n != nil { - if !l.yield(len(n)) { + switch b := l.data[l.i]; { + case numFirstByte(b): + i := l.i + 1 + for ; i < len(l.data) && numTailByte(l.data[i]); i++ { + } + if !l.yield(i - l.i) { return } continue - } - if n := fieldRE.Find(l.data[l.i:]); n != nil { - if !l.yield(len(n)) { + case fieldFirstByte(b): + i := l.i + 1 + for ; i < len(l.data) && fieldTailByte(l.data[i]); i++ { + } + if !l.yield(i - l.i) { return } continue |