2 files changed, 31 insertions, 27 deletions
diff --git a/src/lexer.rs b/src/lexer.rs
index 4130760..397dfda 100644
--- a/src/lexer.rs
+++ b/src/lexer.rs
@@ -5,11 +5,11 @@ pub struct Lexer<'a> {
 impl<'a> Lexer<'a> {
     fn skip_whitespace(&mut self) {
         while let Some(b' ') = self.buf.as_bytes().first() {
-            self.buf = &self.buf[1..]
+            self.buf = &self.buf[1..];
         }
     }
 
-    fn parse_num(&mut self) -> Option<&'a str> {
+    fn parse_num(&mut self) -> &'a str {
         let mut i = 0;
         while matches!(self.buf.as_bytes().get(i), Some(&b) if b.is_ascii_digit()) {
             i += 1;
@@ -23,7 +23,7 @@ impl<'a> Lexer<'a> {
         let (Some(b'e') | Some(b'E')) = self.buf.as_bytes().get(i) else {
             let t = &self.buf[..i];
             self.buf = &self.buf[i..];
-            return Some(t);
+            return t;
         };
         i += 1;
         if let Some(b'-') = self.buf.as_bytes().get(i) {
@@ -34,7 +34,7 @@ impl<'a> Lexer<'a> {
         }
         let t = &self.buf[..i];
         self.buf = &self.buf[i..];
-        Some(t)
+        t
     }
 }
 
@@ -43,32 +43,31 @@ impl<'a> Iterator for Lexer<'a> {
 
     fn next(&mut self) -> Option<&'a str> {
         self.skip_whitespace();
-        let &first_byte = self.buf.as_bytes().first()?;
-        if first_byte.is_ascii_digit() || first_byte == b'.' {
-            return self.parse_num();
+        if self.buf.starts_with("//") {
+            self.buf = &self.buf[2..];
+            return Some("//");
         }
-        match first_byte {
-            b'/' => {
-                if let Some(b'/') = self.buf.as_bytes().get(1) {
-                    self.buf = &self.buf[2..];
-                    return Some("//");
+        Some(match self.buf.as_bytes().first()? {
+            b'0'..=b'9' | b'.' => self.parse_num(),
+            b'a'..=b'z' | b'A'..=b'Z' => {
+                let mut i = 1;
+                while let Some(b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9') = self.buf.as_bytes().get(i)
+                {
+                    i += 1;
                 }
-                self.buf = &self.buf[1..];
-                return Some("/");
+                let t = &self.buf[..i];
+                self.buf = &self.buf[i..];
+                t
             }
-            b'+' | b'-' | b'*' | b'%' | b'^' | b'(' | b')' => {
-                let t = &self.buf[..1];
-                self.buf = &self.buf[1..];
-                return Some(t);
+            _ => {
+                let mut i = 1;
+                while !self.buf.is_char_boundary(i) {
+                    i += 1;
+                }
+                let t = &self.buf[..i];
+                self.buf = &self.buf[i..];
+                t
             }
-            _ => (),
-        }
-        let mut i = 0;
-        while let Some(b'a'..=b'z' | b'A'..=b'Z' | b'0'..=b'9') = self.buf.as_bytes().get(i) {
-            i += 1;
-        }
-        let t = &self.buf[..i];
-        self.buf = &self.buf[i..];
-        Some(t)
+        })
     }
 }
diff --git a/src/parser.rs b/src/parser.rs
index 4cd75be..a31b4f9 100644
--- a/src/parser.rs
+++ b/src/parser.rs
@@ -489,4 +489,9 @@ mod tests {
     fn parse_unmatched_op() {
         assert_eq!(parse("2^2+"), None);
     }
+
+    #[test]
+    fn parse_unicode() {
+        assert_eq!(parse("你好"), None);
+    }
 }