lexer: add floats

src/lexer/mod.rs

@@ -131,6 +131,7 @@ pub enum TokenKind {
 #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord)]
 pub enum Value {
     Int,
+    Float,
     Str(String),
 }
 
@@ -371,24 +372,27 @@ impl Cursor<'_> {
     }
 
     fn number(&mut self) -> TokenKind {
-        match self.first() {
+        let value_kind = match self.first() {
             'b' => {
                 self.bump();
                 self.eat_binary_digits();
+                Value::Int
             }
             'o' => {
                 self.bump();
                 self.eat_octal_digits();
+                Value::Int
             }
             'x' => {
                 self.bump();
                 self.eat_hex_digits();
+                Value::Int
             }
             _ => {
-                self.eat_digits();
+                self.eat_digits()
             }
         };
-        TokenKind::Literal(Value::Int)
+        TokenKind::Literal(value_kind)
     }
 
     fn string(&mut self, end: char) -> Result<TokenKind, String> {
@@ -434,21 +438,21 @@ impl Cursor<'_> {
         TokenKind::Comment
     }
 
-    fn eat_digits(&mut self) -> bool {
-        let mut has_digits = false;
+    fn eat_digits(&mut self) -> Value {
+        let mut value_kind = Value::Int;
         loop {
             match self.first() {
-                '_' => {
+                '0'..='9' | '_' => {
                     self.bump();
                 }
-                '0'..='9' => {
-                    has_digits = true;
+                '.' => {
+                    value_kind = Value::Float;
                     self.bump();
                 }
                 _ => break,
             }
         }
-        has_digits
+        value_kind
     }
 
     fn eat_binary_digits(&mut self) -> bool {

src/lexer/tests.rs

@@ -342,3 +342,42 @@ fn fib() {}
         }
     );
 }
+
+#[test]
+fn test_floats() {
+    let mut tokens = tokenize("1.23").unwrap().into_iter();
+
+    assert_eq!(
+        tokens.next().unwrap(),
+        Token {
+            len: 4,
+            kind: TokenKind::Literal(Value::Float),
+            raw: "1.23".to_owned(),
+            pos: Position {
+                raw: 3,
+                line: 1,
+                offset: 3
+            }
+        }
+    );
+}
+
+#[test]
+#[ignore]
+fn test_negative_floats() {
+    let mut tokens = tokenize("-1.23").unwrap().into_iter();
+
+    assert_eq!(
+        tokens.next().unwrap(),
+        Token {
+            len: 4,
+            kind: TokenKind::Literal(Value::Float),
+            raw: "-1.23".to_owned(),
+            pos: Position {
+                raw: 4,
+                line: 1,
+                offset: 4
+            }
+        }
+    );
+}