scanner: finish off with identifiers and keywords

src/scanner.zig

@@ -152,17 +152,39 @@ pub const Scanner = struct {
         }
     }
 
+    /// "triple" version of makeMatchToken.
+    /// Required per vlang's tokens.
+    fn makeTripleMatchToken(
+        self: *Scanner,
+        char1: u8,
+        ttype1: TokenType,
+        char2: u8,
+        ttype2: TokenType,
+        fallback: TokenType,
+    ) Token {
+        if (self.match(char1)) {
+            return self.makeToken(ttype1);
+        } else if (self.match(char2)) {
+            return self.makeToken(ttype2);
+        } else {
+            return self.makeToken(fallback);
+        }
+    }
+
+    /// Peek at the current character in the scanner
     fn peek(self: *Scanner) u8 {
         if (self.isAtEnd()) return 0;
         if (self.current == 0) return 0;
         return self.source[self.current - 1];
     }
 
+    /// Peek at the next character in the scanner
     fn peekNext(self: *Scanner) u8 {
         if (self.current > self.source.len) return 0;
         return self.source[self.current];
     }
 
+    /// Skip all whitespace, but increments Scanner.line when finding a newline.
     fn skipWhitespace(self: *Scanner) void {
         while (true) {
             var c = self.peek();
@@ -229,21 +251,25 @@ pub const Scanner = struct {
         );
     }
 
-    fn makeTripleMatchToken(
+    /// Either a keyword or an identifier come out of this.
-        self: *Scanner,
+    fn doIdentifier(self: *Scanner) Token {
-        char1: u8,
+        while (isAlphaNumeric(self.peek())) {
-        ttype1: TokenType,
+            _ = self.advance();
-        char2,
-        ttype2: TokenType,
-        fallback: TokenType,
-    ) Token {
-        if (self.match(char1)) {
-            return self.makeToken(ttype1);
-        } else if (self.match(char2)) {
-            return self.makeToken(ttype2);
-        } else {
-            return self.makeToken(fallback);
         }
+
+        // after reading the identifier, we check
+        // if it is any of our keywords, if it is, then we add
+        // the specificed keyword type. if not, just .IDENTIFIER
+        var toktype: TokenType = undefined;
+        var ttype_opt = getKeyword(self.currentLexeme());
+
+        if (ttype_opt) |ttype| {
+            toktype = ttype;
+        } else {
+            toktype = TokenType.Identifier;
+        }
+
+        return self.makeToken(toktype);
     }
 
     pub fn nextToken(self: *Scanner) !?Token {
@@ -254,6 +280,7 @@ pub const Scanner = struct {
 
         var c = self.advance();
         if (isDigit(c)) return self.doNumber();
+        if (isAlpha(c)) return self.doIdentifier();
 
         var token: ?Token = switch (c) {
             '(' => self.makeToken(.LeftParen),