scanner: add keyword handling (copied off the old scanner)

as with most things, lol

src/compiler.zig

@@ -23,7 +23,7 @@ pub const Compiler = struct {
     }
 
     pub fn compile(self: *Compiler) !void {
-        var scanr = scanner.Scanner.init(self.allocator, self.src);
+        var scanr = try scanner.Scanner.init(self.allocator, self.src);
         var line: usize = 0;
         while (true) {
             var token_opt = scanr.scanToken() catch |err| {

src/new_scanner.zig

@@ -11,8 +11,76 @@ pub const TokenError = error{
     Unterminated,
 };
 
+fn isDigit(char: u8) bool {
+    return char >= '0' and char <= '9';
+}
+
+fn isAlpha(c: u8) bool {
+    return (c >= 'a' and c <= 'z') or
+        (c >= 'A' and c <= 'Z') or
+        c == '_';
+}
+
+fn isAlphaNumeric(char: u8) bool {
+    return isAlpha(char) or isDigit(char);
+}
+
+pub const KeywordMap = std.AutoHashMap([]const u8, u6);
+
+/// The book does say that C doesn't have hashmaps. but Zig does. and I can
+/// use it here.
+fn initKeywordMap(allocator: *std.mem.Allocator) !KeywordMap {
+    var map = KeywordMap.init(allocator);
+
+    const keywords = [][]const u8{
+        "and"[0..],
+        "class"[0..],
+        "else"[0..],
+        "false"[0..],
+        "for"[0..],
+        "fun"[0..],
+        "if"[0..],
+        "nil"[0..],
+        "or"[0..],
+        "print"[0..],
+        "return"[0..],
+        "super"[0..],
+        "this"[0..],
+        "true"[0..],
+        "var"[0..],
+        "while"[0..],
+    };
+
+    const tags = []TokenType{
+        TokenType.AND,
+        TokenType.CLASS,
+        TokenType.ELSE,
+        TokenType.FALSE,
+        TokenType.FOR,
+        TokenType.FUN,
+        TokenType.IF,
+        TokenType.NIL,
+        TokenType.OR,
+        TokenType.PRINT,
+        TokenType.RETURN,
+        TokenType.SUPER,
+        TokenType.THIS,
+        TokenType.TRUE,
+        TokenType.VAR,
+        TokenType.WHILE,
+    };
+
+    for (keywords) |keyword, idx| {
+        var tag = @enumToInt(tags[idx]);
+        _ = try map.put(keyword, tag);
+    }
+
+    return map;
+}
+
 pub const Scanner = struct {
     source: []const u8,
+    keywords: KeywordMap,
 
     start: usize = 0,
     current: usize = 0,
@@ -20,10 +88,11 @@ pub const Scanner = struct {
 
     allocator: *Allocator,
 
-    pub fn init(allocator: *Allocator, data: []const u8) Scanner {
+    pub fn init(allocator: *Allocator, data: []const u8) !Scanner {
         return Scanner{
-            .allocator = allocator,
             .source = data,
+            .keywords = try initKeywordMap(allocator),
+            .allocator = allocator,
         };
     }
 
@@ -117,12 +186,55 @@ pub const Scanner = struct {
         return self.makeToken(.STRING);
     }
 
+    /// Consume a number
+    fn doNumber(self: *Scanner) Token {
+        while (isDigit(self.peek())) {
+            _ = self.advance();
+        }
+
+        // check if its a number like 12.34, where the '.' character
+        // exists and the one next to it is a digit.
+        if (self.peek() == '.' and isDigit(self.peekNext())) {
+            _ = self.advance();
+
+            while (isDigit(self.peek())) {
+                _ = self.advance();
+            }
+        }
+
+        return self.makeToken(.NUMBER);
+    }
+
+    /// Either a keyword or an identifier come out of this.
+    fn doIdentifier(self: *Scanner) Token {
+        while (isAlphaNumeric(self.peek())) {
+            _ = self.advance();
+        }
+
+        // after reading the identifier, we check
+        // if it is any of our keywords, if it is, then we add
+        // the specificed keyword type. if not, just .IDENTIFIER
+        var text = self.source[self.start..self.current];
+        var type_opt = self.keywords.get(text);
+        var toktype: TokenType = undefined;
+
+        if (type_opt) |kv| {
+            toktype = @intToEnum(TokenType, kv.value);
+        } else {
+            toktype = TokenType.IDENTIFIER;
+        }
+
+        return self.makeToken(toktype);
+    }
+
     pub fn scanToken(self: *Scanner) !?Token {
         self.skipWhitespace();
         self.start = self.current;
         if (self.isAtEnd()) return self.makeToken(TokenType.EOF);
 
         var c = self.advance();
+        if (isAlpha(c)) return self.doIdentifier();
+        if (isDigit(c)) return self.doNumber();
 
         var token = switch (c) {
             '(' => self.makeToken(.LEFT_PAREN),

src/scanner.zig

@@ -20,7 +20,6 @@ fn isAlphaNumeric(char: u8) bool {
     return isAlpha(char) or isDigit(char);
 }
 
-// hashmaps don't work on HashMaps for some reason. anyways.
 pub const KeywordMap = std.AutoHashMap([]const u8, u6);
 
 fn initKeywordMap(allocator: *std.mem.Allocator) !KeywordMap {