diff --git a/src/compiler.zig b/src/compiler.zig index f0771ff..fd96032 100644 --- a/src/compiler.zig +++ b/src/compiler.zig @@ -23,7 +23,7 @@ pub const Compiler = struct { } pub fn compile(self: *Compiler) !void { - var scanr = scanner.Scanner.init(self.allocator, self.src); + var scanr = try scanner.Scanner.init(self.allocator, self.src); var line: usize = 0; while (true) { var token_opt = scanr.scanToken() catch |err| { diff --git a/src/new_scanner.zig b/src/new_scanner.zig index b2005f7..8a1c436 100644 --- a/src/new_scanner.zig +++ b/src/new_scanner.zig @@ -11,8 +11,76 @@ pub const TokenError = error{ Unterminated, }; +fn isDigit(char: u8) bool { + return char >= '0' and char <= '9'; +} + +fn isAlpha(c: u8) bool { + return (c >= 'a' and c <= 'z') or + (c >= 'A' and c <= 'Z') or + c == '_'; +} + +fn isAlphaNumeric(char: u8) bool { + return isAlpha(char) or isDigit(char); +} + +pub const KeywordMap = std.AutoHashMap([]const u8, u6); + +/// The book does say that C doesn't have hashmaps. but Zig does. and I can +/// use it here. +fn initKeywordMap(allocator: *std.mem.Allocator) !KeywordMap { + var map = KeywordMap.init(allocator); + + const keywords = [][]const u8{ + "and"[0..], + "class"[0..], + "else"[0..], + "false"[0..], + "for"[0..], + "fun"[0..], + "if"[0..], + "nil"[0..], + "or"[0..], + "print"[0..], + "return"[0..], + "super"[0..], + "this"[0..], + "true"[0..], + "var"[0..], + "while"[0..], + }; + + const tags = []TokenType{ + TokenType.AND, + TokenType.CLASS, + TokenType.ELSE, + TokenType.FALSE, + TokenType.FOR, + TokenType.FUN, + TokenType.IF, + TokenType.NIL, + TokenType.OR, + TokenType.PRINT, + TokenType.RETURN, + TokenType.SUPER, + TokenType.THIS, + TokenType.TRUE, + TokenType.VAR, + TokenType.WHILE, + }; + + for (keywords) |keyword, idx| { + var tag = @enumToInt(tags[idx]); + _ = try map.put(keyword, tag); + } + + return map; +} + pub const Scanner = struct { source: []const u8, + keywords: KeywordMap, start: usize = 0, current: usize = 0, @@ -20,10 +88,11 @@ pub const Scanner = struct { allocator: *Allocator, - pub fn init(allocator: *Allocator, data: []const u8) Scanner { + pub fn init(allocator: *Allocator, data: []const u8) !Scanner { return Scanner{ - .allocator = allocator, .source = data, + .keywords = try initKeywordMap(allocator), + .allocator = allocator, }; } @@ -117,12 +186,55 @@ pub const Scanner = struct { return self.makeToken(.STRING); } + /// Consume a number + fn doNumber(self: *Scanner) Token { + while (isDigit(self.peek())) { + _ = self.advance(); + } + + // check if its a number like 12.34, where the '.' character + // exists and the one next to it is a digit. + if (self.peek() == '.' and isDigit(self.peekNext())) { + _ = self.advance(); + + while (isDigit(self.peek())) { + _ = self.advance(); + } + } + + return self.makeToken(.NUMBER); + } + + /// Either a keyword or an identifier come out of this. + fn doIdentifier(self: *Scanner) Token { + while (isAlphaNumeric(self.peek())) { + _ = self.advance(); + } + + // after reading the identifier, we check + // if it is any of our keywords, if it is, then we add + // the specificed keyword type. if not, just .IDENTIFIER + var text = self.source[self.start..self.current]; + var type_opt = self.keywords.get(text); + var toktype: TokenType = undefined; + + if (type_opt) |kv| { + toktype = @intToEnum(TokenType, kv.value); + } else { + toktype = TokenType.IDENTIFIER; + } + + return self.makeToken(toktype); + } + pub fn scanToken(self: *Scanner) !?Token { self.skipWhitespace(); self.start = self.current; if (self.isAtEnd()) return self.makeToken(TokenType.EOF); var c = self.advance(); + if (isAlpha(c)) return self.doIdentifier(); + if (isDigit(c)) return self.doNumber(); var token = switch (c) { '(' => self.makeToken(.LEFT_PAREN), diff --git a/src/scanner.zig b/src/scanner.zig index ed5ec0e..b83a7a4 100644 --- a/src/scanner.zig +++ b/src/scanner.zig @@ -20,7 +20,6 @@ fn isAlphaNumeric(char: u8) bool { return isAlpha(char) or isDigit(char); } -// hashmaps don't work on HashMaps for some reason. anyways. pub const KeywordMap = std.AutoHashMap([]const u8, u6); fn initKeywordMap(allocator: *std.mem.Allocator) !KeywordMap {