diff --git a/src/main.zig b/src/main.zig index 3423204..3dd56e1 100644 --- a/src/main.zig +++ b/src/main.zig @@ -9,7 +9,7 @@ fn run(allocator: *Allocator, data: []u8) !void { var stdout_file = try std.io.getStdOut(); const stdout = &stdout_file.outStream().stream; - var scanner = Scanner.init(allocator, data); + var scanner = try Scanner.init(allocator, data); var tokens = try scanner.scanTokens(); var it = tokens.iterator(); @@ -21,6 +21,9 @@ fn run(allocator: *Allocator, data: []u8) !void { .Slice => |value| { try value.printToken(stdout); }, + .Number => |value| { + try value.printToken(stdout); + }, } hadError = false; diff --git a/src/scanner.zig b/src/scanner.zig index 8bbc5a8..f2073d1 100644 --- a/src/scanner.zig +++ b/src/scanner.zig @@ -4,19 +4,88 @@ const token = @import("token.zig"); const main = @import("main.zig"); const TokenList = std.ArrayList(token.Token); +const TokenType = token.TokenType; + +fn isDigit(char: u8) bool { + return char >= '0' and char <= '9'; +} + +fn isAlpha(c: u8) bool { + return (c >= 'a' and c <= 'z') or + (c >= 'A' and c <= 'Z') or + c == '_'; +} + +fn isAlphaNumeric(char: u8) bool { + return isAlpha(char) or isDigit(char); +} + +// hashmaps don't work on HashMaps for some reason. anyways. +pub const KeywordMap = std.AutoHashMap([]const u8, u6); + +fn initKeywordMap(allocator: *std.mem.Allocator) !KeywordMap { + var map = KeywordMap.init(allocator); + + const keywords = [][]const u8{ + "and"[0..], + "class"[0..], + "else"[0..], + "false"[0..], + "for"[0..], + "fun"[0..], + "if"[0..], + "nil"[0..], + "or"[0..], + "print"[0..], + "return"[0..], + "super"[0..], + "this"[0..], + "true"[0..], + "var"[0..], + "while"[0..], + }; + + const tags = []TokenType{ + TokenType.AND, + TokenType.CLASS, + TokenType.ELSE, + TokenType.FALSE, + TokenType.FOR, + TokenType.FUN, + TokenType.IF, + TokenType.NIL, + TokenType.OR, + TokenType.PRINT, + TokenType.RETURN, + TokenType.SUPER, + TokenType.THIS, + TokenType.TRUE, + TokenType.VAR, + TokenType.WHILE, + }; + + for (keywords) |keyword, idx| { + var tag = @enumToInt(tags[idx]); + _ = try map.put(keyword, tag); + } + + return map; +} pub const Scanner = struct { source: []u8, tokens: TokenList, + keywords: KeywordMap, start: usize = 0, current: usize = 0, line: usize = 1, - pub fn init(allocator: *std.mem.Allocator, data: []u8) Scanner { + pub fn init(allocator: *std.mem.Allocator, data: []u8) !Scanner { return Scanner{ .source = data, .tokens = TokenList.init(allocator), + .keywords = try initKeywordMap(allocator), }; } @@ -55,6 +124,18 @@ pub const Scanner = struct { }); } + /// Keep in mind Lox only has a single number type and that is a float one. + fn addNumberToken(self: *Scanner, ttype: token.TokenType, num: f32) !void { + try self.addToken(token.Token{ + .Number = token.NumberToken.init( + ttype, + self.currentLexeme(), + self.line, + num, + ), + }); + } + fn addToken( self: *Scanner, tok: token.Token, @@ -114,6 +195,58 @@ pub const Scanner = struct { ); } + fn peekNext(self: *Scanner) u8 { + if (self.current + 1 >= self.source.len) return 0; + return self.source[self.current + 1]; + } + + /// Consume a number + fn doNumber(self: *Scanner) !void { + while (isDigit(self.peek())) { + _ = self.advance(); + } + + // check if its a number like 12.34, where the '.' character + // exists and the one next to it is a digit. + if (self.peek() == '.' and isDigit(self.peekNext())) { + _ = self.advance(); + + while (isDigit(self.peek())) { + _ = self.advance(); + } + } + + // after going through all of the number, we can just use fmt.parseFloat + + var num = try std.fmt.parseFloat( + f32, + self.source[self.start..self.current], + ); + + try self.addNumberToken(.NUMBER, num); + } + + fn doIdentifier(self: *Scanner) !void { + while (isAlphaNumeric(self.peek())) { + _ = self.advance(); + } + + // after reading the identifier, we check + // if it is any of our keywords, if it is, then we add + // the specificed keyword type. if not, just .IDENTIFIER + var text = self.source[self.start..self.current]; + var type_opt = self.keywords.get(text); + var toktype: TokenType = undefined; + + if (type_opt) |kv| { + toktype = @intToEnum(TokenType, kv.value); + } else { + toktype = TokenType.IDENTIFIER; + } + + try self.addSimpleToken(toktype); + } + /// Scan through our tokens and add them to the Scanner's token list. fn scanToken(self: *Scanner) !void { var c = self.advance(); @@ -154,7 +287,13 @@ pub const Scanner = struct { '"' => try self.doString(), else => { - try main.doError(self.line, "Unexpected character"); + if (isDigit(c)) { + try self.doNumber(); + } else if (isAlpha(c)) { + try self.doIdentifier(); + } else { + try main.doError(self.line, "Unexpected character"); + } }, } } diff --git a/src/token.zig b/src/token.zig index 4460dad..9e88ee1 100644 --- a/src/token.zig +++ b/src/token.zig @@ -1,6 +1,6 @@ const std = @import("std"); -pub const TokenType = enum { +pub const TokenType = enum(u6) { // Single-character tokens. LEFT_PAREN, RIGHT_PAREN, @@ -98,8 +98,10 @@ pub fn TokenFactory( pub const SimpleToken = TokenFactory(void); pub const SliceToken = TokenFactory([]u8); +pub const NumberToken = TokenFactory(f32); pub const Token = union(enum) { Simple: SimpleToken, Slice: SliceToken, + Number: NumberToken, };