From d1db7a0bd976416ea5f38744f0075fa2d47ba56e Mon Sep 17 00:00:00 2001 From: Luna Date: Fri, 31 May 2019 21:46:18 -0300 Subject: [PATCH] add consumption of comments and strings --- src/main.zig | 10 +++++- src/scanner.zig | 88 +++++++++++++++++++++++++++++++++++++++++++++++++ src/token.zig | 26 +++++++++++---- 3 files changed, 116 insertions(+), 8 deletions(-) diff --git a/src/main.zig b/src/main.zig index 71cfd6b..3423204 100644 --- a/src/main.zig +++ b/src/main.zig @@ -14,7 +14,15 @@ fn run(allocator: *Allocator, data: []u8) !void { var it = tokens.iterator(); while (it.next()) |token| { - try token.Simple.printToken(stdout); + switch (token) { + .Simple => |value| { + try value.printToken(stdout); + }, + .Slice => |value| { + try value.printToken(stdout); + }, + } + hadError = false; } } diff --git a/src/scanner.zig b/src/scanner.zig index c4f9100..8bbc5a8 100644 --- a/src/scanner.zig +++ b/src/scanner.zig @@ -44,6 +44,17 @@ pub const Scanner = struct { }); } + fn addSliceToken(self: *Scanner, ttype: token.TokenType, slice: []u8) !void { + try self.addToken(token.Token{ + .Slice = token.SliceToken.init( + ttype, + self.currentLexeme(), + self.line, + slice, + ), + }); + } + fn addToken( self: *Scanner, tok: token.Token, @@ -51,6 +62,59 @@ pub const Scanner = struct { try self.tokens.append(tok); } + /// Check if the next character matches what is expected. + fn match(self: *Scanner, expected: u8) bool { + if (self.isAtEnd()) return false; + if (self.source[self.current] != expected) return false; + + self.current += 1; + return true; + } + + /// Add a SimpleToken of type_match if the next character is + /// `expected`. Adds a SimpleToken of type_nomatch when it is not. + fn addMatchToken( + self: *Scanner, + expected: u8, + type_match: token.TokenType, + type_nomatch: token.TokenType, + ) !void { + if (self.match(expected)) { + try self.addSimpleToken(type_match); + } else { + try self.addSimpleToken(type_nomatch); + } + } + + fn peek(self: *Scanner) u8 { + if (self.isAtEnd()) return 0; + return self.source[self.current]; + } + + fn doString(self: *Scanner) !void { + // consume entire string + while (self.peek() != '"' and !self.isAtEnd()) { + if (self.peek() == '\n') self.line += 1; + _ = self.advance(); + } + + // unterminated string. + if (self.isAtEnd()) { + try main.doError(self.line, "Unterminated string."); + return; + } + + // the closing ". + _ = self.advance(); + + // trim the surrounding quotes. + try self.addSliceToken( + .STRING, + self.source[self.start + 1 .. self.current - 1], + ); + } + + /// Scan through our tokens and add them to the Scanner's token list. fn scanToken(self: *Scanner) !void { var c = self.advance(); @@ -65,6 +129,30 @@ pub const Scanner = struct { '+' => try self.addSimpleToken(.PLUS), ';' => try self.addSimpleToken(.SEMICOLON), '*' => try self.addSimpleToken(.STAR), + + '!' => try self.addMatchToken('=', .BANG_EQUAL, .BANG), + '=' => try self.addMatchToken('=', .EQUAL_EQUAL, .EQUAL), + '<' => try self.addMatchToken('=', .LESS_EQUAL, .LESS), + '>' => try self.addMatchToken('=', .GREATER_EQUAL, .GREATER), + + '/' => blk: { + // consume comments + if (self.match('/')) { + while (self.peek() != '\n' and !self.isAtEnd()) { + _ = self.advance(); + } + } else { + try self.addSimpleToken(.SLASH); + } + }, + + ' ', '\r', '\t' => blk: {}, + '\n' => blk: { + self.line += 1; + }, + + '"' => try self.doString(), + else => { try main.doError(self.line, "Unexpected character"); }, diff --git a/src/token.zig b/src/token.zig index 0ce1283..4460dad 100644 --- a/src/token.zig +++ b/src/token.zig @@ -76,18 +76,30 @@ pub fn TokenFactory( } pub fn printToken(self: Self, stdout: var) !void { - try stdout.print( - "Token(type={x}, lexeme='{}', line={}\n", - self.ttype, - self.lexeme, - self.line, - ); + if (T == void) { + try stdout.print( + "Token(type={x}, lexeme='{}', line={})\n", + self.ttype, + self.lexeme, + self.line, + ); + } else { + try stdout.print( + "Token(type={x}, lexeme='{}', line={} literal='{}')\n", + self.ttype, + self.lexeme, + self.line, + self.literal, + ); + } } }; } pub const SimpleToken = TokenFactory(void); +pub const SliceToken = TokenFactory([]u8); -pub const Token = union { +pub const Token = union(enum) { Simple: SimpleToken, + Slice: SliceToken, };