From 285f0b841068b333b2bea821f084a531f5052617 Mon Sep 17 00:00:00 2001 From: Luna Date: Tue, 4 Jun 2019 17:52:37 -0300 Subject: [PATCH 1/2] add string support --- src/scanner.zig | 42 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/src/scanner.zig b/src/scanner.zig index 2b7b46b..0c74a70 100644 --- a/src/scanner.zig +++ b/src/scanner.zig @@ -116,10 +116,16 @@ pub const Scanner = struct { }; } - fn makeTokenAdvance(self: *Scanner, ttype: TokenType) Token { - var tok = self.makeToken(ttype); - self.current += 1; - return tok; + fn makeTokenLexeme( + self: *Scanner, + ttype: TokenType, + lexeme: []const u8, + ) Token { + return Token{ + .ttype = ttype, + .lexeme = lexeme, + .line = self.line, + }; } /// Check if the next character matches what is expected. @@ -198,6 +204,31 @@ pub const Scanner = struct { return self.makeToken(ttype); } + /// Consume a string. stop_char is used to determine + /// if the string is a single quote or double quote string + fn doString(self: *Scanner, stop_char: u8) !Token { + // consume entire string + while (self.peekNext() != stop_char and !self.isAtEnd()) { + if (self.peek() == '\n') self.line += 1; + _ = self.advance(); + } + + // unterminated string. + if (self.isAtEnd()) { + return ScannerError.Unterminated; + } + + // the closing character of the string + _ = self.advance(); + + // remove the starting and ending chars of the string + const lexeme = self.currentLexeme(); + return self.makeTokenLexeme( + .String, + lexeme[1 .. lexeme.len - 1], + ); + } + pub fn nextToken(self: *Scanner) !?Token { self.skipWhitespace(); self.start = self.current; @@ -270,6 +301,9 @@ pub const Scanner = struct { } }, + '\'' => try self.doString('\''), + '"' => try self.doString('"'), + else => return ScannerError.Unexpected, }; From 498ea72da43a139ad9882d57af7b0aadbc32370c Mon Sep 17 00:00:00 2001 From: Luna Date: Tue, 4 Jun 2019 18:07:46 -0300 Subject: [PATCH 2/2] add support for ++ and += --- src/scanner.zig | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/src/scanner.zig b/src/scanner.zig index 0c74a70..48fd2f9 100644 --- a/src/scanner.zig +++ b/src/scanner.zig @@ -229,6 +229,23 @@ pub const Scanner = struct { ); } + fn makeTripleMatchToken( + self: *Scanner, + char1: u8, + ttype1: TokenType, + char2, + ttype2: TokenType, + fallback: TokenType, + ) Token { + if (self.match(char1)) { + return self.makeToken(ttype1); + } else if (self.match(char2)) { + return self.makeToken(ttype2); + } else { + return self.makeToken(fallback); + } + } + pub fn nextToken(self: *Scanner) !?Token { self.skipWhitespace(); self.start = self.current; @@ -255,26 +272,14 @@ pub const Scanner = struct { '$' => self.makeToken(.DollarSign), '-' => self.makeToken(.Minus), - '+' => self.makeToken(.Plus), '*' => self.makeToken(.Star), + '%' => self.makeToken(.Modulo), '!' => self.makeMatchToken('=', .BangEqual, .Bang), '=' => self.makeMatchToken('=', .EqualEqual, .Equal), - - // there can be three tokens from a < - // - <, which is LessThan - // - <=, which is LessEqual - // - <<, which is LeftDoubleChevron - '<' => blk: { - if (self.match('=')) { - break :blk self.makeToken(.LessEqual); - } else if (self.match('<')) { - break :blk self.makeToken(.LeftDoubleChevron); - } else { - break :blk self.makeToken(.Less); - } - }, '>' => self.makeMatchToken('=', .GreaterEqual, .Greater), + '+' => self.makeTripleMatchToken('+', .PlusPlus, '=', .PlusEqual, .Plus), + '<' => self.makeTripleMatchToken('=', .LessEqual, '<', .LeftDoubleChevron, .Less), '/' => blk: { var next = self.peekNext();