diff --git a/src/scanner.zig b/src/scanner.zig index 48fd2f9..2b7b46b 100644 --- a/src/scanner.zig +++ b/src/scanner.zig @@ -116,16 +116,10 @@ pub const Scanner = struct { }; } - fn makeTokenLexeme( - self: *Scanner, - ttype: TokenType, - lexeme: []const u8, - ) Token { - return Token{ - .ttype = ttype, - .lexeme = lexeme, - .line = self.line, - }; + fn makeTokenAdvance(self: *Scanner, ttype: TokenType) Token { + var tok = self.makeToken(ttype); + self.current += 1; + return tok; } /// Check if the next character matches what is expected. @@ -204,48 +198,6 @@ pub const Scanner = struct { return self.makeToken(ttype); } - /// Consume a string. stop_char is used to determine - /// if the string is a single quote or double quote string - fn doString(self: *Scanner, stop_char: u8) !Token { - // consume entire string - while (self.peekNext() != stop_char and !self.isAtEnd()) { - if (self.peek() == '\n') self.line += 1; - _ = self.advance(); - } - - // unterminated string. - if (self.isAtEnd()) { - return ScannerError.Unterminated; - } - - // the closing character of the string - _ = self.advance(); - - // remove the starting and ending chars of the string - const lexeme = self.currentLexeme(); - return self.makeTokenLexeme( - .String, - lexeme[1 .. lexeme.len - 1], - ); - } - - fn makeTripleMatchToken( - self: *Scanner, - char1: u8, - ttype1: TokenType, - char2, - ttype2: TokenType, - fallback: TokenType, - ) Token { - if (self.match(char1)) { - return self.makeToken(ttype1); - } else if (self.match(char2)) { - return self.makeToken(ttype2); - } else { - return self.makeToken(fallback); - } - } - pub fn nextToken(self: *Scanner) !?Token { self.skipWhitespace(); self.start = self.current; @@ -272,14 +224,26 @@ pub const Scanner = struct { '$' => self.makeToken(.DollarSign), '-' => self.makeToken(.Minus), + '+' => self.makeToken(.Plus), '*' => self.makeToken(.Star), - '%' => self.makeToken(.Modulo), '!' => self.makeMatchToken('=', .BangEqual, .Bang), '=' => self.makeMatchToken('=', .EqualEqual, .Equal), + + // there can be three tokens from a < + // - <, which is LessThan + // - <=, which is LessEqual + // - <<, which is LeftDoubleChevron + '<' => blk: { + if (self.match('=')) { + break :blk self.makeToken(.LessEqual); + } else if (self.match('<')) { + break :blk self.makeToken(.LeftDoubleChevron); + } else { + break :blk self.makeToken(.Less); + } + }, '>' => self.makeMatchToken('=', .GreaterEqual, .Greater), - '+' => self.makeTripleMatchToken('+', .PlusPlus, '=', .PlusEqual, .Plus), - '<' => self.makeTripleMatchToken('=', .LessEqual, '<', .LeftDoubleChevron, .Less), '/' => blk: { var next = self.peekNext(); @@ -306,9 +270,6 @@ pub const Scanner = struct { } }, - '\'' => try self.doString('\''), - '"' => try self.doString('"'), - else => return ScannerError.Unexpected, };