const std = @import("std"); const tokens = @import("token.zig"); const Token = tokens.Token; const TokenType = tokens.TokenType; const Allocator = std.mem.Allocator; pub const TokenError = error{ Unexpected, Unterminated, }; fn isDigit(char: u8) bool { return char >= '0' and char <= '9'; } fn isAlpha(c: u8) bool { return (c >= 'a' and c <= 'z') or (c >= 'A' and c <= 'Z') or c == '_'; } fn isAlphaNumeric(char: u8) bool { return isAlpha(char) or isDigit(char); } pub const Scanner = struct { source: []const u8, start: usize = 0, current: usize = 0, line: usize = 1, allocator: *Allocator, pub fn init(allocator: *Allocator, data: []const u8) Scanner { return Scanner{ .allocator = allocator, .source = data, }; } fn isAtEnd(self: *Scanner) bool { return self.current >= self.source.len; } fn advance(self: *Scanner) u8 { self.current += 1; return self.source[self.current - 1]; } pub fn currentLexeme(self: *Scanner) []const u8 { return self.source[self.start..self.current]; } fn makeToken(self: *Scanner, ttype: TokenType) Token { return Token{ .ttype = ttype, .lexeme = self.currentLexeme(), .line = self.line, }; } /// Check if the next character matches what is expected. fn match(self: *Scanner, expected: u8) bool { if (self.isAtEnd()) return false; if (self.source[self.current] != expected) return false; self.current += 1; return true; } /// Add a SimpleToken of type_match if the next character is /// `expected`. Adds a SimpleToken of type_nomatch when it is not. fn makeMatchToken( self: *Scanner, expected: u8, type_match: TokenType, type_nomatch: TokenType, ) Token { if (self.match(expected)) { return self.makeToken(type_match); } else { return self.makeToken(type_nomatch); } } fn peek(self: *Scanner) u8 { if (self.isAtEnd()) return 0; return self.source[self.current]; } fn peekNext(self: *Scanner) u8 { if (self.isAtEnd()) return 0; return self.source[self.current - 1]; } fn skipWhitespace(self: *Scanner) void { while (true) { var c = self.peek(); switch (c) { ' ', '\r', '\t' => blk: { _ = self.advance(); }, '\n' => blk: { self.line += 1; _ = self.advance(); }, else => return, } } } fn doString(self: *Scanner) !Token { // consume entire string while (self.peek() != '"' and !self.isAtEnd()) { if (self.peek() == '\n') self.line += 1; _ = self.advance(); } // unterminated string. if (self.isAtEnd()) { return TokenError.Unterminated; } // the closing ". _ = self.advance(); // trim the surrounding quotes. return self.makeToken(.STRING); } /// Consume a number fn doNumber(self: *Scanner) Token { while (isDigit(self.peek())) { _ = self.advance(); } // check if its a number like 12.34, where the '.' character // exists and the one next to it is a digit. if (self.peek() == '.' and isDigit(self.peekNext())) { _ = self.advance(); while (isDigit(self.peek())) { _ = self.advance(); } } return self.makeToken(.NUMBER); } pub fn scanToken(self: *Scanner) !?Token { self.skipWhitespace(); self.start = self.current; if (self.isAtEnd()) return self.makeToken(TokenType.EOF); var c = self.advance(); if (isDigit(c)) return self.doNumber(); var token = switch (c) { '(' => self.makeToken(.LEFT_PAREN), ')' => self.makeToken(.RIGHT_PAREN), '{' => self.makeToken(.LEFT_BRACE), '}' => self.makeToken(.RIGHT_BRACE), ',' => self.makeToken(.COMMA), '.' => self.makeToken(.DOT), '-' => self.makeToken(.MINUS), '+' => self.makeToken(.PLUS), ';' => self.makeToken(.SEMICOLON), '*' => self.makeToken(.STAR), '!' => self.makeMatchToken('=', .BANG_EQUAL, .BANG), '=' => self.makeMatchToken('=', .EQUAL_EQUAL, .EQUAL), '<' => self.makeMatchToken('=', .LESS_EQUAL, .LESS), '>' => self.makeMatchToken('=', .GREATER_EQUAL, .GREATER), '/' => blk: { if (self.peekNext() == '/') { while (self.peek() != '\n' and !self.isAtEnd()) { _ = self.advance(); } break :blk null; } else { break :blk self.makeToken(.SLASH); } }, '"' => try self.doString(), else => return TokenError.Unexpected, }; return token; } };