scanner: fix peekNext

src/compiler.zig

@@ -26,17 +26,29 @@ pub const Compiler = struct {
         var scanr = scanner.Scanner.init(self.allocator, self.src);
         var line: usize = 0;
         while (true) {
-            var token = scanr.scanToken();
+            var token_opt = scanr.scanToken() catch |err| {
+                std.debug.warn("Scan Error: {x}\n", err);
+                std.debug.warn(
+                    "line: {}, cur lexeme: {}\n",
+                    scanr.line,
+                    scanr.currentLexeme(),
+                );
+                break;
+            };
 
-            if (token.line != line) {
-                try self.stdout.print("{} ", token.line);
-                line = token.line;
+            if (token_opt) |token| {
+                if (token.line != line) {
+                    try self.stdout.print("{} ", token.line);
+                    line = token.line;
+                } else {
+                    try self.stdout.print("  | ");
+                }
+
+                try self.stdout.print("{} '{}'\n", token.ttype, token.lexeme);
+                if (token.ttype == TokenType.EOF) break;
             } else {
-                try self.stdout.print("  | ");
+                break;
             }
-
-            try self.stdout.print("{} '{}'\n", token.ttype, token.lexeme);
-            if (token.ttype == TokenType.EOF) break;
         }
     }
 };

src/new_scanner.zig

@@ -1,8 +1,16 @@
 const std = @import("std");
 const tokens = @import("token.zig");
 
+const Token = tokens.Token;
+const TokenType = tokens.TokenType;
+
 const Allocator = std.mem.Allocator;
 
+pub const TokenError = error{
+    Unexpected,
+    Unterminated,
+};
+
 pub const Scanner = struct {
     source: []const u8,
 
@@ -19,7 +27,136 @@ pub const Scanner = struct {
         };
     }
 
-    pub fn scanToken(self: *Scanner) tokens.Token {
-        return tokens.Token{};
+    fn isAtEnd(self: *Scanner) bool {
+        return self.current >= self.source.len;
+    }
+
+    fn advance(self: *Scanner) u8 {
+        self.current += 1;
+        return self.source[self.current - 1];
+    }
+
+    pub fn currentLexeme(self: *Scanner) []const u8 {
+        return self.source[self.start..self.current];
+    }
+
+    fn makeToken(self: *Scanner, ttype: TokenType) Token {
+        return Token{
+            .ttype = ttype,
+            .lexeme = self.currentLexeme(),
+            .line = self.line,
+        };
+    }
+
+    /// Check if the next character matches what is expected.
+    fn match(self: *Scanner, expected: u8) bool {
+        if (self.isAtEnd()) return false;
+        if (self.source[self.current] != expected) return false;
+
+        self.current += 1;
+        return true;
+    }
+
+    /// Add a SimpleToken of type_match if the next character is
+    /// `expected`. Adds a SimpleToken of type_nomatch when it is not.
+    fn makeMatchToken(
+        self: *Scanner,
+        expected: u8,
+        type_match: TokenType,
+        type_nomatch: TokenType,
+    ) Token {
+        if (self.match(expected)) {
+            return self.makeToken(type_match);
+        } else {
+            return self.makeToken(type_nomatch);
+        }
+    }
+
+    fn peek(self: *Scanner) u8 {
+        if (self.isAtEnd()) return 0;
+        return self.source[self.current];
+    }
+
+    fn peekNext(self: *Scanner) u8 {
+        if (self.isAtEnd()) return 0;
+        return self.source[self.current - 1];
+    }
+
+    fn skipWhitespace(self: *Scanner) void {
+        while (true) {
+            var c = self.peek();
+            switch (c) {
+                ' ', '\r', '\t' => blk: {
+                    _ = self.advance();
+                },
+                '\n' => blk: {
+                    self.line += 1;
+                    _ = self.advance();
+                },
+                else => return,
+            }
+        }
+    }
+
+    fn doString(self: *Scanner) !Token {
+        // consume entire string
+        while (self.peek() != '"' and !self.isAtEnd()) {
+            if (self.peek() == '\n') self.line += 1;
+            _ = self.advance();
+        }
+
+        // unterminated string.
+        if (self.isAtEnd()) {
+            return TokenError.Unterminated;
+        }
+
+        // the closing ".
+        _ = self.advance();
+
+        // trim the surrounding quotes.
+        return self.makeToken(.STRING);
+    }
+
+    pub fn scanToken(self: *Scanner) !?Token {
+        self.skipWhitespace();
+        self.start = self.current;
+        if (self.isAtEnd()) return self.makeToken(TokenType.EOF);
+
+        var c = self.advance();
+
+        var token = switch (c) {
+            '(' => self.makeToken(.LEFT_PAREN),
+            ')' => self.makeToken(.RIGHT_PAREN),
+            '{' => self.makeToken(.LEFT_BRACE),
+            '}' => self.makeToken(.RIGHT_BRACE),
+            ',' => self.makeToken(.COMMA),
+            '.' => self.makeToken(.DOT),
+            '-' => self.makeToken(.MINUS),
+            '+' => self.makeToken(.PLUS),
+            ';' => self.makeToken(.SEMICOLON),
+            '*' => self.makeToken(.STAR),
+
+            '!' => self.makeMatchToken('=', .BANG_EQUAL, .BANG),
+            '=' => self.makeMatchToken('=', .EQUAL_EQUAL, .EQUAL),
+            '<' => self.makeMatchToken('=', .LESS_EQUAL, .LESS),
+            '>' => self.makeMatchToken('=', .GREATER_EQUAL, .GREATER),
+
+            '/' => blk: {
+                if (self.peekNext() == '/') {
+                    while (self.peek() != '\n' and !self.isAtEnd()) {
+                        _ = self.advance();
+                    }
+                    break :blk null;
+                } else {
+                    break :blk self.makeToken(.SLASH);
+                }
+            },
+
+            '"' => try self.doString(),
+
+            else => return TokenError.Unexpected,
+        };
+
+        return token;
     }
 };

src/scanner.zig

@@ -98,7 +98,7 @@ pub const Scanner = struct {
         return self.source[self.current - 1];
     }
 
-    fn currentLexeme(self: *Scanner) []u8 {
+    pub fn currentLexeme(self: *Scanner) []u8 {
         return self.source[self.start..self.current];
     }