scanner: add basic error handling, strings, comments

src/compiler.zig

@@ -26,8 +26,17 @@ pub const Compiler = struct {
         var scanr = scanner.Scanner.init(self.allocator, self.src);
         var line: usize = 0;
         while (true) {
-            var token = try scanr.scanToken();
+            var token_opt = scanr.scanToken() catch |err| {
+                std.debug.warn("Scan Error: {x}\n", err);
+                std.debug.warn(
+                    "line: {}, cur lexeme: {}\n",
+                    scanr.line,
+                    scanr.currentLexeme(),
+                );
+                break;
+            };
 
+            if (token_opt) |token| {
                 if (token.line != line) {
                     try self.stdout.print("{} ", token.line);
                     line = token.line;
@@ -37,6 +46,9 @@ pub const Compiler = struct {
 
                 try self.stdout.print("{} '{}'\n", token.ttype, token.lexeme);
                 if (token.ttype == TokenType.EOF) break;
+            } else {
+                break;
+            }
         }
     }
 };

src/new_scanner.zig

@@ -36,10 +36,14 @@ pub const Scanner = struct {
         return self.source[self.current - 1];
     }
 
+    pub fn currentLexeme(self: *Scanner) []const u8 {
+        return self.source[self.start..self.current];
+    }
+
     fn makeToken(self: *Scanner, ttype: TokenType) Token {
         return Token{
             .ttype = ttype,
-            .lexeme = self.source[self.start..self.current],
+            .lexeme = self.currentLexeme(),
             .line = self.line,
         };
     }
@@ -73,6 +77,11 @@ pub const Scanner = struct {
         return self.source[self.current];
     }
 
+    fn peekNext(self: *Scanner) u8 {
+        if (self.isAtEnd()) return 0;
+        return self.source[self.current + 1];
+    }
+
     fn skipWhitespace(self: *Scanner) void {
         while (true) {
             var c = self.peek();
@@ -89,7 +98,26 @@ pub const Scanner = struct {
         }
     }
 
-    pub fn scanToken(self: *Scanner) !tokens.Token {
+    fn doString(self: *Scanner) !Token {
+        // consume entire string
+        while (self.peek() != '"' and !self.isAtEnd()) {
+            if (self.peek() == '\n') self.line += 1;
+            _ = self.advance();
+        }
+
+        // unterminated string.
+        if (self.isAtEnd()) {
+            return TokenError.Unterminated;
+        }
+
+        // the closing ".
+        _ = self.advance();
+
+        // trim the surrounding quotes.
+        return self.makeToken(.STRING);
+    }
+
+    pub fn scanToken(self: *Scanner) !?Token {
         self.skipWhitespace();
         self.start = self.current;
         if (self.isAtEnd()) return self.makeToken(TokenType.EOF);
@@ -113,6 +141,19 @@ pub const Scanner = struct {
             '<' => self.makeMatchToken('=', .LESS_EQUAL, .LESS),
             '>' => self.makeMatchToken('=', .GREATER_EQUAL, .GREATER),
 
+            '/' => blk: {
+                if (self.peekNext() == '/') {
+                    while (self.peek() != '\n' and !self.isAtEnd()) {
+                        _ = self.advance();
+                    }
+                    break :blk null;
+                } else {
+                    break :blk self.makeToken(.SLASH);
+                }
+            },
+
+            '"' => try self.doString(),
+
             else => return TokenError.Unexpected,
         };
 

src/scanner.zig

@@ -98,7 +98,7 @@ pub const Scanner = struct {
         return self.source[self.current - 1];
     }
 
-    fn currentLexeme(self: *Scanner) []u8 {
+    pub fn currentLexeme(self: *Scanner) []u8 {
         return self.source[self.start..self.current];
     }