scanner: add basic tokens and matched-tokens

src/compiler.zig

@@ -26,7 +26,7 @@ pub const Compiler = struct {
         var scanr = scanner.Scanner.init(self.allocator, self.src);
         var line: usize = 0;
         while (true) {
-            var token = scanr.scanToken();
+            var token = try scanr.scanToken();
 
             if (token.line != line) {
                 try self.stdout.print("{} ", token.line);

src/new_scanner.zig

@@ -1,8 +1,16 @@
 const std = @import("std");
 const tokens = @import("token.zig");
 
+const Token = tokens.Token;
+const TokenType = tokens.TokenType;
+
 const Allocator = std.mem.Allocator;
 
+pub const TokenError = error{
+    Unexpected,
+    Unterminated,
+};
+
 pub const Scanner = struct {
     source: []const u8,
 
@@ -19,7 +27,74 @@ pub const Scanner = struct {
         };
     }
 
-    pub fn scanToken(self: *Scanner) tokens.Token {
-        return tokens.Token{};
+    fn isAtEnd(self: *Scanner) bool {
+        return self.current >= self.source.len;
+    }
+
+    fn advance(self: *Scanner) u8 {
+        self.current += 1;
+        return self.source[self.current - 1];
+    }
+
+    fn makeToken(self: *Scanner, ttype: TokenType) Token {
+        return Token{
+            .ttype = ttype,
+            .lexeme = self.source[self.start..self.current],
+            .line = self.line,
+        };
+    }
+
+    /// Check if the next character matches what is expected.
+    fn match(self: *Scanner, expected: u8) bool {
+        if (self.isAtEnd()) return false;
+        if (self.source[self.current] != expected) return false;
+
+        self.current += 1;
+        return true;
+    }
+
+    /// Add a SimpleToken of type_match if the next character is
+    /// `expected`. Adds a SimpleToken of type_nomatch when it is not.
+    fn makeMatchToken(
+        self: *Scanner,
+        expected: u8,
+        type_match: TokenType,
+        type_nomatch: TokenType,
+    ) Token {
+        if (self.match(expected)) {
+            return self.makeToken(type_match);
+        } else {
+            return self.makeToken(type_nomatch);
+        }
+    }
+
+    pub fn scanToken(self: *Scanner) !tokens.Token {
+        self.start = self.current;
+
+        if (self.isAtEnd()) return self.makeToken(TokenType.EOF);
+
+        var c = self.advance();
+
+        var token = switch (c) {
+            '(' => self.makeToken(.LEFT_PAREN),
+            ')' => self.makeToken(.RIGHT_PAREN),
+            '{' => self.makeToken(.LEFT_BRACE),
+            '}' => self.makeToken(.RIGHT_BRACE),
+            ',' => self.makeToken(.COMMA),
+            '.' => self.makeToken(.DOT),
+            '-' => self.makeToken(.MINUS),
+            '+' => self.makeToken(.PLUS),
+            ';' => self.makeToken(.SEMICOLON),
+            '*' => self.makeToken(.STAR),
+
+            '!' => self.makeMatchToken('=', .BANG_EQUAL, .BANG),
+            '=' => self.makeMatchToken('=', .EQUAL_EQUAL, .EQUAL),
+            '<' => self.makeMatchToken('=', .LESS_EQUAL, .LESS),
+            '>' => self.makeMatchToken('=', .GREATER_EQUAL, .GREATER),
+
+            else => return TokenError.Unexpected,
+        };
+
+        return token;
     }
 };