scanner: add basic tokens and matched-tokens

This commit is contained in:
Luna 2019-06-01 16:50:48 -03:00
parent 6b9cc575d9
commit 2c7cf356b3
2 changed files with 78 additions and 3 deletions

View file

@ -26,7 +26,7 @@ pub const Compiler = struct {
var scanr = scanner.Scanner.init(self.allocator, self.src); var scanr = scanner.Scanner.init(self.allocator, self.src);
var line: usize = 0; var line: usize = 0;
while (true) { while (true) {
var token = scanr.scanToken(); var token = try scanr.scanToken();
if (token.line != line) { if (token.line != line) {
try self.stdout.print("{} ", token.line); try self.stdout.print("{} ", token.line);

View file

@ -1,8 +1,16 @@
const std = @import("std"); const std = @import("std");
const tokens = @import("token.zig"); const tokens = @import("token.zig");
const Token = tokens.Token;
const TokenType = tokens.TokenType;
const Allocator = std.mem.Allocator; const Allocator = std.mem.Allocator;
pub const TokenError = error{
Unexpected,
Unterminated,
};
pub const Scanner = struct { pub const Scanner = struct {
source: []const u8, source: []const u8,
@ -19,7 +27,74 @@ pub const Scanner = struct {
}; };
} }
pub fn scanToken(self: *Scanner) tokens.Token { fn isAtEnd(self: *Scanner) bool {
return tokens.Token{}; return self.current >= self.source.len;
}
fn advance(self: *Scanner) u8 {
self.current += 1;
return self.source[self.current - 1];
}
fn makeToken(self: *Scanner, ttype: TokenType) Token {
return Token{
.ttype = ttype,
.lexeme = self.source[self.start..self.current],
.line = self.line,
};
}
/// Check if the next character matches what is expected.
fn match(self: *Scanner, expected: u8) bool {
if (self.isAtEnd()) return false;
if (self.source[self.current] != expected) return false;
self.current += 1;
return true;
}
/// Add a SimpleToken of type_match if the next character is
/// `expected`. Adds a SimpleToken of type_nomatch when it is not.
fn makeMatchToken(
self: *Scanner,
expected: u8,
type_match: TokenType,
type_nomatch: TokenType,
) Token {
if (self.match(expected)) {
return self.makeToken(type_match);
} else {
return self.makeToken(type_nomatch);
}
}
pub fn scanToken(self: *Scanner) !tokens.Token {
self.start = self.current;
if (self.isAtEnd()) return self.makeToken(TokenType.EOF);
var c = self.advance();
var token = switch (c) {
'(' => self.makeToken(.LEFT_PAREN),
')' => self.makeToken(.RIGHT_PAREN),
'{' => self.makeToken(.LEFT_BRACE),
'}' => self.makeToken(.RIGHT_BRACE),
',' => self.makeToken(.COMMA),
'.' => self.makeToken(.DOT),
'-' => self.makeToken(.MINUS),
'+' => self.makeToken(.PLUS),
';' => self.makeToken(.SEMICOLON),
'*' => self.makeToken(.STAR),
'!' => self.makeMatchToken('=', .BANG_EQUAL, .BANG),
'=' => self.makeMatchToken('=', .EQUAL_EQUAL, .EQUAL),
'<' => self.makeMatchToken('=', .LESS_EQUAL, .LESS),
'>' => self.makeMatchToken('=', .GREATER_EQUAL, .GREATER),
else => return TokenError.Unexpected,
};
return token;
} }
}; };