2019-06-01 19:21:36 +00:00
|
|
|
const std = @import("std");
|
2019-06-01 19:37:24 +00:00
|
|
|
const tokens = @import("token.zig");
|
2019-06-01 19:21:36 +00:00
|
|
|
|
2019-06-01 19:50:48 +00:00
|
|
|
const Token = tokens.Token;
|
|
|
|
const TokenType = tokens.TokenType;
|
|
|
|
|
2019-06-01 19:21:36 +00:00
|
|
|
const Allocator = std.mem.Allocator;
|
|
|
|
|
2019-06-01 19:50:48 +00:00
|
|
|
pub const TokenError = error{
|
|
|
|
Unexpected,
|
|
|
|
Unterminated,
|
|
|
|
};
|
|
|
|
|
2019-06-01 20:13:50 +00:00
|
|
|
fn isDigit(char: u8) bool {
|
|
|
|
return char >= '0' and char <= '9';
|
|
|
|
}
|
|
|
|
|
|
|
|
fn isAlpha(c: u8) bool {
|
|
|
|
return (c >= 'a' and c <= 'z') or
|
|
|
|
(c >= 'A' and c <= 'Z') or
|
|
|
|
c == '_';
|
|
|
|
}
|
|
|
|
|
|
|
|
fn isAlphaNumeric(char: u8) bool {
|
|
|
|
return isAlpha(char) or isDigit(char);
|
|
|
|
}
|
|
|
|
|
2019-06-01 19:21:36 +00:00
|
|
|
pub const Scanner = struct {
|
|
|
|
source: []const u8,
|
2019-06-01 19:37:24 +00:00
|
|
|
|
|
|
|
start: usize = 0,
|
|
|
|
current: usize = 0,
|
|
|
|
line: usize = 1,
|
|
|
|
|
2019-06-01 19:21:36 +00:00
|
|
|
allocator: *Allocator,
|
|
|
|
|
|
|
|
pub fn init(allocator: *Allocator, data: []const u8) Scanner {
|
|
|
|
return Scanner{
|
|
|
|
.allocator = allocator,
|
|
|
|
.source = data,
|
|
|
|
};
|
|
|
|
}
|
2019-06-01 19:37:24 +00:00
|
|
|
|
2019-06-01 19:50:48 +00:00
|
|
|
fn isAtEnd(self: *Scanner) bool {
|
|
|
|
return self.current >= self.source.len;
|
|
|
|
}
|
|
|
|
|
|
|
|
fn advance(self: *Scanner) u8 {
|
|
|
|
self.current += 1;
|
|
|
|
return self.source[self.current - 1];
|
|
|
|
}
|
|
|
|
|
2019-06-01 20:07:22 +00:00
|
|
|
pub fn currentLexeme(self: *Scanner) []const u8 {
|
|
|
|
return self.source[self.start..self.current];
|
|
|
|
}
|
|
|
|
|
2019-06-01 19:50:48 +00:00
|
|
|
fn makeToken(self: *Scanner, ttype: TokenType) Token {
|
|
|
|
return Token{
|
|
|
|
.ttype = ttype,
|
2019-06-01 20:07:22 +00:00
|
|
|
.lexeme = self.currentLexeme(),
|
2019-06-01 19:50:48 +00:00
|
|
|
.line = self.line,
|
|
|
|
};
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Check if the next character matches what is expected.
|
|
|
|
fn match(self: *Scanner, expected: u8) bool {
|
|
|
|
if (self.isAtEnd()) return false;
|
|
|
|
if (self.source[self.current] != expected) return false;
|
|
|
|
|
|
|
|
self.current += 1;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
|
|
|
/// Add a SimpleToken of type_match if the next character is
|
|
|
|
/// `expected`. Adds a SimpleToken of type_nomatch when it is not.
|
|
|
|
fn makeMatchToken(
|
|
|
|
self: *Scanner,
|
|
|
|
expected: u8,
|
|
|
|
type_match: TokenType,
|
|
|
|
type_nomatch: TokenType,
|
|
|
|
) Token {
|
|
|
|
if (self.match(expected)) {
|
|
|
|
return self.makeToken(type_match);
|
|
|
|
} else {
|
|
|
|
return self.makeToken(type_nomatch);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-06-01 19:54:15 +00:00
|
|
|
fn peek(self: *Scanner) u8 {
|
|
|
|
if (self.isAtEnd()) return 0;
|
|
|
|
return self.source[self.current];
|
|
|
|
}
|
|
|
|
|
2019-06-01 20:07:22 +00:00
|
|
|
fn peekNext(self: *Scanner) u8 {
|
|
|
|
if (self.isAtEnd()) return 0;
|
2019-06-01 20:12:00 +00:00
|
|
|
return self.source[self.current - 1];
|
2019-06-01 20:07:22 +00:00
|
|
|
}
|
|
|
|
|
2019-06-01 19:54:15 +00:00
|
|
|
fn skipWhitespace(self: *Scanner) void {
|
|
|
|
while (true) {
|
|
|
|
var c = self.peek();
|
|
|
|
switch (c) {
|
|
|
|
' ', '\r', '\t' => blk: {
|
|
|
|
_ = self.advance();
|
|
|
|
},
|
|
|
|
'\n' => blk: {
|
|
|
|
self.line += 1;
|
|
|
|
_ = self.advance();
|
|
|
|
},
|
|
|
|
else => return,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-06-01 20:07:22 +00:00
|
|
|
fn doString(self: *Scanner) !Token {
|
|
|
|
// consume entire string
|
|
|
|
while (self.peek() != '"' and !self.isAtEnd()) {
|
|
|
|
if (self.peek() == '\n') self.line += 1;
|
|
|
|
_ = self.advance();
|
|
|
|
}
|
|
|
|
|
|
|
|
// unterminated string.
|
|
|
|
if (self.isAtEnd()) {
|
|
|
|
return TokenError.Unterminated;
|
|
|
|
}
|
|
|
|
|
|
|
|
// the closing ".
|
|
|
|
_ = self.advance();
|
|
|
|
|
|
|
|
// trim the surrounding quotes.
|
|
|
|
return self.makeToken(.STRING);
|
|
|
|
}
|
|
|
|
|
2019-06-01 20:13:50 +00:00
|
|
|
/// Consume a number
|
|
|
|
fn doNumber(self: *Scanner) Token {
|
|
|
|
while (isDigit(self.peek())) {
|
|
|
|
_ = self.advance();
|
|
|
|
}
|
|
|
|
|
|
|
|
// check if its a number like 12.34, where the '.' character
|
|
|
|
// exists and the one next to it is a digit.
|
|
|
|
if (self.peek() == '.' and isDigit(self.peekNext())) {
|
|
|
|
_ = self.advance();
|
|
|
|
|
|
|
|
while (isDigit(self.peek())) {
|
|
|
|
_ = self.advance();
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return self.makeToken(.NUMBER);
|
|
|
|
}
|
|
|
|
|
2019-06-01 20:15:27 +00:00
|
|
|
fn doIdentifier(self: *Scanner) Token {
|
|
|
|
while (isAlphaNumeric(self.peek())) {
|
|
|
|
_ = self.advance();
|
|
|
|
}
|
|
|
|
|
|
|
|
return self.makeToken(.IDENTIFIER);
|
|
|
|
}
|
|
|
|
|
2019-06-01 20:07:22 +00:00
|
|
|
pub fn scanToken(self: *Scanner) !?Token {
|
2019-06-01 19:54:15 +00:00
|
|
|
self.skipWhitespace();
|
2019-06-01 19:50:48 +00:00
|
|
|
self.start = self.current;
|
|
|
|
if (self.isAtEnd()) return self.makeToken(TokenType.EOF);
|
|
|
|
|
|
|
|
var c = self.advance();
|
2019-06-01 20:15:27 +00:00
|
|
|
if (isAlpha(c)) return self.doIdentifier();
|
2019-06-01 20:13:50 +00:00
|
|
|
if (isDigit(c)) return self.doNumber();
|
2019-06-01 19:50:48 +00:00
|
|
|
|
|
|
|
var token = switch (c) {
|
|
|
|
'(' => self.makeToken(.LEFT_PAREN),
|
|
|
|
')' => self.makeToken(.RIGHT_PAREN),
|
|
|
|
'{' => self.makeToken(.LEFT_BRACE),
|
|
|
|
'}' => self.makeToken(.RIGHT_BRACE),
|
|
|
|
',' => self.makeToken(.COMMA),
|
|
|
|
'.' => self.makeToken(.DOT),
|
|
|
|
'-' => self.makeToken(.MINUS),
|
|
|
|
'+' => self.makeToken(.PLUS),
|
|
|
|
';' => self.makeToken(.SEMICOLON),
|
|
|
|
'*' => self.makeToken(.STAR),
|
|
|
|
|
|
|
|
'!' => self.makeMatchToken('=', .BANG_EQUAL, .BANG),
|
|
|
|
'=' => self.makeMatchToken('=', .EQUAL_EQUAL, .EQUAL),
|
|
|
|
'<' => self.makeMatchToken('=', .LESS_EQUAL, .LESS),
|
|
|
|
'>' => self.makeMatchToken('=', .GREATER_EQUAL, .GREATER),
|
|
|
|
|
2019-06-01 20:07:22 +00:00
|
|
|
'/' => blk: {
|
|
|
|
if (self.peekNext() == '/') {
|
|
|
|
while (self.peek() != '\n' and !self.isAtEnd()) {
|
|
|
|
_ = self.advance();
|
|
|
|
}
|
|
|
|
break :blk null;
|
|
|
|
} else {
|
|
|
|
break :blk self.makeToken(.SLASH);
|
|
|
|
}
|
|
|
|
},
|
|
|
|
|
|
|
|
'"' => try self.doString(),
|
|
|
|
|
2019-06-01 19:50:48 +00:00
|
|
|
else => return TokenError.Unexpected,
|
|
|
|
};
|
|
|
|
|
|
|
|
return token;
|
2019-06-01 19:37:24 +00:00
|
|
|
}
|
2019-06-01 19:21:36 +00:00
|
|
|
};
|