scanner: add basic error handling, strings, comments

This commit is contained in:
Luna 2019-06-01 17:07:22 -03:00
parent f4f1fe1fbc
commit 27b04e1612
3 changed files with 64 additions and 11 deletions

View file

@ -26,17 +26,29 @@ pub const Compiler = struct {
var scanr = scanner.Scanner.init(self.allocator, self.src);
var line: usize = 0;
while (true) {
var token = try scanr.scanToken();
var token_opt = scanr.scanToken() catch |err| {
std.debug.warn("Scan Error: {x}\n", err);
std.debug.warn(
"line: {}, cur lexeme: {}\n",
scanr.line,
scanr.currentLexeme(),
);
break;
};
if (token.line != line) {
try self.stdout.print("{} ", token.line);
line = token.line;
if (token_opt) |token| {
if (token.line != line) {
try self.stdout.print("{} ", token.line);
line = token.line;
} else {
try self.stdout.print(" | ");
}
try self.stdout.print("{} '{}'\n", token.ttype, token.lexeme);
if (token.ttype == TokenType.EOF) break;
} else {
try self.stdout.print(" | ");
break;
}
try self.stdout.print("{} '{}'\n", token.ttype, token.lexeme);
if (token.ttype == TokenType.EOF) break;
}
}
};

View file

@ -36,10 +36,14 @@ pub const Scanner = struct {
return self.source[self.current - 1];
}
pub fn currentLexeme(self: *Scanner) []const u8 {
return self.source[self.start..self.current];
}
fn makeToken(self: *Scanner, ttype: TokenType) Token {
return Token{
.ttype = ttype,
.lexeme = self.source[self.start..self.current],
.lexeme = self.currentLexeme(),
.line = self.line,
};
}
@ -73,6 +77,11 @@ pub const Scanner = struct {
return self.source[self.current];
}
fn peekNext(self: *Scanner) u8 {
if (self.isAtEnd()) return 0;
return self.source[self.current + 1];
}
fn skipWhitespace(self: *Scanner) void {
while (true) {
var c = self.peek();
@ -89,7 +98,26 @@ pub const Scanner = struct {
}
}
pub fn scanToken(self: *Scanner) !tokens.Token {
fn doString(self: *Scanner) !Token {
// consume entire string
while (self.peek() != '"' and !self.isAtEnd()) {
if (self.peek() == '\n') self.line += 1;
_ = self.advance();
}
// unterminated string.
if (self.isAtEnd()) {
return TokenError.Unterminated;
}
// the closing ".
_ = self.advance();
// trim the surrounding quotes.
return self.makeToken(.STRING);
}
pub fn scanToken(self: *Scanner) !?Token {
self.skipWhitespace();
self.start = self.current;
if (self.isAtEnd()) return self.makeToken(TokenType.EOF);
@ -113,6 +141,19 @@ pub const Scanner = struct {
'<' => self.makeMatchToken('=', .LESS_EQUAL, .LESS),
'>' => self.makeMatchToken('=', .GREATER_EQUAL, .GREATER),
'/' => blk: {
if (self.peekNext() == '/') {
while (self.peek() != '\n' and !self.isAtEnd()) {
_ = self.advance();
}
break :blk null;
} else {
break :blk self.makeToken(.SLASH);
}
},
'"' => try self.doString(),
else => return TokenError.Unexpected,
};

View file

@ -98,7 +98,7 @@ pub const Scanner = struct {
return self.source[self.current - 1];
}
fn currentLexeme(self: *Scanner) []u8 {
pub fn currentLexeme(self: *Scanner) []u8 {
return self.source[self.start..self.current];
}