add number and comment support

This commit is contained in:
Luna 2019-06-04 17:24:07 -03:00
parent 1453bef6c5
commit 765cef87db
2 changed files with 49 additions and 5 deletions

View file

@ -16,7 +16,7 @@ fn run(allocator: *Allocator, data: []u8) !void {
var scanner = scanners.Scanner.init(allocator, data);
while (true) {
var tok = scanner.nextToken() catch |err| {
var tok_opt = scanner.nextToken() catch |err| {
try stdout.print(
"error at '{}': {}\n",
scanner.currentLexeme(),
@ -25,8 +25,11 @@ fn run(allocator: *Allocator, data: []u8) !void {
return Result.CompileError;
};
if (tok.ttype == .EOF) break;
try stdout.print("{x}\n", tok);
if (tok_opt) |tok| {
if (tok.ttype == .EOF) break;
try stdout.print("{x}\n", tok);
}
}
return Result.Ok;

View file

@ -173,15 +173,40 @@ pub const Scanner = struct {
}
}
pub fn nextToken(self: *Scanner) !Token {
/// Consume a number.
/// Returns either an Integer or a Float token. Proper typing
/// of the number (i32 i64 u32 u64 f32 f64) are for the parser.
fn doNumber(self: *Scanner) Token {
var ttype = TokenType.Integer;
while (isDigit(self.peek())) {
_ = self.advance();
}
// check if its a number like 12.34, where the '.' character
// exists and the one next to it is a digit.
if (self.peek() == '.' and isDigit(self.peekNext())) {
ttype = TokenType.Float;
_ = self.advance();
while (isDigit(self.peek())) {
_ = self.advance();
}
}
return self.makeToken(ttype);
}
pub fn nextToken(self: *Scanner) !?Token {
self.skipWhitespace();
self.start = self.current;
if (self.isAtEnd()) return self.makeToken(TokenType.EOF);
var c = self.advance();
if (isDigit(c)) return self.doNumber();
var token = switch (c) {
var token: ?Token = switch (c) {
'(' => self.makeToken(.LeftParen),
')' => self.makeToken(.RightParen),
'{' => self.makeToken(.LeftBrace),
@ -197,6 +222,10 @@ pub const Scanner = struct {
'?' => self.makeToken(.QuestionMark),
'$' => self.makeToken(.DollarSign),
'-' => self.makeToken(.Minus),
'+' => self.makeToken(.Plus),
'*' => self.makeToken(.Star),
'!' => self.makeMatchToken('=', .BangEqual, .Bang),
'=' => self.makeMatchToken('=', .EqualEqual, .Equal),
@ -215,6 +244,18 @@ pub const Scanner = struct {
},
'>' => self.makeMatchToken('=', .GreaterEqual, .Greater),
'/' => blk: {
if (self.peekNext() == '/') {
while (self.peek() != '\n' and !self.isAtEnd()) {
_ = self.advance();
}
break :blk null;
} else {
break :blk self.makeToken(.Slash);
}
},
else => return ScannerError.Unexpected,
};