scanner: finish off with identifiers and keywords

This commit is contained in:
Luna 2019-06-04 21:16:25 -03:00
parent 498ea72da4
commit a44f04c7c8

View file

@ -152,17 +152,39 @@ pub const Scanner = struct {
} }
} }
/// "triple" version of makeMatchToken.
/// Required per vlang's tokens.
fn makeTripleMatchToken(
self: *Scanner,
char1: u8,
ttype1: TokenType,
char2: u8,
ttype2: TokenType,
fallback: TokenType,
) Token {
if (self.match(char1)) {
return self.makeToken(ttype1);
} else if (self.match(char2)) {
return self.makeToken(ttype2);
} else {
return self.makeToken(fallback);
}
}
/// Peek at the current character in the scanner
fn peek(self: *Scanner) u8 { fn peek(self: *Scanner) u8 {
if (self.isAtEnd()) return 0; if (self.isAtEnd()) return 0;
if (self.current == 0) return 0; if (self.current == 0) return 0;
return self.source[self.current - 1]; return self.source[self.current - 1];
} }
/// Peek at the next character in the scanner
fn peekNext(self: *Scanner) u8 { fn peekNext(self: *Scanner) u8 {
if (self.current > self.source.len) return 0; if (self.current > self.source.len) return 0;
return self.source[self.current]; return self.source[self.current];
} }
/// Skip all whitespace, but increments Scanner.line when finding a newline.
fn skipWhitespace(self: *Scanner) void { fn skipWhitespace(self: *Scanner) void {
while (true) { while (true) {
var c = self.peek(); var c = self.peek();
@ -229,21 +251,25 @@ pub const Scanner = struct {
); );
} }
fn makeTripleMatchToken( /// Either a keyword or an identifier come out of this.
self: *Scanner, fn doIdentifier(self: *Scanner) Token {
char1: u8, while (isAlphaNumeric(self.peek())) {
ttype1: TokenType, _ = self.advance();
char2,
ttype2: TokenType,
fallback: TokenType,
) Token {
if (self.match(char1)) {
return self.makeToken(ttype1);
} else if (self.match(char2)) {
return self.makeToken(ttype2);
} else {
return self.makeToken(fallback);
} }
// after reading the identifier, we check
// if it is any of our keywords, if it is, then we add
// the specificed keyword type. if not, just .IDENTIFIER
var toktype: TokenType = undefined;
var ttype_opt = getKeyword(self.currentLexeme());
if (ttype_opt) |ttype| {
toktype = ttype;
} else {
toktype = TokenType.Identifier;
}
return self.makeToken(toktype);
} }
pub fn nextToken(self: *Scanner) !?Token { pub fn nextToken(self: *Scanner) !?Token {
@ -254,6 +280,7 @@ pub const Scanner = struct {
var c = self.advance(); var c = self.advance();
if (isDigit(c)) return self.doNumber(); if (isDigit(c)) return self.doNumber();
if (isAlpha(c)) return self.doIdentifier();
var token: ?Token = switch (c) { var token: ?Token = switch (c) {
'(' => self.makeToken(.LeftParen), '(' => self.makeToken(.LeftParen),