scanner: finish off with identifiers and keywords

This commit is contained in:
Luna 2019-06-04 21:16:25 -03:00
parent 498ea72da4
commit a44f04c7c8

View file

@ -152,17 +152,39 @@ pub const Scanner = struct {
}
}
/// "triple" version of makeMatchToken.
/// Required per vlang's tokens.
fn makeTripleMatchToken(
self: *Scanner,
char1: u8,
ttype1: TokenType,
char2: u8,
ttype2: TokenType,
fallback: TokenType,
) Token {
if (self.match(char1)) {
return self.makeToken(ttype1);
} else if (self.match(char2)) {
return self.makeToken(ttype2);
} else {
return self.makeToken(fallback);
}
}
/// Peek at the current character in the scanner
fn peek(self: *Scanner) u8 {
if (self.isAtEnd()) return 0;
if (self.current == 0) return 0;
return self.source[self.current - 1];
}
/// Peek at the next character in the scanner
fn peekNext(self: *Scanner) u8 {
if (self.current > self.source.len) return 0;
return self.source[self.current];
}
/// Skip all whitespace, but increments Scanner.line when finding a newline.
fn skipWhitespace(self: *Scanner) void {
while (true) {
var c = self.peek();
@ -229,21 +251,25 @@ pub const Scanner = struct {
);
}
fn makeTripleMatchToken(
self: *Scanner,
char1: u8,
ttype1: TokenType,
char2,
ttype2: TokenType,
fallback: TokenType,
) Token {
if (self.match(char1)) {
return self.makeToken(ttype1);
} else if (self.match(char2)) {
return self.makeToken(ttype2);
} else {
return self.makeToken(fallback);
/// Either a keyword or an identifier come out of this.
fn doIdentifier(self: *Scanner) Token {
while (isAlphaNumeric(self.peek())) {
_ = self.advance();
}
// after reading the identifier, we check
// if it is any of our keywords, if it is, then we add
// the specificed keyword type. if not, just .IDENTIFIER
var toktype: TokenType = undefined;
var ttype_opt = getKeyword(self.currentLexeme());
if (ttype_opt) |ttype| {
toktype = ttype;
} else {
toktype = TokenType.Identifier;
}
return self.makeToken(toktype);
}
pub fn nextToken(self: *Scanner) !?Token {
@ -254,6 +280,7 @@ pub const Scanner = struct {
var c = self.advance();
if (isDigit(c)) return self.doNumber();
if (isAlpha(c)) return self.doIdentifier();
var token: ?Token = switch (c) {
'(' => self.makeToken(.LeftParen),