ast_nodes: add ReturnType, Identifier, Import

handling of ReturnType follows the hello.jt rule, that it's implicitly
void.

 - add Reader.peek, a repr, Reader.expect_val, Reader.ignore
 - add _fn_ret_type for ReturnType generation
 - fix some things when reading named functions
 - add read_import
 - add handler for unexpected reserved word
 - add some ast cleanups on a read_loop
 - add ast skip when token is None
This commit is contained in:
Luna 2019-03-10 02:38:52 -03:00
parent b7d5646bad
commit 92a365d908
2 changed files with 130 additions and 14 deletions

View file

@ -6,8 +6,24 @@ class TypedVar:
name: str name: str
@dataclass
class ReturnType:
type_: str
@dataclass @dataclass
class Function: class Function:
name: str name: str
arguments: str arguments: str
ret_type: ReturnType
block: list block: list
@dataclass
class Identifier:
name: str
@dataclass
class Import:
module: str

View file

@ -1,7 +1,9 @@
from typing import Optional, Any, List from typing import Optional, Any, List
from jortsc.parser.lexer import Token, TokenType from jortsc.parser.lexer import Token, TokenType
from jortsc.parser.ast_nodes import Function, TypedVar from jortsc.parser.ast_nodes import (
Function, TypedVar, Identifier, Import, ReturnType
)
class ParseError(Exception): class ParseError(Exception):
@ -15,13 +17,20 @@ class Reader:
self.tokens = tokens self.tokens = tokens
self.cur = 0 self.cur = 0
def next(self) -> Optional[Token]: def __repr__(self):
"""Fetch the current token then skip to the next one.""" return f'<Reader cur={self.cur} cur_tok={self.peek()}>'
def peek(self) -> Optional[Token]:
"""Peek at the current token."""
try: try:
token = self.tokens[self.cur] token = self.tokens[self.cur]
return token
except IndexError: except IndexError:
return None return None
def next(self) -> Optional[Token]:
"""Fetch the current token then skip to the next one."""
token = self.peek()
self.cur += 1 self.cur += 1
return token return token
@ -30,10 +39,23 @@ class Reader:
token = self.next() token = self.next()
if token.type_ != token_type: if token.type_ != token_type:
raise ParseError(f'Expected {token_type}, got {token.type_}') raise ParseError(f'Expected {token_type}, got '
f'{token.type_} {token.value!r}')
return token return token
def expect_val(self, value: str) -> Token:
"""Check the next token to see if it matches against a given value,
instead of a type."""
token = self.next()
if token.value != value:
raise ParseError(f'Expected {value!r}, got '
f'{token.type_} {token.value!r}')
return token
def next_safe(self) -> Token: def next_safe(self) -> Token:
"""'Safe' version of next(). """'Safe' version of next().
@ -46,6 +68,14 @@ class Reader:
return token return token
def ignore(self, token_type: TokenType):
"""Only increase self.cur if token_type is the upcoming token."""
try:
assert self.tokens[self.cur].type_ == token_type
self.cur += 1
except AssertionError:
pass
def _fn_read_args(reader: Reader, cur: List = None) -> List: def _fn_read_args(reader: Reader, cur: List = None) -> List:
"""Recursively read the arguments of the function.""" """Recursively read the arguments of the function."""
@ -68,6 +98,18 @@ def _fn_read_args(reader: Reader, cur: List = None) -> List:
return _fn_read_args(reader, cur) return _fn_read_args(reader, cur)
def _fn_ret_type(reader: Reader) -> ReturnType:
"""Fetch the return type of a function. Defaults to void."""
try:
reader.expect_val('->')
except ParseError:
return ReturnType('void')
reader.ignore(TokenType.whitespace)
token = reader.expect(TokenType.identifier)
return ReturnType(token.value)
def read_function(reader: Reader): def read_function(reader: Reader):
"""Read a function block.""" """Read a function block."""
reader.expect(TokenType.whitespace) reader.expect(TokenType.whitespace)
@ -81,19 +123,49 @@ def read_function(reader: Reader):
if token.type_ == TokenType.identifier: if token.type_ == TokenType.identifier:
fn_name = token.value fn_name = token.value
reader.expect(TokenType.whitespace)
reader.expect_val('(')
fn_args = _fn_read_args(reader) fn_args = _fn_read_args(reader)
reader.expect(TokenType.whitespace)
fn_ret_type = _fn_ret_type(reader)
# only skip whitespace if we see it
reader.ignore(TokenType.whitespace)
block = read_start(reader) block = read_start(reader)
elif token.value == '(': elif token.value == '(':
fn_args = _fn_read_args(reader) fn_args = _fn_read_args(reader)
fn_ret_type = _fn_ret_type(reader)
block = read_start(reader) block = read_start(reader)
return Function(fn_name, fn_args, block) print('final function', fn_name, fn_args, fn_ret_type, block)
return Function(fn_name, fn_args, fn_ret_type, block)
def read_import(reader):
"""Read an import"""
reader.expect(TokenType.whitespace)
module = reader.next_safe()
return Import(module.value)
HANDLERS = {
'fn': read_function,
'import': read_import,
}
def read_reserved(token: Token, reader: Reader): def read_reserved(token: Token, reader: Reader):
"""Read reserved statements.""" """Read reserved statements."""
if token.value == 'fn': try:
return read_function(reader) handler = HANDLERS[token.value]
except KeyError:
raise ParseError(f'Unexpected reserved word {token.value!r}')
return handler(reader)
def read_start(reader: Reader): def read_start(reader: Reader):
@ -101,29 +173,57 @@ def read_start(reader: Reader):
print('reader', reader) print('reader', reader)
token = reader.next() token = reader.next()
if token is None:
return []
ast = [] ast = []
res = [] res = []
print('cur', token) print('cur', token)
# TODO: statement handling
if token.type_ == TokenType.reserved:
res = read_reserved(token, reader)
# handle blocks # handle blocks
elif token.value == '{': if token.value == '{':
while True: while True:
token = reader.peek() token = reader.next()
if token.value == '}': if token.value == '}':
break break
res.extend(read_start(reader)) res.extend(read_start(reader))
# import, fn, etc
elif token.type_ == TokenType.reserved:
res = read_reserved(token, reader)
elif token.type_ == TokenType.comment:
return []
elif token.type_ == TokenType.identifier:
res = [Identifier(token.value)]
ast.append(res) ast.append(res)
return ast return ast
def read_loop(reader: Reader):
final_ast = []
while True:
ast = read_start(reader)
if not ast:
break
inner = ast[0]
if not inner:
break
final_ast.append(ast)
return final_ast
def syntatic(tokens: List[Token]): def syntatic(tokens: List[Token]):
"""Create an AST out of the tokens.""" """Create an AST out of the tokens."""
return read_start(Reader(tokens)) return read_loop(Reader(tokens))