From 92a365d908520e4373a2f509ea4e9499a2db0bb5 Mon Sep 17 00:00:00 2001 From: Luna Date: Sun, 10 Mar 2019 02:38:52 -0300 Subject: [PATCH] ast_nodes: add ReturnType, Identifier, Import handling of ReturnType follows the hello.jt rule, that it's implicitly void. - add Reader.peek, a repr, Reader.expect_val, Reader.ignore - add _fn_ret_type for ReturnType generation - fix some things when reading named functions - add read_import - add handler for unexpected reserved word - add some ast cleanups on a read_loop - add ast skip when token is None --- jortsc/parser/ast_nodes.py | 16 +++++ jortsc/parser/syntatic.py | 128 +++++++++++++++++++++++++++++++++---- 2 files changed, 130 insertions(+), 14 deletions(-) diff --git a/jortsc/parser/ast_nodes.py b/jortsc/parser/ast_nodes.py index 4d72ec2..18c4d7f 100644 --- a/jortsc/parser/ast_nodes.py +++ b/jortsc/parser/ast_nodes.py @@ -6,8 +6,24 @@ class TypedVar: name: str +@dataclass +class ReturnType: + type_: str + + @dataclass class Function: name: str arguments: str + ret_type: ReturnType block: list + + +@dataclass +class Identifier: + name: str + + +@dataclass +class Import: + module: str diff --git a/jortsc/parser/syntatic.py b/jortsc/parser/syntatic.py index 7906127..4db92bd 100644 --- a/jortsc/parser/syntatic.py +++ b/jortsc/parser/syntatic.py @@ -1,7 +1,9 @@ from typing import Optional, Any, List from jortsc.parser.lexer import Token, TokenType -from jortsc.parser.ast_nodes import Function, TypedVar +from jortsc.parser.ast_nodes import ( + Function, TypedVar, Identifier, Import, ReturnType +) class ParseError(Exception): @@ -15,13 +17,20 @@ class Reader: self.tokens = tokens self.cur = 0 - def next(self) -> Optional[Token]: - """Fetch the current token then skip to the next one.""" + def __repr__(self): + return f'' + + def peek(self) -> Optional[Token]: + """Peek at the current token.""" try: token = self.tokens[self.cur] + return token except IndexError: return None + def next(self) -> Optional[Token]: + """Fetch the current token then skip to the next one.""" + token = self.peek() self.cur += 1 return token @@ -30,10 +39,23 @@ class Reader: token = self.next() if token.type_ != token_type: - raise ParseError(f'Expected {token_type}, got {token.type_}') + raise ParseError(f'Expected {token_type}, got ' + f'{token.type_} {token.value!r}') return token + def expect_val(self, value: str) -> Token: + """Check the next token to see if it matches against a given value, + instead of a type.""" + token = self.next() + + if token.value != value: + raise ParseError(f'Expected {value!r}, got ' + f'{token.type_} {token.value!r}') + + return token + + def next_safe(self) -> Token: """'Safe' version of next(). @@ -46,6 +68,14 @@ class Reader: return token + def ignore(self, token_type: TokenType): + """Only increase self.cur if token_type is the upcoming token.""" + try: + assert self.tokens[self.cur].type_ == token_type + self.cur += 1 + except AssertionError: + pass + def _fn_read_args(reader: Reader, cur: List = None) -> List: """Recursively read the arguments of the function.""" @@ -68,6 +98,18 @@ def _fn_read_args(reader: Reader, cur: List = None) -> List: return _fn_read_args(reader, cur) +def _fn_ret_type(reader: Reader) -> ReturnType: + """Fetch the return type of a function. Defaults to void.""" + try: + reader.expect_val('->') + except ParseError: + return ReturnType('void') + + reader.ignore(TokenType.whitespace) + token = reader.expect(TokenType.identifier) + return ReturnType(token.value) + + def read_function(reader: Reader): """Read a function block.""" reader.expect(TokenType.whitespace) @@ -81,19 +123,49 @@ def read_function(reader: Reader): if token.type_ == TokenType.identifier: fn_name = token.value + + reader.expect(TokenType.whitespace) + reader.expect_val('(') + fn_args = _fn_read_args(reader) + + reader.expect(TokenType.whitespace) + fn_ret_type = _fn_ret_type(reader) + + # only skip whitespace if we see it + reader.ignore(TokenType.whitespace) block = read_start(reader) elif token.value == '(': fn_args = _fn_read_args(reader) + fn_ret_type = _fn_ret_type(reader) block = read_start(reader) - return Function(fn_name, fn_args, block) + print('final function', fn_name, fn_args, fn_ret_type, block) + + return Function(fn_name, fn_args, fn_ret_type, block) + + +def read_import(reader): + """Read an import""" + reader.expect(TokenType.whitespace) + module = reader.next_safe() + return Import(module.value) + + +HANDLERS = { + 'fn': read_function, + 'import': read_import, +} def read_reserved(token: Token, reader: Reader): """Read reserved statements.""" - if token.value == 'fn': - return read_function(reader) + try: + handler = HANDLERS[token.value] + except KeyError: + raise ParseError(f'Unexpected reserved word {token.value!r}') + + return handler(reader) def read_start(reader: Reader): @@ -101,29 +173,57 @@ def read_start(reader: Reader): print('reader', reader) token = reader.next() + + if token is None: + return [] + ast = [] res = [] print('cur', token) - # TODO: statement handling - if token.type_ == TokenType.reserved: - res = read_reserved(token, reader) - # handle blocks - elif token.value == '{': + if token.value == '{': while True: - token = reader.peek() + token = reader.next() if token.value == '}': break res.extend(read_start(reader)) + # import, fn, etc + elif token.type_ == TokenType.reserved: + res = read_reserved(token, reader) + + elif token.type_ == TokenType.comment: + return [] + + elif token.type_ == TokenType.identifier: + res = [Identifier(token.value)] + ast.append(res) return ast +def read_loop(reader: Reader): + final_ast = [] + + while True: + ast = read_start(reader) + + if not ast: + break + + inner = ast[0] + + if not inner: + break + + final_ast.append(ast) + + return final_ast + def syntatic(tokens: List[Token]): """Create an AST out of the tokens.""" - return read_start(Reader(tokens)) + return read_loop(Reader(tokens))