ast_nodes: add ReturnType, Identifier, Import

handling of ReturnType follows the hello.jt rule, that it's implicitly
void.

 - add Reader.peek, a repr, Reader.expect_val, Reader.ignore
 - add _fn_ret_type for ReturnType generation
 - fix some things when reading named functions
 - add read_import
 - add handler for unexpected reserved word
 - add some ast cleanups on a read_loop
 - add ast skip when token is None

jortsc/parser/ast_nodes.py

@@ -6,8 +6,24 @@ class TypedVar:
     name: str
 
 
+@dataclass
+class ReturnType:
+    type_: str
+
+
 @dataclass
 class Function:
     name: str
     arguments: str
+    ret_type: ReturnType
     block: list
+
+
+@dataclass
+class Identifier:
+    name: str
+
+
+@dataclass
+class Import:
+    module: str

jortsc/parser/syntatic.py

@@ -1,7 +1,9 @@
 from typing import Optional, Any, List
 
 from jortsc.parser.lexer import Token, TokenType
-from jortsc.parser.ast_nodes import Function, TypedVar
+from jortsc.parser.ast_nodes import (
+    Function, TypedVar, Identifier, Import, ReturnType
+)
 
 
 class ParseError(Exception):
@@ -15,13 +17,20 @@ class Reader:
         self.tokens = tokens
         self.cur = 0
 
-    def next(self) -> Optional[Token]:
-        """Fetch the current token then skip to the next one."""
+    def __repr__(self):
+        return f'<Reader cur={self.cur} cur_tok={self.peek()}>'
+
+    def peek(self) -> Optional[Token]:
+        """Peek at the current token."""
         try:
             token = self.tokens[self.cur]
+            return token
         except IndexError:
             return None
 
+    def next(self) -> Optional[Token]:
+        """Fetch the current token then skip to the next one."""
+        token = self.peek()
         self.cur += 1
         return token
 
@@ -30,10 +39,23 @@ class Reader:
         token = self.next()
 
         if token.type_ != token_type:
-            raise ParseError(f'Expected {token_type}, got {token.type_}')
+            raise ParseError(f'Expected {token_type}, got '
+                             f'{token.type_} {token.value!r}')
 
         return token
 
+    def expect_val(self, value: str) -> Token:
+        """Check the next token to see if it matches against a given value,
+        instead of a type."""
+        token = self.next()
+
+        if token.value != value:
+            raise ParseError(f'Expected {value!r}, got '
+                             f'{token.type_} {token.value!r}')
+
+        return token
+
+
     def next_safe(self) -> Token:
         """'Safe' version of next().
 
@@ -46,6 +68,14 @@ class Reader:
 
         return token
 
+    def ignore(self, token_type: TokenType):
+        """Only increase self.cur if token_type is the upcoming token."""
+        try:
+            assert self.tokens[self.cur].type_ == token_type
+            self.cur += 1
+        except AssertionError:
+            pass
+
 
 def _fn_read_args(reader: Reader, cur: List = None) -> List:
     """Recursively read the arguments of the function."""
@@ -68,6 +98,18 @@ def _fn_read_args(reader: Reader, cur: List = None) -> List:
     return _fn_read_args(reader, cur)
 
 
+def _fn_ret_type(reader: Reader) -> ReturnType:
+    """Fetch the return type of a function. Defaults to void."""
+    try:
+        reader.expect_val('->')
+    except ParseError:
+        return ReturnType('void')
+
+    reader.ignore(TokenType.whitespace)
+    token = reader.expect(TokenType.identifier)
+    return ReturnType(token.value)
+
+
 def read_function(reader: Reader):
     """Read a function block."""
     reader.expect(TokenType.whitespace)
@@ -81,19 +123,49 @@ def read_function(reader: Reader):
 
     if token.type_ == TokenType.identifier:
         fn_name = token.value
+
+        reader.expect(TokenType.whitespace)
+        reader.expect_val('(')
+
         fn_args = _fn_read_args(reader)
+
+        reader.expect(TokenType.whitespace)
+        fn_ret_type = _fn_ret_type(reader)
+
+        # only skip whitespace if we see it
+        reader.ignore(TokenType.whitespace)
         block = read_start(reader)
     elif token.value == '(':
         fn_args = _fn_read_args(reader)
+        fn_ret_type = _fn_ret_type(reader)
         block = read_start(reader)
 
-    return Function(fn_name, fn_args, block)
+    print('final function', fn_name, fn_args, fn_ret_type, block)
+
+    return Function(fn_name, fn_args, fn_ret_type, block)
+
+
+def read_import(reader):
+    """Read an import"""
+    reader.expect(TokenType.whitespace)
+    module = reader.next_safe()
+    return Import(module.value)
+
+
+HANDLERS = {
+    'fn': read_function,
+    'import': read_import,
+}
 
 
 def read_reserved(token: Token, reader: Reader):
     """Read reserved statements."""
-    if token.value == 'fn':
-        return read_function(reader)
+    try:
+        handler = HANDLERS[token.value]
+    except KeyError:
+        raise ParseError(f'Unexpected reserved word {token.value!r}')
+
+    return handler(reader)
 
 
 def read_start(reader: Reader):
@@ -101,29 +173,57 @@ def read_start(reader: Reader):
     print('reader', reader)
 
     token = reader.next()
+
+    if token is None:
+        return []
+
     ast = []
     res = []
 
     print('cur', token)
 
-    # TODO: statement handling
-    if token.type_ == TokenType.reserved:
-        res = read_reserved(token, reader)
-
     # handle blocks
-    elif token.value == '{':
+    if token.value == '{':
         while True:
-            token = reader.peek()
+            token = reader.next()
 
             if token.value == '}':
                 break
 
             res.extend(read_start(reader))
 
+    # import, fn, etc
+    elif token.type_ == TokenType.reserved:
+        res = read_reserved(token, reader)
+
+    elif token.type_ == TokenType.comment:
+        return []
+
+    elif token.type_ == TokenType.identifier:
+        res = [Identifier(token.value)]
+
     ast.append(res)
     return ast
 
+def read_loop(reader: Reader):
+    final_ast = []
+
+    while True:
+        ast = read_start(reader)
+
+        if not ast:
+            break
+
+        inner = ast[0]
+
+        if not inner:
+            break
+
+        final_ast.append(ast)
+
+    return final_ast
+
 
 def syntatic(tokens: List[Token]):
     """Create an AST out of the tokens."""
-    return read_start(Reader(tokens))
+    return read_loop(Reader(tokens))