From ac23c0d2d7283e3f89b7ac2cf7ba08c16c7cd27d Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 1 Jul 2019 15:06:53 -0300 Subject: [PATCH 1/4] fix identifier lexemes --- src/scanner.zig | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/scanner.zig b/src/scanner.zig index 491f1f2..56084a5 100644 --- a/src/scanner.zig +++ b/src/scanner.zig @@ -110,6 +110,10 @@ pub const Scanner = struct { return self.source[self.current - 1]; } + fn rollback(self: *Scanner) void { + self.current -= 1; + } + pub fn currentLexeme(self: *Scanner) []const u8 { return self.source[self.start..self.current]; } @@ -257,6 +261,9 @@ pub const Scanner = struct { toktype = TokenType.Identifier; } + // ugly hack. + self.rollback(); + return self.makeToken(toktype); } From b591ecdf9b5b13c4e2f8ab0ced0acd70358c87a7 Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 1 Jul 2019 15:07:17 -0300 Subject: [PATCH 2/4] fix identifier -> keyword conversion --- src/scanner.zig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/scanner.zig b/src/scanner.zig index 56084a5..613b096 100644 --- a/src/scanner.zig +++ b/src/scanner.zig @@ -249,6 +249,9 @@ pub const Scanner = struct { _ = self.advance(); } + // ugly hack. + self.rollback(); + // after reading the identifier, we check // if it is any of our keywords, if it is, then we add // the specificed keyword type. if not, just .IDENTIFIER @@ -261,9 +264,6 @@ pub const Scanner = struct { toktype = TokenType.Identifier; } - // ugly hack. - self.rollback(); - return self.makeToken(toktype); } From e2438f143b722d944e778cf4ead13673d45fc816 Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 1 Jul 2019 15:25:07 -0300 Subject: [PATCH 3/4] work towards FnDecl ast node --- src/ast.zig | 2 +- src/parser.zig | 34 ++++++++++++++++++++++++++-------- 2 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/ast.zig b/src/ast.zig index 4de4f03..a2b1834 100644 --- a/src/ast.zig +++ b/src/ast.zig @@ -10,7 +10,7 @@ pub const NodeType = enum { }; pub const FnDecl = struct { - func_name: []const u8, + func_name: Token, }; pub const Node = union(NodeType) { diff --git a/src/parser.zig b/src/parser.zig index e43588e..6142051 100644 --- a/src/parser.zig +++ b/src/parser.zig @@ -90,26 +90,44 @@ pub const Parser = struct { return Result.CompileError; } - fn mkFnDecl(self: *Parser, name: []const u8) !*ast.Node { - var node = try self.allocator.create(Node.FnDecl); - node.* = Node.FnDecl{ .name = name }; + fn consumeSingle(self: *Parser, ttype: TokenType) !Token { + if (self.check(ttype)) return self.advance(); + + var buf_main: [1000]u8 = undefined; + var buf = try std.fmt.bufPrint( + buf_main[0..], + "expected {}, got {}", + ttype, + self.peek().ttype, + ); + try self.tokenError(self.peek(), buf); + + return Result.CompileError; + } + + fn mkFnDecl(self: *Parser, name: Token) !*ast.Node { + var node = try self.allocator.create(Node); + node.* = Node{ .FnDecl = ast.FnDecl{ .func_name = name } }; return node; } fn functionDecl(self: *Parser) !*ast.Node { // get the name var name = try self.consume(.Identifier, "expected function name"); + _ = try self.consumeSingle(.LeftParen); return try self.mkFnDecl(name); } - fn processToken(self: *Parser, token: Token) Result!ast.Node { - switch (token.ttype) { - //.Fn => try self.functionDecl(), + fn processToken(self: *Parser, token: Token) !*ast.Node { + var node = switch (token.ttype) { + .Fn => try self.functionDecl(), else => blk: { try self.doError("TODO handle {}\n", token.ttype); return Result.CompileError; }, - } + }; + + return node; } pub fn parse(self: *Parser) !*ast.Node { @@ -128,7 +146,7 @@ pub const Parser = struct { if (token.ttype == .EOF) break; var node = try self.processToken(token); - try root.Root.append(&node); + try root.Root.append(node); } else { continue; } From 4f7478d7b2f77a2e1f48514e779daf410cad5046 Mon Sep 17 00:00:00 2001 From: Luna Date: Mon, 1 Jul 2019 15:39:48 -0300 Subject: [PATCH 4/4] basic fndecl ast node working --- src/parser.zig | 76 +++++++++++++++++++++++--------------------------- 1 file changed, 35 insertions(+), 41 deletions(-) diff --git a/src/parser.zig b/src/parser.zig index 6142051..76b8c59 100644 --- a/src/parser.zig +++ b/src/parser.zig @@ -17,7 +17,6 @@ pub const Parser = struct { scanner: *Scanner, tokens: []Token = undefined, - current: usize = 0, pub fn init(allocator: *Allocator, scanner: *Scanner) Parser { return Parser{ .allocator = allocator, .scanner = scanner }; @@ -32,11 +31,11 @@ pub const Parser = struct { } fn peek(self: *Parser) Token { - return self.tokens[self.current]; + return self.tokens[self.tokens.len - 1]; } fn previous(self: *Parser) Token { - return self.tokens[self.current - 1]; + return self.tokens[self.tokens.len - 2]; } fn tokenError(self: *Parser, token: Token, msg: []const u8) Result!void { @@ -53,45 +52,47 @@ pub const Parser = struct { return self.peek().ttype == .EOF; } - fn advance(self: *Parser) Token { - if (!self.isAtEnd()) self.current += 1; - return self.previous(); - } - fn check(self: *Parser, ttype: TokenType) bool { if (self.isAtEnd()) return false; return self.peek().ttype == ttype; } - fn match(self: *Parser, ttypes: []TokenType) bool { - for (ttypes) |ttype| { - if (self.check(ttype)) { - _ = self.advance(); - return true; + fn nextToken(self: *Parser) !Token { + var token: Token = undefined; + + while (true) { + var next_token_opt = try self.scanner.nextToken(); + if (next_token_opt) |token_nice| { + token = token_nice; + break; } } - return false; + self.tokens = try self.allocator.realloc( + self.tokens, + self.tokens.len + 1, + ); + + self.tokens[self.tokens.len - 1] = token; + return token; } - fn matchSingle(self: *Parser, ttype: TokenType) bool { - if (self.check(ttype)) { - _ = self.advance(); - return true; - } - - return false; - } - - fn consume(self: *Parser, ttype: TokenType, comptime msg: []const u8) Result!Token { - if (self.check(ttype)) return self.advance(); + fn consume(self: *Parser, ttype: TokenType, comptime msg: []const u8) !Token { + if (self.check(ttype)) return try self.nextToken(); try self.tokenError(self.peek(), msg); return Result.CompileError; } fn consumeSingle(self: *Parser, ttype: TokenType) !Token { - if (self.check(ttype)) return self.advance(); + std.debug.warn("consume {}, has {}\n", ttype, self.peek().ttype); + + if (self.check(ttype)) { + var cur = self.peek(); + _ = try self.nextToken(); + std.debug.warn("now has {}\n", self.peek()); + return cur; + } var buf_main: [1000]u8 = undefined; var buf = try std.fmt.bufPrint( @@ -112,9 +113,10 @@ pub const Parser = struct { } fn functionDecl(self: *Parser) !*ast.Node { - // get the name - var name = try self.consume(.Identifier, "expected function name"); + _ = try self.consumeSingle(.Fn); + var name = try self.consumeSingle(.Identifier); _ = try self.consumeSingle(.LeftParen); + return try self.mkFnDecl(name); } @@ -132,24 +134,16 @@ pub const Parser = struct { pub fn parse(self: *Parser) !*ast.Node { self.tokens = try self.allocator.alloc(Token, 0); - var i: usize = 0; var root = try ast.mkRoot(self.allocator); while (true) { - var tok_opt = try self.scanner.nextToken(); + var token = try self.nextToken(); - if (tok_opt) |token| { - self.tokens = try self.allocator.realloc(self.tokens, i + 1); - self.tokens[i] = token; - i += 1; + if (token.ttype == .EOF) break; - if (token.ttype == .EOF) break; - - var node = try self.processToken(token); - try root.Root.append(node); - } else { - continue; - } + var node = try self.processToken(token); + std.debug.warn("{}\n", node.*); + try root.Root.append(node); } return root;