add draft scanner, remove messy Token union, replace by struct

src/compiler.zig

@@ -1,17 +1,42 @@
 const std = @import("std");
 const scanner = @import("new_scanner.zig");
+const vm = @import("vm.zig");
 
 const Allocator = std.mem.Allocator;
+const TokenType = @import("token.zig").TokenType;
 
 pub const Compiler = struct {
     src: []const u8,
+    stdout: vm.StdOut,
     allocator: *Allocator,
 
-    pub fn init(allocator: *Allocator, source: []const u8) Compiler {
+    pub fn init(
-        return Compiler{ .src = source, .allocator = allocator };
+        allocator: *Allocator,
+        stdout: vm.StdOut,
+        source: []const u8,
+    ) Compiler {
+        return Compiler{
+            .src = source,
+            .allocator = allocator,
+            .stdout = stdout,
+        };
     }
 
-    pub fn compile(self: *Compiler) void {
+    pub fn compile(self: *Compiler) !void {
         var scanr = scanner.Scanner.init(self.allocator, self.src);
+        var line: usize = 0;
+        while (true) {
+            var token = scanr.scanToken();
+
+            if (token.line != line) {
+                try self.stdout.print("{} ", token.line);
+                line = token.line;
+            } else {
+                try self.stdout.print("  | ");
+            }
+
+            try self.stdout.print("{} '{}'\n", token.ttype, token.lexeme);
+            if (token.ttype == TokenType.EOF) break;
+        }
     }
 };

src/main.zig

@@ -17,7 +17,7 @@ fn run(allocator: *Allocator, data: []u8) !InterpretResult {
     const stdout = &stdout_file.outStream().stream;
 
     var vmach = try vm.VM.init(allocator, stdout, data, true);
-    return vmach.interpret();
+    return try vmach.interpret();
 }
 
 pub fn doError(line: usize, message: []const u8) !void {

src/new_scanner.zig

@@ -1,9 +1,15 @@
 const std = @import("std");
+const tokens = @import("token.zig");
 
 const Allocator = std.mem.Allocator;
 
 pub const Scanner = struct {
     source: []const u8,
+
+    start: usize = 0,
+    current: usize = 0,
+    line: usize = 1,
+
     allocator: *Allocator,
 
     pub fn init(allocator: *Allocator, data: []const u8) Scanner {
@@ -12,4 +18,8 @@ pub const Scanner = struct {
             .source = data,
         };
     }
+
+    pub fn scanToken(self: *Scanner) tokens.Token {
+        return tokens.Token{};
+    }
 };

src/token.zig

@@ -50,58 +50,8 @@ pub const TokenType = enum(u6) {
     EOF,
 };
 
-pub fn TokenFactory(
+pub const Token = struct {
-    comptime T: type,
+    ttype: TokenType = TokenType.EOF,
-) type {
+    lexeme: []const u8 = ""[0..],
-    return struct {
+    line: usize = 0,
-        const Self = @This();
-
-        ttype: TokenType,
-        lexeme: []u8,
-        line: usize,
-        literal: T,
-
-        pub fn init(
-            ttype: TokenType,
-            lexeme: []u8,
-            line: usize,
-            literal: T,
-        ) Self {
-            return Self{
-                .ttype = ttype,
-                .lexeme = lexeme,
-                .line = line,
-                .literal = literal,
-            };
-        }
-
-        pub fn printToken(self: Self, stdout: var) !void {
-            if (T == void) {
-                try stdout.print(
-                    "Token(type={x}, lexeme='{}', line={})\n",
-                    self.ttype,
-                    self.lexeme,
-                    self.line,
-                );
-            } else {
-                try stdout.print(
-                    "Token(type={x}, lexeme='{}', line={} literal='{}')\n",
-                    self.ttype,
-                    self.lexeme,
-                    self.line,
-                    self.literal,
-                );
-            }
-        }
-    };
-}
-
-pub const SimpleToken = TokenFactory(void);
-pub const SliceToken = TokenFactory([]u8);
-pub const NumberToken = TokenFactory(f32);
-
-pub const Token = union(enum) {
-    Simple: SimpleToken,
-    Slice: SliceToken,
-    Number: NumberToken,
 };

src/vm.zig

@@ -7,7 +7,7 @@ const Chunk = chunk.Chunk;
 const Value = value.Value;
 const Compiler = compiler.Compiler;
 
-const StdOut = *std.io.OutStream(std.fs.File.WriteError);
+pub const StdOut = *std.io.OutStream(std.fs.File.WriteError);
 
 pub const InterpretResult = enum {
     Ok,
@@ -159,14 +159,14 @@ pub const VM = struct {
         }
     }
 
-    pub fn interpret(self: *VM) InterpretResult {
+    pub fn interpret(self: *VM) !InterpretResult {
         //self.ip = 0;
         //self.debug("VM start\n");
         //var res = try self.run();
         //self.debug("VM end\n");
         //return res;
-        var cmpr = Compiler.init(self.allocator, self.src);
+        var cmpr = Compiler.init(self.allocator, self.stdout, self.src);
-        cmpr.compile();
+        try cmpr.compile();
         return InterpretResult.Ok;
     }