Refactor tokenizer

spec/ameba/rules/large_numbers_spec.cr

@@ -1,17 +1,17 @@
 require "../../spec_helper"
 
+module Ameba
+  subject = Rules::LargeNumbers.new
+
   private def it_transforms(number, expected)
     it "transforms large number #{number}" do
-    s = Ameba::Source.new number
-    Ameba::Rules::LargeNumbers.new.catch(s).should_not be_valid
+      s = Source.new number
+      Rules::LargeNumbers.new.catch(s).should_not be_valid
       s.errors.first.message.should contain expected
     end
   end
 
-module Ameba::Rules
-  subject = LargeNumbers.new
-
-  describe LargeNumbers do
+  describe Rules::LargeNumbers do
     it "passes if large number does not require underscore" do
       s = Source.new %q(
         1 2 3 4 5 6 7 8 9 10 11 12 13 14 15

spec/ameba/tokenizer_spec.cr

@@ -1,30 +1,32 @@
 require "../spec_helper"
 
+module Ameba
   private def it_tokenizes(str, expected)
     it "tokenizes #{str}" do
       ([] of Symbol).tap do |token_types|
-      Ameba::Tokenizer.new(Ameba::Source.new str).run do |token|
-        token_types << token.type
-      end.should be_true
+        Tokenizer.new(Source.new str)
+                 .run { |token| token_types << token.type }
+                 .should be_true
       end.should eq expected
     end
   end
 
-module Ameba
   describe Tokenizer do
     describe "#run" do
-      it_tokenizes %("string"), %i(STRING)
-      it_tokenizes %(100), %i(NUMBER)
-      it_tokenizes %('a'), %i(CHAR)
-      it_tokenizes %([]), %i([])
-      it_tokenizes %([] of String), %i([] SPACE IDENT SPACE CONST)
-      it_tokenizes %q("str #{3}"), %i(STRING NUMBER)
+      it_tokenizes %("string"), %i(DELIMITER_START STRING DELIMITER_END EOF)
+      it_tokenizes %(100), %i(NUMBER EOF)
+      it_tokenizes %('a'), %i(CHAR EOF)
+      it_tokenizes %([]), %i([] EOF)
+      it_tokenizes %([] of String), %i([] SPACE IDENT SPACE CONST EOF)
+      it_tokenizes %q("str #{3}"), %i(
+        DELIMITER_START STRING INTERPOLATION_START NUMBER } DELIMITER_END EOF
+      )
 
       it_tokenizes %(%w(1 2)),
-        %i(STRING_ARRAY_START STRING STRING STRING_ARRAY_END)
+        %i(STRING_ARRAY_START STRING STRING STRING_ARRAY_END EOF)
 
       it_tokenizes %(%i(one two)),
-        %i(SYMBOL_ARRAY_START STRING STRING STRING_ARRAY_END)
+        %i(SYMBOL_ARRAY_START STRING STRING STRING_ARRAY_END EOF)
 
       it_tokenizes %(
           class A
@@ -32,11 +34,11 @@ module Ameba
               puts "hello"
             end
           end
-      ), [
-        :NEWLINE, :SPACE, :IDENT, :SPACE, :CONST, :NEWLINE, :SPACE, :IDENT,
-        :SPACE, :IDENT, :NEWLINE, :SPACE, :IDENT, :SPACE, :STRING, :NEWLINE,
-        :SPACE, :IDENT, :NEWLINE, :SPACE, :IDENT, :NEWLINE, :SPACE,
-      ]
+      ), %i(
+        NEWLINE SPACE IDENT SPACE CONST NEWLINE SPACE IDENT SPACE IDENT
+        NEWLINE SPACE IDENT SPACE DELIMITER_START STRING DELIMITER_END
+        NEWLINE SPACE IDENT NEWLINE SPACE IDENT NEWLINE SPACE EOF
+      )
     end
   end
 end

src/ameba/tokenizer.cr

@@ -22,19 +22,17 @@ module Ameba
                                  &block : Crystal::Token -> _)
       while true
         token = @lexer.next_token
+        block.call token
+
         case token.type
         when :DELIMITER_START
           run_delimiter_state lexer, token, &block
         when :STRING_ARRAY_START, :SYMBOL_ARRAY_START
-          block.call token
           run_array_state lexer, token, &block
         when :EOF
           break
         when :"}"
           break if break_on_rcurly
-          block.call token
-        else
-          block.call token
         end
       end
     end
@@ -42,6 +40,8 @@ module Ameba
     private def run_delimiter_state(lexer, token, &block : Crystal::Token -> _)
       while true
         token = @lexer.next_string_token(token.delimiter_state)
+        block.call token
+
         case token.type
         when :DELIMITER_END
           break
@@ -49,8 +49,6 @@ module Ameba
           run_normal_state lexer, break_on_rcurly: true, &block
         when :EOF
           break
-        else
-          block.call token
         end
       end
     end
@@ -58,15 +56,13 @@ module Ameba
     private def run_array_state(lexer, token, &block : Crystal::Token -> _)
       while true
         lexer.next_string_array_token
+        block.call token
 
         case token.type
         when :STRING_ARRAY_END
-          block.call token
           break
         when :EOF
-          raise "Unterminated symbol array literal"
-        else
-          block.call token
+          break
         end
       end
     end