Refactor tokenizer

This commit is contained in:
Vitalii Elenhaupt 2017-11-04 17:38:04 +02:00
parent fedc29ceb6
commit e383ec17c2
No known key found for this signature in database
GPG key ID: 7558EF3A4056C706
3 changed files with 45 additions and 47 deletions

View file

@ -1,17 +1,17 @@
require "../../spec_helper"
module Ameba
subject = Rules::LargeNumbers.new
private def it_transforms(number, expected)
it "transforms large number #{number}" do
s = Ameba::Source.new number
Ameba::Rules::LargeNumbers.new.catch(s).should_not be_valid
s = Source.new number
Rules::LargeNumbers.new.catch(s).should_not be_valid
s.errors.first.message.should contain expected
end
end
module Ameba::Rules
subject = LargeNumbers.new
describe LargeNumbers do
describe Rules::LargeNumbers do
it "passes if large number does not require underscore" do
s = Source.new %q(
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15

View file

@ -1,30 +1,32 @@
require "../spec_helper"
module Ameba
private def it_tokenizes(str, expected)
it "tokenizes #{str}" do
([] of Symbol).tap do |token_types|
Ameba::Tokenizer.new(Ameba::Source.new str).run do |token|
token_types << token.type
end.should be_true
Tokenizer.new(Source.new str)
.run { |token| token_types << token.type }
.should be_true
end.should eq expected
end
end
module Ameba
describe Tokenizer do
describe "#run" do
it_tokenizes %("string"), %i(STRING)
it_tokenizes %(100), %i(NUMBER)
it_tokenizes %('a'), %i(CHAR)
it_tokenizes %([]), %i([])
it_tokenizes %([] of String), %i([] SPACE IDENT SPACE CONST)
it_tokenizes %q("str #{3}"), %i(STRING NUMBER)
it_tokenizes %("string"), %i(DELIMITER_START STRING DELIMITER_END EOF)
it_tokenizes %(100), %i(NUMBER EOF)
it_tokenizes %('a'), %i(CHAR EOF)
it_tokenizes %([]), %i([] EOF)
it_tokenizes %([] of String), %i([] SPACE IDENT SPACE CONST EOF)
it_tokenizes %q("str #{3}"), %i(
DELIMITER_START STRING INTERPOLATION_START NUMBER } DELIMITER_END EOF
)
it_tokenizes %(%w(1 2)),
%i(STRING_ARRAY_START STRING STRING STRING_ARRAY_END)
%i(STRING_ARRAY_START STRING STRING STRING_ARRAY_END EOF)
it_tokenizes %(%i(one two)),
%i(SYMBOL_ARRAY_START STRING STRING STRING_ARRAY_END)
%i(SYMBOL_ARRAY_START STRING STRING STRING_ARRAY_END EOF)
it_tokenizes %(
class A
@ -32,11 +34,11 @@ module Ameba
puts "hello"
end
end
), [
:NEWLINE, :SPACE, :IDENT, :SPACE, :CONST, :NEWLINE, :SPACE, :IDENT,
:SPACE, :IDENT, :NEWLINE, :SPACE, :IDENT, :SPACE, :STRING, :NEWLINE,
:SPACE, :IDENT, :NEWLINE, :SPACE, :IDENT, :NEWLINE, :SPACE,
]
), %i(
NEWLINE SPACE IDENT SPACE CONST NEWLINE SPACE IDENT SPACE IDENT
NEWLINE SPACE IDENT SPACE DELIMITER_START STRING DELIMITER_END
NEWLINE SPACE IDENT NEWLINE SPACE IDENT NEWLINE SPACE EOF
)
end
end
end

View file

@ -22,19 +22,17 @@ module Ameba
&block : Crystal::Token -> _)
while true
token = @lexer.next_token
block.call token
case token.type
when :DELIMITER_START
run_delimiter_state lexer, token, &block
when :STRING_ARRAY_START, :SYMBOL_ARRAY_START
block.call token
run_array_state lexer, token, &block
when :EOF
break
when :"}"
break if break_on_rcurly
block.call token
else
block.call token
end
end
end
@ -42,6 +40,8 @@ module Ameba
private def run_delimiter_state(lexer, token, &block : Crystal::Token -> _)
while true
token = @lexer.next_string_token(token.delimiter_state)
block.call token
case token.type
when :DELIMITER_END
break
@ -49,8 +49,6 @@ module Ameba
run_normal_state lexer, break_on_rcurly: true, &block
when :EOF
break
else
block.call token
end
end
end
@ -58,15 +56,13 @@ module Ameba
private def run_array_state(lexer, token, &block : Crystal::Token -> _)
while true
lexer.next_string_array_token
block.call token
case token.type
when :STRING_ARRAY_END
block.call token
break
when :EOF
raise "Unterminated symbol array literal"
else
block.call token
break
end
end
end