diff --git a/spec/helpers/example.cr b/spec/helpers/example.cr new file mode 100644 index 0000000..8ad8d46 --- /dev/null +++ b/spec/helpers/example.cr @@ -0,0 +1,71 @@ +require "ecr" +require "json" +require "./result" + +module Spectator::SpecHelpers + # Wrapper for compiling and running an example at runtime and getting a result. + class Example + # Creates the example. + # The *spec_helper_path* is the path to spec_helper.cr file. + # The name or ID of the example is given by *example_id*. + # Lastly, the source code for the example is given by *example_code*. + def initialize(@spec_helper_path : String, @example_id : String, @example_code : String) + end + + # Instructs the Crystal compiler to compile the test. + # Returns an instance of `JSON::Any`. + # This will be the outcome and information about the test. + # Output will be surpressed for the test. + # If an error occurs while attempting to compile and run the test, an error will be raised. + def compile + # Create a temporary file containing the test. + with_tempfile do |source_file| + args = ["run", "--no-color", source_file, "--", "--json"] + Process.run(crystal_executable, args) do |process| + JSON.parse(process.output) + rescue JSON::ParseException + raise "Compilation of example #{@example_id} failed\n\n#{process.error.gets_to_end}" + end + end + end + + # Same as `#compile`, but returns the result of the first example in the test. + # Returns a `SpectatorHelpers::Result` instance. + def result + output = compile + example = output["examples"][0] + Result.from_json_any(example) + end + + # Constructs the string representation of the example. + # This produces the Crystal source code. + # *io* is the file handle to write to. + # The *dir* is the directory of the file being written to. + # This is needed to resolve the relative path to the spec_helper.cr file. + private def write(io, dir) + spec_helper_path = Path[@spec_helper_path].relative_to(dir) + ECR.embed(__DIR__ + "/example.ecr", io) + end + + # Creates a temporary file containing the compilable example code. + # Yields the path of the temporary file. + # Ensures the file is deleted after it is done being used. + private def with_tempfile + tempfile = File.tempfile("_#{@example_id}_spec.cr") do |file| + dir = File.dirname(file.path) + write(file, dir) + end + + begin + yield tempfile.path + ensure + tempfile.delete + end + end + + # Attempts to find the Crystal compiler on the system or raises an error. + private def crystal_executable + Process.find_executable("crystal") || raise("Could not find Crystal compiler") + end + end +end diff --git a/spec/helpers/example.ecr b/spec/helpers/example.ecr new file mode 100644 index 0000000..53355bf --- /dev/null +++ b/spec/helpers/example.ecr @@ -0,0 +1,5 @@ +require "<%= spec_helper_path %>" + +Spectator.describe "<%= @example_id %>" do + <%= @example_code %> +end diff --git a/spec/helpers/result.cr b/spec/helpers/result.cr new file mode 100644 index 0000000..76aa48d --- /dev/null +++ b/spec/helpers/result.cr @@ -0,0 +1,59 @@ +module Spectator::SpecHelpers + # Information about an example compiled and run at runtime. + class Result + # Status of the example after running. + enum Outcome + Success + Failure + Error + Unknown + end + + # Full name and description of the example. + getter name : String + + # Status of the example after running. + getter outcome : Outcome + + # Creates the result. + def initialize(@name, @outcome) + end + + # Checks if the example was successful. + def success? + outcome.success? + end + + # :ditto: + def successful? + outcome.success? + end + + # Checks if the example failed, but did not error. + def failure? + outcome.failure? + end + + # Checks if the example encountered an error. + def error? + outcome.error? + end + + # Extracts the result information from a `JSON::Any` object. + def self.from_json_any(object : JSON::Any) + name = object["name"].as_s + outcome = parse_outcome_string(object["result"].as_s) + new(name, outcome) + end + + # Converts a result string, such as "fail" to an enum value. + private def self.parse_outcome_string(string) + case string + when /success/i then Outcome::Success + when /fail/i then Outcome::Failure + when /error/i then Outcome::Error + else Outcome::Unknown + end + end + end +end diff --git a/spec/runtime_example_spec.cr b/spec/runtime_example_spec.cr new file mode 100644 index 0000000..a6a4ecd --- /dev/null +++ b/spec/runtime_example_spec.cr @@ -0,0 +1,42 @@ +require "./spec_helper" + +# This is a meta test that ensures specs can be compiled and run at runtime. +# The purpose of this is to report an error if this process fails. +# Other tests will fail, but display a different name/description of the test. +# This clearly indicates that runtime testing failed. +# +# Runtime compilation is used to get output of tests as well as check syntax. +# Some specs are too complex to be ran normally. +# Additionally, this allows examples to easily check specific failure cases. +# Plus, it makes testing user-reported issues easy. +Spectator.describe "Runtime compilation" do + given_example passing_example do + it "does something" do + expect(true).to be_true + end + end + + it "can compile and retrieve the result of an example" do + expect(passing_example).to be_successful + end + + given_example failing_example do + it "does something" do + expect(true).to be_false + end + end + + it "detects failed examples" do + expect(failing_example).to be_failure + end + + given_example malformed_example do + it "does something" do + asdf + end + end + + it "raises on compilation errors" do + expect { malformed_example }.to raise_error(/compilation/i) + end +end diff --git a/spec/spec_helper.cr b/spec/spec_helper.cr index f8047da..865a6b3 100644 --- a/spec/spec_helper.cr +++ b/spec/spec_helper.cr @@ -1,4 +1,5 @@ require "../src/spectator" +require "./helpers/**" macro it_fails(description = nil, &block) it {{description}} do @@ -11,3 +12,17 @@ end macro specify_fails(description = nil, &block) it_fails {{description}} {{block}} end + +# Defines an example ("it" block) that is lazily compiled. +# When the example is referenced with *id*, it will be compiled and the results retrieved. +# The value returned by *id* will be a `Spectator::SpecHelpers::Result`. +# This allows the test result to be inspected. +macro given_example(id, &block) + let({{id}}) do + ::Spectator::SpecHelpers::Example.new( + {{__FILE__}}, + {{id.id.stringify}}, + {{block.body.stringify}} + ).result + end +end