Initial runtime test compilation

Allows for compiling single examples at runtime.

spec/helpers/example.cr

@@ -0,0 +1,71 @@
+require "ecr"
+require "json"
+require "./result"
+
+module Spectator::SpecHelpers
+  # Wrapper for compiling and running an example at runtime and getting a result.
+  class Example
+    # Creates the example.
+    # The *spec_helper_path* is the path to spec_helper.cr file.
+    # The name or ID of the example is given by *example_id*.
+    # Lastly, the source code for the example is given by *example_code*.
+    def initialize(@spec_helper_path : String, @example_id : String, @example_code : String)
+    end
+
+    # Instructs the Crystal compiler to compile the test.
+    # Returns an instance of `JSON::Any`.
+    # This will be the outcome and information about the test.
+    # Output will be surpressed for the test.
+    # If an error occurs while attempting to compile and run the test, an error will be raised.
+    def compile
+      # Create a temporary file containing the test.
+      with_tempfile do |source_file|
+        args = ["run", "--no-color", source_file, "--", "--json"]
+        Process.run(crystal_executable, args) do |process|
+          JSON.parse(process.output)
+        rescue JSON::ParseException
+          raise "Compilation of example #{@example_id} failed\n\n#{process.error.gets_to_end}"
+        end
+      end
+    end
+
+    # Same as `#compile`, but returns the result of the first example in the test.
+    # Returns a `SpectatorHelpers::Result` instance.
+    def result
+      output = compile
+      example = output["examples"][0]
+      Result.from_json_any(example)
+    end
+
+    # Constructs the string representation of the example.
+    # This produces the Crystal source code.
+    # *io* is the file handle to write to.
+    # The *dir* is the directory of the file being written to.
+    # This is needed to resolve the relative path to the spec_helper.cr file.
+    private def write(io, dir)
+      spec_helper_path = Path[@spec_helper_path].relative_to(dir)
+      ECR.embed(__DIR__ + "/example.ecr", io)
+    end
+
+    # Creates a temporary file containing the compilable example code.
+    # Yields the path of the temporary file.
+    # Ensures the file is deleted after it is done being used.
+    private def with_tempfile
+      tempfile = File.tempfile("_#{@example_id}_spec.cr") do |file|
+        dir = File.dirname(file.path)
+        write(file, dir)
+      end
+
+      begin
+        yield tempfile.path
+      ensure
+        tempfile.delete
+      end
+    end
+
+    # Attempts to find the Crystal compiler on the system or raises an error.
+    private def crystal_executable
+      Process.find_executable("crystal") || raise("Could not find Crystal compiler")
+    end
+  end
+end

spec/helpers/example.ecr

@@ -0,0 +1,5 @@
+require "<%= spec_helper_path %>"
+
+Spectator.describe "<%= @example_id %>" do
+  <%= @example_code %>
+end

spec/helpers/result.cr

@@ -0,0 +1,59 @@
+module Spectator::SpecHelpers
+  # Information about an example compiled and run at runtime.
+  class Result
+    # Status of the example after running.
+    enum Outcome
+      Success
+      Failure
+      Error
+      Unknown
+    end
+
+    # Full name and description of the example.
+    getter name : String
+
+    # Status of the example after running.
+    getter outcome : Outcome
+
+    # Creates the result.
+    def initialize(@name, @outcome)
+    end
+
+    # Checks if the example was successful.
+    def success?
+      outcome.success?
+    end
+
+    # :ditto:
+    def successful?
+      outcome.success?
+    end
+
+    # Checks if the example failed, but did not error.
+    def failure?
+      outcome.failure?
+    end
+
+    # Checks if the example encountered an error.
+    def error?
+      outcome.error?
+    end
+
+    # Extracts the result information from a `JSON::Any` object.
+    def self.from_json_any(object : JSON::Any)
+      name = object["name"].as_s
+      outcome = parse_outcome_string(object["result"].as_s)
+      new(name, outcome)
+    end
+
+    # Converts a result string, such as "fail" to an enum value.
+    private def self.parse_outcome_string(string)
+      case string
+      when /success/i then Outcome::Success
+      when /fail/i then Outcome::Failure
+      when /error/i then Outcome::Error
+      else Outcome::Unknown
+      end
+    end
+  end
+end

spec/runtime_example_spec.cr

@@ -0,0 +1,42 @@
+require "./spec_helper"
+
+# This is a meta test that ensures specs can be compiled and run at runtime.
+# The purpose of this is to report an error if this process fails.
+# Other tests will fail, but display a different name/description of the test.
+# This clearly indicates that runtime testing failed.
+#
+# Runtime compilation is used to get output of tests as well as check syntax.
+# Some specs are too complex to be ran normally.
+# Additionally, this allows examples to easily check specific failure cases.
+# Plus, it makes testing user-reported issues easy.
+Spectator.describe "Runtime compilation" do
+  given_example passing_example do
+    it "does something" do
+      expect(true).to be_true
+    end
+  end
+
+  it "can compile and retrieve the result of an example" do
+    expect(passing_example).to be_successful
+  end
+
+  given_example failing_example do
+    it "does something" do
+      expect(true).to be_false
+    end
+  end
+
+  it "detects failed examples" do
+    expect(failing_example).to be_failure
+  end
+
+  given_example malformed_example do
+    it "does something" do
+      asdf
+    end
+  end
+
+  it "raises on compilation errors" do
+    expect { malformed_example }.to raise_error(/compilation/i)
+  end
+end

spec/spec_helper.cr

@@ -1,4 +1,5 @@
 require "../src/spectator"
+require "./helpers/**"
 
 macro it_fails(description = nil, &block)
   it {{description}} do
@@ -11,3 +12,17 @@ end
 macro specify_fails(description = nil, &block)
   it_fails {{description}} {{block}}
 end
+
+# Defines an example ("it" block) that is lazily compiled.
+# When the example is referenced with *id*, it will be compiled and the results retrieved.
+# The value returned by *id* will be a `Spectator::SpecHelpers::Result`.
+# This allows the test result to be inspected.
+macro given_example(id, &block)
+  let({{id}}) do
+    ::Spectator::SpecHelpers::Example.new(
+      {{__FILE__}},
+      {{id.id.stringify}},
+      {{block.body.stringify}}
+    ).result
+  end
+end