diff --git a/src/spectator/formatting/components.cr b/src/spectator/formatting/components.cr index 9c7c7b0..9d888f3 100644 --- a/src/spectator/formatting/components.cr +++ b/src/spectator/formatting/components.cr @@ -1,4 +1,4 @@ -require "./components/*" +require "./components/**" module Spectator::Formatting # Namespace for snippets of text displayed in console output. diff --git a/src/spectator/formatting/components/junit/root.cr b/src/spectator/formatting/components/junit/root.cr new file mode 100644 index 0000000..f38f968 --- /dev/null +++ b/src/spectator/formatting/components/junit/root.cr @@ -0,0 +1,39 @@ +require "./test_suite" + +module Spectator::Formatting::Components::JUnit + # Root node of the JUnit XML document. + # This is the "testsuites" element and all of its children. + struct Root + # Creates the root element. + def initialize(@runtime : Time::Span, @suites : Array(TestSuite), *, + @total : Int32, @failures : Int32, @errors : Int32) + end + + # Constructs the element from a report. + def self.from_report(report) + hostname = System.hostname + counts = report.counts + suites = report.examples.group_by(&.location.file) + suites = suites.map do |file, examples| + TestSuite.from_examples(file, examples, hostname) + end + + new(report.runtime, suites, + total: counts.total, + failures: counts.fail, + errors: counts.error) + end + + # Produces the XML fragment. + def to_xml(xml) + xml.element("testsuites", + tests: @total, + failures: @failures, + errors: @errors, + time: @runtime.total_seconds, + name: "Spec") do + @suites.each(&.to_xml(xml)) + end + end + end +end diff --git a/src/spectator/formatting/components/junit/test_case.cr b/src/spectator/formatting/components/junit/test_case.cr new file mode 100644 index 0000000..f57cd5f --- /dev/null +++ b/src/spectator/formatting/components/junit/test_case.cr @@ -0,0 +1,86 @@ +require "xml" +require "../../../example" + +module Spectator::Formatting::Components::JUnit + # Test case node of the JUnit XML document. + # This is the "testsuite" element and all of its children. + struct TestCase + # Creates the test case element. + def initialize(@class_name : String, @example : Example) + end + + # Produces the XML fragment. + def to_xml(xml) + result = @example.result + xml.element("testcase", + name: @example, + assertions: result.expectations.size, + classname: @class_name, + status: result.accept(StatusVisitor), + time: result.elapsed.total_seconds) do + visitor = ElementVisitor.new(xml) + result.accept(visitor) + end + end + + # Picks the status string for a result. + private module StatusVisitor + extend self + + # Returns "PASS". + def pass(_result) + "PASS" + end + + # Returns "FAIL". + def fail(_result) + "FAIL" + end + + # :ditto: + def error(result) + fail(result) + end + + # Returns "TODO". + def pending(_result) + "TODO" + end + end + + # Result visitor that adds elements to the test case node depending on the result. + private struct ElementVisitor + # Creates the visitor. + def initialize(@xml : XML::Builder) + end + + # Does nothing. + def pass(_result) + # ... + end + + # Adds a failure element to the test case node. + def fail(result) + error = result.error + @xml.element("failure", message: error.message, type: error.class) do + # TODO: Add match-data as text to node. + end + end + + # Adds an error element to the test case node. + def error(result) + error = result.error + @xml.element("error", message: error.message, type: error.class) do + if backtrace = error.backtrace + @xml.text(backtrace.join("\n")) + end + end + end + + # Adds a skipped element to the test case node. + def pending(_result) + @xml.element("skipped") # TODO: Populate message attribute with reason from result. + end + end + end +end diff --git a/src/spectator/formatting/components/junit/test_suite.cr b/src/spectator/formatting/components/junit/test_suite.cr new file mode 100644 index 0000000..0c16809 --- /dev/null +++ b/src/spectator/formatting/components/junit/test_suite.cr @@ -0,0 +1,104 @@ +require "./test_case" + +module Spectator::Formatting::Components::JUnit + # Test suite node of the JUnit XML document. + # This is the "testsuite" element and all of its children. + struct TestSuite + # Amounts for each type of test result. + record Counts, total : Int32, failures : Int32, errors : Int32, skipped : Int32 + + # Creates the test suite element. + def initialize(@package : String, @name : String, @cases : Array(TestCase), + @time : Time::Span, @counts : Counts, @hostname : String) + end + + # Constructs the test suite element from a collection of tests. + # The *examples* should all come from the same *file*. + def self.from_examples(file, examples, hostname) + package, name = package_name_from_file(file) + counts = count_examples(examples) + time = examples.sum(&.result.elapsed) + cases = examples.map { |example| TestCase.new(name, example) } + new(package, name, cases, time, counts, hostname) + end + + # Constructs a package and suite name from a file path. + private def self.package_name_from_file(file) + path = Path.new(file.to_s) + name = path.stem + directory = path.relative_to(Dir.current).dirname + package = directory.gsub(File::SEPARATOR, '.') + {package, name} + end + + # Counts the number of examples for each result type. + private def self.count_examples(examples) + visitor = CountVisitor.new + + # Iterate through each example and count the number of each type of result. + # Don't count examples that haven't run (indicated by `Node#finished?`). + # This typically happens in fail-fast mode. + examples.each do |example| + example.result.accept(visitor) if example.finished? + end + + visitor.counts + end + + # Produces the XML fragment. + def to_xml(xml) + xml.element("testsuite", + package: @package, + name: @name, + tests: @counts.total, + failures: @counts.failures, + errors: @counts.errors, + skipped: @counts.skipped, + time: @time.total_seconds, + hostname: @hostname) do + @cases.each(&.to_xml(xml)) + end + end + + # Totals up the number of each type of result. + # Defines methods for the different types of results. + # Call `#counts` to retrieve the `Counts` instance. + private class CountVisitor + @pass = 0 + @failures = 0 + @errors = 0 + @skipped = 0 + + # Increments the number of passing examples. + def pass(_result) + @pass += 1 + end + + # Increments the number of failing (non-error) examples. + def fail(_result) + @failures += 1 + end + + # Increments the number of error (and failed) examples. + def error(result) + fail(result) + @errors += 1 + end + + # Increments the number of pending (skipped) examples. + def pending(_result) + @skipped += 1 + end + + # Produces the total counts. + def counts + Counts.new( + total: @pass + @failures + @skipped, + failures: @failures, + errors: @errors, + skipped: @skipped + ) + end + end + end +end diff --git a/src/spectator/formatting/junit_formatter.cr b/src/spectator/formatting/junit_formatter.cr index 32cec45..a777371 100644 --- a/src/spectator/formatting/junit_formatter.cr +++ b/src/spectator/formatting/junit_formatter.cr @@ -1,8 +1,51 @@ +require "xml" require "./formatter" module Spectator::Formatting + # Produces a JUnit compatible XML file containing the test results. class JUnitFormatter < Formatter - def initialize(output_dir) + # Default XML file name. + private OUTPUT_FILE = "output.xml" + + # XML builder for the entire document. + private getter! xml : XML::Builder + + # Output stream for the XML file. + private getter! io : IO + + # Creates the formatter. + # The *output_path* can be a directory or path of an XML file. + # If the former, then an "output.xml" file will be generated in the specified directory. + def initialize(output_path = OUTPUT_FILE) + @output_path = if output_path.ends_with?(".xml") + output_path + else + File.join(output_path, OUTPUT_FILE) + end + end + + # Prepares the formatter for writing. + def start(_notification) + @io = io = File.open(@output_path, "w") + @xml = xml = XML::Builder.new(io) + xml.start_document("1.0", "UTF-8") + end + + # Invoked after testing completes with summarized information from the test suite. + # Unfortunately, the JUnit specification is not conducive to streaming data. + # All results are gathered at the end, then the report is generated. + def dump_summary(notification) + report = notification.report + root = Components::JUnit::Root.from_report(report) + root.to_xml(xml) + end + + # Invoked at the end of the program. + # Allows the formatter to perform any cleanup and teardown. + def close + xml.end_document + xml.flush + io.close end end end diff --git a/src/spectator/report.cr b/src/spectator/report.cr index d565531..93a88ea 100644 --- a/src/spectator/report.cr +++ b/src/spectator/report.cr @@ -22,6 +22,9 @@ module Spectator end end + # Retrieves all examples that were planned to run as part of the suite. + getter examples : Array(Example) + # Total length of time it took to execute the test suite. # This includes examples, hooks, and framework processes. getter runtime : Time::Span