From a8e2e5c02158b9e80e819c5b4592099709af8bed Mon Sep 17 00:00:00 2001 From: Michael Miller Date: Mon, 27 Jul 2020 10:36:53 -0600 Subject: [PATCH 01/13] Store random seed --- src/spectator/config.cr | 4 ++++ src/spectator/config_builder.cr | 4 ++++ src/spectator/report.cr | 6 +++++- src/spectator/runner.cr | 2 +- 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/spectator/config.cr b/src/spectator/config.cr index b7a1eda..a44564b 100644 --- a/src/spectator/config.cr +++ b/src/spectator/config.cr @@ -19,6 +19,9 @@ module Spectator # Indicates whether tests are run in a random order. getter? randomize : Bool + # Random seed used for number generation. + getter! random_seed : UInt64? + # Indicates whether profiling information should be displayed. getter? profile : Bool @@ -33,6 +36,7 @@ module Spectator @dry_run = builder.dry_run? @random = builder.random @randomize = builder.randomize? + @random_seed = builder.seed? @profile = builder.profile? @example_filter = builder.example_filter end diff --git a/src/spectator/config_builder.cr b/src/spectator/config_builder.cr index c650945..33dfb82 100644 --- a/src/spectator/config_builder.cr +++ b/src/spectator/config_builder.cr @@ -90,8 +90,12 @@ module Spectator @dry_run end + # Seed used for random number generation. + getter! seed : UInt64? + # Sets the seed for the random number generator. def seed=(seed) + @seed = seed @random = Random.new(seed) end diff --git a/src/spectator/report.cr b/src/spectator/report.cr index 54cbb95..9ce1ebd 100644 --- a/src/spectator/report.cr +++ b/src/spectator/report.cr @@ -25,12 +25,16 @@ module Spectator # This will be greater than zero only in fail-fast mode. getter remaining_count + # Random seed used to determine test ordering. + getter! random_seed : UInt64? + # Creates the report. # The *results* are from running the examples in the test suite. # The *runtime* is the total time it took to execute the suite. # The *remaining_count* is the number of tests skipped due to fail-fast. # The *fail_blank* flag indicates whether it is a failure if there were no tests run. - def initialize(@results : Array(Result), @runtime, @remaining_count = 0, @fail_blank = false) + # The *random_seed* is the seed used for random number generation. + def initialize(@results : Array(Result), @runtime, @remaining_count = 0, @fail_blank = false, @random_seed = nil) @results.each do |result| case result when SuccessfulResult diff --git a/src/spectator/runner.cr b/src/spectator/runner.cr index 8bde725..1c1c5d8 100644 --- a/src/spectator/runner.cr +++ b/src/spectator/runner.cr @@ -25,7 +25,7 @@ module Spectator # Generate a report and pass it along to the formatter. remaining = @suite.size - results.size - report = Report.new(results, elapsed, remaining, @config.fail_blank?) + report = Report.new(results, elapsed, remaining, @config.fail_blank?, @config.random_seed?) @config.each_formatter(&.end_suite(report, profile(report))) !report.failed? From 7aa8cb14a4974c5b270ee15f1cad131877fe998e Mon Sep 17 00:00:00 2001 From: Michael Miller Date: Mon, 27 Jul 2020 10:37:08 -0600 Subject: [PATCH 02/13] Force seed to UInt64 --- src/spectator/command_line_arguments_config_source.cr | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/spectator/command_line_arguments_config_source.cr b/src/spectator/command_line_arguments_config_source.cr index 7b31e99..ce3c7e5 100644 --- a/src/spectator/command_line_arguments_config_source.cr +++ b/src/spectator/command_line_arguments_config_source.cr @@ -61,7 +61,7 @@ module Spectator private def seed_option(parser, builder) parser.on("--seed INTEGER", "Set the seed for the random number generator (implies -r)") do |seed| builder.randomize - builder.seed = seed.to_i + builder.seed = seed.to_u64 end end @@ -74,7 +74,7 @@ module Spectator when /^rand/ builder.randomize parts = method.split(':', 2) - builder.seed = parts[1].to_i if parts.size > 1 + builder.seed = parts[1].to_u64 if parts.size > 1 else nil end From ba29c1e032acb6f8c2766e492bbca2999913e4e2 Mon Sep 17 00:00:00 2001 From: Michael Miller Date: Mon, 27 Jul 2020 12:10:39 -0600 Subject: [PATCH 03/13] Show random seed Only works when a seed specified, not when a random seed is used. --- src/spectator/formatting/random_seed_text.cr | 14 ++++++++++++++ src/spectator/formatting/suite_summary.cr | 3 +++ 2 files changed, 17 insertions(+) create mode 100644 src/spectator/formatting/random_seed_text.cr diff --git a/src/spectator/formatting/random_seed_text.cr b/src/spectator/formatting/random_seed_text.cr new file mode 100644 index 0000000..998fed1 --- /dev/null +++ b/src/spectator/formatting/random_seed_text.cr @@ -0,0 +1,14 @@ +module Spectator::Formatting + # Text displayed when using a random seed. + private struct RandomSeedText + # Creates the text object. + def initialize(@seed : UInt64) + end + + # Appends the command to the output. + def to_s(io) + io << "Randomized with seed " + io << @seed + end + end +end diff --git a/src/spectator/formatting/suite_summary.cr b/src/spectator/formatting/suite_summary.cr index 3d2dcfd..8ff10cf 100644 --- a/src/spectator/formatting/suite_summary.cr +++ b/src/spectator/formatting/suite_summary.cr @@ -49,6 +49,9 @@ module Spectator::Formatting private def stats(report) @io.puts Runtime.new(report.runtime) @io.puts StatsCounter.new(report).color + if (seed = report.random_seed?) + @io.puts RandomSeedText.new(seed) + end end # Produces the skipped tests text if fail-fast is enabled and tests were omitted. From 8fc2c93960f6bab1ac01cf92f9d001a3a3da2661 Mon Sep 17 00:00:00 2001 From: Michael Miller Date: Mon, 27 Jul 2020 13:32:39 -0600 Subject: [PATCH 04/13] Track seed used when just -r is provided --- src/spectator/config_builder.cr | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/spectator/config_builder.cr b/src/spectator/config_builder.cr index 33dfb82..42544eb 100644 --- a/src/spectator/config_builder.cr +++ b/src/spectator/config_builder.cr @@ -11,6 +11,11 @@ module Spectator # Random number generator to use. protected getter random = Random::DEFAULT + def initialize + @seed = seed = @random.rand(UInt64) + @random.new_seed(seed) + end + @primary_formatter : Formatting::Formatter? @additional_formatters = [] of Formatting::Formatter @fail_fast = false From e168b0ac7a5bfde608d09e3909bdc259d0325955 Mon Sep 17 00:00:00 2001 From: Michael Miller Date: Mon, 27 Jul 2020 13:32:49 -0600 Subject: [PATCH 05/13] Blank line before seed output --- src/spectator/formatting/suite_summary.cr | 1 + 1 file changed, 1 insertion(+) diff --git a/src/spectator/formatting/suite_summary.cr b/src/spectator/formatting/suite_summary.cr index 8ff10cf..183a78e 100644 --- a/src/spectator/formatting/suite_summary.cr +++ b/src/spectator/formatting/suite_summary.cr @@ -50,6 +50,7 @@ module Spectator::Formatting @io.puts Runtime.new(report.runtime) @io.puts StatsCounter.new(report).color if (seed = report.random_seed?) + @io.puts @io.puts RandomSeedText.new(seed) end end From e5ed6418db28619f17e94b4adec7addfad5fc323 Mon Sep 17 00:00:00 2001 From: Michael Miller Date: Mon, 27 Jul 2020 13:37:41 -0600 Subject: [PATCH 06/13] Only include seed if report if randomized --- src/spectator/runner.cr | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/spectator/runner.cr b/src/spectator/runner.cr index 1c1c5d8..d5c50f9 100644 --- a/src/spectator/runner.cr +++ b/src/spectator/runner.cr @@ -25,7 +25,8 @@ module Spectator # Generate a report and pass it along to the formatter. remaining = @suite.size - results.size - report = Report.new(results, elapsed, remaining, @config.fail_blank?, @config.random_seed?) + seed = (@config.random_seed? if @config.randomize?) + report = Report.new(results, elapsed, remaining, @config.fail_blank?, seed) @config.each_formatter(&.end_suite(report, profile(report))) !report.failed? From 4e50108fb91c6becedbce3193e9f2017ef40cafc Mon Sep 17 00:00:00 2001 From: Michael Miller Date: Mon, 27 Jul 2020 13:39:57 -0600 Subject: [PATCH 07/13] Use smaller range for default random seed RSpec seems to use a smaller range, something like 65k (UInt16). --- src/spectator/config_builder.cr | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/spectator/config_builder.cr b/src/spectator/config_builder.cr index 42544eb..d65ffc1 100644 --- a/src/spectator/config_builder.cr +++ b/src/spectator/config_builder.cr @@ -12,7 +12,7 @@ module Spectator protected getter random = Random::DEFAULT def initialize - @seed = seed = @random.rand(UInt64) + @seed = seed = @random.rand(UInt16).to_u64 @random.new_seed(seed) end From ac9a0cec4f4f57b6612058156ba038c5b7d64641 Mon Sep 17 00:00:00 2001 From: Michael Miller Date: Mon, 27 Jul 2020 13:42:55 -0600 Subject: [PATCH 08/13] Bump version to 0.9.21 Reference latest Crystal. --- shard.yml | 4 ++-- src/spectator.cr | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/shard.yml b/shard.yml index b34a636..14630ad 100644 --- a/shard.yml +++ b/shard.yml @@ -1,12 +1,12 @@ name: spectator -version: 0.9.20 +version: 0.9.21 description: | A feature-rich spec testing framework for Crystal with similarities to RSpec. authors: - Michael Miller -crystal: 0.34.0 +crystal: 0.35.1 license: MIT diff --git a/src/spectator.cr b/src/spectator.cr index 9c4907c..66939af 100644 --- a/src/spectator.cr +++ b/src/spectator.cr @@ -6,7 +6,7 @@ module Spectator extend self # Current version of the Spectator library. - VERSION = "0.9.20" + VERSION = "0.9.21" # Top-level describe method. # All specs in a file must be wrapped in this call. From 6e7f094a4b2ba997eabc19a96018af801b24cfad Mon Sep 17 00:00:00 2001 From: Michael Miller Date: Mon, 27 Jul 2020 13:47:40 -0600 Subject: [PATCH 09/13] Update Ameba --- shard.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shard.yml b/shard.yml index 14630ad..0257cdc 100644 --- a/shard.yml +++ b/shard.yml @@ -13,4 +13,4 @@ license: MIT development_dependencies: ameba: github: crystal-ameba/ameba - version: ~> 0.12.0 + version: ~> 0.13.1 From 3e3be77d300e489375d6c38b64124eb719088bfc Mon Sep 17 00:00:00 2001 From: Michael Miller Date: Thu, 6 Aug 2020 19:16:05 -0600 Subject: [PATCH 10/13] Ignore Crystal version for nightly builds --- .gitlab-ci.yml | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8a58eae..630d59b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -10,23 +10,25 @@ cache: before_script: - crystal -v # Print out Crystal version for debugging - - shards spec: script: - - crystal spec --error-on-warnings + - shards + - crystal spec --error-on-warnings style: script: - - bin/ameba - - crystal tool format --check + - shards + - bin/ameba + - crystal tool format --check nightly: image: "crystallang/crystal:nightly" allow_failure: true script: - - crystal spec --error-on-warnings - - crystal tool format --check + - shards --ignore-crystal-version + - crystal spec --error-on-warnings + - crystal tool format --check pages: stage: deploy From 9c14c69bba96357f8164a052da55ba7370111a43 Mon Sep 17 00:00:00 2001 From: Michael Miller Date: Thu, 6 Aug 2020 19:20:18 -0600 Subject: [PATCH 11/13] Change "ditto" to ":ditto:" --- src/spectator.cr | 2 +- src/spectator/dsl/assertions.cr | 2 +- src/spectator/dsl/matchers.cr | 4 ++-- src/spectator/expectations/expectation_partial.cr | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/spectator.cr b/src/spectator.cr index 66939af..4ed295d 100644 --- a/src/spectator.cr +++ b/src/spectator.cr @@ -45,7 +45,7 @@ module Spectator end end - # ditto + # :ditto: macro context(description, &block) describe({{description}}) {{block}} end diff --git a/src/spectator/dsl/assertions.cr b/src/spectator/dsl/assertions.cr index aa94b13..11b5ffe 100644 --- a/src/spectator/dsl/assertions.cr +++ b/src/spectator/dsl/assertions.cr @@ -207,7 +207,7 @@ module Spectator raise ExampleFailed.new(reason) end - # ditto + # :ditto: @[AlwaysInline] def fail fail("Example failed") diff --git a/src/spectator/dsl/matchers.cr b/src/spectator/dsl/matchers.cr index 3a2a703..bbf2f62 100644 --- a/src/spectator/dsl/matchers.cr +++ b/src/spectator/dsl/matchers.cr @@ -518,7 +518,7 @@ module Spectator ::Spectator::Matchers::HaveKeyMatcher.new(%test_value) end - # ditto + # :ditto: macro has_key(expected) have_key({{expected}}) end @@ -536,7 +536,7 @@ module Spectator ::Spectator::Matchers::HaveValueMatcher.new(%test_value) end - # ditto + # :ditto: macro has_value(expected) have_value({{expected}}) end diff --git a/src/spectator/expectations/expectation_partial.cr b/src/spectator/expectations/expectation_partial.cr index 6a38301..c837c86 100644 --- a/src/spectator/expectations/expectation_partial.cr +++ b/src/spectator/expectations/expectation_partial.cr @@ -52,7 +52,7 @@ module Spectator::Expectations stubs.each { |stub| to_not(stub) } end - # ditto + # :ditto: @[AlwaysInline] def not_to(matcher) : Nil to_not(matcher) @@ -86,7 +86,7 @@ module Spectator::Expectations to_not(stub) end - # ditto + # :ditto: @[AlwaysInline] def never_to(matcher) : Nil to_never(matcher) From 8f0718db86e0b7475d3f33f90b84a0a6cd8b4158 Mon Sep 17 00:00:00 2001 From: Michael Miller Date: Tue, 11 Aug 2020 14:36:50 -0600 Subject: [PATCH 12/13] Handle splat in macro for matcher DSL Should fix https://github.com/icy-arctic-fox/spectator/issues/8 --- src/spectator/dsl/matchers.cr | 45 +++++++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 10 deletions(-) diff --git a/src/spectator/dsl/matchers.cr b/src/spectator/dsl/matchers.cr index bbf2f62..a9f1030 100644 --- a/src/spectator/dsl/matchers.cr +++ b/src/spectator/dsl/matchers.cr @@ -450,8 +450,13 @@ module Spectator # expect(%i[a b c]).to contain(:a, :b) # ``` macro contain(*expected) - %test_value = ::Spectator::TestValue.new({{expected}}, {{expected.splat.stringify}}) - ::Spectator::Matchers::ContainMatcher.new(%test_value) + {% if expected.id.starts_with?("{*") %} + %test_value = ::Spectator::TestValue.new({{expected.id[2...-1]}}, {{expected.splat.stringify}}) + ::Spectator::Matchers::ContainMatcher.new(%test_value) + {% else %} + %test_value = ::Spectator::TestValue.new({{expected}}, {{expected.splat.stringify}}) + ::Spectator::Matchers::ContainMatcher.new(%test_value) + {% end %} end # Indicates that some range (or collection) should contain another value. @@ -471,8 +476,13 @@ module Spectator # expect(..100).to contain(0, 50) # ``` macro cover(*expected) - %test_value = ::Spectator::TestValue.new({{expected}}, {{expected.splat.stringify}}) - ::Spectator::Matchers::ContainMatcher.new(%test_value) + {% if expected.id.starts_with?("{*") %} + %test_value = ::Spectator::TestValue.new({{expected.id[2...-1]}}, {{expected.splat.stringify}}) + ::Spectator::Matchers::ContainMatcher.new(%test_value) + {% else %} + %test_value = ::Spectator::TestValue.new({{expected}}, {{expected.splat.stringify}}) + ::Spectator::Matchers::ContainMatcher.new(%test_value) + {% end %} end # Indicates that some value or set should contain another value. @@ -501,8 +511,13 @@ module Spectator # expect(%w[FOO BAR BAZ]).to have(/foo/i, String) # ``` macro have(*expected) - %test_value = ::Spectator::TestValue.new({{expected}}, {{expected.splat.stringify}}) - ::Spectator::Matchers::HaveMatcher.new(%test_value) + {% if expected.id.starts_with?("{*") %} + %test_value = ::Spectator::TestValue.new({{expected.id[2...-1]}}, {{expected.splat.stringify}}) + ::Spectator::Matchers::HaveMatcher.new(%test_value) + {% else %} + %test_value = ::Spectator::TestValue.new({{expected}}, {{expected.splat.stringify}}) + ::Spectator::Matchers::HaveMatcher.new(%test_value) + {% end %} end # Indicates that some set, such as a `Hash`, has a given key. @@ -548,8 +563,13 @@ module Spectator # expect([1, 2, 3]).to contain_exactly(3, 2, 1) # ``` macro contain_exactly(*expected) - %test_value = ::Spectator::TestValue.new(({{expected}}).to_a, {{expected.stringify}}) - ::Spectator::Matchers::ArrayMatcher.new(%test_value) + {% if expected.id.starts_with?("{*") %} + %test_value = ::Spectator::TestValue.new(({{expected.id[2...-1]}}).to_a, {{expected.stringify}}) + ::Spectator::Matchers::ArrayMatcher.new(%test_value) + {% else %} + %test_value = ::Spectator::TestValue.new(({{expected}}).to_a, {{expected.stringify}}) + ::Spectator::Matchers::ArrayMatcher.new(%test_value) + {% end %} end # Indicates that some set should contain the same values in any order as another set. @@ -597,8 +617,13 @@ module Spectator # expect(%i[a b c]).to have_attributes(size: 1..5, first: Symbol) # ``` macro have_attributes(**expected) - %test_value = ::Spectator::TestValue.new({{expected}}, {{expected.double_splat.stringify}}) - ::Spectator::Matchers::AttributesMatcher.new(%test_value) + {% if expected.id.starts_with?("{**") %} + %test_value = ::Spectator::TestValue.new({{expected.id[3...-1]}}, {{expected.double_splat.stringify}}) + ::Spectator::Matchers::AttributesMatcher.new(%test_value) + {% else %} + %test_value = ::Spectator::TestValue.new({{expected}}, {{expected.double_splat.stringify}}) + ::Spectator::Matchers::AttributesMatcher.new(%test_value) + {% end %} end # Verifies that all elements of a collection satisfy some matcher. From ce248de0d9725ae723eb4e0f6bc1a853f3ae20fb Mon Sep 17 00:00:00 2001 From: Michael Miller Date: Tue, 11 Aug 2020 14:37:23 -0600 Subject: [PATCH 13/13] Bump version to 0.9.22 --- shard.yml | 2 +- src/spectator.cr | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/shard.yml b/shard.yml index 0257cdc..a6ba14f 100644 --- a/shard.yml +++ b/shard.yml @@ -1,5 +1,5 @@ name: spectator -version: 0.9.21 +version: 0.9.22 description: | A feature-rich spec testing framework for Crystal with similarities to RSpec. diff --git a/src/spectator.cr b/src/spectator.cr index 4ed295d..77c333e 100644 --- a/src/spectator.cr +++ b/src/spectator.cr @@ -6,7 +6,7 @@ module Spectator extend self # Current version of the Spectator library. - VERSION = "0.9.21" + VERSION = "0.9.22" # Top-level describe method. # All specs in a file must be wrapped in this call.