diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0e0494a..9d379bd 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,6 +4,6 @@ repos: - id: rubocop name: Rubocop language: system - entry: bundle exec rubocop + entry: bundle exec rake rubocop types: [ruby] exclude: \.lock diff --git a/.rubocop.yml b/.rubocop.yml index 308b92b..f7e14e5 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -4,6 +4,7 @@ plugins: AllCops: TargetRubyVersion: 3.1 NewCops: enable + SuggestExtensions: false Exclude: - bin/**/* - vendor/**/* diff --git a/.ruby-version b/.ruby-version index 4d54dad..d13e837 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -4.0.2 +4.0.6 diff --git a/allure-cucumber/lib/allure_cucumber/formatter.rb b/allure-cucumber/lib/allure_cucumber/formatter.rb index b233323..c9ce002 100644 --- a/allure-cucumber/lib/allure_cucumber/formatter.rb +++ b/allure-cucumber/lib/allure_cucumber/formatter.rb @@ -35,6 +35,8 @@ def initialize(config) config.on_event(:test_step_started) { |event| on_test_step_started(event) } config.on_event(:test_step_finished) { |event| on_test_step_finished(event) } config.on_event(:test_case_finished) { |event| on_test_case_finished(event) } + + register_test_run_hook_finished(config) end # Clean test result directory before starting run @@ -50,6 +52,17 @@ def on_test_run_started(_event) # @return [void] def on_test_run_finished(_event) lifecycle.write_environment + lifecycle.write_globals + end + + # Handle a global hook finishing + # @param [Cucumber::Events::TestRunHookFinished] event + # @return [void] + def on_test_run_hook_finished(event) + return unless event.test_result.failed? + + details = Allure::ResultUtils.status_details(event.test_result.exception) + lifecycle.add_global_error(message: details.message, trace: details.trace) end # Handle test case started event @@ -101,6 +114,15 @@ def on_test_case_finished(event) attr_reader :lifecycle, :cucumber_model + # Register global hook finished event + # @param [Cucumber::Configuration] config + # @return [void] + def register_test_run_hook_finished(config) + return unless defined?(Cucumber::Events::TestRunHookFinished) + + config.on_event(:test_run_hook_finished) { |event| on_test_run_hook_finished(event) } + end + # Is hook fixture like Before, After or Step as AfterStep # @param [String] text # @return [boolean] diff --git a/allure-cucumber/spec/cucumber_helper.rb b/allure-cucumber/spec/cucumber_helper.rb index 181d1e9..076c89f 100644 --- a/allure-cucumber/spec/cucumber_helper.rb +++ b/allure-cucumber/spec/cucumber_helper.rb @@ -74,8 +74,8 @@ def initialize(tmp_dir) @tmp_dir = tmp_dir end - def execute(feature) - setup(feature) + def execute(feature, support_code: ENV) + setup(feature, support_code) Cucumber::Cli::Main.new( [feature_file, *args], @@ -95,11 +95,11 @@ def all_output [@stdout.string, @stderr.string].reject(&:empty?).join("\n") end - def setup(feature) + def setup(feature, support_code) FileUtils.rm_rf(tmp_dir) write_file(feature_file, feature) - write_file("#{tmp_dir}/features/support/env.rb", ENV) + write_file("#{tmp_dir}/features/support/env.rb", support_code) write_file("#{tmp_dir}/features/step_definitions/step_defs.rb", STEPS) end diff --git a/allure-cucumber/spec/spec_helper.rb b/allure-cucumber/spec/spec_helper.rb index 10f8571..da8327b 100644 --- a/allure-cucumber/spec/spec_helper.rb +++ b/allure-cucumber/spec/spec_helper.rb @@ -44,7 +44,9 @@ RSpec.shared_context("cucumber runner") do let!(:test_tmp_dir) { |e| "tmp/#{e.full_description.tr(' ', '_')}" } - def run_cucumber_cli(feature) - Thread.new { CucumberHelper.new(test_tmp_dir).execute(feature) }.join + def run_cucumber_cli(feature, support_code: CucumberHelper::ENV) + Thread.new do + CucumberHelper.new(test_tmp_dir).execute(feature, support_code: support_code) + end.join end end diff --git a/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb b/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb index bc411df..90e50c0 100644 --- a/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb +++ b/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb @@ -14,7 +14,8 @@ FEATURE end - it "creates environment.properties file" do + it "writes run-level files" do expect(lifecycle).to have_received(:write_environment).once + expect(lifecycle).to have_received(:write_globals).once end end diff --git a/allure-cucumber/spec/unit/formatter_test_run_hook_finished_spec.rb b/allure-cucumber/spec/unit/formatter_test_run_hook_finished_spec.rb new file mode 100644 index 0000000..b6efd18 --- /dev/null +++ b/allure-cucumber/spec/unit/formatter_test_run_hook_finished_spec.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +describe "on_test_run_hook_finished" do + include_context "allure mock" + include_context "cucumber runner" + + let(:feature) do + <<~FEATURE + Feature: Global hooks + + Scenario: passing scenario + Given a is 5 + FEATURE + end + + def run_with_global_hooks(hooks) + run_cucumber_cli(feature, support_code: "#{CucumberHelper::ENV}\n#{hooks}") + end + + it "adds a failed BeforeAll hook to globals" do + run_with_global_hooks(<<~RUBY) + BeforeAll do + raise "BeforeAll failed" + end + RUBY + + expect(lifecycle).to have_received(:add_global_error).with( + message: "BeforeAll failed", + trace: a_string_including("env.rb") + ).once + end + + it "adds a failed AfterAll hook to globals" do + run_with_global_hooks(<<~RUBY) + AfterAll do + raise "AfterAll failed" + end + RUBY + + expect(lifecycle).to have_received(:add_global_error).with( + message: "AfterAll failed", + trace: a_string_including("env.rb") + ).once + end + + it "ignores passed global hooks" do + run_with_global_hooks(<<~RUBY) + BeforeAll do + end + + AfterAll do + end + RUBY + + expect(lifecycle).not_to have_received(:add_global_error) + end + + it "adds every failed global hook" do + errors = [] + allow(lifecycle).to receive(:add_global_error) { |**details| errors << details } + + run_with_global_hooks(<<~RUBY) + BeforeAll do + raise "BeforeAll failed" + end + + AfterAll do + raise "First AfterAll failed" + end + + AfterAll do + raise "Second AfterAll failed" + end + RUBY + + expect(errors.map { |error| error[:message] }).to eq( + ["BeforeAll failed", "First AfterAll failed", "Second AfterAll failed"] + ) + end + + it "captures the error before globals are written" do + run_with_global_hooks(<<~RUBY) + AfterAll do + raise "AfterAll failed" + end + RUBY + + expect(lifecycle).to have_received(:add_global_error).once.ordered + expect(lifecycle).to have_received(:write_globals).once.ordered + end +end diff --git a/allure-rspec/lib/allure_rspec/formatter.rb b/allure-rspec/lib/allure_rspec/formatter.rb index e475a00..e6f1f39 100644 --- a/allure-rspec/lib/allure_rspec/formatter.rb +++ b/allure-rspec/lib/allure_rspec/formatter.rb @@ -15,6 +15,8 @@ class RSpecFormatter < RSpec::Core::Formatters::BaseFormatter pending: Allure::Status::SKIPPED, passed: Allure::Status::PASSED }.freeze + ANSI_ESCAPE_PATTERN = /\e\[(\d+)(?:;\d+)*m/ + SUITE_HOOK_FAILURE_PATTERN = /\AAn error occurred in an? `(?:before|after)\(:suite\)` hook\.\z/ RSpec::Core::Formatters.register( self, @@ -23,7 +25,8 @@ class RSpecFormatter < RSpec::Core::Formatters::BaseFormatter :example_group_started, :example_group_finished, :example_started, - :example_finished + :example_finished, + :message ) RSpec.configure do |config| @@ -58,6 +61,7 @@ def start(_start_notification) # @return [void] def stop(_stop_notification) lifecycle.write_environment + lifecycle.write_globals end # Starts example group @@ -92,6 +96,17 @@ def example_group_finished(_example_group_notification) lifecycle.stop_test_container end + # Capture errors raised by before(:suite) and after(:suite) hooks + # @param [RSpec::Core::Notifications::MessageNotification] notification + # @return [void] + def message(notification) + formatted_error = notification.message.to_s.gsub(ANSI_ESCAPE_PATTERN, "").strip + first_line = formatted_error.lines.first&.chomp + return unless SUITE_HOOK_FAILURE_PATTERN.match?(first_line.to_s) + + lifecycle.add_global_error(message: first_line, trace: formatted_error) + end + private attr_reader :lifecycle, :allure_config @@ -125,7 +140,7 @@ def update_test_proc(result) test_case.stage = Allure::Stage::FINISHED test_case.status = status(result) test_case.status_details.message = status_detail.message - test_case.status_details.trace = status_detail.trace&.gsub(/\e\[(\d+)(?:;\d+)*m/, "") + test_case.status_details.trace = status_detail.trace&.gsub(ANSI_ESCAPE_PATTERN, "") end end end diff --git a/allure-rspec/spec/unit/formatter_message_spec.rb b/allure-rspec/spec/unit/formatter_message_spec.rb new file mode 100644 index 0000000..0165183 --- /dev/null +++ b/allure-rspec/spec/unit/formatter_message_spec.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +describe "message" do + include_context "allure mock" + include_context "rspec runner" + + let(:passing_example) do + <<~SPEC + describe "Suite" do + it "passes" do + end + end + SPEC + end + + def run_with_suite_hooks(hooks) + run_rspec("#{hooks}\n#{passing_example}") + end + + it "adds a before(:suite) failure to globals" do + run_with_suite_hooks(<<~SPEC) + RSpec.configure do |config| + config.before(:suite) { raise "before suite failed" } + end + SPEC + + expect(lifecycle).to have_received(:add_global_error).with( + message: "An error occurred in a `before(:suite)` hook.", + trace: a_string_including("RuntimeError", "before suite failed") + ).once + end + + it "adds an after(:suite) failure to globals" do + run_with_suite_hooks(<<~SPEC) + RSpec.configure do |config| + config.after(:suite) { raise "after suite failed" } + end + SPEC + + expect(lifecycle).to have_received(:add_global_error).with( + message: "An error occurred in an `after(:suite)` hook.", + trace: a_string_including("RuntimeError", "after suite failed") + ).once + end + + it "adds every failed suite hook" do + errors = [] + allow(lifecycle).to receive(:add_global_error) { |**details| errors << details } + + run_with_suite_hooks(<<~SPEC) + RSpec.configure do |config| + config.after(:suite) { raise "first after suite failed" } + config.after(:suite) { raise "second after suite failed" } + end + SPEC + + expect(errors.map { |error| error[:trace] }).to contain_exactly( + a_string_including("first after suite failed"), + a_string_including("second after suite failed") + ) + end + + it "removes ANSI formatting from a suite-hook failure" do + error = nil + allow(lifecycle).to receive(:add_global_error) { |**details| error = details } + + run_rspec(<<~SPEC) + RSpec.configuration.reporter.message( + "\e[31mAn error occurred in a `before(:suite)` hook.\e[0m\n" \ + "\e[31mRuntimeError: colored failure\e[0m" + ) + + #{passing_example} + SPEC + + expect(error).to include(message: "An error occurred in a `before(:suite)` hook.") + expect(error[:trace]).to include("RuntimeError: colored failure") + expect(error[:trace]).not_to include("\e[") + end + + it "ignores an ordinary reporter message" do + run_rspec(<<~SPEC) + RSpec.configuration.reporter.message("ordinary reporter message") + + #{passing_example} + SPEC + + expect(lifecycle).not_to have_received(:add_global_error) + end + + it "ignores a load error" do + run_rspec(<<~SPEC) + raise "spec load failed" + SPEC + + expect(lifecycle).not_to have_received(:add_global_error) + end + + it "captures the error before globals are written" do + run_with_suite_hooks(<<~SPEC) + RSpec.configure do |config| + config.after(:suite) { raise "after suite failed" } + end + SPEC + + expect(lifecycle).to have_received(:add_global_error).once.ordered + expect(lifecycle).to have_received(:write_globals).once.ordered + end +end diff --git a/allure-rspec/spec/unit/formatter_stop_spec.rb b/allure-rspec/spec/unit/formatter_stop_spec.rb index 6f92942..0df5aa6 100644 --- a/allure-rspec/spec/unit/formatter_stop_spec.rb +++ b/allure-rspec/spec/unit/formatter_stop_spec.rb @@ -14,7 +14,8 @@ SPEC end - it "creates environment.properties file" do + it "writes run-level files" do expect(lifecycle).to have_received(:write_environment).once + expect(lifecycle).to have_received(:write_globals).once end end diff --git a/allure-ruby-commons/lib/allure_ruby_commons/allure_lifecycle.rb b/allure-ruby-commons/lib/allure_ruby_commons/allure_lifecycle.rb index 69b2bce..4f7aad5 100644 --- a/allure-ruby-commons/lib/allure_ruby_commons/allure_lifecycle.rb +++ b/allure-ruby-commons/lib/allure_ruby_commons/allure_lifecycle.rb @@ -14,13 +14,14 @@ class AllureLifecycle # rubocop:disable Metrics/ClassLength def initialize(config = Config.instance) @test_context = [] @step_context = [] + @globals = Globals.new @config = config @logger = config.logger end attr_reader :config - def_delegators :file_writer, :write_attachment, :write_globals + def_delegators :file_writer, :write_attachment # Start test result container # @param [Allure::TestResultContainer] test_result_container @@ -231,7 +232,7 @@ def add_global_attachment(name:, source:, type:) logger.debug { "Adding global attachment '#{name}'" } write_attachment(source, attachment) - write_globals(Globals.new(attachments: [attachment])) + @globals.add_attachment(attachment) end # Add run-level error @@ -246,7 +247,7 @@ def add_global_error(**details) error = ResultUtils.prepare_global_error(timestamp: ResultUtils.timestamp, **details) logger.debug { "Adding global error '#{error.message}'" } - write_globals(Globals.new(errors: [error])) + @globals.add_error(error) end # Add environment.properties file @@ -270,6 +271,12 @@ def write_categories(categories = config.categories) file_writer.write_categories(categories) end + # Write global errors and attachments + # @return [void] + def write_globals + file_writer.write_globals(@globals) + end + # Add step to current fixture|step|test case # @param [Allure::StepResult] step_result # @return [Allure::StepResult] diff --git a/allure-ruby-commons/lib/allure_ruby_commons/model/globals.rb b/allure-ruby-commons/lib/allure_ruby_commons/model/globals.rb index eb11569..4ea4bb7 100644 --- a/allure-ruby-commons/lib/allure_ruby_commons/model/globals.rb +++ b/allure-ruby-commons/lib/allure_ruby_commons/model/globals.rb @@ -13,5 +13,19 @@ def initialize(attachments: [], errors: []) end attr_accessor :attachments, :errors + + # Add an error to the globals + # @param [Allure::GlobalError] error + # @return [void] + def add_error(error) + @errors.push(error) + end + + # Add an attachment to the globals + # @param [Allure::GlobalAttachment] attachment + # @return [void] + def add_attachment(attachment) + @attachments.push(attachment) + end end end diff --git a/allure-ruby-commons/spec/unit/allure_spec.rb b/allure-ruby-commons/spec/unit/allure_spec.rb index 1ec6b58..f1c4bef 100644 --- a/allure-ruby-commons/spec/unit/allure_spec.rb +++ b/allure-ruby-commons/spec/unit/allure_spec.rb @@ -123,6 +123,9 @@ def lifecycle args = { name: "Global attach", source: "Some string", type: Allure::ContentType::TXT } allure.add_global_attachment(**args) + expect(file_writer).not_to have_received(:write_globals) + lifecycle.write_globals + expect(file_writer).to have_received(:write_attachment).with(args[:source], kind_of(Allure::GlobalAttachment)) expect(file_writer).to have_received(:write_globals).with(kind_of(Allure::Globals)) end @@ -130,6 +133,9 @@ def lifecycle it "adds global error" do allure.add_global_error(message: "Global failure", trace: "trace line") + expect(file_writer).not_to have_received(:write_globals) + lifecycle.write_globals + expect(file_writer).to have_received(:write_globals).with(kind_of(Allure::Globals)) do |globals| error = globals.errors.first diff --git a/allure-ruby-commons/spec/unit/attachment_spec.rb b/allure-ruby-commons/spec/unit/attachment_spec.rb index 644ba00..ae99e43 100644 --- a/allure-ruby-commons/spec/unit/attachment_spec.rb +++ b/allure-ruby-commons/spec/unit/attachment_spec.rb @@ -92,9 +92,12 @@ end end - it "adds global attachment as a globals chunk" do + it "adds global attachment to the globals chunk" do lifecycle.add_global_attachment(**attach_opts) + expect(file_writer).not_to have_received(:write_globals) + lifecycle.write_globals + aggregate_failures "Global attachment should be written" do expect(@test_case.attachments).to be_empty expect(file_writer).to have_received(:write_attachment).with( diff --git a/allure-ruby-commons/spec/unit/lifecycle_spec.rb b/allure-ruby-commons/spec/unit/lifecycle_spec.rb index ba443b8..ef2500e 100644 --- a/allure-ruby-commons/spec/unit/lifecycle_spec.rb +++ b/allure-ruby-commons/spec/unit/lifecycle_spec.rb @@ -93,9 +93,12 @@ end describe "#add_global_attachment" do - it "writes a run-level attachment without an active test" do + it "buffers a run-level attachment without an active test" do lifecycle.add_global_attachment(name: "Global attachment", source: "payload", type: Allure::ContentType::TXT) + expect(file_writer).not_to have_received(:write_globals) + lifecycle.write_globals + expect(file_writer).to have_received(:write_attachment).with("payload", kind_of(Allure::GlobalAttachment)) expect(file_writer).to have_received(:write_globals).with(kind_of(Allure::Globals)) do |globals| attachment = globals.attachments.first @@ -112,9 +115,12 @@ end describe "#add_global_error" do - it "writes a run-level error without an active test" do + it "buffers a run-level error without an active test" do lifecycle.add_global_error(message: "Global failure", trace: "trace line") + expect(file_writer).not_to have_received(:write_globals) + lifecycle.write_globals + expect(file_writer).to have_received(:write_globals).with(kind_of(Allure::Globals)) do |globals| error = globals.errors.first @@ -128,4 +134,23 @@ end end end + + describe "#write_globals" do + it "writes all accumulated globals in one chunk" do + lifecycle.add_global_attachment(name: "First attachment", source: "first", type: Allure::ContentType::TXT) + lifecycle.add_global_attachment(name: "Second attachment", source: "second", type: Allure::ContentType::TXT) + lifecycle.add_global_error(message: "First failure") + lifecycle.add_global_error(message: "Second failure") + + expect(file_writer).not_to have_received(:write_globals) + lifecycle.write_globals + + expect(file_writer).to have_received(:write_globals).with(kind_of(Allure::Globals)).once do |globals| + aggregate_failures do + expect(globals.attachments.map(&:name)).to eq(["First attachment", "Second attachment"]) + expect(globals.errors.map(&:message)).to eq(["First failure", "Second failure"]) + end + end + end + end end