From b593491cee116b766ee43f6b0593e704a9c7ca71 Mon Sep 17 00:00:00 2001 From: epszaw Date: Thu, 20 Aug 2026 17:08:13 +0200 Subject: [PATCH] treat failed fixtures as global errors --- .../lib/allure_cucumber/global_hook.rb | 53 ++++++++++++ .../spec/unit/formatter_global_hook_spec.rb | 51 ++++++++++++ allure-rspec/lib/allure_rspec/formatter.rb | 1 + allure-rspec/lib/allure_rspec/global_hook.rb | 80 +++++++++++++++++++ .../spec/unit/formatter_global_hook_spec.rb | 50 ++++++++++++ .../allure_ruby_commons/allure_lifecycle.rb | 32 +++++--- .../spec/unit/test_fixture_spec.rb | 63 ++++++++++++++- 7 files changed, 318 insertions(+), 12 deletions(-) create mode 100644 allure-cucumber/lib/allure_cucumber/global_hook.rb create mode 100644 allure-cucumber/spec/unit/formatter_global_hook_spec.rb create mode 100644 allure-rspec/lib/allure_rspec/global_hook.rb create mode 100644 allure-rspec/spec/unit/formatter_global_hook_spec.rb diff --git a/allure-cucumber/lib/allure_cucumber/global_hook.rb b/allure-cucumber/lib/allure_cucumber/global_hook.rb new file mode 100644 index 00000000..0f858dc4 --- /dev/null +++ b/allure-cucumber/lib/allure_cucumber/global_hook.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +require "cucumber/glue/hook" + +module AllureCucumber + # Captures Cucumber hooks outside the lifecycle of any scenario + module GlobalHook + HOOK_HANDLERS = { + "BeforeAll" => :start_prepare_fixture, + "AfterAll" => :start_tear_down_fixture + }.freeze + + def self.install + hook_class = Cucumber::Glue::Hook + hook_class.prepend(Hook) unless hook_class < Hook + end + + # Adds reporting around Cucumber BeforeAll and AfterAll hooks + module Hook + def invoke(pseudo_method, arguments, &) + handler = HOOK_HANDLERS[pseudo_method] + return super unless handler + + start_global_fixture(pseudo_method, handler) + result = super + Allure.lifecycle.update_fixture { |fixture| fixture.status = Allure::Status::PASSED } + result + rescue Exception => e # rubocop:disable Lint/RescueException + update_failed_global_fixture(e) if handler + raise + ensure + Allure.lifecycle.stop_fixture if handler + end + + private + + def start_global_fixture(pseudo_method, handler) + source = location.to_s.split("/").last + name = "#{pseudo_method} hook (#{source})" + Allure.lifecycle.public_send(handler, Allure::FixtureResult.new(name: name)) + end + + def update_failed_global_fixture(exception) + Allure.lifecycle.update_fixture do |fixture| + fixture.status = Allure::ResultUtils.status(exception) + fixture.status_details = Allure::ResultUtils.status_details(exception) + end + end + end + end +end + +AllureCucumber::GlobalHook.install diff --git a/allure-cucumber/spec/unit/formatter_global_hook_spec.rb b/allure-cucumber/spec/unit/formatter_global_hook_spec.rb new file mode 100644 index 00000000..ff83549c --- /dev/null +++ b/allure-cucumber/spec/unit/formatter_global_hook_spec.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true + +describe "global hooks" do + include_context "allure mock" + include_context "cucumber runner" + + let(:feature) do + <<~FEATURE + Feature: Simple feature + + Scenario: Add a to b + Given a is 5 + FEATURE + end + + it "reports a failed BeforeAll hook through a global fixture" do + stub_const("CucumberHelper::ENV", <<~RUBY) + #{CucumberHelper::ENV} + BeforeAll { raise "BeforeAll failed" } + RUBY + + run_cucumber_cli(feature) + + expect(lifecycle).to have_received(:start_prepare_fixture).once + expect_failed_fixture("BeforeAll failed") + expect(lifecycle).to have_received(:stop_fixture).once + end + + it "reports a failed AfterAll hook through a global fixture" do + stub_const("CucumberHelper::ENV", <<~RUBY) + #{CucumberHelper::ENV} + AfterAll { raise "AfterAll failed" } + RUBY + + run_cucumber_cli(feature) + + expect(lifecycle).to have_received(:start_tear_down_fixture).once + expect_failed_fixture("AfterAll failed") + expect(lifecycle).to have_received(:stop_fixture).once + end + + def expect_failed_fixture(message) + expect(lifecycle).to have_received(:update_fixture).once do |&update| + fixture = Allure::FixtureResult.new + update.call(fixture) + + expect(fixture.status).to eq(Allure::Status::BROKEN) + expect(fixture.status_details.message).to eq(message) + end + end +end diff --git a/allure-rspec/lib/allure_rspec/formatter.rb b/allure-rspec/lib/allure_rspec/formatter.rb index e475a00f..66a0fd72 100644 --- a/allure-rspec/lib/allure_rspec/formatter.rb +++ b/allure-rspec/lib/allure_rspec/formatter.rb @@ -37,6 +37,7 @@ class RSpecFormatter < RSpec::Core::Formatters::BaseFormatter def initialize(output) super + GlobalHook.install @allure_config = AllureRspec.configuration Allure.lifecycle = @lifecycle = Allure::AllureLifecycle.new(@allure_config) end diff --git a/allure-rspec/lib/allure_rspec/global_hook.rb b/allure-rspec/lib/allure_rspec/global_hook.rb new file mode 100644 index 00000000..23fe14bb --- /dev/null +++ b/allure-rspec/lib/allure_rspec/global_hook.rb @@ -0,0 +1,80 @@ +# frozen_string_literal: true + +module AllureRspec + # Captures suite hook results outside the lifecycle of any example + module GlobalHook + HOOK_TYPES = { + before: ["Before suite hook", :start_prepare_fixture], + after: ["After suite hook", :start_tear_down_fixture] + }.freeze + + class << self + def install + RSpec::Core::Hooks::BeforeHook.prepend(BeforeHook) unless RSpec::Core::Hooks::BeforeHook < BeforeHook + RSpec::Core::Hooks::AfterHook.prepend(AfterHook) unless RSpec::Core::Hooks::AfterHook < AfterHook + end + + def start(hook, type) + name, handler = HOOK_TYPES.fetch(type) + location = hook.block.source_location&.join(":") + fixture_name = location ? "#{name} (#{location})" : name + fixture = Allure::FixtureResult.new(name: fixture_name) + Allure.lifecycle.public_send(handler, fixture) + end + + def pass + Allure.lifecycle.update_fixture { |fixture| fixture.status = Allure::Status::PASSED } + end + + def fail(exception) + Allure.lifecycle.update_fixture do |fixture| + fixture.status = Allure::ResultUtils.status(exception) + fixture.status_details = Allure::ResultUtils.status_details(exception) + end + end + + def stop + Allure.lifecycle.stop_fixture + end + + def suite_hook?(hook, type) + hooks = RSpec.configuration.instance_variable_get(:"@#{type}_suite_hooks") + hooks&.include?(hook) + end + end + + # Adds reporting around RSpec before(:suite) hooks + module BeforeHook + def run(example) + global_hook = GlobalHook.suite_hook?(self, :before) + return super unless global_hook + + GlobalHook.start(self, :before) + super + GlobalHook.pass + rescue RSpec::Support::AllExceptionsExceptOnesWeMustNotRescue => e + GlobalHook.fail(e) + raise + ensure + GlobalHook.stop if global_hook + end + end + + # Adds reporting around RSpec after(:suite) hooks + module AfterHook + def run(example) + global_hook = GlobalHook.suite_hook?(self, :after) + return super unless global_hook + + GlobalHook.start(self, :after) + example.instance_exec(example, &block) + GlobalHook.pass + rescue RSpec::Support::AllExceptionsExceptOnesWeMustNotRescue => e + GlobalHook.fail(e) + example.set_exception(e) + ensure + GlobalHook.stop if global_hook + end + end + end +end diff --git a/allure-rspec/spec/unit/formatter_global_hook_spec.rb b/allure-rspec/spec/unit/formatter_global_hook_spec.rb new file mode 100644 index 00000000..749d338b --- /dev/null +++ b/allure-rspec/spec/unit/formatter_global_hook_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +describe "global hooks" do + include_context "allure mock" + include_context "rspec runner" + + it "reports a failed before(:suite) hook through a global fixture" do + run_rspec(<<~SPEC) + RSpec.configure do |config| + config.before(:suite) { raise "Before suite failed" } + end + + describe "Suite" do + it("spec") { } + end + SPEC + + expect(lifecycle).to have_received(:start_prepare_fixture).once + expect(lifecycle).to have_received(:update_fixture).once do |&update| + fixture = Allure::FixtureResult.new + update.call(fixture) + + expect(fixture.status).to eq(Allure::Status::BROKEN) + expect(fixture.status_details.message).to eq("Before suite failed") + end + expect(lifecycle).to have_received(:stop_fixture).once + end + + it "reports a failed after(:suite) hook through a global fixture" do + run_rspec(<<~SPEC) + RSpec.configure do |config| + config.after(:suite) { raise "After suite failed" } + end + + describe "Suite" do + it("spec") { } + end + SPEC + + expect(lifecycle).to have_received(:start_tear_down_fixture).once + expect(lifecycle).to have_received(:update_fixture).once do |&update| + fixture = Allure::FixtureResult.new + update.call(fixture) + + expect(fixture.status).to eq(Allure::Status::BROKEN) + expect(fixture.status_details.message).to eq("After suite failed") + end + expect(lifecycle).to have_received(:stop_fixture).once + end +end diff --git a/allure-ruby-commons/lib/allure_ruby_commons/allure_lifecycle.rb b/allure-ruby-commons/lib/allure_ruby_commons/allure_lifecycle.rb index 69b2bcec..d5157e2d 100644 --- a/allure-ruby-commons/lib/allure_ruby_commons/allure_lifecycle.rb +++ b/allure-ruby-commons/lib/allure_ruby_commons/allure_lifecycle.rb @@ -148,8 +148,8 @@ def stop_test_step # @param [Allure::FixtureResult] fixture_result # @return [Allure::FixtureResult] def start_prepare_fixture(fixture_result) - start_fixture(fixture_result) || return - current_test_result_container.befores.push(fixture_result) + start_fixture(fixture_result) + current_test_result_container&.befores&.push(fixture_result) @current_fixture = fixture_result end @@ -157,8 +157,8 @@ def start_prepare_fixture(fixture_result) # @param [Allure::FixtureResult] fixture_result # @return [Allure::FixtureResult] def start_tear_down_fixture(fixture_result) - start_fixture(fixture_result) || return - current_test_result_container.afters.push(fixture_result) + start_fixture(fixture_result) + current_test_result_container&.afters&.push(fixture_result) @current_fixture = fixture_result end @@ -167,14 +167,11 @@ def start_tear_down_fixture(fixture_result) # @return [Allure::FixtureResult] def start_fixture(fixture_result) clear_step_context - unless current_test_result_container - logger.error("Could not start fixture, test container is not started") - return false - end - logger.debug { "Starting fixture: #{fixture_result.name}" } fixture_result.start = ResultUtils.timestamp fixture_result.stage = Stage::RUNNING + @global_fixture = current_test_result_container.nil? + fixture_result end # @example Update current fixture @@ -198,6 +195,7 @@ def stop_fixture logger.debug { "Stopping fixture: #{@current_fixture.name}" } @current_fixture.stop = ResultUtils.timestamp @current_fixture.stage = Stage::FINISHED + write_global_fixture_error if @global_fixture clear_current_fixture clear_step_context end @@ -323,6 +321,22 @@ def clear_current_test_case def clear_current_fixture @current_fixture = nil + @global_fixture = false + end + + def write_global_fixture_error + return unless [Status::FAILED, Status::BROKEN].include?(@current_fixture.status) + + details = @current_fixture.status_details + return unless details.message || details.trace + + add_global_error( + known: details.known, + muted: details.muted, + flaky: details.flaky, + message: details.message, + trace: details.trace + ) end end end diff --git a/allure-ruby-commons/spec/unit/test_fixture_spec.rb b/allure-ruby-commons/spec/unit/test_fixture_spec.rb index 6d7b9509..013518a6 100644 --- a/allure-ruby-commons/spec/unit/test_fixture_spec.rb +++ b/allure-ruby-commons/spec/unit/test_fixture_spec.rb @@ -44,13 +44,70 @@ expect(fixture_result.stage).to eq(Allure::Stage::FINISHED) end end + + it "keeps a failed fixture on its test container" do + fixture = start_fixture("Prepare fixture", "prepare") + lifecycle.update_fixture do |result| + result.status = Allure::Status::BROKEN + result.status_details.message = "Fixture failed" + end + lifecycle.stop_fixture + + aggregate_failures do + expect(@result_container.befores).to contain_exactly(fixture) + expect(file_writer).not_to have_received(:write_globals) + end + end end - context "logs error message" do - it "no running container" do - start_fixture("Prepare fixture", "prepare") + context "without a running test container" do + it "writes a failed prepare fixture as a global error" do + fixture = start_fixture("Global prepare fixture", "prepare") + details = Allure::StatusDetails.new( + known: true, + muted: true, + flaky: true, + message: "Global fixture failed", + trace: "trace line" + ) + lifecycle.update_fixture do |result| + result.status = Allure::Status::FAILED + result.status_details = details + end + lifecycle.stop_fixture + + aggregate_failures do + expect(fixture.stage).to eq(Allure::Stage::FINISHED) + expect(file_writer).to have_received(:write_globals).with(kind_of(Allure::Globals)) do |globals| + error = globals.errors.first + + expect(globals.attachments).to be_empty + expect(error.known).to be(true) + expect(error.muted).to be(true) + expect(error.flaky).to be(true) + expect(error.message).to eq("Global fixture failed") + expect(error.trace).to eq("trace line") + end + end + end + + it "does not write a passed prepare fixture as a global error" do + start_fixture("Global prepare fixture", "prepare") + lifecycle.update_fixture { |fixture| fixture.status = Allure::Status::PASSED } + lifecycle.stop_fixture + + expect(file_writer).not_to have_received(:write_globals) end + it "does not write an error without failure details" do + start_fixture("Global prepare fixture", "prepare") + lifecycle.stop_fixture + + expect(file_writer).not_to have_received(:write_globals) + end + end + + context "logs error message" do it "no running fixture" do start_test_container("Test container")