Treat failed fixtures as global errors

allure-cucumber/lib/allure_cucumber/global_hook.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require "cucumber/glue/hook"
+
+module AllureCucumber
+  # Captures Cucumber hooks outside the lifecycle of any scenario
+  module GlobalHook
+    HOOK_HANDLERS = {
+      "BeforeAll" => :start_prepare_fixture,
+      "AfterAll" => :start_tear_down_fixture
+    }.freeze
+
+    def self.install
+      hook_class = Cucumber::Glue::Hook
+      hook_class.prepend(Hook) unless hook_class < Hook
+    end
+
+    # Adds reporting around Cucumber BeforeAll and AfterAll hooks
+    module Hook
+      def invoke(pseudo_method, arguments, &)
+        handler = HOOK_HANDLERS[pseudo_method]
+        return super unless handler
+
+        start_global_fixture(pseudo_method, handler)
+        result = super
+        Allure.lifecycle.update_fixture { |fixture| fixture.status = Allure::Status::PASSED }
+        result
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        update_failed_global_fixture(e) if handler
+        raise
+      ensure
+        Allure.lifecycle.stop_fixture if handler
+      end
+
+      private
+
+      def start_global_fixture(pseudo_method, handler)
+        source = location.to_s.split("/").last
+        name = "#{pseudo_method} hook (#{source})"
+        Allure.lifecycle.public_send(handler, Allure::FixtureResult.new(name: name))
+      end
+
+      def update_failed_global_fixture(exception)
+        Allure.lifecycle.update_fixture do |fixture|
+          fixture.status = Allure::ResultUtils.status(exception)
+          fixture.status_details = Allure::ResultUtils.status_details(exception)
+        end
+      end
+    end
+  end
+end
+
+AllureCucumber::GlobalHook.install

allure-cucumber/spec/unit/formatter_global_hook_spec.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+describe "global hooks" do
+  include_context "allure mock"
+  include_context "cucumber runner"
+
+  let(:feature) do
+    <<~FEATURE
+      Feature: Simple feature
+
+      Scenario: Add a to b
+        Given a is 5
+    FEATURE
+  end
+
+  it "reports a failed BeforeAll hook through a global fixture" do
+    stub_const("CucumberHelper::ENV", <<~RUBY)
+      #{CucumberHelper::ENV}
+      BeforeAll { raise "BeforeAll failed" }
+    RUBY
+
+    run_cucumber_cli(feature)
+
+    expect(lifecycle).to have_received(:start_prepare_fixture).once
+    expect_failed_fixture("BeforeAll failed")
+    expect(lifecycle).to have_received(:stop_fixture).once
+  end
+
+  it "reports a failed AfterAll hook through a global fixture" do
+    stub_const("CucumberHelper::ENV", <<~RUBY)
+      #{CucumberHelper::ENV}
+      AfterAll { raise "AfterAll failed" }
+    RUBY
+
+    run_cucumber_cli(feature)
+
+    expect(lifecycle).to have_received(:start_tear_down_fixture).once
+    expect_failed_fixture("AfterAll failed")
+    expect(lifecycle).to have_received(:stop_fixture).once
+  end
+
+  def expect_failed_fixture(message)
+    expect(lifecycle).to have_received(:update_fixture).once do |&update|
+      fixture = Allure::FixtureResult.new
+      update.call(fixture)
+
+      expect(fixture.status).to eq(Allure::Status::BROKEN)
+      expect(fixture.status_details.message).to eq(message)
+    end
+  end
+end

allure-rspec/lib/allure_rspec/formatter.rb

@@ -37,6 +37,7 @@ class RSpecFormatter < RSpec::Core::Formatters::BaseFormatter
     def initialize(output)
       super
 
+      GlobalHook.install
       @allure_config = AllureRspec.configuration
       Allure.lifecycle = @lifecycle = Allure::AllureLifecycle.new(@allure_config)
     end

allure-rspec/lib/allure_rspec/global_hook.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module AllureRspec
+  # Captures suite hook results outside the lifecycle of any example
+  module GlobalHook
+    HOOK_TYPES = {
+      before: ["Before suite hook", :start_prepare_fixture],
+      after: ["After suite hook", :start_tear_down_fixture]
+    }.freeze
+
+    class << self
+      def install
+        RSpec::Core::Hooks::BeforeHook.prepend(BeforeHook) unless RSpec::Core::Hooks::BeforeHook < BeforeHook
+        RSpec::Core::Hooks::AfterHook.prepend(AfterHook) unless RSpec::Core::Hooks::AfterHook < AfterHook
+      end
+
+      def start(hook, type)
+        name, handler = HOOK_TYPES.fetch(type)
+        location = hook.block.source_location&.join(":")
+        fixture_name = location ? "#{name} (#{location})" : name
+        fixture = Allure::FixtureResult.new(name: fixture_name)
+        Allure.lifecycle.public_send(handler, fixture)
+      end
+
+      def pass
+        Allure.lifecycle.update_fixture { |fixture| fixture.status = Allure::Status::PASSED }
+      end
+
+      def fail(exception)
+        Allure.lifecycle.update_fixture do |fixture|
+          fixture.status = Allure::ResultUtils.status(exception)
+          fixture.status_details = Allure::ResultUtils.status_details(exception)
+        end
+      end
+
+      def stop
+        Allure.lifecycle.stop_fixture
+      end
+
+      def suite_hook?(hook, type)
+        hooks = RSpec.configuration.instance_variable_get(:"@#{type}_suite_hooks")
+        hooks&.include?(hook)
+      end
+    end
+
+    # Adds reporting around RSpec before(:suite) hooks
+    module BeforeHook
+      def run(example)
+        global_hook = GlobalHook.suite_hook?(self, :before)
+        return super unless global_hook
+
+        GlobalHook.start(self, :before)
+        super
+        GlobalHook.pass
+      rescue RSpec::Support::AllExceptionsExceptOnesWeMustNotRescue => e
+        GlobalHook.fail(e)
+        raise
+      ensure
+        GlobalHook.stop if global_hook
+      end
+    end
+
+    # Adds reporting around RSpec after(:suite) hooks
+    module AfterHook
+      def run(example)
+        global_hook = GlobalHook.suite_hook?(self, :after)
+        return super unless global_hook
+
+        GlobalHook.start(self, :after)
+        example.instance_exec(example, &block)
+        GlobalHook.pass
+      rescue RSpec::Support::AllExceptionsExceptOnesWeMustNotRescue => e
+        GlobalHook.fail(e)
+        example.set_exception(e)
+      ensure
+        GlobalHook.stop if global_hook
+      end
+    end
+  end
+end

allure-rspec/spec/unit/formatter_global_hook_spec.rb

@@ -0,0 +1,50 @@
+# frozen_string_literal: true
+
+describe "global hooks" do
+  include_context "allure mock"
+  include_context "rspec runner"
+
+  it "reports a failed before(:suite) hook through a global fixture" do
+    run_rspec(<<~SPEC)
+      RSpec.configure do |config|
+        config.before(:suite) { raise "Before suite failed" }
+      end
+
+      describe "Suite" do
+        it("spec") { }
+      end
+    SPEC
+
+    expect(lifecycle).to have_received(:start_prepare_fixture).once
+    expect(lifecycle).to have_received(:update_fixture).once do |&update|
+      fixture = Allure::FixtureResult.new
+      update.call(fixture)
+
+      expect(fixture.status).to eq(Allure::Status::BROKEN)
+      expect(fixture.status_details.message).to eq("Before suite failed")
+    end
+    expect(lifecycle).to have_received(:stop_fixture).once
+  end
+
+  it "reports a failed after(:suite) hook through a global fixture" do
+    run_rspec(<<~SPEC)
+      RSpec.configure do |config|
+        config.after(:suite) { raise "After suite failed" }
+      end
+
+      describe "Suite" do
+        it("spec") { }
+      end
+    SPEC
+
+    expect(lifecycle).to have_received(:start_tear_down_fixture).once
+    expect(lifecycle).to have_received(:update_fixture).once do |&update|
+      fixture = Allure::FixtureResult.new
+      update.call(fixture)
+
+      expect(fixture.status).to eq(Allure::Status::BROKEN)
+      expect(fixture.status_details.message).to eq("After suite failed")
+    end
+    expect(lifecycle).to have_received(:stop_fixture).once
+  end
+end

allure-ruby-commons/lib/allure_ruby_commons/allure_lifecycle.rb

@@ -148,17 +148,17 @@ def stop_test_step
     # @param [Allure::FixtureResult] fixture_result
     # @return [Allure::FixtureResult]
     def start_prepare_fixture(fixture_result)
-      start_fixture(fixture_result) || return
-      current_test_result_container.befores.push(fixture_result)
+      start_fixture(fixture_result)
+      current_test_result_container&.befores&.push(fixture_result)
       @current_fixture = fixture_result
     end
 
     # Start tear down fixture
     # @param [Allure::FixtureResult] fixture_result
     # @return [Allure::FixtureResult]
     def start_tear_down_fixture(fixture_result)
-      start_fixture(fixture_result) || return
-      current_test_result_container.afters.push(fixture_result)
+      start_fixture(fixture_result)
+      current_test_result_container&.afters&.push(fixture_result)
       @current_fixture = fixture_result
     end
 
@@ -167,14 +167,11 @@ def start_tear_down_fixture(fixture_result)
     # @return [Allure::FixtureResult]
     def start_fixture(fixture_result)
       clear_step_context
-      unless current_test_result_container
-        logger.error("Could not start fixture, test container is not started")
-        return false
-      end
-
       logger.debug { "Starting fixture: #{fixture_result.name}" }
       fixture_result.start = ResultUtils.timestamp
       fixture_result.stage = Stage::RUNNING
+      @global_fixture = current_test_result_container.nil?
+      fixture_result
     end
 
     # @example Update current fixture
@@ -198,6 +195,7 @@ def stop_fixture
       logger.debug { "Stopping fixture: #{@current_fixture.name}" }
       @current_fixture.stop = ResultUtils.timestamp
       @current_fixture.stage = Stage::FINISHED
+      write_global_fixture_error if @global_fixture
       clear_current_fixture
       clear_step_context
     end
@@ -323,6 +321,22 @@ def clear_current_test_case
 
     def clear_current_fixture
       @current_fixture = nil
+      @global_fixture = false
+    end
+
+    def write_global_fixture_error
+      return unless [Status::FAILED, Status::BROKEN].include?(@current_fixture.status)
+
+      details = @current_fixture.status_details
+      return unless details.message || details.trace
+
+      add_global_error(
+        known: details.known,
+        muted: details.muted,
+        flaky: details.flaky,
+        message: details.message,
+        trace: details.trace
+      )
     end
   end
 end

allure-ruby-commons/spec/unit/test_fixture_spec.rb

@@ -44,13 +44,70 @@
         expect(fixture_result.stage).to eq(Allure::Stage::FINISHED)
       end
     end
+
+    it "keeps a failed fixture on its test container" do
+      fixture = start_fixture("Prepare fixture", "prepare")
+      lifecycle.update_fixture do |result|
+        result.status = Allure::Status::BROKEN
+        result.status_details.message = "Fixture failed"
+      end
+      lifecycle.stop_fixture
+
+      aggregate_failures do
+        expect(@result_container.befores).to contain_exactly(fixture)
+        expect(file_writer).not_to have_received(:write_globals)
+      end
+    end
   end
 
-  context "logs error message" do
-    it "no running container" do
-      start_fixture("Prepare fixture", "prepare")
+  context "without a running test container" do
+    it "writes a failed prepare fixture as a global error" do
+      fixture = start_fixture("Global prepare fixture", "prepare")
+      details = Allure::StatusDetails.new(
+        known: true,
+        muted: true,
+        flaky: true,
+        message: "Global fixture failed",
+        trace: "trace line"
+      )
+      lifecycle.update_fixture do |result|
+        result.status = Allure::Status::FAILED
+        result.status_details = details
+      end
+      lifecycle.stop_fixture
+
+      aggregate_failures do
+        expect(fixture.stage).to eq(Allure::Stage::FINISHED)
+        expect(file_writer).to have_received(:write_globals).with(kind_of(Allure::Globals)) do |globals|
+          error = globals.errors.first
+
+          expect(globals.attachments).to be_empty
+          expect(error.known).to be(true)
+          expect(error.muted).to be(true)
+          expect(error.flaky).to be(true)
+          expect(error.message).to eq("Global fixture failed")
+          expect(error.trace).to eq("trace line")
+        end
+      end
+    end
+
+    it "does not write a passed prepare fixture as a global error" do
+      start_fixture("Global prepare fixture", "prepare")
+      lifecycle.update_fixture { |fixture| fixture.status = Allure::Status::PASSED }
+      lifecycle.stop_fixture
+
+      expect(file_writer).not_to have_received(:write_globals)
     end
 
+    it "does not write an error without failure details" do
+      start_fixture("Global prepare fixture", "prepare")
+      lifecycle.stop_fixture
+
+      expect(file_writer).not_to have_received(:write_globals)
+    end
+  end
+
+  context "logs error message" do
     it "no running fixture" do
       start_test_container("Test container")