From 87ab8ff62d6d0aa11afae0b9afb6586b67867d80 Mon Sep 17 00:00:00 2001 From: andrejs Date: Tue, 1 Sep 2026 16:56:53 +0300 Subject: [PATCH 1/3] Ensure globals are written once per test when test run finishes --- .ruby-version | 2 +- .../lib/allure_cucumber/formatter.rb | 1 + .../unit/formatter_test_run_finished_spec.rb | 3 +- allure-rspec/lib/allure_rspec/formatter.rb | 1 + allure-rspec/spec/unit/formatter_stop_spec.rb | 3 +- .../allure_ruby_commons/allure_lifecycle.rb | 13 +++++++-- .../lib/allure_ruby_commons/model/globals.rb | 14 +++++++++ allure-ruby-commons/spec/unit/allure_spec.rb | 6 ++++ .../spec/unit/attachment_spec.rb | 5 +++- .../spec/unit/lifecycle_spec.rb | 29 +++++++++++++++++-- 10 files changed, 68 insertions(+), 9 deletions(-) diff --git a/.ruby-version b/.ruby-version index 4d54dadd..d13e837c 100644 --- a/.ruby-version +++ b/.ruby-version @@ -1 +1 @@ -4.0.2 +4.0.6 diff --git a/allure-cucumber/lib/allure_cucumber/formatter.rb b/allure-cucumber/lib/allure_cucumber/formatter.rb index b2333234..e0079f14 100644 --- a/allure-cucumber/lib/allure_cucumber/formatter.rb +++ b/allure-cucumber/lib/allure_cucumber/formatter.rb @@ -50,6 +50,7 @@ def on_test_run_started(_event) # @return [void] def on_test_run_finished(_event) lifecycle.write_environment + lifecycle.write_globals end # Handle test case started event diff --git a/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb b/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb index bc411dfc..90e50c00 100644 --- a/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb +++ b/allure-cucumber/spec/unit/formatter_test_run_finished_spec.rb @@ -14,7 +14,8 @@ FEATURE end - it "creates environment.properties file" do + it "writes run-level files" do expect(lifecycle).to have_received(:write_environment).once + expect(lifecycle).to have_received(:write_globals).once end end diff --git a/allure-rspec/lib/allure_rspec/formatter.rb b/allure-rspec/lib/allure_rspec/formatter.rb index e475a00f..d1c69774 100644 --- a/allure-rspec/lib/allure_rspec/formatter.rb +++ b/allure-rspec/lib/allure_rspec/formatter.rb @@ -58,6 +58,7 @@ def start(_start_notification) # @return [void] def stop(_stop_notification) lifecycle.write_environment + lifecycle.write_globals end # Starts example group diff --git a/allure-rspec/spec/unit/formatter_stop_spec.rb b/allure-rspec/spec/unit/formatter_stop_spec.rb index 6f929426..0df5aa61 100644 --- a/allure-rspec/spec/unit/formatter_stop_spec.rb +++ b/allure-rspec/spec/unit/formatter_stop_spec.rb @@ -14,7 +14,8 @@ SPEC end - it "creates environment.properties file" do + it "writes run-level files" do expect(lifecycle).to have_received(:write_environment).once + expect(lifecycle).to have_received(:write_globals).once end end diff --git a/allure-ruby-commons/lib/allure_ruby_commons/allure_lifecycle.rb b/allure-ruby-commons/lib/allure_ruby_commons/allure_lifecycle.rb index 69b2bcec..4f7aad52 100644 --- a/allure-ruby-commons/lib/allure_ruby_commons/allure_lifecycle.rb +++ b/allure-ruby-commons/lib/allure_ruby_commons/allure_lifecycle.rb @@ -14,13 +14,14 @@ class AllureLifecycle # rubocop:disable Metrics/ClassLength def initialize(config = Config.instance) @test_context = [] @step_context = [] + @globals = Globals.new @config = config @logger = config.logger end attr_reader :config - def_delegators :file_writer, :write_attachment, :write_globals + def_delegators :file_writer, :write_attachment # Start test result container # @param [Allure::TestResultContainer] test_result_container @@ -231,7 +232,7 @@ def add_global_attachment(name:, source:, type:) logger.debug { "Adding global attachment '#{name}'" } write_attachment(source, attachment) - write_globals(Globals.new(attachments: [attachment])) + @globals.add_attachment(attachment) end # Add run-level error @@ -246,7 +247,7 @@ def add_global_error(**details) error = ResultUtils.prepare_global_error(timestamp: ResultUtils.timestamp, **details) logger.debug { "Adding global error '#{error.message}'" } - write_globals(Globals.new(errors: [error])) + @globals.add_error(error) end # Add environment.properties file @@ -270,6 +271,12 @@ def write_categories(categories = config.categories) file_writer.write_categories(categories) end + # Write global errors and attachments + # @return [void] + def write_globals + file_writer.write_globals(@globals) + end + # Add step to current fixture|step|test case # @param [Allure::StepResult] step_result # @return [Allure::StepResult] diff --git a/allure-ruby-commons/lib/allure_ruby_commons/model/globals.rb b/allure-ruby-commons/lib/allure_ruby_commons/model/globals.rb index eb115699..4ea4bb77 100644 --- a/allure-ruby-commons/lib/allure_ruby_commons/model/globals.rb +++ b/allure-ruby-commons/lib/allure_ruby_commons/model/globals.rb @@ -13,5 +13,19 @@ def initialize(attachments: [], errors: []) end attr_accessor :attachments, :errors + + # Add an error to the globals + # @param [Allure::GlobalError] error + # @return [void] + def add_error(error) + @errors.push(error) + end + + # Add an attachment to the globals + # @param [Allure::GlobalAttachment] attachment + # @return [void] + def add_attachment(attachment) + @attachments.push(attachment) + end end end diff --git a/allure-ruby-commons/spec/unit/allure_spec.rb b/allure-ruby-commons/spec/unit/allure_spec.rb index 1ec6b586..f1c4bef2 100644 --- a/allure-ruby-commons/spec/unit/allure_spec.rb +++ b/allure-ruby-commons/spec/unit/allure_spec.rb @@ -123,6 +123,9 @@ def lifecycle args = { name: "Global attach", source: "Some string", type: Allure::ContentType::TXT } allure.add_global_attachment(**args) + expect(file_writer).not_to have_received(:write_globals) + lifecycle.write_globals + expect(file_writer).to have_received(:write_attachment).with(args[:source], kind_of(Allure::GlobalAttachment)) expect(file_writer).to have_received(:write_globals).with(kind_of(Allure::Globals)) end @@ -130,6 +133,9 @@ def lifecycle it "adds global error" do allure.add_global_error(message: "Global failure", trace: "trace line") + expect(file_writer).not_to have_received(:write_globals) + lifecycle.write_globals + expect(file_writer).to have_received(:write_globals).with(kind_of(Allure::Globals)) do |globals| error = globals.errors.first diff --git a/allure-ruby-commons/spec/unit/attachment_spec.rb b/allure-ruby-commons/spec/unit/attachment_spec.rb index 644ba00b..ae99e43d 100644 --- a/allure-ruby-commons/spec/unit/attachment_spec.rb +++ b/allure-ruby-commons/spec/unit/attachment_spec.rb @@ -92,9 +92,12 @@ end end - it "adds global attachment as a globals chunk" do + it "adds global attachment to the globals chunk" do lifecycle.add_global_attachment(**attach_opts) + expect(file_writer).not_to have_received(:write_globals) + lifecycle.write_globals + aggregate_failures "Global attachment should be written" do expect(@test_case.attachments).to be_empty expect(file_writer).to have_received(:write_attachment).with( diff --git a/allure-ruby-commons/spec/unit/lifecycle_spec.rb b/allure-ruby-commons/spec/unit/lifecycle_spec.rb index ba443b85..ef2500e8 100644 --- a/allure-ruby-commons/spec/unit/lifecycle_spec.rb +++ b/allure-ruby-commons/spec/unit/lifecycle_spec.rb @@ -93,9 +93,12 @@ end describe "#add_global_attachment" do - it "writes a run-level attachment without an active test" do + it "buffers a run-level attachment without an active test" do lifecycle.add_global_attachment(name: "Global attachment", source: "payload", type: Allure::ContentType::TXT) + expect(file_writer).not_to have_received(:write_globals) + lifecycle.write_globals + expect(file_writer).to have_received(:write_attachment).with("payload", kind_of(Allure::GlobalAttachment)) expect(file_writer).to have_received(:write_globals).with(kind_of(Allure::Globals)) do |globals| attachment = globals.attachments.first @@ -112,9 +115,12 @@ end describe "#add_global_error" do - it "writes a run-level error without an active test" do + it "buffers a run-level error without an active test" do lifecycle.add_global_error(message: "Global failure", trace: "trace line") + expect(file_writer).not_to have_received(:write_globals) + lifecycle.write_globals + expect(file_writer).to have_received(:write_globals).with(kind_of(Allure::Globals)) do |globals| error = globals.errors.first @@ -128,4 +134,23 @@ end end end + + describe "#write_globals" do + it "writes all accumulated globals in one chunk" do + lifecycle.add_global_attachment(name: "First attachment", source: "first", type: Allure::ContentType::TXT) + lifecycle.add_global_attachment(name: "Second attachment", source: "second", type: Allure::ContentType::TXT) + lifecycle.add_global_error(message: "First failure") + lifecycle.add_global_error(message: "Second failure") + + expect(file_writer).not_to have_received(:write_globals) + lifecycle.write_globals + + expect(file_writer).to have_received(:write_globals).with(kind_of(Allure::Globals)).once do |globals| + aggregate_failures do + expect(globals.attachments.map(&:name)).to eq(["First attachment", "Second attachment"]) + expect(globals.errors.map(&:message)).to eq(["First failure", "Second failure"]) + end + end + end + end end From 7a2f3676081d7993f9549881845d188c7eccab2a Mon Sep 17 00:00:00 2001 From: andrejs Date: Wed, 2 Sep 2026 10:02:15 +0300 Subject: [PATCH 2/3] Save global hook errors in globals object --- .../lib/allure_cucumber/formatter.rb | 21 ++++ allure-cucumber/spec/cucumber_helper.rb | 12 +- allure-cucumber/spec/spec_helper.rb | 6 +- .../formatter_test_run_hook_finished_spec.rb | 91 +++++++++++++++ allure-rspec/lib/allure_rspec/formatter.rb | 18 ++- .../spec/unit/formatter_message_spec.rb | 109 ++++++++++++++++++ 6 files changed, 249 insertions(+), 8 deletions(-) create mode 100644 allure-cucumber/spec/unit/formatter_test_run_hook_finished_spec.rb create mode 100644 allure-rspec/spec/unit/formatter_message_spec.rb diff --git a/allure-cucumber/lib/allure_cucumber/formatter.rb b/allure-cucumber/lib/allure_cucumber/formatter.rb index e0079f14..c9ce0029 100644 --- a/allure-cucumber/lib/allure_cucumber/formatter.rb +++ b/allure-cucumber/lib/allure_cucumber/formatter.rb @@ -35,6 +35,8 @@ def initialize(config) config.on_event(:test_step_started) { |event| on_test_step_started(event) } config.on_event(:test_step_finished) { |event| on_test_step_finished(event) } config.on_event(:test_case_finished) { |event| on_test_case_finished(event) } + + register_test_run_hook_finished(config) end # Clean test result directory before starting run @@ -53,6 +55,16 @@ def on_test_run_finished(_event) lifecycle.write_globals end + # Handle a global hook finishing + # @param [Cucumber::Events::TestRunHookFinished] event + # @return [void] + def on_test_run_hook_finished(event) + return unless event.test_result.failed? + + details = Allure::ResultUtils.status_details(event.test_result.exception) + lifecycle.add_global_error(message: details.message, trace: details.trace) + end + # Handle test case started event # @param [Cucumber::Events::TestCaseStarted] event # @return [void] @@ -102,6 +114,15 @@ def on_test_case_finished(event) attr_reader :lifecycle, :cucumber_model + # Register global hook finished event + # @param [Cucumber::Configuration] config + # @return [void] + def register_test_run_hook_finished(config) + return unless defined?(Cucumber::Events::TestRunHookFinished) + + config.on_event(:test_run_hook_finished) { |event| on_test_run_hook_finished(event) } + end + # Is hook fixture like Before, After or Step as AfterStep # @param [String] text # @return [boolean] diff --git a/allure-cucumber/spec/cucumber_helper.rb b/allure-cucumber/spec/cucumber_helper.rb index 181d1e9e..21d87473 100644 --- a/allure-cucumber/spec/cucumber_helper.rb +++ b/allure-cucumber/spec/cucumber_helper.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +# rubocop:disable Style/OneClassPerFile + require "cucumber/cli/main" class MockKernel @@ -74,8 +76,8 @@ def initialize(tmp_dir) @tmp_dir = tmp_dir end - def execute(feature) - setup(feature) + def execute(feature, support_code: ENV) + setup(feature, support_code) Cucumber::Cli::Main.new( [feature_file, *args], @@ -95,11 +97,11 @@ def all_output [@stdout.string, @stderr.string].reject(&:empty?).join("\n") end - def setup(feature) + def setup(feature, support_code) FileUtils.rm_rf(tmp_dir) write_file(feature_file, feature) - write_file("#{tmp_dir}/features/support/env.rb", ENV) + write_file("#{tmp_dir}/features/support/env.rb", support_code) write_file("#{tmp_dir}/features/step_definitions/step_defs.rb", STEPS) end @@ -121,3 +123,5 @@ def args ] end end + +# rubocop:enable Style/OneClassPerFile diff --git a/allure-cucumber/spec/spec_helper.rb b/allure-cucumber/spec/spec_helper.rb index 10f8571b..da8327b6 100644 --- a/allure-cucumber/spec/spec_helper.rb +++ b/allure-cucumber/spec/spec_helper.rb @@ -44,7 +44,9 @@ RSpec.shared_context("cucumber runner") do let!(:test_tmp_dir) { |e| "tmp/#{e.full_description.tr(' ', '_')}" } - def run_cucumber_cli(feature) - Thread.new { CucumberHelper.new(test_tmp_dir).execute(feature) }.join + def run_cucumber_cli(feature, support_code: CucumberHelper::ENV) + Thread.new do + CucumberHelper.new(test_tmp_dir).execute(feature, support_code: support_code) + end.join end end diff --git a/allure-cucumber/spec/unit/formatter_test_run_hook_finished_spec.rb b/allure-cucumber/spec/unit/formatter_test_run_hook_finished_spec.rb new file mode 100644 index 00000000..b6efd18d --- /dev/null +++ b/allure-cucumber/spec/unit/formatter_test_run_hook_finished_spec.rb @@ -0,0 +1,91 @@ +# frozen_string_literal: true + +describe "on_test_run_hook_finished" do + include_context "allure mock" + include_context "cucumber runner" + + let(:feature) do + <<~FEATURE + Feature: Global hooks + + Scenario: passing scenario + Given a is 5 + FEATURE + end + + def run_with_global_hooks(hooks) + run_cucumber_cli(feature, support_code: "#{CucumberHelper::ENV}\n#{hooks}") + end + + it "adds a failed BeforeAll hook to globals" do + run_with_global_hooks(<<~RUBY) + BeforeAll do + raise "BeforeAll failed" + end + RUBY + + expect(lifecycle).to have_received(:add_global_error).with( + message: "BeforeAll failed", + trace: a_string_including("env.rb") + ).once + end + + it "adds a failed AfterAll hook to globals" do + run_with_global_hooks(<<~RUBY) + AfterAll do + raise "AfterAll failed" + end + RUBY + + expect(lifecycle).to have_received(:add_global_error).with( + message: "AfterAll failed", + trace: a_string_including("env.rb") + ).once + end + + it "ignores passed global hooks" do + run_with_global_hooks(<<~RUBY) + BeforeAll do + end + + AfterAll do + end + RUBY + + expect(lifecycle).not_to have_received(:add_global_error) + end + + it "adds every failed global hook" do + errors = [] + allow(lifecycle).to receive(:add_global_error) { |**details| errors << details } + + run_with_global_hooks(<<~RUBY) + BeforeAll do + raise "BeforeAll failed" + end + + AfterAll do + raise "First AfterAll failed" + end + + AfterAll do + raise "Second AfterAll failed" + end + RUBY + + expect(errors.map { |error| error[:message] }).to eq( + ["BeforeAll failed", "First AfterAll failed", "Second AfterAll failed"] + ) + end + + it "captures the error before globals are written" do + run_with_global_hooks(<<~RUBY) + AfterAll do + raise "AfterAll failed" + end + RUBY + + expect(lifecycle).to have_received(:add_global_error).once.ordered + expect(lifecycle).to have_received(:write_globals).once.ordered + end +end diff --git a/allure-rspec/lib/allure_rspec/formatter.rb b/allure-rspec/lib/allure_rspec/formatter.rb index d1c69774..e6f1f39a 100644 --- a/allure-rspec/lib/allure_rspec/formatter.rb +++ b/allure-rspec/lib/allure_rspec/formatter.rb @@ -15,6 +15,8 @@ class RSpecFormatter < RSpec::Core::Formatters::BaseFormatter pending: Allure::Status::SKIPPED, passed: Allure::Status::PASSED }.freeze + ANSI_ESCAPE_PATTERN = /\e\[(\d+)(?:;\d+)*m/ + SUITE_HOOK_FAILURE_PATTERN = /\AAn error occurred in an? `(?:before|after)\(:suite\)` hook\.\z/ RSpec::Core::Formatters.register( self, @@ -23,7 +25,8 @@ class RSpecFormatter < RSpec::Core::Formatters::BaseFormatter :example_group_started, :example_group_finished, :example_started, - :example_finished + :example_finished, + :message ) RSpec.configure do |config| @@ -93,6 +96,17 @@ def example_group_finished(_example_group_notification) lifecycle.stop_test_container end + # Capture errors raised by before(:suite) and after(:suite) hooks + # @param [RSpec::Core::Notifications::MessageNotification] notification + # @return [void] + def message(notification) + formatted_error = notification.message.to_s.gsub(ANSI_ESCAPE_PATTERN, "").strip + first_line = formatted_error.lines.first&.chomp + return unless SUITE_HOOK_FAILURE_PATTERN.match?(first_line.to_s) + + lifecycle.add_global_error(message: first_line, trace: formatted_error) + end + private attr_reader :lifecycle, :allure_config @@ -126,7 +140,7 @@ def update_test_proc(result) test_case.stage = Allure::Stage::FINISHED test_case.status = status(result) test_case.status_details.message = status_detail.message - test_case.status_details.trace = status_detail.trace&.gsub(/\e\[(\d+)(?:;\d+)*m/, "") + test_case.status_details.trace = status_detail.trace&.gsub(ANSI_ESCAPE_PATTERN, "") end end end diff --git a/allure-rspec/spec/unit/formatter_message_spec.rb b/allure-rspec/spec/unit/formatter_message_spec.rb new file mode 100644 index 00000000..01651833 --- /dev/null +++ b/allure-rspec/spec/unit/formatter_message_spec.rb @@ -0,0 +1,109 @@ +# frozen_string_literal: true + +describe "message" do + include_context "allure mock" + include_context "rspec runner" + + let(:passing_example) do + <<~SPEC + describe "Suite" do + it "passes" do + end + end + SPEC + end + + def run_with_suite_hooks(hooks) + run_rspec("#{hooks}\n#{passing_example}") + end + + it "adds a before(:suite) failure to globals" do + run_with_suite_hooks(<<~SPEC) + RSpec.configure do |config| + config.before(:suite) { raise "before suite failed" } + end + SPEC + + expect(lifecycle).to have_received(:add_global_error).with( + message: "An error occurred in a `before(:suite)` hook.", + trace: a_string_including("RuntimeError", "before suite failed") + ).once + end + + it "adds an after(:suite) failure to globals" do + run_with_suite_hooks(<<~SPEC) + RSpec.configure do |config| + config.after(:suite) { raise "after suite failed" } + end + SPEC + + expect(lifecycle).to have_received(:add_global_error).with( + message: "An error occurred in an `after(:suite)` hook.", + trace: a_string_including("RuntimeError", "after suite failed") + ).once + end + + it "adds every failed suite hook" do + errors = [] + allow(lifecycle).to receive(:add_global_error) { |**details| errors << details } + + run_with_suite_hooks(<<~SPEC) + RSpec.configure do |config| + config.after(:suite) { raise "first after suite failed" } + config.after(:suite) { raise "second after suite failed" } + end + SPEC + + expect(errors.map { |error| error[:trace] }).to contain_exactly( + a_string_including("first after suite failed"), + a_string_including("second after suite failed") + ) + end + + it "removes ANSI formatting from a suite-hook failure" do + error = nil + allow(lifecycle).to receive(:add_global_error) { |**details| error = details } + + run_rspec(<<~SPEC) + RSpec.configuration.reporter.message( + "\e[31mAn error occurred in a `before(:suite)` hook.\e[0m\n" \ + "\e[31mRuntimeError: colored failure\e[0m" + ) + + #{passing_example} + SPEC + + expect(error).to include(message: "An error occurred in a `before(:suite)` hook.") + expect(error[:trace]).to include("RuntimeError: colored failure") + expect(error[:trace]).not_to include("\e[") + end + + it "ignores an ordinary reporter message" do + run_rspec(<<~SPEC) + RSpec.configuration.reporter.message("ordinary reporter message") + + #{passing_example} + SPEC + + expect(lifecycle).not_to have_received(:add_global_error) + end + + it "ignores a load error" do + run_rspec(<<~SPEC) + raise "spec load failed" + SPEC + + expect(lifecycle).not_to have_received(:add_global_error) + end + + it "captures the error before globals are written" do + run_with_suite_hooks(<<~SPEC) + RSpec.configure do |config| + config.after(:suite) { raise "after suite failed" } + end + SPEC + + expect(lifecycle).to have_received(:add_global_error).once.ordered + expect(lifecycle).to have_received(:write_globals).once.ordered + end +end From 50f8325204effa15d0448e8c312ae5b292cad7cc Mon Sep 17 00:00:00 2001 From: andrejs Date: Wed, 2 Sep 2026 10:41:12 +0300 Subject: [PATCH 3/3] Disable rubocop extension suggestion --- .pre-commit-config.yaml | 2 +- .rubocop.yml | 1 + allure-cucumber/spec/cucumber_helper.rb | 4 ---- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0e0494ad..9d379bd6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -4,6 +4,6 @@ repos: - id: rubocop name: Rubocop language: system - entry: bundle exec rubocop + entry: bundle exec rake rubocop types: [ruby] exclude: \.lock diff --git a/.rubocop.yml b/.rubocop.yml index 308b92bd..f7e14e5f 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -4,6 +4,7 @@ plugins: AllCops: TargetRubyVersion: 3.1 NewCops: enable + SuggestExtensions: false Exclude: - bin/**/* - vendor/**/* diff --git a/allure-cucumber/spec/cucumber_helper.rb b/allure-cucumber/spec/cucumber_helper.rb index 21d87473..076c89f8 100644 --- a/allure-cucumber/spec/cucumber_helper.rb +++ b/allure-cucumber/spec/cucumber_helper.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -# rubocop:disable Style/OneClassPerFile - require "cucumber/cli/main" class MockKernel @@ -123,5 +121,3 @@ def args ] end end - -# rubocop:enable Style/OneClassPerFile