From 4af951b171a81cd7ee1966c2840c36f9adae86a5 Mon Sep 17 00:00:00 2001 From: Todd Baert Date: Thu, 27 Aug 2026 15:30:11 -0400 Subject: [PATCH 1/4] chore: add e2e test suite Signed-off-by: Todd Baert --- .github/workflows/ruby.yml | 19 +++++ .gitmodules | 3 + .../openfeature-flagd-provider/Gemfile.lock | 36 ++++++++++ providers/openfeature-flagd-provider/Rakefile | 5 ++ .../openfeature-flagd-provider/cucumber.yml | 1 + .../features/step_definitions/flagd_steps.rb | 71 +++++++++++++++++++ .../features/support/env.rb | 63 ++++++++++++++++ .../openfeature-flagd-provider.gemspec | 1 + .../openfeature-flagd-provider/test-harness | 1 + 9 files changed, 200 insertions(+) create mode 100644 providers/openfeature-flagd-provider/cucumber.yml create mode 100644 providers/openfeature-flagd-provider/features/step_definitions/flagd_steps.rb create mode 100644 providers/openfeature-flagd-provider/features/support/env.rb create mode 160000 providers/openfeature-flagd-provider/test-harness diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 1edbf9b..2ce1795 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -44,6 +44,25 @@ jobs: run: bundle exec rubocop - name: Run tests run: bundle exec rspec + + test_flagd_provider_e2e: + runs-on: ubuntu-latest + defaults: + run: + working-directory: ./providers/openfeature-flagd-provider + steps: + - uses: actions/checkout@v7 + with: + submodules: recursive + - name: Set up Ruby + uses: ruby/setup-ruby@v1 + with: + ruby-version: "3.4" + bundler-cache: true + working-directory: ./providers/openfeature-flagd-provider + - name: Run e2e (flagd-testbed / cucumber) + run: bundle exec rake e2e + test_meta_provider: runs-on: ubuntu-latest defaults: diff --git a/.gitmodules b/.gitmodules index 7762457..5c8e777 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,6 @@ [submodule "providers/openfeature-flagd-provider/schemas"] path = providers/openfeature-flagd-provider/schemas url = https://github.com/open-feature/flagd-schemas +[submodule "providers/openfeature-flagd-provider/test-harness"] + path = providers/openfeature-flagd-provider/test-harness + url = https://github.com/open-feature/flagd-testbed.git diff --git a/providers/openfeature-flagd-provider/Gemfile.lock b/providers/openfeature-flagd-provider/Gemfile.lock index 987fbc3..377efb0 100644 --- a/providers/openfeature-flagd-provider/Gemfile.lock +++ b/providers/openfeature-flagd-provider/Gemfile.lock @@ -10,8 +10,37 @@ GEM specs: ast (2.4.3) bigdecimal (4.0.1) + builder (3.3.0) + cucumber (9.2.1) + builder (~> 3.2) + cucumber-ci-environment (> 9, < 11) + cucumber-core (> 13, < 14) + cucumber-cucumber-expressions (~> 17.0) + cucumber-gherkin (> 24, < 28) + cucumber-html-formatter (> 20.3, < 22) + cucumber-messages (> 19, < 25) + diff-lcs (~> 1.5) + mini_mime (~> 1.1) + multi_test (~> 1.1) + sys-uname (~> 1.2) + cucumber-ci-environment (10.0.1) + cucumber-core (13.0.3) + cucumber-gherkin (>= 27, < 28) + cucumber-messages (>= 20, < 23) + cucumber-tag-expressions (> 5, < 7) + cucumber-cucumber-expressions (17.1.0) + bigdecimal + cucumber-gherkin (27.0.0) + cucumber-messages (>= 19.1.4, < 23) + cucumber-html-formatter (21.15.1) + cucumber-messages (> 19, < 28) + cucumber-messages (22.0.0) + cucumber-tag-expressions (6.1.2) diff-lcs (1.5.1) docile (1.4.1) + ffi (1.17.4) + ffi (1.17.4-arm64-darwin) + ffi (1.17.4-x86_64-linux-gnu) google-protobuf (4.34.0) bigdecimal rake (~> 13.3) @@ -35,6 +64,9 @@ GEM json (2.21.2) language_server-protocol (3.17.0.5) lint_roller (1.1.0) + memoist3 (1.0.0) + mini_mime (1.1.5) + multi_test (1.1.0) openfeature-sdk (0.3.1) parallel (1.27.0) parser (3.3.10.2) @@ -95,6 +127,9 @@ GEM standard-performance (1.9.0) lint_roller (~> 1.1) rubocop-performance (~> 1.26.0) + sys-uname (1.5.1) + ffi (~> 1.1) + memoist3 (~> 1.0.0) unicode-display_width (3.2.0) unicode-emoji (~> 4.1) unicode-emoji (4.2.0) @@ -106,6 +141,7 @@ PLATFORMS x86_64-linux DEPENDENCIES + cucumber (~> 9.2) openfeature-flagd-provider! rake (~> 13.0) rspec (~> 3.12.0) diff --git a/providers/openfeature-flagd-provider/Rakefile b/providers/openfeature-flagd-provider/Rakefile index b6ae734..3ac3b79 100644 --- a/providers/openfeature-flagd-provider/Rakefile +++ b/providers/openfeature-flagd-provider/Rakefile @@ -2,7 +2,12 @@ require "bundler/gem_tasks" require "rspec/core/rake_task" +require "cucumber/rake/task" RSpec::Core::RakeTask.new(:spec) +# End-to-end suite: runs the shared flagd-testbed gherkin features against a testbed container. +# Requires Docker; not part of the default task. +Cucumber::Rake::Task.new(:e2e) + task default: :spec diff --git a/providers/openfeature-flagd-provider/cucumber.yml b/providers/openfeature-flagd-provider/cucumber.yml new file mode 100644 index 0000000..1ca1c73 --- /dev/null +++ b/providers/openfeature-flagd-provider/cucumber.yml @@ -0,0 +1 @@ +default: --require features --publish-quiet --tags "not @deprecated and not @no-default-variant and not @fractional-v1 and not @fractional-v3" test-harness/gherkin/evaluation.feature test-harness/gherkin/targeting.feature diff --git a/providers/openfeature-flagd-provider/features/step_definitions/flagd_steps.rb b/providers/openfeature-flagd-provider/features/step_definitions/flagd_steps.rb new file mode 100644 index 0000000..244964a --- /dev/null +++ b/providers/openfeature-flagd-provider/features/step_definitions/flagd_steps.rb @@ -0,0 +1,71 @@ +# frozen_string_literal: true + +module StepHelpers + def cast(type, value) + case type + when "Boolean" then value == "true" + when "Integer" then value.to_i + when "Float" then value.to_f + else value + end + end +end +World(StepHelpers) + +Given(/^an option "[^"]*" of type "[^"]*" with value "[^"]*"$/) do + # no-op: caching/streaming options aren't applicable here +end + +Given(/^a stable flagd provider$/) do + @client = OpenFeature::Flagd::Provider.build_client + @context = {} +end + +Given(/^a (\w+)-flag with key "([^"]*)" and a default value "([^"]*)"$/) do |type, key, default| + @type = type + @key = key + @default = default +end + +Given(/^a context containing a key "([^"]*)", with type "([^"]*)" and with value "([^"]*)"$/) do |key, type, value| + @context[key] = cast(type, value) +end + +Given(/^a context containing a targeting key with value "([^"]*)"$/) do |value| + # the provider maps the "targeting_key" field to flagd's targetingKey + @context["targeting_key"] = value +end + +Given(/^a context containing a nested property with outer key "([^"]*)" and inner key "([^"]*)", with value "([^"]*)"$/) do |outer, inner, value| + (@context[outer] ||= {})[inner] = value +end + +When(/^the flag was evaluated with details$/) do + ctx = @context.empty? ? nil : OpenFeature::SDK::EvaluationContext.new(**@context.transform_keys(&:to_sym)) + + @details = + case @type + when "Boolean" then @client.fetch_boolean_value(flag_key: @key, default_value: cast("Boolean", @default), evaluation_context: ctx) + when "String" then @client.fetch_string_value(flag_key: @key, default_value: @default, evaluation_context: ctx) + when "Integer" then @client.fetch_integer_value(flag_key: @key, default_value: @default.to_i, evaluation_context: ctx) + when "Float" then @client.fetch_float_value(flag_key: @key, default_value: @default.to_f, evaluation_context: ctx) + when "Object" then @client.fetch_object_value(flag_key: @key, default_value: {}, evaluation_context: ctx) + else raise "unsupported flag type: #{@type}" + end +end + +Then(/^the resolved details value should be "([^"]*)"$/) do |value| + expect(@details[:value]).to eq(cast(@type, value)) +end + +Then(/^the reason should be "([^"]*)"$/) do |reason| + expect(@details[:reason]).to eq(reason) +end + +Then(/^the error-code should be "([^"]*)"$/) do |code| + if code.empty? + expect(@details[:error_code]).to be_nil + else + expect(@details[:error_code]).to eq(code) + end +end diff --git a/providers/openfeature-flagd-provider/features/support/env.rb b/providers/openfeature-flagd-provider/features/support/env.rb new file mode 100644 index 0000000..f73d343 --- /dev/null +++ b/providers/openfeature-flagd-provider/features/support/env.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require "net/http" +require "uri" +require "rspec/expectations" +require "open_feature/sdk" +require "openfeature/flagd/provider" + +World(RSpec::Matchers) + +# Boots the shared flagd testbed container and drives its launchpad so flagd is serving the +# standard flag set before the suite runs. Uses the Docker CLI directly (reliable on CI). +module Testbed + VERSION = File.read(File.expand_path("../../test-harness/version.txt", __dir__)).strip + IMAGE = "ghcr.io/open-feature/flagd-testbed:v#{VERSION}" + NAME = "flagd-e2e-ruby-#{Process.pid}" + + module_function + + def start + sh("docker rm -f #{NAME}") + raise "failed to start #{IMAGE} (is docker available?)" unless + sh("docker run -d --name #{NAME} -p 8013:8013 -p 8014:8014 -p 8080:8080 #{IMAGE}") + + # launchpad starts flagd with the default flag set + wait_for { http_code(:post, "http://localhost:8080/start") } + wait_for { http_code(:get, "http://localhost:8014/healthz") == "200" } || + raise("flagd testbed did not become healthy in time") + end + + def stop + sh("docker rm -f #{NAME}") + end + + def sh(cmd) + system(cmd, out: File::NULL, err: File::NULL) + end + + def http_code(verb, url) + uri = URI(url) + res = if verb == :post + Net::HTTP.post(uri, "") + else + Net::HTTP.get_response(uri) + end + res.code + rescue + nil + end + + def wait_for(timeout: 60) + deadline = Time.now + timeout + loop do + return true if yield + return false if Time.now > deadline + + sleep 0.5 + end + end +end + +Testbed.start +at_exit { Testbed.stop } diff --git a/providers/openfeature-flagd-provider/openfeature-flagd-provider.gemspec b/providers/openfeature-flagd-provider/openfeature-flagd-provider.gemspec index 9222695..81520d7 100644 --- a/providers/openfeature-flagd-provider/openfeature-flagd-provider.gemspec +++ b/providers/openfeature-flagd-provider/openfeature-flagd-provider.gemspec @@ -35,6 +35,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency "rake", "~> 13.0" spec.add_development_dependency "rspec", "~> 3.12.0" + spec.add_development_dependency "cucumber", "~> 9.2" spec.add_development_dependency "standard" spec.add_development_dependency "rubocop" spec.add_development_dependency "simplecov", "~> 0.22" diff --git a/providers/openfeature-flagd-provider/test-harness b/providers/openfeature-flagd-provider/test-harness new file mode 160000 index 0000000..6dea1da --- /dev/null +++ b/providers/openfeature-flagd-provider/test-harness @@ -0,0 +1 @@ +Subproject commit 6dea1da12636b9332d9018ae6ff8ed2894742616 From 916c31a0656ffedecc2ddbcb54e050bf2d0d8c8f Mon Sep 17 00:00:00 2001 From: Todd Baert Date: Thu, 27 Aug 2026 15:46:32 -0400 Subject: [PATCH 2/4] fixup: pr feedback Signed-off-by: Todd Baert --- .../features/step_definitions/flagd_steps.rb | 13 +++++++++++-- .../features/support/env.rb | 3 ++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/providers/openfeature-flagd-provider/features/step_definitions/flagd_steps.rb b/providers/openfeature-flagd-provider/features/step_definitions/flagd_steps.rb index 244964a..e688769 100644 --- a/providers/openfeature-flagd-provider/features/step_definitions/flagd_steps.rb +++ b/providers/openfeature-flagd-provider/features/step_definitions/flagd_steps.rb @@ -1,14 +1,22 @@ # frozen_string_literal: true +require "json" + module StepHelpers def cast(type, value) case type when "Boolean" then value == "true" when "Integer" then value.to_i when "Float" then value.to_f + when "Object" then JSON.parse(value) else value end end + + # flagd returns object values as a protobuf Struct; normalise to a Hash for comparison. + def unwrap(value) + value.respond_to?(:to_h) ? value.to_h : value + end end World(StepHelpers) @@ -49,13 +57,14 @@ def cast(type, value) when "String" then @client.fetch_string_value(flag_key: @key, default_value: @default, evaluation_context: ctx) when "Integer" then @client.fetch_integer_value(flag_key: @key, default_value: @default.to_i, evaluation_context: ctx) when "Float" then @client.fetch_float_value(flag_key: @key, default_value: @default.to_f, evaluation_context: ctx) - when "Object" then @client.fetch_object_value(flag_key: @key, default_value: {}, evaluation_context: ctx) + when "Object" then @client.fetch_object_value(flag_key: @key, default_value: cast("Object", @default), evaluation_context: ctx) else raise "unsupported flag type: #{@type}" end end Then(/^the resolved details value should be "([^"]*)"$/) do |value| - expect(@details[:value]).to eq(cast(@type, value)) + actual = (@type == "Object") ? unwrap(@details[:value]) : @details[:value] + expect(actual).to eq(cast(@type, value)) end Then(/^the reason should be "([^"]*)"$/) do |reason| diff --git a/providers/openfeature-flagd-provider/features/support/env.rb b/providers/openfeature-flagd-provider/features/support/env.rb index f73d343..eb33302 100644 --- a/providers/openfeature-flagd-provider/features/support/env.rb +++ b/providers/openfeature-flagd-provider/features/support/env.rb @@ -59,5 +59,6 @@ def wait_for(timeout: 60) end end -Testbed.start +# register teardown before startup so a container is cleaned up even if start raises at_exit { Testbed.stop } +Testbed.start From 9f688e84a621f974318cb159818dcfec0c1956d2 Mon Sep 17 00:00:00 2001 From: Todd Baert Date: Fri, 28 Aug 2026 10:13:40 -0400 Subject: [PATCH 3/4] fixup: use matrix for e2e Signed-off-by: Todd Baert --- .github/workflows/ruby.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 2ce1795..f793132 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -50,6 +50,13 @@ jobs: defaults: run: working-directory: ./providers/openfeature-flagd-provider + strategy: + fail-fast: false + matrix: + ruby-version: + - "3.3" + - "3.4" + - "4.0" steps: - uses: actions/checkout@v7 with: @@ -57,7 +64,7 @@ jobs: - name: Set up Ruby uses: ruby/setup-ruby@v1 with: - ruby-version: "3.4" + ruby-version: ${{ matrix.ruby-version }} bundler-cache: true working-directory: ./providers/openfeature-flagd-provider - name: Run e2e (flagd-testbed / cucumber) From bb79f20f244ad308ff379409d2372682b8a019e7 Mon Sep 17 00:00:00 2001 From: Todd Baert Date: Fri, 28 Aug 2026 11:48:49 -0400 Subject: [PATCH 4/4] fixup: cuke logger Signed-off-by: Todd Baert --- providers/openfeature-flagd-provider/Gemfile.lock | 2 ++ .../openfeature-flagd-provider.gemspec | 1 + 2 files changed, 3 insertions(+) diff --git a/providers/openfeature-flagd-provider/Gemfile.lock b/providers/openfeature-flagd-provider/Gemfile.lock index 377efb0..b541f0c 100644 --- a/providers/openfeature-flagd-provider/Gemfile.lock +++ b/providers/openfeature-flagd-provider/Gemfile.lock @@ -64,6 +64,7 @@ GEM json (2.21.2) language_server-protocol (3.17.0.5) lint_roller (1.1.0) + logger (1.7.0) memoist3 (1.0.0) mini_mime (1.1.5) multi_test (1.1.0) @@ -142,6 +143,7 @@ PLATFORMS DEPENDENCIES cucumber (~> 9.2) + logger openfeature-flagd-provider! rake (~> 13.0) rspec (~> 3.12.0) diff --git a/providers/openfeature-flagd-provider/openfeature-flagd-provider.gemspec b/providers/openfeature-flagd-provider/openfeature-flagd-provider.gemspec index 81520d7..2409c56 100644 --- a/providers/openfeature-flagd-provider/openfeature-flagd-provider.gemspec +++ b/providers/openfeature-flagd-provider/openfeature-flagd-provider.gemspec @@ -36,6 +36,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency "rake", "~> 13.0" spec.add_development_dependency "rspec", "~> 3.12.0" spec.add_development_dependency "cucumber", "~> 9.2" + spec.add_development_dependency "logger" spec.add_development_dependency "standard" spec.add_development_dependency "rubocop" spec.add_development_dependency "simplecov", "~> 0.22"