From 5441ccea4774087da8cfe417812663db260f8335 Mon Sep 17 00:00:00 2001 From: IG Date: Wed, 9 Sep 2026 08:20:30 +0400 Subject: [PATCH 01/26] feat: add the value objects the provider contract is built from --- lib/translation_diff.rb | 5 ++ lib/translation_diff/capabilities.rb | 19 ++++++++ lib/translation_diff/errors.rb | 47 ++++++++++++++++++ lib/translation_diff/translation/request.rb | 12 +++++ lib/translation_diff/translation/response.rb | 22 +++++++++ lib/translation_diff/translation/usage.rb | 14 ++++++ test/translation_diff/capabilities_test.rb | 32 +++++++++++++ test/translation_diff/errors_test.rb | 45 +++++++++++++++++ .../translation/response_test.rb | 48 +++++++++++++++++++ 9 files changed, 244 insertions(+) create mode 100644 lib/translation_diff/capabilities.rb create mode 100644 lib/translation_diff/errors.rb create mode 100644 lib/translation_diff/translation/request.rb create mode 100644 lib/translation_diff/translation/response.rb create mode 100644 lib/translation_diff/translation/usage.rb create mode 100644 test/translation_diff/capabilities_test.rb create mode 100644 test/translation_diff/errors_test.rb create mode 100644 test/translation_diff/translation/response_test.rb diff --git a/lib/translation_diff.rb b/lib/translation_diff.rb index a3a4b94..765e3b2 100644 --- a/lib/translation_diff.rb +++ b/lib/translation_diff.rb @@ -9,6 +9,11 @@ require "translation_diff/version" require "translation_diff/error" +require "translation_diff/errors" +require "translation_diff/capabilities" +require "translation_diff/translation/usage" +require "translation_diff/translation/request" +require "translation_diff/translation/response" require "translation_diff/registry" require "translation_diff/configuration" require "translation_diff/configuration/provider_option_owners" diff --git a/lib/translation_diff/capabilities.rb b/lib/translation_diff/capabilities.rb new file mode 100644 index 0000000..e4401e9 --- /dev/null +++ b/lib/translation_diff/capabilities.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +# What one provider can do and how much it will accept, declared rather than +# discovered. Before this existed, `max_batch_size` was a method, "can it +# detect a language" was `respond_to?(:detect)`, and "does it honour +# notranslate" was not expressed anywhere -- which is how two providers +# shipped with notranslate silently broken. +# +# `html` holds the name of the provider option that turns HTML handling on, +# because every vendor spells it differently (`tag_handling`, `format`, +# `textType`), or :none when the provider has no HTML mode at all. +TranslationDiff::Capabilities = Data.define(:max_request_size, :max_batch_size, + :max_text_size, :html, :notranslate, + :detects_language, :reports_billing) do + def html? = html != :none + def notranslate? = notranslate + def detects_language? = detects_language + def reports_billing? = reports_billing +end diff --git a/lib/translation_diff/errors.rb b/lib/translation_diff/errors.rb new file mode 100644 index 0000000..d837318 --- /dev/null +++ b/lib/translation_diff/errors.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +# One hierarchy for everything that can go wrong, so a caller handles a rate +# limit the same way whichever provider raised it. +# +# The three branches answer three different questions. ConfigurationError +# means the caller set something up wrong and no request was made. +# ProviderError means the service answered and said no. TransportError means +# nobody answered. ResponseError means the answer was well-formed HTTP but +# broke this library's contract. +# +# No error carries the text being translated. Errors are logged, and this +# library handles other people's content. +module TranslationDiff + class ConfigurationError < Error; end + + class ProviderError < Error + attr_reader :provider, :status + + def initialize(message, provider: nil, status: nil) + super(message) + @provider = provider + @status = status + end + end + + class AuthenticationError < ProviderError; end + class QuotaExceededError < ProviderError; end + class InvalidRequestError < ProviderError; end + class ServiceError < ProviderError; end + + class RateLimitError < ProviderError + # Seconds the provider asked us to wait, when it said so at all. Faraday's + # retry middleware honours the header itself; this is for a caller who + # rescues the error after the retries are exhausted and wants to schedule + # its own attempt. + attr_reader :retry_after + + def initialize(message, provider: nil, status: nil, retry_after: nil) + super(message, provider: provider, status: status) + @retry_after = retry_after + end + end + + class TransportError < Error; end + class ResponseError < Error; end +end diff --git a/lib/translation_diff/translation/request.rb b/lib/translation_diff/translation/request.rb new file mode 100644 index 0000000..bf279f5 --- /dev/null +++ b/lib/translation_diff/translation/request.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +# What the pipeline asks a provider for. `from` may be nil, which means the +# provider should detect the source language. `options` carries the per-call +# provider options the caller passed to TranslationDiff.translate, untouched. +module TranslationDiff::Translation + Request = Data.define(:texts, :from, :to, :options) do + def initialize(texts:, from:, to:, options: {}) + super + end + end +end diff --git a/lib/translation_diff/translation/response.rb b/lib/translation_diff/translation/response.rb new file mode 100644 index 0000000..dfcb162 --- /dev/null +++ b/lib/translation_diff/translation/response.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +# What a provider hands back. Built through ::build, which is the only place +# the count is checked: a response with fewer texts than the request shifts +# nils into the results, and they surface much later as a NoMethodError far +# from the provider that caused them. +# +# The check lives here rather than in a base-class method so that it holds +# for every provider, including ones that override #translate outright +# instead of using the HTTP seams. +module TranslationDiff::Translation + Response = Data.define(:texts, :detected_source, :usage) do + def self.build(request:, texts:, detected_source: nil, usage: nil) + if texts.size != request.texts.size + raise TranslationDiff::ResponseError, + "Provider returned #{texts.size} translations for #{request.texts.size} values" + end + + new(texts: texts, detected_source: detected_source, usage: usage) + end + end +end diff --git a/lib/translation_diff/translation/usage.rb b/lib/translation_diff/translation/usage.rb new file mode 100644 index 0000000..c3a2ef9 --- /dev/null +++ b/lib/translation_diff/translation/usage.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# What one provider call cost. `characters` is what we sent and is always +# known; `billed_characters` is what the provider says it charged for and is +# nil for the providers that do not report it. `tokens` and `model` are nil +# for every machine-translation provider and exist so that an LLM-backed +# provider needs no new type. +module TranslationDiff::Translation + Usage = Data.define(:characters, :billed_characters, :tokens, :model) do + def initialize(characters:, billed_characters: nil, tokens: nil, model: nil) + super + end + end +end diff --git a/test/translation_diff/capabilities_test.rb b/test/translation_diff/capabilities_test.rb new file mode 100644 index 0000000..52572fd --- /dev/null +++ b/test/translation_diff/capabilities_test.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require "test_helper" + +class CapabilitiesTest < Minitest::Test + def capabilities(**overrides) + TranslationDiff::Capabilities.new(max_request_size: 5_000, max_batch_size: 128, max_text_size: nil, + html: :format, notranslate: true, detects_language: true, + reports_billing: false, **overrides) + end + + # Every reader asks a yes-or-no question, and Data.define generates plain + # readers. Declaring the predicates once stops the codebase from asking + # `detects_language` in one place and `detects_language?` in another. + def test_it_answers_in_predicates + assert_predicate capabilities, :html? + assert_predicate capabilities, :notranslate? + assert_predicate capabilities, :detects_language? + refute_predicate capabilities, :reports_billing? + end + + # `html` holds the name of the provider option that turns HTML on, which is + # different for every vendor, so :none is the only way to say "cannot". + def test_html_is_false_only_when_the_provider_has_no_html_mode + refute_predicate capabilities(html: :none), :html? + assert_predicate capabilities(html: :tag_handling), :html? + end + + def test_it_is_frozen_so_a_provider_cannot_be_mutated_at_runtime + assert_predicate capabilities, :frozen? + end +end diff --git a/test/translation_diff/errors_test.rb b/test/translation_diff/errors_test.rb new file mode 100644 index 0000000..0893a77 --- /dev/null +++ b/test/translation_diff/errors_test.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require "test_helper" + +class ErrorsTest < Minitest::Test + # A caller who wants to rescue anything this gem raises writes one rescue. + def test_every_error_descends_from_the_common_ancestor + [TranslationDiff::ConfigurationError, TranslationDiff::ProviderError, + TranslationDiff::AuthenticationError, TranslationDiff::RateLimitError, + TranslationDiff::QuotaExceededError, TranslationDiff::InvalidRequestError, + TranslationDiff::ServiceError, TranslationDiff::TransportError, + TranslationDiff::ResponseError].each do |klass| + assert_operator klass, :<, TranslationDiff::Error + end + end + + # Rescuing "the provider said no" must not also catch a local + # configuration mistake or a socket timeout. + def test_provider_errors_are_a_family_of_their_own + [TranslationDiff::AuthenticationError, TranslationDiff::RateLimitError, + TranslationDiff::QuotaExceededError, TranslationDiff::InvalidRequestError, + TranslationDiff::ServiceError].each do |klass| + assert_operator klass, :<, TranslationDiff::ProviderError + end + + refute_operator TranslationDiff::ConfigurationError, :<, TranslationDiff::ProviderError + refute_operator TranslationDiff::TransportError, :<, TranslationDiff::ProviderError + end + + def test_a_provider_error_carries_the_provider_and_the_status + error = TranslationDiff::RateLimitError.new("slow down", provider: :deepl, status: 429, + retry_after: 30) + + assert_equal :deepl, error.provider + assert_equal 429, error.status + assert_equal 30, error.retry_after + assert_equal "slow down", error.message + end + + def test_retry_after_is_nil_when_the_provider_did_not_say + error = TranslationDiff::RateLimitError.new("slow down", provider: :deepl, status: 429) + + assert_nil error.retry_after + end +end diff --git a/test/translation_diff/translation/response_test.rb b/test/translation_diff/translation/response_test.rb new file mode 100644 index 0000000..a5e5589 --- /dev/null +++ b/test/translation_diff/translation/response_test.rb @@ -0,0 +1,48 @@ +# frozen_string_literal: true + +require "test_helper" + +class TranslationResponseTest < Minitest::Test + def request(texts = %w[one two]) + TranslationDiff::Translation::Request.new(texts: texts, from: "en", to: "ru") + end + + def test_a_request_defaults_its_options_to_an_empty_hash + assert_empty TranslationDiff::Translation::Request.new(texts: %w[one], from: nil, to: "ru").options + end + + def test_build_returns_the_texts_it_was_given + response = TranslationDiff::Translation::Response.build(request: request, texts: %w[один два]) + + assert_equal %w[один два], response.texts + end + + # A short response means nils get shifted into the results and surface much + # later as a NoMethodError far from the cause. The check lives in the + # constructor rather than in a base-class method so that it still holds for + # a provider that overrides #translate outright. + def test_build_raises_when_the_provider_returned_the_wrong_number_of_texts + error = assert_raises(TranslationDiff::ResponseError) do + TranslationDiff::Translation::Response.build(request: request, texts: %w[один]) + end + + assert_match(/1/, error.message) + assert_match(/2/, error.message) + end + + def test_detected_source_and_usage_default_to_nil + response = TranslationDiff::Translation::Response.build(request: request, texts: %w[один два]) + + assert_nil response.detected_source + assert_nil response.usage + end + + def test_usage_carries_what_the_provider_reported_and_nil_for_the_rest + usage = TranslationDiff::Translation::Usage.new(characters: 40, billed_characters: 40) + + assert_equal 40, usage.characters + assert_equal 40, usage.billed_characters + assert_nil usage.tokens + assert_nil usage.model + end +end From 6631ef7590767947a31e52e8e796a4b24370b896 Mon Sep 17 00:00:00 2001 From: IG Date: Wed, 9 Sep 2026 08:35:25 +0400 Subject: [PATCH 02/26] feat: give providers a base class and require it in the registry TranslationDiff::Provider replaces the duck-typed provider contract: subclasses declare configuration options/requirements and capabilities, translate a Translation::Request into a Translation::Response, and get a checked, all-or-nothing option registration and a guarded cache_key for free. Providers.register now rejects anything that isn't a Provider subclass, and Null is ported onto the new base class. DeepL and Google still predate Provider (Tasks 4/5 port them), so their unconditional registration at require time is rescued in translation_diff.rb rather than taking the whole library down; they're simply absent from the registry until then. Migrating Null also means every provider built through the registry now speaks the new contract, which request.rb (owned by a later task) doesn't consume yet -- three integration test files that routed through :null needed small local doubles in place of the now-incompatible provider to keep exercising their own behaviour without touching request.rb. --- lib/translation_diff.rb | 23 ++++- lib/translation_diff/provider.rb | 84 +++++++++++++++++++ lib/translation_diff/providers.rb | 46 +++------- lib/translation_diff/providers/null.rb | 23 ++--- test/support/provider_contract.rb | 30 ++++--- test/translation_diff/context_test.rb | 15 +++- test/translation_diff/instrumentation_test.rb | 16 +++- test/translation_diff/provider_test.rb | 78 +++++++++++++++++ test/translation_diff/providers/null_test.rb | 10 ++- test/translation_diff/providers_test.rb | 69 ++++++++------- test/translation_diff/request_test.rb | 36 +++++++- 11 files changed, 332 insertions(+), 98 deletions(-) create mode 100644 lib/translation_diff/provider.rb create mode 100644 test/translation_diff/provider_test.rb diff --git a/lib/translation_diff.rb b/lib/translation_diff.rb index 765e3b2..f7647d1 100644 --- a/lib/translation_diff.rb +++ b/lib/translation_diff.rb @@ -18,10 +18,29 @@ require "translation_diff/configuration" require "translation_diff/configuration/provider_option_owners" +require "translation_diff/provider" require "translation_diff/providers" require "translation_diff/providers/null" -require "translation_diff/providers/deepl" -require "translation_diff/providers/google" + +# DeepL and Google still wrap their vendor SDKs directly instead of +# inheriting Provider -- Tasks 4 and 5 port them. Providers.register now +# raises for exactly that shape of class, which would otherwise take this +# entire require chain, and therefore every caller of this library, down +# with it before either provider is ever used. Until they are ported, +# `:deepl` and `:google` are simply absent from the registry; requesting +# either through TranslationDiff::Providers.build raises the ordinary +# "unknown provider" error instead. +begin + require "translation_diff/providers/deepl" +rescue TranslationDiff::Error + nil +end + +begin + require "translation_diff/providers/google" +rescue TranslationDiff::Error + nil +end require "translation_diff/segmenters" require "translation_diff/segmenters/simple" require "translation_diff/segmenters/pragmatic" diff --git a/lib/translation_diff/provider.rb b/lib/translation_diff/provider.rb new file mode 100644 index 0000000..4ea093c --- /dev/null +++ b/lib/translation_diff/provider.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +# A Provider connects this library to one translation service. It knows where +# to talk, who it is, and what it can do. It knows nothing about HTTP -- that +# is HTTPProvider, which most providers inherit instead. Inheriting Provider +# directly is for services reached some other way: Amazon, whose requests are +# signed rather than merely headed, and an LLM-backed provider that delegates +# to another gem. +# +# Subclass, declare the options you need, then register: +# +# class Acme < TranslationDiff::Provider +# def self.configuration_options = %i[acme_api_key] +# def self.configuration_requirements = %i[acme_api_key] +# def self.capabilities = TranslationDiff::Capabilities.new(...) +# +# def translate(request) = TranslationDiff::Translation::Response.build(...) +# end +# +# TranslationDiff::Providers.register(:acme, Acme) +class TranslationDiff::Provider + # What a provider can do when it says nothing: the least capable thing that + # can still translate. A subclass that forgets to declare its capabilities + # therefore under-promises rather than over-promises -- the failure mode is + # smaller batches, not a rejected request or silently unprotected content. + DEFAULT_CAPABILITIES = TranslationDiff::Capabilities.new( + max_request_size: 1_000, max_batch_size: 1, max_text_size: nil, + html: :none, notranslate: false, detects_language: false, reports_billing: false + ).freeze + + # Stamped by the registry at build time. See #cache_key. + attr_accessor :name + + attr_reader :config + + def initialize(config) + @config = config + ensure_configured! + end + + # Translate a Translation::Request, return a Translation::Response. + def translate(_request) = raise NotImplementedError, "#{self.class} must implement #translate" + + # Return the source language of a sample of text, lowercased. Only called + # when `capabilities.detects_language?`. + def detect(_text) = raise NotImplementedError, "#{self.class} must implement #detect" + + # The segment of every cache key that keeps one provider's translations from + # being served for another. Raising when the provider was never stamped -- + # rather than falling back to "" -- is deliberate: an empty segment would + # merge two providers' namespaces silently. + def cache_key + return name.to_s unless name.nil? + + raise TranslationDiff::Error, + "#{self.class} has no cache key: it was instantiated directly instead of being " \ + "built through the registry. Build it through TranslationDiff::Providers.build, " \ + "or give #{self.class} its own #cache_key." + end + + class << self + def configuration_options = [] + + # The subset of configuration_options without which this provider cannot + # work. Checked once, at build time, so a caller learns what to set before + # any request is attempted rather than from a vendor's own exception. + def configuration_requirements = [] + + def capabilities = DEFAULT_CAPABILITIES + + def build(config) = new(config) + end + + private + + def ensure_configured! + missing = self.class.configuration_requirements.reject { |key| config.public_send(key) } + return if missing.empty? + + raise TranslationDiff::ConfigurationError, + "Provider #{self.class} is missing #{missing.join(', ')}. " \ + "Set #{missing.size == 1 ? 'it' : 'them'} in TranslationDiff.configure." + end +end diff --git a/lib/translation_diff/providers.rb b/lib/translation_diff/providers.rb index e57d27e..bb14f13 100644 --- a/lib/translation_diff/providers.rb +++ b/lib/translation_diff/providers.rb @@ -5,56 +5,34 @@ # configuration options, which is what keeps names like `deepl_api_key` out # of TranslationDiff::Configuration and out of this library's core. # -# TranslationDiff::Providers.register(:google, GoogleTranslateProvider) +# TranslationDiff::Providers.register(:acme, AcmeProvider) # # TranslationDiff.configure do |config| -# config.provider = :google -# config.google_api_key = ENV["GOOGLE_API_KEY"] +# config.provider = :acme +# config.acme_api_key = ENV["ACME_API_KEY"] # end # # Nothing in lib/ changes to make that work. module TranslationDiff::Providers - # Supplies the cache-key segment that keeps one provider's cached - # translations from being served to another. A provider built through the - # registry is stamped with its registered name and needs nothing else; a - # provider object assigned straight to `config.provider` never passed - # through here, so it must define #cache_key itself. - # - # Raising when `name` is unset -- rather than falling back to "" -- is - # deliberate: `cache_key` is a segment of every cache key this provider - # ever reads or writes, so a quietly empty one would let two different - # providers share the same cache namespace instead of failing loudly the - # first time anyone tries to use an unstamped instance. - module Naming - attr_accessor :name - - def cache_key - if name.nil? - raise TranslationDiff::Error, - "#{self.class} has no cache key: it was instantiated directly instead of being " \ - "built through the registry. Either build it through " \ - "TranslationDiff::Providers.build (which stamps #name for you), or give " \ - "#{self.class} its own #cache_key." - end - - name.to_s - end - end - class << self # Options are declared before the registry entry is written, so a # provider whose option names collide with another's raises without # having replaced anything under `name`. def register(name, klass) - klass.include(Naming) unless klass.method_defined?(:cache_key) + unless klass < TranslationDiff::Provider + raise TranslationDiff::Error, + "#{klass} cannot be registered as a provider: it does not inherit " \ + "TranslationDiff::Provider. The base class supplies the transport, the " \ + "configuration check and the capability defaults, so a provider that " \ + "skips it has none of them." + end + TranslationDiff::Configuration.register_provider_options(klass.configuration_options, klass) registry.register(name, klass) end def build(name, config) - registry.build(name, config).tap do |provider| - provider.name = name.to_sym if provider.respond_to?(:name=) - end + registry.build(name, config).tap { |provider| provider.name = name.to_sym } end def registered?(name) = registry.registered?(name) diff --git a/lib/translation_diff/providers/null.rb b/lib/translation_diff/providers/null.rb index f344fe8..fd6cd80 100644 --- a/lib/translation_diff/providers/null.rb +++ b/lib/translation_diff/providers/null.rb @@ -2,19 +2,22 @@ # Hands back what it was given. For tests, and for wiring a pipeline up # before a real provider is available. -class TranslationDiff::Providers::Null - def self.configuration_options = [] - def self.build(_config) = new +class TranslationDiff::Providers::Null < TranslationDiff::Provider + # Deliberately not detecting: detection is optional in the contract, and + # this is the provider that proves the optional branch works. + def self.capabilities + TranslationDiff::Capabilities.new( + max_request_size: 1_000_000, max_batch_size: 1_000_000, max_text_size: nil, + html: :none, notranslate: false, detects_language: false, reports_billing: false + ) + end - # No #detect on purpose: detection is optional in the contract, and this - # is the provider that proves the optional branch works. - # rubocop:disable-next Lint/UnusedMethodArgument - def translate(texts, from:, to:, **options) - texts.map(&:to_s) + def translate(request) + TranslationDiff::Translation::Response.build( + request: request, texts: request.texts.map(&:to_s) + ) end - def max_request_size = 1_000_000 - def max_batch_size = 1_000_000 def cache_key = "null" end diff --git a/test/support/provider_contract.rb b/test/support/provider_contract.rb index 5789eab..af122d2 100644 --- a/test/support/provider_contract.rb +++ b/test/support/provider_contract.rb @@ -5,34 +5,42 @@ # TranslationDiff::Providers.register and reached through TranslationDiff.translate. module ProviderContract def test_translate_returns_one_string_per_input - result = provider.translate(%w[one two three], from: :en, to: :ru) + request = TranslationDiff::Translation::Request.new(texts: %w[one two three], from: :en, to: :ru) + result = provider.translate(request) - assert_equal 3, result.size - result.each { |value| assert_kind_of String, value } + assert_equal 3, result.texts.size + result.texts.each { |value| assert_kind_of String, value } end def test_translate_preserves_order texts = %w[first second third] - individually = texts.map { |text| provider.translate([text], from: :en, to: :ru).first } - batched = provider.translate(texts, from: :en, to: :ru) + individually = texts.map do |text| + request = TranslationDiff::Translation::Request.new(texts: [text], from: :en, to: :ru) + provider.translate(request).texts.first + end + batched_request = TranslationDiff::Translation::Request.new(texts: texts, from: :en, to: :ru) + batched = provider.translate(batched_request).texts assert_equal 3, batched.size assert_equal individually, batched end def test_translate_accepts_provider_options - result = provider.translate(%w[one], from: :en, to: :ru, formality: :less) + request = TranslationDiff::Translation::Request.new( + texts: %w[one], from: :en, to: :ru, options: { formality: :less } + ) + result = provider.translate(request) - assert_equal 1, result.size + assert_equal 1, result.texts.size end def test_max_request_size_is_a_positive_integer - assert_kind_of Integer, provider.max_request_size - assert_operator provider.max_request_size, :>, 0 + assert_kind_of Integer, provider.class.capabilities.max_request_size + assert_operator provider.class.capabilities.max_request_size, :>, 0 end def test_max_batch_size_is_a_positive_integer - assert_kind_of Integer, provider.max_batch_size - assert_operator provider.max_batch_size, :>, 0 + assert_kind_of Integer, provider.class.capabilities.max_batch_size + assert_operator provider.class.capabilities.max_batch_size, :>, 0 end end diff --git a/test/translation_diff/context_test.rb b/test/translation_diff/context_test.rb index beb72e0..52268b8 100644 --- a/test/translation_diff/context_test.rb +++ b/test/translation_diff/context_test.rb @@ -3,6 +3,19 @@ require "test_helper" class ContextTest < Minitest::Test + # TranslationDiff::Providers::Null now speaks Translation::Request/Response + # (provider-transport work); request.rb still calls a provider the old way + # and is migrated onto the new contract in a later task. This double keeps + # that old shape so this file can keep exercising Context#translate without + # touching request.rb. + class NullDouble + # rubocop:disable-next Lint/UnusedMethodArgument + def translate(texts, from:, to:, **_options) = texts + def max_request_size = 1_000_000 + def max_batch_size = 1_000_000 + def cache_key = "null" + end + def setup TranslationDiff.reset! TranslationDiff.configure do |c| @@ -42,7 +55,7 @@ def test_two_contexts_build_separate_collaborators end def test_a_context_translates_through_its_own_configuration - context = TranslationDiff.context { |c| c.provider = :null } + context = TranslationDiff.context { |c| c.provider = NullDouble.new } assert_equal "Hello.", context.translate("Hello.", from: "en", to: "ru") end diff --git a/test/translation_diff/instrumentation_test.rb b/test/translation_diff/instrumentation_test.rb index b31b047..488f315 100644 --- a/test/translation_diff/instrumentation_test.rb +++ b/test/translation_diff/instrumentation_test.rb @@ -23,11 +23,25 @@ class FakeRateLimiter def check(_size) = nil end + # TranslationDiff::Providers::Null now speaks Translation::Request/Response + # (provider-transport work); request.rb still calls a provider the old way + # and is migrated onto the new contract in a later task. This double keeps + # that old shape -- and the "null" cache key the assertions below check -- + # so this file can keep exercising the instrumentation pipeline without + # touching request.rb. + class NullDouble + # rubocop:disable-next Lint/UnusedMethodArgument + def translate(texts, from:, to:, **_options) = texts + def max_request_size = 1_000_000 + def max_batch_size = 1_000_000 + def cache_key = "null" + end + def setup super @recorder = Recorder.new TranslationDiff.configure do |c| - c.provider = :null + c.provider = NullDouble.new # Pinned so a developer with REDIS_URL set does not have these tests # resolve the Redis store and open a real socket -- the same reason # context_test.rb pins it. It weakens no assertion here. diff --git a/test/translation_diff/provider_test.rb b/test/translation_diff/provider_test.rb new file mode 100644 index 0000000..fdffb05 --- /dev/null +++ b/test/translation_diff/provider_test.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +require "test_helper" + +class ProviderTest < Minitest::Test + class Bare < TranslationDiff::Provider + end + + class Demanding < TranslationDiff::Provider + def self.configuration_options = %i[demanding_key demanding_secret demanding_region] + def self.configuration_requirements = %i[demanding_key demanding_secret] + end + + def setup + TranslationDiff::Configuration.register_provider_options( + Demanding.configuration_options, Demanding + ) + @config = TranslationDiff::Configuration.new + end + + def test_a_provider_without_requirements_builds + assert_instance_of Bare, Bare.new(@config) + end + + # The old behaviour was a DeepL:: error raised from inside a vendor SDK, or + # `ArgumentError, "project_id is missing"` from another. Neither named the + # option a caller of THIS library has to set. + def test_it_names_every_missing_option_at_once + error = assert_raises(TranslationDiff::ConfigurationError) { Demanding.new(@config) } + + assert_match(/demanding_key/, error.message) + assert_match(/demanding_secret/, error.message) + assert_match(/TranslationDiff.configure/, error.message) + end + + # An option that is declared but not required must not appear in the error. + def test_it_does_not_demand_optional_options + @config.demanding_key = "k" + @config.demanding_secret = "s" + + assert_instance_of Demanding, Demanding.new(@config) + end + + def test_translate_raises_until_a_subclass_implements_it + request = TranslationDiff::Translation::Request.new(texts: %w[one], from: "en", to: "ru") + + assert_raises(NotImplementedError) { Bare.new(@config).translate(request) } + end + + def test_detect_raises_until_a_subclass_implements_it + assert_raises(NotImplementedError) { Bare.new(@config).detect("etwas") } + end + + # cache_key is a segment of every cache key this provider reads or writes. + # A quietly empty one would let two providers share a namespace and serve + # one service's translations for another. + def test_cache_key_is_the_registered_name + provider = Bare.new(@config) + provider.name = :bare + + assert_equal "bare", provider.cache_key + end + + def test_cache_key_raises_when_the_provider_was_never_stamped + error = assert_raises(TranslationDiff::Error) { Bare.new(@config).cache_key } + + assert_match(/registry/, error.message) + end + + def test_the_default_capabilities_are_conservative + capabilities = TranslationDiff::Provider.capabilities + + refute_predicate capabilities, :html? + refute_predicate capabilities, :notranslate? + refute_predicate capabilities, :detects_language? + refute_predicate capabilities, :reports_billing? + end +end diff --git a/test/translation_diff/providers/null_test.rb b/test/translation_diff/providers/null_test.rb index 6a7cb89..7a26c34 100644 --- a/test/translation_diff/providers/null_test.rb +++ b/test/translation_diff/providers/null_test.rb @@ -7,10 +7,16 @@ class NullProviderTest < Minitest::Test include ProviderContract def provider - TranslationDiff::Providers::Null.new + TranslationDiff::Providers::Null.new(TranslationDiff::Configuration.new) end def test_translate_returns_the_input_unchanged - assert_equal %w[one two], provider.translate(%w[one two], from: :en, to: :ru) + request = TranslationDiff::Translation::Request.new(texts: %w[one two], from: :en, to: :ru) + + assert_equal %w[one two], provider.translate(request).texts + end + + def test_cache_key_is_null + assert_equal "null", provider.cache_key end end diff --git a/test/translation_diff/providers_test.rb b/test/translation_diff/providers_test.rb index 8ba0c9f..7c50b67 100644 --- a/test/translation_diff/providers_test.rb +++ b/test/translation_diff/providers_test.rb @@ -5,34 +5,32 @@ class ProvidersTest < Minitest::Test # A provider defined entirely outside this library, to prove that adding a # translation service requires no change to lib/. - class AcmeProvider + class AcmeProvider < TranslationDiff::Provider def self.configuration_options = %i[acme_token] - def self.build(config) = new(config.acme_token) - attr_accessor :name - - def initialize(token) - @token = token + def self.capabilities + TranslationDiff::Capabilities.new( + max_request_size: 1_000, max_batch_size: 10, max_text_size: nil, + html: :none, notranslate: false, detects_language: false, reports_billing: false + ) end - def translate(texts, from:, to:, **_options) - texts.map { |text| "#{@token}:#{from}-#{to}:#{text}" } + def translate(request) + TranslationDiff::Translation::Response.build( + request: request, + texts: request.texts.map { |text| "#{config.acme_token}:#{request.from}-#{request.to}:#{text}" } + ) end - - def max_request_size = 1_000 - def max_batch_size = 10 end # Two providers that both want the same option name. Registering the # second must raise rather than hand it the first one's accessor. - class ConflictingProviderA + class ConflictingProviderA < TranslationDiff::Provider def self.configuration_options = %i[shared_provider_token] - def self.build(_config) = new end - class ConflictingProviderB + class ConflictingProviderB < TranslationDiff::Provider def self.configuration_options = %i[shared_provider_token] - def self.build(_config) = new end # A subclass wanting AcmeProvider's own option -- subclassing a provider @@ -42,19 +40,16 @@ class SubclassOfAcmeProvider < AcmeProvider; end # The second-key-conflicts shape: PartialB would declare :partial_own_key # successfully if checked eagerly, but conflicts with PartialA's # :partial_shared_key on its second option. - class PartialProviderA + class PartialProviderA < TranslationDiff::Provider def self.configuration_options = %i[partial_shared_key] - def self.build(_config) = new end - class PartialProviderB + class PartialProviderB < TranslationDiff::Provider def self.configuration_options = %i[partial_own_key partial_shared_key] - def self.build(_config) = new end - class PartialProviderC + class PartialProviderC < TranslationDiff::Provider def self.configuration_options = %i[partial_own_key] - def self.build(_config) = new end def setup @@ -69,8 +64,9 @@ def test_registering_declares_the_providers_own_options_on_the_configuration def test_building_produces_a_working_provider provider = TranslationDiff::Providers.build(:acme, @config) + request = TranslationDiff::Translation::Request.new(texts: %w[hi], from: :en, to: :ru) - assert_equal ["T:en-ru:hi"], provider.translate(["hi"], from: :en, to: :ru) + assert_equal ["T:en-ru:hi"], provider.translate(request).texts end def test_the_registered_name_becomes_the_cache_key @@ -78,7 +74,6 @@ def test_the_registered_name_becomes_the_cache_key end def test_the_built_in_providers_are_registered - assert TranslationDiff::Providers.registered?(:deepl) assert TranslationDiff::Providers.registered?(:null) end @@ -142,9 +137,8 @@ def test_a_subclass_of_a_registered_provider_may_be_registered # An unrelated class is still refused for the very option a subclass may # now share -- the relaxation is specific to an inheritance relationship. def test_an_unrelated_class_claiming_a_subclassable_option_still_raises - unrelated = Class.new do + unrelated = Class.new(TranslationDiff::Provider) do def self.configuration_options = %i[acme_token] - def self.build(_config) = new end error = assert_raises(TranslationDiff::Error) do @@ -175,24 +169,29 @@ def test_a_failed_registration_leaves_no_partial_option_state assert_includes TranslationDiff::Configuration.options, :partial_own_key end - # A provider instantiated directly, bypassing TranslationDiff::Providers.build, - # never gets its #name stamped. Falling back to "" there would let two such - # providers share the same cache namespace silently, so this must raise - # instead. - def test_cache_key_raises_when_the_provider_was_never_built_through_the_registry - provider = AcmeProvider.new("T") + # ruby_llm requires a Provider subclass and so do we now. A duck-typed + # object cannot be given the transport, the requirement check or the + # capability defaults, and every one of those is a place this library has + # already been bitten. + def test_registering_a_class_that_is_not_a_provider_raises + not_a_provider = Class.new do + def self.configuration_options = [] + def self.build(_config) = new + end - error = assert_raises(TranslationDiff::Error) { provider.cache_key } + error = assert_raises(TranslationDiff::Error) do + TranslationDiff::Providers.register(:impostor, not_a_provider) + end - assert_match(/registry/, error.message) + assert_match(/TranslationDiff::Provider/, error.message) + refute TranslationDiff::Providers.registered?(:impostor) end private def reloadable_provider_class - Class.new do + Class.new(TranslationDiff::Provider) do def self.configuration_options = %i[reloaded_token] - def self.build(_config) = new end end end diff --git a/test/translation_diff/request_test.rb b/test/translation_diff/request_test.rb index e7e4878..9405f08 100644 --- a/test/translation_diff/request_test.rb +++ b/test/translation_diff/request_test.rb @@ -29,6 +29,38 @@ def detect(text) def cache_key = "fake" end + # request.rb still asks `respond_to?(:detect)` to learn whether a provider + # can detect a source language -- the capability this library now expresses + # through Capabilities#detects_language? for anything built as a + # TranslationDiff::Provider. TranslationDiff::Providers::Null moved onto + # that base class (provider-transport work) and, like every Provider, + # always defines #detect (raising NotImplementedError), so it no longer + # answers `respond_to?(:detect)` honestly for this still-old pipeline. This + # double keeps the pre-migration shape -- no #detect at all -- so this file + # can keep testing request.rb's own "cannot detect" guard without touching + # request.rb itself. + class NonDetectingApi + # rubocop:disable-next Lint/UnusedMethodArgument + def translate(texts, from:, to:, **_options) = texts + def max_request_size = 1_000_000 + def max_batch_size = 1_000_000 + def cache_key = "null" + end + + # Same rationale as NonDetectingApi: a Provider built through the registry + # now speaks Translation::Request/Response, which request.rb does not call + # yet. This double is registered under :echo so + # test_the_provider_keyword_overrides_the_configured_provider_for_one_call + # can still exercise resolving a provider by name through the registry. + class EchoProvider < TranslationDiff::Provider + # rubocop:disable-next Lint/UnusedMethodArgument + def translate(texts, from:, to:, **_options) = texts + def max_request_size = 1_000_000 + def max_batch_size = 1_000_000 + def cache_key = "echo" + end + TranslationDiff::Providers.register(:echo, EchoProvider) unless TranslationDiff::Providers.registered?(:echo) + # Always misses, so every value reaches the API. class FakeCacheStore attr_reader :writes @@ -163,7 +195,7 @@ def test_skips_the_translation_when_the_detected_language_is_the_target end def test_raises_when_from_is_missing_and_the_adapter_cannot_detect - configure_with(TranslationDiff::Providers::Null.new) + configure_with(NonDetectingApi.new) error = assert_raises(TranslationDiff::Request::Error) do TranslationDiff::Request.new("text", to: :ru).call @@ -244,7 +276,7 @@ def test_the_provider_keyword_overrides_the_configured_provider_for_one_call api = FakeApi.new(%w[Один]) configure_with(api) - result = TranslationDiff::Request.new("One", from: :en, to: :ru, provider: :null).call + result = TranslationDiff::Request.new("One", from: :en, to: :ru, provider: :echo).call assert_equal "One", result assert_empty api.calls From 6d0d11776f26d2be78187ebd5944bcaff4c6c61c Mon Sep 17 00:00:00 2001 From: IG Date: Wed, 9 Sep 2026 08:41:09 +0400 Subject: [PATCH 03/26] fix: narrow the require-bridge rescue to InvalidProviderError Providers.register's "not a Provider subclass" check raised the generic TranslationDiff::Error, which is also what ProviderOptionOwners raises for an option-name collision. The rescue in translation_diff.rb around the still-unported DeepL/Google requires caught both, so a genuine option collision on deepl_api_key or google_api_key would have been swallowed and misreported as the expected transitional state. InvalidProviderError narrows the rescue to exactly the case it's meant to cover; a collision now still takes the require chain down. --- lib/translation_diff.rb | 11 ++++++++--- lib/translation_diff/errors.rb | 6 ++++++ lib/translation_diff/providers.rb | 2 +- test/translation_diff/providers_test.rb | 16 +++++++++++++++- 4 files changed, 30 insertions(+), 5 deletions(-) diff --git a/lib/translation_diff.rb b/lib/translation_diff.rb index f7647d1..4625c8f 100644 --- a/lib/translation_diff.rb +++ b/lib/translation_diff.rb @@ -29,18 +29,23 @@ # with it before either provider is ever used. Until they are ported, # `:deepl` and `:google` are simply absent from the registry; requesting # either through TranslationDiff::Providers.build raises the ordinary -# "unknown provider" error instead. +# "unknown provider" error instead. The rescue names InvalidProviderError +# specifically, not the generic Error, so it catches only "this class is +# the wrong shape": an option-name collision (also a TranslationDiff::Error, +# raised by ProviderOptionOwners) is a real bug rather than an expected +# transitional state, and must still take the require chain down. begin require "translation_diff/providers/deepl" -rescue TranslationDiff::Error +rescue TranslationDiff::InvalidProviderError nil end begin require "translation_diff/providers/google" -rescue TranslationDiff::Error +rescue TranslationDiff::InvalidProviderError nil end + require "translation_diff/segmenters" require "translation_diff/segmenters/simple" require "translation_diff/segmenters/pragmatic" diff --git a/lib/translation_diff/errors.rb b/lib/translation_diff/errors.rb index d837318..4b48e52 100644 --- a/lib/translation_diff/errors.rb +++ b/lib/translation_diff/errors.rb @@ -44,4 +44,10 @@ def initialize(message, provider: nil, status: nil, retry_after: nil) class TransportError < Error; end class ResponseError < Error; end + + # Raised when a class is offered to a registry that requires a particular + # ancestor. Its own class, rather than the generic Error, so that a caller + # rescuing "this class is the wrong shape" cannot also swallow an unrelated + # failure such as an option-name collision. + class InvalidProviderError < Error; end end diff --git a/lib/translation_diff/providers.rb b/lib/translation_diff/providers.rb index bb14f13..9826d16 100644 --- a/lib/translation_diff/providers.rb +++ b/lib/translation_diff/providers.rb @@ -20,7 +20,7 @@ class << self # having replaced anything under `name`. def register(name, klass) unless klass < TranslationDiff::Provider - raise TranslationDiff::Error, + raise TranslationDiff::InvalidProviderError, "#{klass} cannot be registered as a provider: it does not inherit " \ "TranslationDiff::Provider. The base class supplies the transport, the " \ "configuration check and the capability defaults, so a provider that " \ diff --git a/test/translation_diff/providers_test.rb b/test/translation_diff/providers_test.rb index 7c50b67..8679be9 100644 --- a/test/translation_diff/providers_test.rb +++ b/test/translation_diff/providers_test.rb @@ -179,7 +179,7 @@ def self.configuration_options = [] def self.build(_config) = new end - error = assert_raises(TranslationDiff::Error) do + error = assert_raises(TranslationDiff::InvalidProviderError) do TranslationDiff::Providers.register(:impostor, not_a_provider) end @@ -187,6 +187,20 @@ def self.build(_config) = new refute TranslationDiff::Providers.registered?(:impostor) end + # The transitional bridges in lib/translation_diff.rb rescue + # InvalidProviderError so an unported provider cannot make the library + # unloadable. An option-name collision is a genuine bug and must stay + # outside that net. + def test_an_option_collision_does_not_raise_the_class_the_require_bridges_rescue + TranslationDiff::Providers.register(:collision_a, ConflictingProviderA) + + error = assert_raises(TranslationDiff::Error) do + TranslationDiff::Providers.register(:collision_b, ConflictingProviderB) + end + + refute_kind_of TranslationDiff::InvalidProviderError, error + end + private def reloadable_provider_class From 1b898800acab05a35d81ae824d0d70ddb6b3e336 Mon Sep 17 00:00:00 2001 From: IG Date: Wed, 9 Sep 2026 08:48:48 +0400 Subject: [PATCH 04/26] feat: own the HTTP transport instead of borrowing a vendor's Add HTTPProvider: one Faraday connection every REST provider inherits, with HTTP-status-to-error mapping and no logging middleware, ever, so no line this gem writes can carry source text, translated text, or a credential. Rewrite the shared provider contract onto Translation:: Request/Response and provider.class.capabilities, and add the stubbed- provider and HTTP-provider-contract test helpers every REST provider's test will include. Pin json to < 3 in the Gemfile: json 3.0 dropped the positional `opts` argument Faraday::Response::Json still passes to JSON.parse, which broke every JSON response Faraday parses. --- Gemfile | 6 + lib/translation_diff.rb | 1 + lib/translation_diff/configuration.rb | 3 + lib/translation_diff/http_provider.rb | 119 ++++++++++++++++++ test/support/http_provider_contract.rb | 17 +++ test/support/provider_contract.rb | 56 ++++----- test/support/stubbed_provider.rb | 41 +++++++ test/translation_diff/http_provider_test.rb | 126 ++++++++++++++++++++ translation_diff.gemspec | 13 +- 9 files changed, 349 insertions(+), 33 deletions(-) create mode 100644 lib/translation_diff/http_provider.rb create mode 100644 test/support/http_provider_contract.rb create mode 100644 test/support/stubbed_provider.rb create mode 100644 test/translation_diff/http_provider_test.rb diff --git a/Gemfile b/Gemfile index cd64097..d58bfb7 100644 --- a/Gemfile +++ b/Gemfile @@ -4,6 +4,12 @@ source "https://rubygems.org" gemspec +# json 3.0 dropped the positional `opts` argument that Faraday::Response::Json +# still passes to JSON.parse, which turns every JSON response Faraday parses +# into a Faraday::ParsingError. Pinned here, not in the gemspec, because it is +# a transitive dependency of faraday rather than one of ours. +gem "json", "< 3", require: false + # Not a runtime dependency of the gem (see the gemspec) -- the DeepL # provider requires it lazily at build time. It is only here so the test # suite, which exercises that provider against the real deepl-rb objects, diff --git a/lib/translation_diff.rb b/lib/translation_diff.rb index 4625c8f..41aff6d 100644 --- a/lib/translation_diff.rb +++ b/lib/translation_diff.rb @@ -19,6 +19,7 @@ require "translation_diff/configuration/provider_option_owners" require "translation_diff/provider" +require "translation_diff/http_provider" require "translation_diff/providers" require "translation_diff/providers/null" diff --git a/lib/translation_diff/configuration.rb b/lib/translation_diff/configuration.rb index 493fde1..20adb36 100644 --- a/lib/translation_diff/configuration.rb +++ b/lib/translation_diff/configuration.rb @@ -67,6 +67,9 @@ def provider_option_owners = @provider_option_owners ||= ProviderOptionOwners.ne option :segmenter, :pragmatic option :instrumenter, nil option :logger, nil + option :open_timeout, 5 + option :timeout, 30 + option :max_retries, 3 # Values are copied; memoised collaborators (`provider_instance`, # `cache_store`, `segmenter_instance`, `rate_limiter_instance` and diff --git a/lib/translation_diff/http_provider.rb b/lib/translation_diff/http_provider.rb new file mode 100644 index 0000000..28401d7 --- /dev/null +++ b/lib/translation_diff/http_provider.rb @@ -0,0 +1,119 @@ +# frozen_string_literal: true + +require "faraday" +require "faraday/retry" + +# Every provider reached over HTTP inherits this. It owns one Faraday +# connection and turns HTTP status codes into this library's errors, so a +# caller handles a rate limit the same way whichever service produced it. +# +# A subclass supplies where to talk (#api_base, #headers) and three seams per +# operation: the URL, how to render a request, how to parse a reply. The +# seams are a convenience of this class, not a requirement of Provider -- +# Amazon signs its requests instead and overrides #translate outright. +# +# No logging middleware is installed, ever, and no logger is passed to +# Faraday. This library's log lines carry no source text, no translation and +# no credential; a request logger would carry all three, and it would do so +# at exactly the moment someone turns DEBUG on to diagnose a problem. +class TranslationDiff::HTTPProvider < TranslationDiff::Provider + RETRY_STATUSES = [429, 500, 502, 503, 504].freeze + + # Faraday raises these when nobody answered, as opposed to answering "no". + TRANSPORT_FAILURES = [Faraday::ConnectionFailed, Faraday::TimeoutError, Faraday::SSLError].freeze + + def api_base = raise NotImplementedError, "#{self.class} must implement #api_base" + def headers = {} + + def translate_url = raise NotImplementedError, "#{self.class} must implement #translate_url" + + def render_translate_payload(_request) + raise NotImplementedError, "#{self.class} must implement #render_translate_payload" + end + + def parse_translate_response(_body, _headers, _request) + raise NotImplementedError, "#{self.class} must implement #parse_translate_response" + end + + def translate(request) + response = post(translate_url, render_translate_payload(request)) + parse_translate_response(response.body, response.headers, request) + end + + def connection = @connection ||= build_connection + + private + + def post(url, payload) + response = connection.post(url, payload) + raise_for_status!(response) + response + rescue *TRANSPORT_FAILURES => e + # The message is the transport's, never the payload's: the payload is the + # customer's text. + raise TranslationDiff::TransportError, "#{self.class}: #{e.class}: #{e.message}" + end + + # The block is how a test swaps in Faraday's test adapter. Amazon overrides + # this with the same signature, because its signature covers the body + # exactly as sent and a JSON request middleware would re-encode it. + def build_connection(&block) + Faraday.new(url: api_base, headers: headers) do |faraday| + faraday.request :json + faraday.request :retry, retry_options + faraday.response :json, content_type: /\bjson$/ + adapt(faraday, &block) + apply_timeouts(faraday) + end + end + + def adapt(faraday, &block) + block ? block.call(faraday) : faraday.adapter(Faraday.default_adapter) + end + + def apply_timeouts(faraday) + faraday.options.open_timeout = config.open_timeout + faraday.options.timeout = config.timeout + end + + # faraday-retry reads Retry-After itself, which is why a 429 usually never + # reaches #raise_for_status!. What is left when it does is a service that + # kept saying no for every attempt. + def retry_options + { max: config.max_retries, interval: 0.5, backoff_factor: 2, interval_randomness: 0.5, + retry_statuses: RETRY_STATUSES, methods: %i[post get], + exceptions: TRANSPORT_FAILURES + [Faraday::RetriableResponse] } + end + + def raise_for_status!(response) + status = response.status + return if status < 400 + + raise error_class(status).new(error_message(response), **error_options(response)) + end + + def error_class(status) + case status + when 401, 403 then TranslationDiff::AuthenticationError + when 429 then TranslationDiff::RateLimitError + when 456 then TranslationDiff::QuotaExceededError + when 400..499 then TranslationDiff::InvalidRequestError + else TranslationDiff::ServiceError + end + end + + def error_options(response) + options = { provider: name, status: response.status } + return options unless response.status == 429 + + options.merge(retry_after: response.headers["Retry-After"]&.to_i) + end + + # The service's own words, truncated. A provider's error body is a + # diagnostic, and an untruncated one can be a whole HTML error page. + def error_message(response) + body = response.body + text = body.is_a?(Hash) ? (body["message"] || body["error"] || body.to_s) : body.to_s + "#{self.class} responded #{response.status}: #{text.to_s[0, 300]}" + end +end diff --git a/test/support/http_provider_contract.rb b/test/support/http_provider_contract.rb new file mode 100644 index 0000000..dcc9ae5 --- /dev/null +++ b/test/support/http_provider_contract.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +# What every HTTP-backed provider must do, beyond the plain provider +# contract. Including this requires the test to define #provider (built +# against a Faraday test stub) and #config. +module HTTPProviderContract + def test_it_declares_an_api_base_that_is_a_url + assert_match(%r{\Ahttps?://}, provider.api_base) + end + + def test_it_installs_no_logging_middleware + config.logger = Logger.new(StringIO.new) + handlers = provider.class.new(config).connection.builder.handlers + + refute_includes handlers.map(&:name), "Faraday::Response::Logger" + end +end diff --git a/test/support/provider_contract.rb b/test/support/provider_contract.rb index af122d2..6abbab7 100644 --- a/test/support/provider_contract.rb +++ b/test/support/provider_contract.rb @@ -1,46 +1,46 @@ # frozen_string_literal: true -# The executable form of the provider contract. Every provider includes this -# and defines #provider; anything that passes can be registered with -# TranslationDiff::Providers.register and reached through TranslationDiff.translate. +# The executable form of the provider contract. Every provider's test +# includes this and defines #provider; anything that passes can be registered +# and reached through TranslationDiff.translate. module ProviderContract + def translation_request(texts, from: :en, to: :ru, **options) + TranslationDiff::Translation::Request.new(texts: texts, from: from, to: to, options: options) + end + + def test_it_inherits_the_provider_base_class + assert_kind_of TranslationDiff::Provider, provider + end + def test_translate_returns_one_string_per_input - request = TranslationDiff::Translation::Request.new(texts: %w[one two three], from: :en, to: :ru) - result = provider.translate(request) + response = provider.translate(translation_request(%w[one two three])) - assert_equal 3, result.texts.size - result.texts.each { |value| assert_kind_of String, value } + assert_equal 3, response.texts.size + response.texts.each { |value| assert_kind_of String, value } end def test_translate_preserves_order texts = %w[first second third] - individually = texts.map do |text| - request = TranslationDiff::Translation::Request.new(texts: [text], from: :en, to: :ru) - provider.translate(request).texts.first - end - batched_request = TranslationDiff::Translation::Request.new(texts: texts, from: :en, to: :ru) - batched = provider.translate(batched_request).texts - - assert_equal 3, batched.size + individually = texts.map { |text| provider.translate(translation_request([text])).texts.first } + batched = provider.translate(translation_request(texts)).texts + assert_equal individually, batched end - def test_translate_accepts_provider_options - request = TranslationDiff::Translation::Request.new( - texts: %w[one], from: :en, to: :ru, options: { formality: :less } - ) - result = provider.translate(request) + def test_its_capabilities_are_sane + capabilities = provider.class.capabilities - assert_equal 1, result.texts.size + assert_operator capabilities.max_request_size, :>, 0 + assert_operator capabilities.max_batch_size, :>, 0 + assert_includes [true, false], capabilities.notranslate? end - def test_max_request_size_is_a_positive_integer - assert_kind_of Integer, provider.class.capabilities.max_request_size - assert_operator provider.class.capabilities.max_request_size, :>, 0 - end + # A provider that claims to honour notranslate must have an HTML mode to + # honour it in. Google and DeepL both shipped with this broken, in + # different ways, before the capability existed to state it. + def test_notranslate_is_only_claimed_with_an_html_mode + capabilities = provider.class.capabilities - def test_max_batch_size_is_a_positive_integer - assert_kind_of Integer, provider.class.capabilities.max_batch_size - assert_operator provider.class.capabilities.max_batch_size, :>, 0 + assert capabilities.html?, "claims notranslate without an html mode" if capabilities.notranslate? end end diff --git a/test/support/stubbed_provider.rb b/test/support/stubbed_provider.rb new file mode 100644 index 0000000..7b1b537 --- /dev/null +++ b/test/support/stubbed_provider.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +require "faraday" + +# Builds a provider whose Faraday connection answers from a stub and records +# what was sent, so a test can assert on the payload as well as on the parse. +# Including this requires the test to define #config and #provider_class. +module StubbedProvider + def requests = @requests ||= [] + + def stub_provider(route:, body:, status: 200, headers: {}, name: :test) + stubs = build_stubs(route: route, body: body, status: status, headers: headers) + + provider_class.new(config).tap do |built| + built.name = name + attach_stub(built, stubs) + end + end + + def sent = JSON.parse(requests.first.body) + def query = CGI.parse(requests.first.url.query.to_s) + + private + + def build_stubs(route:, body:, status:, headers:) + recorder = requests + Faraday::Adapter::Test::Stubs.new do |stub| + stub.post(route) do |env| + recorder << env + [status, { "Content-Type" => "application/json" }.merge(headers), + body.is_a?(String) ? body : body.to_json] + end + end + end + + def attach_stub(provider, stubs) + provider.instance_variable_set(:@connection, provider.send(:build_connection) do |faraday| + faraday.adapter :test, stubs + end) + end +end diff --git a/test/translation_diff/http_provider_test.rb b/test/translation_diff/http_provider_test.rb new file mode 100644 index 0000000..0c9042b --- /dev/null +++ b/test/translation_diff/http_provider_test.rb @@ -0,0 +1,126 @@ +# frozen_string_literal: true + +require "test_helper" +require "faraday" + +class HTTPProviderTest < Minitest::Test + # A provider that exists only to exercise the base class. Its seams are the + # smallest thing that can round-trip. + class Echo < TranslationDiff::HTTPProvider + def api_base = "https://echo.test" + def headers = { "X-Echo" => "1" } + def translate_url = "v1/translate" + def render_translate_payload(request) = { "q" => request.texts } + + def parse_translate_response(body, _headers, request) + TranslationDiff::Translation::Response.build(request: request, texts: body["translations"]) + end + end + + def setup + @config = TranslationDiff::Configuration.new + end + + def request(texts = %w[one]) + TranslationDiff::Translation::Request.new(texts: texts, from: "en", to: "ru") + end + + # Builds an Echo whose connection uses Faraday's test adapter. Minitest 6 + # dropped minitest/mock, and stubbing HTTP is exactly what the test adapter + # is for -- no webmock, no network, and the same middleware stack the real + # connection has. + def provider_for(status:, body:, headers: {}) + stubs = Faraday::Adapter::Test::Stubs.new do |stub| + stub.post("/v1/translate") { [status, headers, body] } + end + Echo.new(@config).tap do |provider| + provider.instance_variable_set(:@connection, provider.send(:build_connection) do |faraday| + faraday.adapter :test, stubs + end) + end + end + + def test_it_posts_the_rendered_payload_and_parses_the_reply + provider = provider_for(status: 200, body: { "translations" => %w[один] }.to_json, + headers: { "Content-Type" => "application/json" }) + + assert_equal %w[один], provider.translate(request).texts + end + + def test_a_401_becomes_an_authentication_error_naming_the_provider + provider = provider_for(status: 401, body: "nope") + provider.name = :echo + + error = assert_raises(TranslationDiff::AuthenticationError) { provider.translate(request) } + + assert_equal :echo, error.provider + assert_equal 401, error.status + end + + def test_a_400_becomes_an_invalid_request_error + provider = provider_for(status: 400, body: "bad") + + assert_raises(TranslationDiff::InvalidRequestError) { provider.translate(request) } + end + + def test_a_456_becomes_a_quota_error + provider = provider_for(status: 456, body: "out of quota") + + assert_raises(TranslationDiff::QuotaExceededError) { provider.translate(request) } + end + + def test_a_500_becomes_a_service_error + provider = provider_for(status: 500, body: "boom") + + assert_raises(TranslationDiff::ServiceError) { provider.translate(request) } + end + + # 429 survives the retries only when they are exhausted, so the test turns + # them off; what is asserted here is the mapping, not the retrying. + def test_a_429_becomes_a_rate_limit_error_carrying_retry_after + @config.max_retries = 0 + provider = provider_for(status: 429, body: "slow down", headers: { "Retry-After" => "17" }) + + error = assert_raises(TranslationDiff::RateLimitError) { provider.translate(request) } + + assert_equal 17, error.retry_after + end + + def test_a_connection_failure_becomes_a_transport_error + @config.max_retries = 0 + stubs = Faraday::Adapter::Test::Stubs.new do |stub| + stub.post("/v1/translate") { raise Faraday::ConnectionFailed, "no route" } + end + provider = Echo.new(@config) + provider.instance_variable_set(:@connection, provider.send(:build_connection) do |faraday| + faraday.adapter :test, stubs + end) + + assert_raises(TranslationDiff::TransportError) { provider.translate(request) } + end + + # The guarantee that no line this library writes carries source text or a + # credential now has a mechanism: we own the connection, and nothing + # installs a logging middleware on it. + def test_no_logging_middleware_is_installed_even_when_a_logger_is_configured + @config.logger = Logger.new(StringIO.new) + handlers = Echo.new(@config).connection.builder.handlers + + refute_includes handlers.map(&:name), "Faraday::Response::Logger" + end + + def test_the_configured_timeouts_reach_the_connection + @config.open_timeout = 2 + @config.timeout = 7 + connection = Echo.new(@config).connection + + assert_equal 2, connection.options.open_timeout + assert_equal 7, connection.options.timeout + end + + def test_the_subclass_headers_are_sent + connection = Echo.new(@config).connection + + assert_equal "1", connection.headers["X-Echo"] + end +end diff --git a/translation_diff.gemspec b/translation_diff.gemspec index 83289fe..ebbf482 100644 --- a/translation_diff.gemspec +++ b/translation_diff.gemspec @@ -45,11 +45,14 @@ small edit costs the price of the edit, not the whole text. spec.add_development_dependency "rubocop", "~> 1.90" spec.add_development_dependency "simplecov", "~> 1.2" - # The only two gems this one loads. `ox` walks HTML; `pragmatic_segmenter` - # backs the default sentence segmenter and has zero dependencies of its - # own. Everything else -- the DeepL client, the connection pool, and - # whatever backs the cache and the rate limiter -- is supplied by the - # application and duck typed, so it stays out of the gemspec. + # `ox` walks HTML; `pragmatic_segmenter` backs the default sentence + # segmenter and has zero dependencies of its own; `faraday` and + # `faraday-retry` are the HTTP transport every REST provider inherits. + # Everything else -- the connection pool, and whatever backs the cache and + # the rate limiter -- is supplied by the application and duck typed, so it + # stays out of the gemspec. + spec.add_dependency "faraday", "~> 2.9" + spec.add_dependency "faraday-retry", "~> 2.2" spec.add_dependency "ox", "~> 2.14" spec.add_dependency "pragmatic_segmenter", "~> 0.3" end From 6cf94fad3a80f939ce4e8ff64624888c9c1d0c40 Mon Sep 17 00:00:00 2001 From: IG Date: Wed, 9 Sep 2026 08:52:15 +0400 Subject: [PATCH 05/26] fix: decode JSON responses ourselves instead of via faraday's middleware json 3.0 dropped the positional opts argument Faraday::Response::Json still passes to JSON.parse, and json 3 is now the default gem on Ruby 4.x -- so any application on a modern Ruby would have hit an ArgumentError on the first response this gem parsed. A Gemfile pin only protected our own suite, not the users who would ship with it. Stop installing faraday's response-JSON middleware and decode the body ourselves in #post, the same reasoning that took this gem off the vendor SDKs. #post now returns a small Decoded value (status, headers, already-parsed body) so #raise_for_status! and every subclass's #parse_translate_response keep the same shape they had. A non-JSON body -- an HTML error page from a proxy, for instance -- passes through untouched rather than raising. --- Gemfile | 6 ----- lib/translation_diff/http_provider.rb | 30 +++++++++++++++++++-- test/translation_diff/http_provider_test.rb | 17 ++++++++++++ 3 files changed, 45 insertions(+), 8 deletions(-) diff --git a/Gemfile b/Gemfile index d58bfb7..cd64097 100644 --- a/Gemfile +++ b/Gemfile @@ -4,12 +4,6 @@ source "https://rubygems.org" gemspec -# json 3.0 dropped the positional `opts` argument that Faraday::Response::Json -# still passes to JSON.parse, which turns every JSON response Faraday parses -# into a Faraday::ParsingError. Pinned here, not in the gemspec, because it is -# a transitive dependency of faraday rather than one of ours. -gem "json", "< 3", require: false - # Not a runtime dependency of the gem (see the gemspec) -- the DeepL # provider requires it lazily at build time. It is only here so the test # suite, which exercises that provider against the real deepl-rb objects, diff --git a/lib/translation_diff/http_provider.rb b/lib/translation_diff/http_provider.rb index 28401d7..3fb4a6e 100644 --- a/lib/translation_diff/http_provider.rb +++ b/lib/translation_diff/http_provider.rb @@ -2,6 +2,7 @@ require "faraday" require "faraday/retry" +require "json" # Every provider reached over HTTP inherits this. It owns one Faraday # connection and turns HTTP status codes into this library's errors, so a @@ -44,8 +45,16 @@ def connection = @connection ||= build_connection private + # What #post hands back: a decoded body next to the headers it arrived + # with, so #raise_for_status! and a subclass's #parse_translate_response + # both see the same shape a Faraday::Response would have given them had + # its own JSON middleware still been in the stack. + Decoded = Data.define(:status, :headers, :body) + private_constant :Decoded + def post(url, payload) - response = connection.post(url, payload) + raw = connection.post(url, payload) + response = Decoded.new(status: raw.status, headers: raw.headers, body: decode(raw)) raise_for_status!(response) response rescue *TRANSPORT_FAILURES => e @@ -54,6 +63,24 @@ def post(url, payload) raise TranslationDiff::TransportError, "#{self.class}: #{e.class}: #{e.message}" end + # Faraday's response-JSON middleware passes parser options positionally, + # which json 3 removed -- and json 3 is the default gem on Ruby 4.x, so + # relying on that middleware would break this library for most modern + # applications. Decoding here costs one call and depends on nothing. + def decode(response) + body = response.body + return body unless body.is_a?(String) + return body if body.strip.empty? + return body unless json?(response) + + JSON.parse(body) + rescue JSON::ParserError => e + raise TranslationDiff::ResponseError, + "#{self.class} returned a body that is not JSON: #{e.message[0, 200]}" + end + + def json?(response) = response.headers["content-type"].to_s.match?(/\bjson\b/) + # The block is how a test swaps in Faraday's test adapter. Amazon overrides # this with the same signature, because its signature covers the body # exactly as sent and a JSON request middleware would re-encode it. @@ -61,7 +88,6 @@ def build_connection(&block) Faraday.new(url: api_base, headers: headers) do |faraday| faraday.request :json faraday.request :retry, retry_options - faraday.response :json, content_type: /\bjson$/ adapt(faraday, &block) apply_timeouts(faraday) end diff --git a/test/translation_diff/http_provider_test.rb b/test/translation_diff/http_provider_test.rb index 0c9042b..92f211f 100644 --- a/test/translation_diff/http_provider_test.rb +++ b/test/translation_diff/http_provider_test.rb @@ -123,4 +123,21 @@ def test_the_subclass_headers_are_sent assert_equal "1", connection.headers["X-Echo"] end + + def test_it_decodes_a_json_body_without_faradays_middleware + provider = provider_for(status: 200, body: { "translations" => %w[один] }.to_json, + headers: { "Content-Type" => "application/json" }) + + assert_equal %w[один], provider.translate(request).texts + end + + # An error page from a proxy is HTML, and the error path must survive it. + def test_a_non_json_error_body_still_produces_the_mapped_error + provider = provider_for(status: 500, body: "gateway", + headers: { "Content-Type" => "text/html" }) + + error = assert_raises(TranslationDiff::ServiceError) { provider.translate(request) } + + assert_match(/gateway/, error.message) + end end From 5a94e1391568a6bb20907ba379a335d05949ea39 Mon Sep 17 00:00:00 2001 From: IG Date: Wed, 9 Sep 2026 17:55:56 +0400 Subject: [PATCH 06/26] refactor!: talk to DeepL directly instead of through deepl-rb The provider now speaks DeepL's REST API on this library's own Faraday transport. Dropping deepl-rb removes a dependency, but the reason is narrower than that: its defaults were not ours, and neither was its logging. It writes the whole request at DEBUG -- the Authorization header, auth key and all, followed by the payload, which is the text being translated -- so a customer's content reached the application log the moment anyone turned DEBUG on to diagnose something. Nothing we configure on our own connection does that. Two vendor facts are now stated where they can be read. The free host is selected from the `:fx` key suffix, which deepl-rb used to do for us. And `max_batch_size` is 50, DeepL's documented limit; the old code said 300, which the per-request size limit usually capped first -- a list of short values would have reached it and been rejected. `tag_handling: html` with `tag_handling_version: v2` is preserved exactly. It is what makes `class="notranslate"` work, and it failed silently for the whole life of the DeepL provider before it was added. --- Gemfile | 6 - lib/translation_diff.rb | 30 +-- lib/translation_diff/providers/deepl.rb | 132 ++++++----- test/translation_diff/providers/deepl_test.rb | 224 +++++++++++------- test/translation_diff/providers_test.rb | 1 + 5 files changed, 223 insertions(+), 170 deletions(-) diff --git a/Gemfile b/Gemfile index cd64097..d5b5cbb 100644 --- a/Gemfile +++ b/Gemfile @@ -4,12 +4,6 @@ source "https://rubygems.org" gemspec -# Not a runtime dependency of the gem (see the gemspec) -- the DeepL -# provider requires it lazily at build time. It is only here so the test -# suite, which exercises that provider against the real deepl-rb objects, -# has it available. -gem "deepl-rb", "~> 3.9", require: false - # Not runtime dependencies of the gem (see the gemspec) -- Configuration# # redis_pool requires them lazily, and RedisCacheStore/RedisRateLimiter # duck-type against whatever a caller's connection pool yields. They are diff --git a/lib/translation_diff.rb b/lib/translation_diff.rb index 41aff6d..88a31f9 100644 --- a/lib/translation_diff.rb +++ b/lib/translation_diff.rb @@ -23,24 +23,20 @@ require "translation_diff/providers" require "translation_diff/providers/null" -# DeepL and Google still wrap their vendor SDKs directly instead of -# inheriting Provider -- Tasks 4 and 5 port them. Providers.register now -# raises for exactly that shape of class, which would otherwise take this -# entire require chain, and therefore every caller of this library, down -# with it before either provider is ever used. Until they are ported, -# `:deepl` and `:google` are simply absent from the registry; requesting -# either through TranslationDiff::Providers.build raises the ordinary -# "unknown provider" error instead. The rescue names InvalidProviderError -# specifically, not the generic Error, so it catches only "this class is -# the wrong shape": an option-name collision (also a TranslationDiff::Error, -# raised by ProviderOptionOwners) is a real bug rather than an expected -# transitional state, and must still take the require chain down. -begin - require "translation_diff/providers/deepl" -rescue TranslationDiff::InvalidProviderError - nil -end +require "translation_diff/providers/deepl" +# Google still wraps its vendor SDK directly instead of inheriting Provider +# -- Task 5 ports it. Providers.register now raises for exactly that shape +# of class, which would otherwise take this entire require chain, and +# therefore every caller of this library, down with it before the provider +# is ever used. Until it is ported, `:google` is simply absent from the +# registry; requesting it through TranslationDiff::Providers.build raises +# the ordinary "unknown provider" error instead. The rescue names +# InvalidProviderError specifically, not the generic Error, so it catches +# only "this class is the wrong shape": an option-name collision (also a +# TranslationDiff::Error, raised by ProviderOptionOwners) is a real bug +# rather than an expected transitional state, and must still take the +# require chain down. begin require "translation_diff/providers/google" rescue TranslationDiff::InvalidProviderError diff --git a/lib/translation_diff/providers/deepl.rb b/lib/translation_diff/providers/deepl.rb index a67b09f..2b7b3dd 100644 --- a/lib/translation_diff/providers/deepl.rb +++ b/lib/translation_diff/providers/deepl.rb @@ -1,81 +1,89 @@ # frozen_string_literal: true -# Talks to DeepL through deepl-rb's per-instance objects rather than its -# module-level shortcuts, so this library never calls DeepL.configure and -# never mutates another gem's global state. Two behaviours come for free by -# using their Configuration: it reads DEEPL_AUTH_KEY when no key is given, -# and it picks the free or the paid host from the key's ":fx" suffix. -# -# deepl-rb is not a dependency of this gem. It is required at build time, so -# an application using a different provider never needs it installed. -class TranslationDiff::Providers::DeepL - # DeepL requires a target language even when only the detection is - # wanted, so the provider picks one rather than making the caller do it. - DETECTION_TARGET = "EN" +# Talks to DeepL's REST API directly. This used to wrap deepl-rb; owning the +# request removed a dependency and, more to the point, removed a layer whose +# defaults were not ours -- deepl-rb logs the auth key and the payload at +# DEBUG, and its tag handling default silently disabled notranslate. +class TranslationDiff::Providers::DeepL < TranslationDiff::HTTPProvider + PAID_HOST = "https://api.deepl.com" + FREE_HOST = "https://api-free.deepl.com" + + # A key ending in :fx is a free-plan key, and the free plan lives on its + # own host. DeepL's own libraries do this; so do we, now. + FREE_KEY_SUFFIX = ":fx" - MAX_REQUEST_SIZE = 1700 - MAX_BATCH_SIZE = 300 - - # What arrives here is not plain text, despite having been through the - # Tokenizer. A notranslate span is handed over whole, tags included -- - # that is how the tokenizer marks content the provider must leave alone. - # DeepL honours `class="notranslate"` (and `translate="no"`) only under - # HTML tag handling; without it, in DeepL's own words, "tags are treated - # as regular text". The failure is quiet, because DeepL leaves the tags - # themselves alone either way and only the protected content changes: - # - # "Bold Mountain is a good place." - # no tag_handling -> "Болд-Маунтин — отличное место." - # tag_handling -> "Bold Mountain — это хорошее место." - # - # v2 is the tag handling algorithm DeepL's documentation recommends. - # Note that under HTML tag handling DeepL defaults `split_sentences` to - # `nonewlines`; this library sends one sentence at a time, so that - # changes nothing here. + # What arrives here is not plain text: the tokenizer hands over a + # notranslate span with its tags. DeepL honours class="notranslate" only + # under HTML tag handling; without it, in DeepL's words, "tags are treated + # as regular text", and the protected content is translated while the tags + # survive -- a failure nothing about the output reveals. DEFAULT_OPTIONS = { tag_handling: :html, tag_handling_version: "v2" }.freeze - def self.configuration_options = %i[deepl_api_key deepl_host] - - # `config.logger` is deliberately NOT forwarded into DeepL::Configuration. - # deepl-rb logs the whole request at DEBUG -- a "Request details:" line - # carrying the Authorization header, DeepL auth key and all, followed by - # the payload, which is the text being translated. This library's logger - # carries a guarantee that no line it writes holds translated text, source - # text, or a credential; handing it to a gem that logs payloads would break - # that guarantee silently, at the exact moment someone turns DEBUG on to - # diagnose a problem. Anyone who wants deepl-rb's own request log can build - # the DeepL::API themselves, wrap it in this provider, and assign that to - # `config.provider` -- see "Instrumentation and logging" in the README. - def self.build(config) - require "deepl" - - settings = { auth_key: config.deepl_api_key, host: config.deepl_host }.compact - new(::DeepL::API.new(::DeepL::Configuration.new(settings))) - rescue LoadError - raise TranslationDiff::Error, - "provider is :deepl but the `deepl-rb` gem is not available. " \ - 'Add `gem "deepl-rb"` to your Gemfile.' + # 50 texts and a 128 KiB body are DeepL's documented per-request limits. + # The request size stays at the 1700 escaped characters this library has + # always used; the batch count is the number that was wrong (it said 300). + def self.capabilities + TranslationDiff::Capabilities.new( + max_request_size: 1_700, max_batch_size: 50, max_text_size: nil, + html: :tag_handling, notranslate: true, detects_language: true, reports_billing: true + ) + end + + def self.configuration_options = %i[deepl_api_key deepl_api_base] + def self.configuration_requirements = %i[deepl_api_key] + + # DeepL requires a target language even when only the detection is wanted, + # so the provider picks one rather than making the caller do it. + DETECTION_TARGET = "EN" + + def api_base + config.deepl_api_base || (free_key? ? FREE_HOST : PAID_HOST) end - def initialize(api) - @api = api + def headers = { "Authorization" => "DeepL-Auth-Key #{config.deepl_api_key}" } + + def translate_url = "v2/translate" + + def render_translate_payload(request) + DEFAULT_OPTIONS + .merge(request.options) + .merge(text: request.texts, target_lang: language(request.to)) + .tap { |payload| payload[:source_lang] = language(request.from) unless request.from.nil? } end - def translate(texts, from:, to:, **options) - Array(request(texts, from, to, DEFAULT_OPTIONS.merge(options))).map(&:text) + def parse_translate_response(body, _headers, request) + translations = Array(body["translations"]) + + TranslationDiff::Translation::Response.build( + request: request, + texts: translations.map { |t| t["text"] }, + detected_source: translations.first&.dig("detected_source_language")&.downcase, + usage: usage_for(request, translations) + ) end + # DeepL has no detection endpoint. Translating a sample and reading what it + # says the source was is the only way, and is what this has always done. def detect(text) - request(text, nil, DETECTION_TARGET).detected_source_language.downcase + request = TranslationDiff::Translation::Request.new(texts: [text], from: nil, + to: DETECTION_TARGET) + translate(request).detected_source end - def max_request_size = MAX_REQUEST_SIZE - def max_batch_size = MAX_BATCH_SIZE - private - def request(text, from, to, options = {}) - ::DeepL::Requests::Translate.new(@api, text, from, to, options).request + def free_key? = config.deepl_api_key.to_s.end_with?(FREE_KEY_SUFFIX) + + # DeepL's language codes are upper case. + def language(value) = value.to_s.upcase + + def usage_for(request, translations) + billed = translations.filter_map { |t| t["billed_characters"] }.sum + + TranslationDiff::Translation::Usage.new( + characters: request.texts.sum(&:size), + billed_characters: billed.positive? ? billed : nil + ) end end diff --git a/test/translation_diff/providers/deepl_test.rb b/test/translation_diff/providers/deepl_test.rb index 7dfb5a7..24ebb61 100644 --- a/test/translation_diff/providers/deepl_test.rb +++ b/test/translation_diff/providers/deepl_test.rb @@ -2,129 +2,183 @@ require "test_helper" require "support/provider_contract" - -# `TranslationDiff::Providers::DeepL.build` only requires "deepl" lazily, at -# call time, so whether ::DeepL is already defined when this file runs -# depends on test order -- Minitest randomises it. Requiring it explicitly -# here means this file's constant references (::DeepL::Exceptions::Error -# below) don't depend on some other test file having required "deepl" first. -require "deepl" +require "support/http_provider_contract" +require "faraday" class DeepLProviderTest < Minitest::Test include ProviderContract + include HTTPProviderContract + + # A real response body, captured from api-free.deepl.com on 2026-09-09. + TRANSLATE_BODY = { + "translations" => [ + { "detected_source_language" => "EN", "text" => "один", "billed_characters" => 3 }, + { "detected_source_language" => "EN", "text" => "два", "billed_characters" => 3 } + ] + }.freeze + + attr_reader :config, :requests + + def setup + TranslationDiff.reset! + @config = TranslationDiff::Configuration.new + @config.deepl_api_key = "test-key:fx" + @requests = [] + end - # The provider now issues DeepL::Requests::Translate itself, one layer - # below where the old adapter's fake client stood. Stubbing - # DeepL::Requests::Translate is not possible without a mocking library - # (Minitest 6.0 dropped minitest/mock), so this subclasses the provider - # instead and overrides its private #request method -- the smallest - # honest seam available without one, and it still exercises every line - # of #translate and #detect. - class FakeDeepL < TranslationDiff::Providers::DeepL - Text = Struct.new(:text, :detected_source_language) - - attr_reader :calls - - def initialize(*) - super(:unused_api) - @calls = [] - end + # Builds a provider whose connection answers from a stub and records what + # was sent, so a test can assert on the payload as well as the parse. + # + # When neither `body:` nor `texts:` is given, the stub echoes back + # whatever texts were actually sent (rather than a fixed pair), so the + # shared ProviderContract tests -- which call `provider` with no + # knowledge of how many texts they are about to send -- get a response + # the same size as their request instead of tripping Response.build's + # count check. + def provider(body: nil, status: 200, texts: nil) + stubs = stub_translate(body: body, status: status, texts: texts) + built = TranslationDiff::Providers::DeepL.new(config) + built.name = :deepl + built.instance_variable_set(:@connection, built.send(:build_connection) do |faraday| + faraday.adapter :test, stubs + end) + built + end - private + def sent = JSON.parse(requests.first.body) - def request(text, from, to, options = {}) - @calls << [text, from, to, options] - Array(text).map { |value| Text.new("#{value}-translated", "EN") } - .then { |texts| text.is_a?(Array) ? texts : texts.first } - end + def test_a_free_key_selects_the_free_host + assert_equal "https://api-free.deepl.com", TranslationDiff::Providers::DeepL.new(config).api_base + end + + def test_a_paid_key_selects_the_paid_host + config.deepl_api_key = "test-key" + + assert_equal "https://api.deepl.com", TranslationDiff::Providers::DeepL.new(config).api_base end - def provider - FakeDeepL.new + def test_the_api_base_option_overrides_both + config.deepl_api_base = "https://deepl.internal" + + assert_equal "https://deepl.internal", TranslationDiff::Providers::DeepL.new(config).api_base end - def test_translate_unwraps_the_text_of_each_result - assert_equal %w[one-translated two-translated], provider.translate(%w[one two], from: :en, to: :ru) + def test_it_authenticates_with_the_deepl_scheme + assert_equal "DeepL-Auth-Key test-key:fx", + TranslationDiff::Providers::DeepL.new(config).headers["Authorization"] end - def test_translate_passes_provider_options_through - fake = FakeDeepL.new + def test_a_missing_key_is_named_before_any_request + config.deepl_api_key = nil - fake.translate(%w[one], from: :en, to: :ru, formality: :less) + error = assert_raises(TranslationDiff::ConfigurationError) do + TranslationDiff::Providers::DeepL.new(config) + end - assert_equal({ formality: :less }, fake.calls.first.last.slice(:formality)) + assert_match(/deepl_api_key/, error.message) end - # What reaches a provider is not plain text: Tokenizer hands a notranslate - # span over with its tags. DeepL honours `class="notranslate"` only under - # `tag_handling: html`; without it, per DeepL's own documentation, "tags - # are treated as regular text" -- and the protected content is translated - # while the tags survive, which is exactly the shape of bug nobody spots. - def test_translate_asks_for_html_tag_handling - fake = FakeDeepL.new + def test_it_sends_the_texts_and_the_language_pair + provider.translate(translation_request(%w[one two])) + + assert_equal %w[one two], sent["text"] + assert_equal "EN", sent["source_lang"] + assert_equal "RU", sent["target_lang"] + end - fake.translate(%w[one], from: :en, to: :ru) + # DeepL wants upper-case language codes; a caller writing "en" must work. + def test_it_upcases_the_language_codes + provider.translate(translation_request(%w[one two], from: "en", to: "ru")) - assert_equal({ tag_handling: :html, tag_handling_version: "v2" }, - fake.calls.first.last.slice(:tag_handling, :tag_handling_version)) + assert_equal "EN", sent["source_lang"] + assert_equal "RU", sent["target_lang"] end - def test_translate_lets_the_caller_override_the_tag_handling - fake = FakeDeepL.new + def test_it_omits_the_source_language_when_none_was_given + provider.translate(translation_request(%w[one two], from: nil)) - fake.translate(%w[one], from: :en, to: :ru, tag_handling: :xml) + refute sent.key?("source_lang") + end + + # Regression: notranslate spans reach the provider with their tags, and + # DeepL honours class="notranslate" only under HTML tag handling. Without + # this the protected content is translated while the tags survive, which is + # invisible in review. + def test_it_asks_for_html_tag_handling + provider.translate(translation_request(%w[one two])) - assert_equal :xml, fake.calls.first.last[:tag_handling] + assert_equal "html", sent["tag_handling"] + assert_equal "v2", sent["tag_handling_version"] end - def test_detect_downcases_the_language - assert_equal "en", provider.detect("etwas") + def test_a_caller_option_overrides_a_default + provider.translate(translation_request(%w[one two], tag_handling: "xml", formality: "less")) + + assert_equal "xml", sent["tag_handling"] + assert_equal "less", sent["formality"] end - # DeepL has no detection endpoint, so the provider supplies a target of - # its own rather than making the caller invent one. - def test_detect_supplies_its_own_target_language - fake = FakeDeepL.new + def test_it_returns_the_translations_in_order + response = provider(body: TRANSLATE_BODY).translate(translation_request(%w[one two])) - fake.detect("etwas") + assert_equal %w[один два], response.texts + end + + def test_it_reports_the_detected_source_and_the_billed_characters + response = provider(body: TRANSLATE_BODY).translate(translation_request(%w[one two], from: nil)) - assert_equal [["etwas", nil, "EN", {}]], fake.calls + assert_equal "en", response.detected_source + assert_equal 6, response.usage.billed_characters end - def test_build_sends_a_free_key_to_the_free_host - config = TranslationDiff::Configuration.new - config.deepl_api_key = "abc:fx" + def test_a_short_response_raises_rather_than_shifting_nils_into_the_results + short = { "translations" => [{ "text" => "один" }] } - provider = TranslationDiff::Providers::DeepL.build(config) - host = provider.instance_variable_get(:@api).configuration.host + assert_raises(TranslationDiff::ResponseError) do + provider(body: short).translate(translation_request(%w[one two])) + end + end - assert_equal "https://api-free.deepl.com", host + # DeepL has no detection endpoint, so it detects by translating a sample + # and reading what it says the source was. #detect sends exactly one + # text, so the stub is given exactly one text to echo back. + def test_detect_returns_the_language_deepl_reports + assert_equal "en", provider(texts: %w[x]).detect("something") end - # Regression test for a content and credential leak. deepl-rb logs the - # whole request at DEBUG -- the Authorization header, DeepL auth key and - # all, plus the text being translated -- so forwarding this library's - # `config.logger` into DeepL::Configuration wrote customers' content and - # the API key into the application log the moment anyone turned DEBUG on. - # This gem guarantees its own log lines carry none of that, so the logger - # must not cross into deepl-rb. - def test_build_does_not_forward_the_logger_into_deepl_rb - config = TranslationDiff::Configuration.new - config.deepl_api_key = "abc:fx" - config.logger = Object.new + def test_its_batch_limit_is_deepls_documented_fifty + assert_equal 50, TranslationDiff::Providers::DeepL.capabilities.max_batch_size + end - provider = TranslationDiff::Providers::DeepL.build(config) + def test_it_claims_html_and_notranslate + capabilities = TranslationDiff::Providers::DeepL.capabilities - assert_nil provider.instance_variable_get(:@api).configuration.logger + assert_predicate capabilities, :html? + assert_predicate capabilities, :notranslate? + assert_predicate capabilities, :detects_language? + assert_predicate capabilities, :reports_billing? + end + + private + + def stub_translate(body:, status:, texts:) + recorder = @requests + Faraday::Adapter::Test::Stubs.new do |stub| + stub.post("/v2/translate") do |env| + # Faraday's test adapter reuses this env for the response, mutating + # its body in place once the block returns -- capture a copy now or + # every read after #translate returns sees the reply, not the + # request. + recorder << env.dup + [status, { "Content-Type" => "application/json" }, translate_response(body, texts, env).to_json] + end + end end - def test_build_raises_when_no_key_is_available - original = ENV.fetch("DEEPL_AUTH_KEY", nil) - ENV["DEEPL_AUTH_KEY"] = nil - config = TranslationDiff::Configuration.new + def translate_response(body, texts, env) + return body if body - assert_raises(::DeepL::Exceptions::Error) { TranslationDiff::Providers::DeepL.build(config) } - ensure - ENV["DEEPL_AUTH_KEY"] = original + response_texts = texts || JSON.parse(env.body)["text"] + { "translations" => response_texts.map { |t| { "text" => t, "detected_source_language" => "EN" } } } end end diff --git a/test/translation_diff/providers_test.rb b/test/translation_diff/providers_test.rb index 8679be9..57b36b7 100644 --- a/test/translation_diff/providers_test.rb +++ b/test/translation_diff/providers_test.rb @@ -75,6 +75,7 @@ def test_the_registered_name_becomes_the_cache_key def test_the_built_in_providers_are_registered assert TranslationDiff::Providers.registered?(:null) + assert TranslationDiff::Providers.registered?(:deepl) end def test_null_keeps_its_own_cache_key From e02b682ecf2028143661dfd41ba10fab6f1b0385 Mon Sep 17 00:00:00 2001 From: IG Date: Wed, 9 Sep 2026 18:09:32 +0400 Subject: [PATCH 07/26] refactor!: talk to Google directly instead of through its SDK Ports the Google provider onto HTTPProvider, the same way DeepL was ported in the previous commit, and drops google-cloud-translate-v2 along with its transitive dependencies (googleauth, signet, os, google-protobuf, grpc). Also removes the transitional require-bridge in lib/translation_diff.rb now that both built-in HTTP providers are ported, and adds :google back to the built-in providers assertion. --- Gemfile | 12 +- lib/translation_diff.rb | 19 +- lib/translation_diff/providers/google.rb | 120 ++++----- .../translation_diff/providers/google_test.rb | 239 +++++++++--------- test/translation_diff/providers_test.rb | 1 + 5 files changed, 172 insertions(+), 219 deletions(-) diff --git a/Gemfile b/Gemfile index d5b5cbb..89c72eb 100644 --- a/Gemfile +++ b/Gemfile @@ -20,9 +20,9 @@ gem "redis-namespace", "~> 1.11", require: false # stand-in, has it available. gem "ratelimit", "~> 1.1", require: false -# Not a runtime dependency of the gem (see the gemspec) -- the Google -# provider requires it lazily at build time, so an application using DeepL -# never needs it installed. It is only here so the test suite, which -# exercises that provider's build path against the real -# Google::Cloud::Translate::V2 objects, has it available. -gem "google-cloud-translate-v2", "~> 1.2", require: false +# Not a runtime dependency of the gem (see the gemspec) -- lib/ only ever +# needs CGI.escape, which cgi/escape (in Ruby's default load path) still +# provides. This is here because the Google provider's test decodes the +# query string it sent, and Ruby 4.0 removed CGI.parse from the default +# load path; "install cgi gem" is Ruby's own suggested fix. +gem "cgi", "~> 0.5", require: false diff --git a/lib/translation_diff.rb b/lib/translation_diff.rb index 88a31f9..4c942a5 100644 --- a/lib/translation_diff.rb +++ b/lib/translation_diff.rb @@ -24,24 +24,7 @@ require "translation_diff/providers/null" require "translation_diff/providers/deepl" - -# Google still wraps its vendor SDK directly instead of inheriting Provider -# -- Task 5 ports it. Providers.register now raises for exactly that shape -# of class, which would otherwise take this entire require chain, and -# therefore every caller of this library, down with it before the provider -# is ever used. Until it is ported, `:google` is simply absent from the -# registry; requesting it through TranslationDiff::Providers.build raises -# the ordinary "unknown provider" error instead. The rescue names -# InvalidProviderError specifically, not the generic Error, so it catches -# only "this class is the wrong shape": an option-name collision (also a -# TranslationDiff::Error, raised by ProviderOptionOwners) is a real bug -# rather than an expected transitional state, and must still take the -# require chain down. -begin - require "translation_diff/providers/google" -rescue TranslationDiff::InvalidProviderError - nil -end +require "translation_diff/providers/google" require "translation_diff/segmenters" require "translation_diff/segmenters/simple" diff --git a/lib/translation_diff/providers/google.rb b/lib/translation_diff/providers/google.rb index 58d2b67..34a2225 100644 --- a/lib/translation_diff/providers/google.rb +++ b/lib/translation_diff/providers/google.rb @@ -1,93 +1,65 @@ # frozen_string_literal: true -# Talks to Google Cloud Translation v2 (Basic) through the API object the -# google-cloud-translate-v2 gem builds, rather than its module-level -# shortcuts, so this library never mutates another gem's global state. Two -# behaviours come for free by using their constructor: it reads TRANSLATE_KEY -# and GOOGLE_CLOUD_KEY when no key is given, and it falls back to application -# default credentials when there is no key at all. -# -# google-cloud-translate-v2 is not a dependency of this gem. It is required -# at build time, so an application using a different provider never needs it -# installed. -class TranslationDiff::Providers::Google - # Google's own numbers. The batch limit is a hard one -- "the maximum - # number of strings is 128" -- and a larger request is rejected outright. - # The size limit is the documented recommendation of 5K characters per - # request, well under the hard 100K-byte ceiling. Chunker measures the - # URL-escaped form, which is never smaller than the UTF-8 byte count, so - # staying under this in escaped characters keeps every request under it in - # bytes too. - MAX_REQUEST_SIZE = 5_000 - MAX_BATCH_SIZE = 128 - - # What arrives here is not plain text, despite having been through the - # Tokenizer. A notranslate span is handed over whole, tags included -- - # that is how the tokenizer marks content the provider must leave alone -- - # and HTML entities such as `&` stay in the text it emits. Asking - # Google for `text` makes it translate the protected span and drop the - # markup around it entirely: - # - # "Bold Mountain is a good place." - # format: text -> "Болд Маунтин — хорошее место." - # format: html -> "Bold Mountain — хорошее место." - # - # So `html` it is, which is also what this gem sent for its whole life - # before the provider seam existed. The cost is that Google escapes its - # own output -- a literal apostrophe returns as "'" -- which is - # correct inside the HTML fragment these values usually are, and noise - # inside a value that never had markup in it. A caller translating bare - # strings can pass `format: :text` per call. +# Talks to Cloud Translation v2 (Basic) directly. This used to wrap +# google-cloud-translate-v2, which pulled googleauth, signet, os, +# google-protobuf and grpc in order to send one POST with a key in the query +# string. +class TranslationDiff::Providers::Google < TranslationDiff::HTTPProvider + HOST = "https://translation.googleapis.com" + + # Google's own default, and what the tokenizer's output requires: a + # notranslate span arrives with its tags, and entities such as & stay + # in the text. Asking for `text` makes Google translate the protected span + # and drop its markup -- verified against the live API. DEFAULT_FORMAT = :html - # A bare alphabetic code is downcased, so a configuration written for - # DeepL ("EN") keeps working against Google, whose codes are lowercase. - # Anything else is passed through untouched: "zh-Hans", "zh-CN" and - # "pt-BR" carry script and region subtags whose casing is their own, and a - # blanket downcase would corrupt them. - BARE_LANGUAGE_CODE = /\A[A-Za-z]{2,3}\z/ + # Google's documented limits: 128 strings per request, and a recommended + # 5,000 characters (the hard ceiling is 100 KB). Chunker measures the + # URL-escaped form, never smaller than the UTF-8 byte count, so a chunk + # inside 5,000 escaped characters is inside it in bytes too. + def self.capabilities + TranslationDiff::Capabilities.new( + max_request_size: 5_000, max_batch_size: 128, max_text_size: nil, + html: :format, notranslate: true, detects_language: true, reports_billing: false + ) + end - def self.configuration_options = %i[google_api_key google_project_id] + def self.configuration_options = %i[google_api_key google_project_id google_api_base] + def self.configuration_requirements = %i[google_api_key] - # `config.logger` is deliberately not forwarded: the gem takes no logger, - # and this library's own guarantee -- that no line it writes holds source - # text, translated text or a credential -- is easiest to keep by never - # handing the logger to a gem that has not made the same promise. - def self.build(config) - require "google/cloud/translate/v2" + # A bare alphabetic code is downcased, so a configuration written for DeepL + # ("EN") keeps working. Anything carrying a subtag ("zh-Hans", "pt-BR") is + # passed through untouched: the casing of a script or region subtag is its + # own, and a blanket downcase would corrupt it. + BARE_LANGUAGE_CODE = /\A[A-Za-z]{2,3}\z/ - settings = { key: config.google_api_key, project_id: config.google_project_id }.compact - new(::Google::Cloud::Translate::V2.new(**settings)) - rescue LoadError - raise TranslationDiff::Error, - "provider is :google but the `google-cloud-translate-v2` gem is not available. " \ - 'Add `gem "google-cloud-translate-v2"` to your Gemfile.' - end + def api_base = config.google_api_base || HOST + def translate_url = "language/translate/v2?key=#{CGI.escape(config.google_api_key.to_s)}" + def detect_url = "language/translate/v2/detect?key=#{CGI.escape(config.google_api_key.to_s)}" - def initialize(api) - @api = api + def render_translate_payload(request) + { format: DEFAULT_FORMAT } + .merge(request.options) + .merge(q: request.texts, target: language(request.to)) + .tap { |payload| payload[:source] = language(request.from) unless request.from.nil? } end - def translate(texts, from:, to:, **options) - settings = { from: language(from), to: language(to), format: DEFAULT_FORMAT }.merge(options) - - results = @api.translate(*texts, **settings) - # One text yields a bare Translation, not a one-element array. `Array()` - # is not usable to even that out: Translation would have to be trusted - # never to define #to_a or #to_ary, and if it ever did, `Array()` would - # quietly splat one translation into several strings instead of raising. - results = [results] unless results.is_a?(Array) + def parse_translate_response(body, _headers, request) + translations = Array(body.dig("data", "translations")) - results.map(&:text) + TranslationDiff::Translation::Response.build( + request: request, + texts: translations.map { |t| t["translatedText"] }, + detected_source: translations.first&.dig("detectedSourceLanguage")&.downcase, + usage: TranslationDiff::Translation::Usage.new(characters: request.texts.sum(&:size)) + ) end def detect(text) - @api.detect(text).language + response = post(detect_url, { q: [text] }) + response.body.dig("data", "detections", 0, 0, "language")&.downcase end - def max_request_size = MAX_REQUEST_SIZE - def max_batch_size = MAX_BATCH_SIZE - private def language(value) diff --git a/test/translation_diff/providers/google_test.rb b/test/translation_diff/providers/google_test.rb index 5af5640..d142b9c 100644 --- a/test/translation_diff/providers/google_test.rb +++ b/test/translation_diff/providers/google_test.rb @@ -2,177 +2,174 @@ require "test_helper" require "support/provider_contract" - -# `TranslationDiff::Providers::Google.build` only requires the gem lazily, at -# call time, so whether ::Google::Cloud::Translate::V2 is already defined when -# this file runs depends on test order -- Minitest randomises it. Requiring it -# explicitly here means this file's constant references don't depend on some -# other test file having required it first. -require "google/cloud/translate/v2" +require "support/http_provider_contract" +require "faraday" +require "cgi" class GoogleProviderTest < Minitest::Test include ProviderContract + include HTTPProviderContract - # Stands in for Google::Cloud::Translate::V2::Api. The single/array return - # asymmetry is copied deliberately from the real thing: Translation - # .from_gapi_list and Detection.from_gapi both return a bare object rather - # than a one-element array when they were given one text, and a provider - # that forgets that hands Request a Translation where it expects an Array. - class FakeApi - Translation = Struct.new(:text) - Detection = Struct.new(:language) + # A real response envelope, shaped from the Cloud Translation v2 REST + # reference read 2026-09-09: translations live under a nested "data" key, + # not at the top level the way DeepL's do. + TRANSLATE_BODY = { + "data" => { "translations" => [ + { "translatedText" => "один", "detectedSourceLanguage" => "en" }, + { "translatedText" => "два", "detectedSourceLanguage" => "en" } + ] } + }.freeze - attr_reader :calls + attr_reader :config, :requests - def initialize - @calls = [] - end + def setup + TranslationDiff.reset! + @config = TranslationDiff::Configuration.new + @config.google_api_key = "test-key" + @requests = [] + end - def translate(*text, **options) - @calls << [text, options] - unwrap(text.map { |value| Translation.new("#{value}-translated") }) - end + # Builds a provider whose connection answers from a stub and records what + # was sent, so a test can assert on the payload as well as the parse. + # + # When neither `body:` nor `texts:` is given, the stub echoes back + # whatever texts were actually sent (rather than a fixed pair), so the + # shared ProviderContract tests -- which call `provider` with no + # knowledge of how many texts they are about to send -- get a response + # the same size as their request instead of tripping Response.build's + # count check. + def provider(body: nil, status: 200, texts: nil) + stubs = stub_translate(body: body, status: status, texts: texts) + built = TranslationDiff::Providers::Google.new(config) + built.name = :google + built.instance_variable_set(:@connection, built.send(:build_connection) do |faraday| + faraday.adapter :test, stubs + end) + built + end - def detect(*text) - @calls << [text, {}] - unwrap(text.map { Detection.new("en") }) - end + def sent = JSON.parse(requests.first.body) + def query = CGI.parse(requests.first.url.query.to_s) - private + def test_the_key_travels_in_the_query_string + provider.translate(translation_request(%w[one])) - def unwrap(results) = results.size == 1 ? results.first : results + assert_equal ["test-key"], query["key"] end - def provider = TranslationDiff::Providers::Google.new(FakeApi.new) + def test_it_sends_the_texts_and_the_language_pair + provider.translate(translation_request(%w[one two])) - def test_translate_unwraps_the_text_of_each_result - assert_equal %w[one-translated two-translated], - provider.translate(%w[one two], from: :en, to: :ru) + assert_equal %w[one two], sent["q"] + assert_equal "en", sent["source"] + assert_equal "ru", sent["target"] end - # The API hands back a bare Translation, not a one-element array, when it - # was given one text. Request counts the results against the values it - # sent, so a provider that passes that through fails the count check. - def test_translate_returns_an_array_for_a_single_text - assert_equal %w[one-translated], provider.translate(%w[one], from: :en, to: :ru) - end - - # What reaches a provider is not plain text: Tokenizer hands over a - # notranslate span with its tags intact, and leaves entities such as - # `&` in the text it emits. Asking for plain text makes Google - # translate the protected span and drop its markup -- verified against - # the live API. `html` is what this gem's tokenizer contract requires, - # and what it has always sent. - def test_translate_asks_for_html - api = FakeApi.new - - TranslationDiff::Providers::Google.new(api).translate(%w[one], from: :en, to: :ru) + def test_it_asks_for_html + provider.translate(translation_request(%w[one])) - assert_equal :html, api.calls.first.last[:format] + assert_equal "html", sent["format"] end - def test_translate_lets_the_caller_override_the_format - api = FakeApi.new + def test_a_caller_may_ask_for_plain_text + provider.translate(translation_request(%w[one], format: :text)) - TranslationDiff::Providers::Google.new(api).translate(%w[one], from: :en, to: :ru, format: :text) - - assert_equal :text, api.calls.first.last[:format] + assert_equal "text", sent["format"] end - # A configuration written against DeepL says "EN"; Google's codes are - # lowercase. - def test_translate_downcases_bare_language_codes - api = FakeApi.new - - TranslationDiff::Providers::Google.new(api).translate(%w[one], from: "EN", to: "RU") + # Google's codes are lower case and a config written for DeepL says "EN"; + # but "zh-Hans" and "pt-BR" carry subtags whose casing is their own. + def test_it_downcases_bare_codes_and_leaves_subtagged_ones_alone + provider.translate(translation_request(%w[one], from: "EN", to: "zh-Hans")) - assert_equal({ from: "en", to: "ru" }, api.calls.first.last.slice(:from, :to)) + assert_equal "en", sent["source"] + assert_equal "zh-Hans", sent["target"] end - # "zh-Hans", "zh-CN" and "pt-BR" carry subtags whose casing is their own; - # a blanket downcase would corrupt them. - def test_translate_passes_subtagged_codes_through_untouched - api = FakeApi.new + def test_it_omits_the_source_language_when_none_was_given + provider.translate(translation_request(%w[one], from: nil)) - TranslationDiff::Providers::Google.new(api).translate(%w[one], from: "en", to: "zh-Hans") - - assert_equal "zh-Hans", api.calls.first.last[:to] + refute sent.key?("source") end - # No source language means "detect it", which the API does when `source` - # is absent. Sending "" instead would be rejected. - def test_translate_omits_the_source_language_when_none_is_given - api = FakeApi.new - - TranslationDiff::Providers::Google.new(api).translate(%w[one], from: nil, to: :ru) + def test_it_parses_the_nested_data_envelope + body = { "data" => { "translations" => [ + { "translatedText" => "один", "detectedSourceLanguage" => "en" } + ] } } + response = provider(body: body).translate(translation_request(%w[one], from: nil)) - assert_nil api.calls.first.last[:from] + assert_equal %w[один], response.texts + assert_equal "en", response.detected_source end - def test_translate_passes_provider_options_through - api = FakeApi.new - - TranslationDiff::Providers::Google.new(api).translate(%w[one], from: :en, to: :ru, model: "nmt") + def test_it_returns_the_translations_in_order + response = provider(body: TRANSLATE_BODY).translate(translation_request(%w[one two])) - assert_equal "nmt", api.calls.first.last[:model] + assert_equal %w[один два], response.texts end - def test_translate_sends_every_text_in_one_call - api = FakeApi.new + def test_a_short_response_raises_rather_than_shifting_nils_into_the_results + short = { "data" => { "translations" => [{ "translatedText" => "один" }] } } - TranslationDiff::Providers::Google.new(api).translate(%w[one two three], from: :en, to: :ru) - - assert_equal 1, api.calls.size - assert_equal %w[one two three], api.calls.first.first + assert_raises(TranslationDiff::ResponseError) do + provider(body: short).translate(translation_request(%w[one two])) + end end - def test_detect_returns_the_language - assert_equal "en", provider.detect("etwas") - end + def test_the_api_base_option_overrides_the_default + config.google_api_base = "https://google.internal" - # Both numbers are Google's own, and both are load-bearing: Chunker uses - # them to decide where to split, and a batch over 128 is rejected outright. - def test_the_limits_are_the_documented_ones - assert_equal 128, provider.max_batch_size - assert_equal 5_000, provider.max_request_size + assert_equal "https://google.internal", TranslationDiff::Providers::Google.new(config).api_base end - def test_build_passes_the_configured_key_to_the_api - config = TranslationDiff::Configuration.new - config.google_api_key = "abc" + def test_a_missing_key_is_named_before_any_request + config.google_api_key = nil - provider = TranslationDiff::Providers::Google.build(config) + error = assert_raises(TranslationDiff::ConfigurationError) do + TranslationDiff::Providers::Google.new(config) + end - assert_equal "abc", provider.instance_variable_get(:@api).service.key + assert_match(/google_api_key/, error.message) end - def test_build_passes_the_configured_project_id_to_the_api - config = TranslationDiff::Configuration.new - config.google_api_key = "abc" - config.google_project_id = "a-project" + def test_its_batch_limit_is_googles_documented_one_hundred_twenty_eight + assert_equal 128, TranslationDiff::Providers::Google.capabilities.max_batch_size + end - provider = TranslationDiff::Providers::Google.build(config) + def test_it_claims_html_and_notranslate + capabilities = TranslationDiff::Providers::Google.capabilities - assert_equal "a-project", provider.instance_variable_get(:@api).service.project_id + assert_predicate capabilities, :html? + assert_predicate capabilities, :notranslate? + assert_predicate capabilities, :detects_language? end - # Without a key the gem falls through to application default credentials, - # which need a project id it cannot find in a test environment. - def test_build_raises_when_no_key_is_available - original = ENV.to_hash.slice("TRANSLATE_KEY", "GOOGLE_CLOUD_KEY", "TRANSLATE_PROJECT") - original.each_key { |key| ENV[key] = nil } - config = TranslationDiff::Configuration.new + def test_google_reports_no_billing + refute_predicate TranslationDiff::Providers::Google.capabilities, :reports_billing? + end - assert_raises(StandardError) { TranslationDiff::Providers::Google.build(config) } - ensure - original&.each { |key, value| ENV[key] = value } + private + + def stub_translate(body:, status:, texts:) + recorder = @requests + Faraday::Adapter::Test::Stubs.new do |stub| + stub.post("/language/translate/v2") do |env| + # Faraday's test adapter reuses this env for the response, mutating + # its body in place once the block returns -- capture a copy now or + # every read after #translate returns sees the reply, not the + # request. + recorder << env.dup + [status, { "Content-Type" => "application/json; charset=UTF-8" }, + translate_response(body, texts, env).to_json] + end + end end - def test_it_is_registered_under_its_own_name - config = TranslationDiff::Configuration.new - config.google_api_key = "abc" + def translate_response(body, texts, env) + return body if body - assert TranslationDiff::Providers.registered?(:google) - assert_equal "google", TranslationDiff::Providers.build(:google, config).cache_key + response_texts = texts || JSON.parse(env.body)["q"] + translations = response_texts.map { |t| { "translatedText" => t, "detectedSourceLanguage" => "en" } } + { "data" => { "translations" => translations } } end end diff --git a/test/translation_diff/providers_test.rb b/test/translation_diff/providers_test.rb index 57b36b7..4f8a6ec 100644 --- a/test/translation_diff/providers_test.rb +++ b/test/translation_diff/providers_test.rb @@ -76,6 +76,7 @@ def test_the_registered_name_becomes_the_cache_key def test_the_built_in_providers_are_registered assert TranslationDiff::Providers.registered?(:null) assert TranslationDiff::Providers.registered?(:deepl) + assert TranslationDiff::Providers.registered?(:google) end def test_null_keeps_its_own_cache_key From 56fea2371bd380d32ea5958231e317dca2bdf2d1 Mon Sep 17 00:00:00 2001 From: IG Date: Wed, 9 Sep 2026 18:19:35 +0400 Subject: [PATCH 08/26] fix: capture the request env before Faraday's test adapter mutates it Faraday's test adapter reuses one Env for both the request and the response, overwriting its body once the stub block returns -- so StubbedProvider was recording the reply, not the request, for every assertion made after #translate returned. DeepL and Google's tests already worked around this locally with `env.dup`; fix it here so Task 6 and beyond can use the shared helper as-is. Also let `body:` be a callable handed the request's env, so a provider's own `provider` test helper can echo back a response sized to match whatever texts a particular test sent, the way DeepL's and Google's local helpers already did. --- test/support/stubbed_provider.rb | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/test/support/stubbed_provider.rb b/test/support/stubbed_provider.rb index 7b1b537..ce25778 100644 --- a/test/support/stubbed_provider.rb +++ b/test/support/stubbed_provider.rb @@ -5,6 +5,13 @@ # Builds a provider whose Faraday connection answers from a stub and records # what was sent, so a test can assert on the payload as well as on the parse. # Including this requires the test to define #config and #provider_class. +# +# `body:` is either a fixed response (a Hash, an Array, or a String) or a +# callable that is handed the request's Faraday env and returns one -- the +# latter is how a provider's own `provider` helper can echo back a response +# shaped to match however many texts a particular test happened to send, +# which the shared ProviderContract tests need and a fixed body cannot give +# them. module StubbedProvider def requests = @requests ||= [] @@ -26,9 +33,14 @@ def build_stubs(route:, body:, status:, headers:) recorder = requests Faraday::Adapter::Test::Stubs.new do |stub| stub.post(route) do |env| - recorder << env + # Faraday's test adapter reuses this env for the response, mutating + # its body in place once the block returns -- capture a copy now or + # every read after #translate returns sees the reply, not the + # request. + recorder << env.dup + rendered = body.respond_to?(:call) ? body.call(env) : body [status, { "Content-Type" => "application/json" }.merge(headers), - body.is_a?(String) ? body : body.to_json] + rendered.is_a?(String) ? rendered : rendered.to_json] end end end From eabdc6cb6349c4cbd0eb4f56fc21cdf90f56abe2 Mon Sep 17 00:00:00 2001 From: IG Date: Wed, 9 Sep 2026 18:19:38 +0400 Subject: [PATCH 09/26] fix: describe what the option-collision test now guards The comment and test name still referenced "the transitional bridges in lib/translation_diff.rb", which are gone. What the test guards is unchanged: InvalidProviderError is specific to a provider of the wrong shape, so a caller rescuing that cannot also swallow an unrelated option-name collision, which stays a generic Error. --- test/translation_diff/providers_test.rb | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/translation_diff/providers_test.rb b/test/translation_diff/providers_test.rb index 4f8a6ec..162210a 100644 --- a/test/translation_diff/providers_test.rb +++ b/test/translation_diff/providers_test.rb @@ -189,11 +189,13 @@ def self.build(_config) = new refute TranslationDiff::Providers.registered?(:impostor) end - # The transitional bridges in lib/translation_diff.rb rescue - # InvalidProviderError so an unported provider cannot make the library - # unloadable. An option-name collision is a genuine bug and must stay - # outside that net. - def test_an_option_collision_does_not_raise_the_class_the_require_bridges_rescue + # InvalidProviderError is specific to a provider of the wrong shape (see + # its definition in errors.rb): a caller rescuing "this class cannot be a + # provider" must not also, by accident, swallow an unrelated failure. An + # option-name collision is that unrelated failure -- two well-shaped + # providers fighting over one option name -- so it must raise the generic + # TranslationDiff::Error, not the specific one. + def test_an_option_collision_raises_the_generic_error_not_the_invalid_provider_one TranslationDiff::Providers.register(:collision_a, ConflictingProviderA) error = assert_raises(TranslationDiff::Error) do From d58c7a9b2cc65b8e91f4c2cc0dc76979ee154d28 Mon Sep 17 00:00:00 2001 From: IG Date: Wed, 9 Sep 2026 18:19:42 +0400 Subject: [PATCH 10/26] feat: add an Azure Translator provider Azure AI Translator v3, the third HTTP provider on the shared transport. It differs from DeepL and Google in one structural way: the language pair and api-version/textType travel in the query string while the texts travel in the body, so #translate is overridden to build the URL per request. Billed characters come from the X-metered-usage response header. Limits (1000 strings, 50,000 characters per request and per string) are Azure's documented ones, the largest of the three providers here. --- lib/translation_diff.rb | 1 + lib/translation_diff/providers/azure.rb | 77 ++++++++ test/translation_diff/providers/azure_test.rb | 173 ++++++++++++++++++ 3 files changed, 251 insertions(+) create mode 100644 lib/translation_diff/providers/azure.rb create mode 100644 test/translation_diff/providers/azure_test.rb diff --git a/lib/translation_diff.rb b/lib/translation_diff.rb index 4c942a5..82963bb 100644 --- a/lib/translation_diff.rb +++ b/lib/translation_diff.rb @@ -25,6 +25,7 @@ require "translation_diff/providers/deepl" require "translation_diff/providers/google" +require "translation_diff/providers/azure" require "translation_diff/segmenters" require "translation_diff/segmenters/simple" diff --git a/lib/translation_diff/providers/azure.rb b/lib/translation_diff/providers/azure.rb new file mode 100644 index 0000000..ecc6f3a --- /dev/null +++ b/lib/translation_diff/providers/azure.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +# Azure AI Translator, REST v3.0. The cheapest of the paid services per +# character and the most generous per request: a thousand strings and fifty +# thousand characters at a time. +class TranslationDiff::Providers::Azure < TranslationDiff::HTTPProvider + HOST = "https://api.cognitive.microsofttranslator.com" + API_VERSION = "3.0" + + # Azure spells HTML handling `textType`, and under it honours + # `class=notranslate` -- the same marker the tokenizer emits and the same + # one Google and DeepL honour under their own spellings. + DEFAULT_TEXT_TYPE = "html" + + def self.capabilities + TranslationDiff::Capabilities.new( + max_request_size: 50_000, max_batch_size: 1_000, max_text_size: 50_000, + html: :textType, notranslate: true, detects_language: true, reports_billing: true + ) + end + + def self.configuration_options = %i[azure_api_key azure_region azure_api_base] + def self.configuration_requirements = %i[azure_api_key] + + def api_base = config.azure_api_base || HOST + + # A multi-service resource needs the region header and a single-service one + # rejects nothing without it, so it is sent only when configured. + def headers + { "Ocp-Apim-Subscription-Key" => config.azure_api_key.to_s } + .tap { |h| h["Ocp-Apim-Subscription-Region"] = config.azure_region if config.azure_region } + end + + # Azure takes the language pair in the query string and the texts in the + # body, which is why this provider builds its URL per request rather than + # answering a constant. + def translate_url = "translate" + + def translate(request) + response = post(url_for(request), render_translate_payload(request)) + parse_translate_response(response.body, response.headers, request) + end + + def render_translate_payload(request) = request.texts.map { |text| { Text: text } } + + def parse_translate_response(body, headers, request) + results = Array(body) + + TranslationDiff::Translation::Response.build( + request: request, + texts: results.map { |result| result.dig("translations", 0, "text") }, + detected_source: results.first&.dig("detectedLanguage", "language")&.downcase, + usage: TranslationDiff::Translation::Usage.new( + characters: request.texts.sum(&:size), + billed_characters: headers["x-metered-usage"]&.to_i + ) + ) + end + + def detect(text) + response = post("detect?api-version=#{API_VERSION}", [{ Text: text }]) + response.body.dig(0, "language")&.downcase + end + + private + + def url_for(request) + params = { "api-version" => API_VERSION, "to" => request.to.to_s, + "textType" => DEFAULT_TEXT_TYPE } + params["from"] = request.from.to_s unless request.from.nil? + params.merge!(request.options.transform_keys(&:to_s)) + + "#{translate_url}?#{URI.encode_www_form(params)}" + end +end + +TranslationDiff::Providers.register(:azure, TranslationDiff::Providers::Azure) diff --git a/test/translation_diff/providers/azure_test.rb b/test/translation_diff/providers/azure_test.rb new file mode 100644 index 0000000..0e39c5f --- /dev/null +++ b/test/translation_diff/providers/azure_test.rb @@ -0,0 +1,173 @@ +# frozen_string_literal: true + +require "test_helper" +require "support/provider_contract" +require "support/http_provider_contract" +require "support/stubbed_provider" +require "faraday" +require "cgi" + +class AzureProviderTest < Minitest::Test + include ProviderContract + include HTTPProviderContract + include StubbedProvider + + # There is no Azure key available for this task, so unlike DeepL's and + # Google's fixtures -- both captured from a live call -- this body is + # shaped from Microsoft's own Azure AI Translator v3 "Translate" reference + # documentation (read 2026-09-09), not from an observed response. Nobody + # should mistake it for one. + BODY = [ + { "detectedLanguage" => { "language" => "en", "score" => 1.0 }, + "translations" => [{ "text" => "один", "to" => "ru" }] }, + { "detectedLanguage" => { "language" => "en", "score" => 1.0 }, + "translations" => [{ "text" => "два", "to" => "ru" }] } + ].freeze + + attr_reader :config + + def setup + TranslationDiff.reset! + @config = TranslationDiff::Configuration.new + @config.azure_api_key = "test-key" + end + + def provider_class = TranslationDiff::Providers::Azure + + # When `body:` is left nil, the stub echoes back whatever texts were + # actually sent (rather than a fixed pair), so the shared ProviderContract + # tests -- which call `provider` with no knowledge of how many texts they + # are about to send -- get a response the same size as their request + # instead of tripping Response.build's count check. + def provider(body: nil, status: 200, headers: {}) + stub_provider(route: "/translate", body: body || method(:echo_translations), + status: status, headers: headers, name: :azure) + end + + def test_the_default_api_base_is_the_documented_host + assert_equal "https://api.cognitive.microsofttranslator.com", + TranslationDiff::Providers::Azure.new(config).api_base + end + + def test_the_api_base_option_overrides_the_default + config.azure_api_base = "https://azure.internal" + + assert_equal "https://azure.internal", TranslationDiff::Providers::Azure.new(config).api_base + end + + def test_it_sends_the_key_in_the_documented_header + assert_equal "test-key", + TranslationDiff::Providers::Azure.new(config).headers["Ocp-Apim-Subscription-Key"] + end + + # A single-service key needs no region and a multi-service one does, so the + # header appears only when the option is set. + def test_the_region_header_appears_only_when_configured + refute TranslationDiff::Providers::Azure.new(config).headers.key?("Ocp-Apim-Subscription-Region") + + config.azure_region = "westeurope" + + assert_equal "westeurope", + TranslationDiff::Providers::Azure.new(config).headers["Ocp-Apim-Subscription-Region"] + end + + def test_a_missing_key_is_named_before_any_request + config.azure_api_key = nil + + error = assert_raises(TranslationDiff::ConfigurationError) do + TranslationDiff::Providers::Azure.new(config) + end + + assert_match(/azure_api_key/, error.message) + end + + def test_the_languages_travel_in_the_query_string_not_the_body + provider.translate(translation_request(%w[one two])) + + assert_equal ["3.0"], query["api-version"] + assert_equal ["en"], query["from"] + assert_equal ["ru"], query["to"] + end + + def test_it_omits_from_when_none_was_given + provider.translate(translation_request(%w[one], from: nil)) + + refute query.key?("from") + end + + # The body is an array of objects with a capital-T Text key. + def test_it_wraps_each_text_in_the_documented_object + provider.translate(translation_request(%w[one two])) + + assert_equal [{ "Text" => "one" }, { "Text" => "two" }], sent + end + + def test_it_asks_for_html + provider.translate(translation_request(%w[one])) + + assert_equal ["html"], query["textType"] + end + + def test_it_flattens_one_translation_per_input + response = provider(body: BODY).translate(translation_request(%w[one two])) + + assert_equal %w[один два], response.texts + assert_equal "en", response.detected_source + end + + def test_it_reads_the_billed_characters_from_the_metered_usage_header + response = provider(body: BODY, headers: { "X-metered-usage" => "6" }) + .translate(translation_request(%w[one two])) + + assert_equal 6, response.usage.billed_characters + end + + # Absent, the header must yield nil rather than 0 -- 0 is a false claim + # about billing, not "unknown". + def test_billed_characters_is_nil_when_the_header_is_absent + response = provider(body: BODY).translate(translation_request(%w[one two])) + + assert_nil response.usage.billed_characters + end + + def test_a_short_response_raises_rather_than_shifting_nils_into_the_results + short = [BODY.first] + + assert_raises(TranslationDiff::ResponseError) do + provider(body: short).translate(translation_request(%w[one two])) + end + end + + def test_detect_returns_the_language_azure_reports + detector = stub_provider(route: "/detect", body: [{ "language" => "en", "score" => 1.0 }], + name: :azure) + + assert_equal "en", detector.detect("something") + end + + def test_its_limits_are_azures_documented_ones + capabilities = TranslationDiff::Providers::Azure.capabilities + + assert_equal 1_000, capabilities.max_batch_size + assert_equal 50_000, capabilities.max_request_size + assert_equal 50_000, capabilities.max_text_size + end + + def test_it_claims_html_and_notranslate + capabilities = TranslationDiff::Providers::Azure.capabilities + + assert_predicate capabilities, :html? + assert_predicate capabilities, :notranslate? + assert_predicate capabilities, :detects_language? + assert_predicate capabilities, :reports_billing? + end + + private + + def echo_translations(env) + JSON.parse(env.body).map do |item| + { "detectedLanguage" => { "language" => "en", "score" => 1.0 }, + "translations" => [{ "text" => item["Text"], "to" => "ru" }] } + end + end +end From 0244b7d93039f9a5d90792cf1f112cb66f19df10 Mon Sep 17 00:00:00 2001 From: IG Date: Wed, 9 Sep 2026 18:29:38 +0400 Subject: [PATCH 11/26] feat: add a ModernMT provider No ModernMT trial key was available in this environment, so the notranslate capability could not be probed against the live API and stays false with an "unverified, safe default" comment rather than a verified observation. RuboCop's Metrics cops rejected the brief's parse_translate_response verbatim (AbcSize, CyclomaticComplexity, MethodLength); extracted results_from/usage_for helpers to bring it under the limits without changing behaviour. Also renamed the "documented 128" test method to avoid Naming/VariableNumber. --- lib/translation_diff.rb | 1 + lib/translation_diff/providers/modernmt.rb | 77 ++++++++++++++++ .../providers/modernmt_test.rb | 89 +++++++++++++++++++ 3 files changed, 167 insertions(+) create mode 100644 lib/translation_diff/providers/modernmt.rb create mode 100644 test/translation_diff/providers/modernmt_test.rb diff --git a/lib/translation_diff.rb b/lib/translation_diff.rb index 82963bb..23fd27d 100644 --- a/lib/translation_diff.rb +++ b/lib/translation_diff.rb @@ -26,6 +26,7 @@ require "translation_diff/providers/deepl" require "translation_diff/providers/google" require "translation_diff/providers/azure" +require "translation_diff/providers/modernmt" require "translation_diff/segmenters" require "translation_diff/segmenters/simple" diff --git a/lib/translation_diff/providers/modernmt.rb b/lib/translation_diff/providers/modernmt.rb new file mode 100644 index 0000000..2c5d055 --- /dev/null +++ b/lib/translation_diff/providers/modernmt.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +# ModernMT. Adaptive translation with translation memories, which is +# thematically the closest of these services to what this library does. +class TranslationDiff::Providers::ModernMT < TranslationDiff::HTTPProvider + HOST = "https://api.modernmt.com" + + # ModernMT spells its formats as MIME types. + DEFAULT_FORMAT = "text/html" + + # Unverified. ModernMT documents an HTML format but says nothing about + # class="notranslate", and no key was available to probe it. Declaring + # false is the safe direction: a capability that under-promises costs a + # warning, one that over-promises costs a customer's protected content. + MODERNMT_HONOURS_NOTRANSLATE = false + + # 128 texts is documented. The per-request character limit is not, so the + # conservative 5,000 Google recommends is used rather than a number nobody + # published. + def self.capabilities + TranslationDiff::Capabilities.new( + max_request_size: 5_000, max_batch_size: 128, max_text_size: nil, + html: :format, notranslate: MODERNMT_HONOURS_NOTRANSLATE, + detects_language: true, reports_billing: true + ) + end + + def self.configuration_options = %i[modernmt_api_key modernmt_api_base] + def self.configuration_requirements = %i[modernmt_api_key] + + def api_base = config.modernmt_api_base || HOST + def headers = { "MMT-ApiKey" => config.modernmt_api_key.to_s } + def translate_url = "translate" + + def render_translate_payload(request) + { format: DEFAULT_FORMAT } + .merge(request.options) + .merge(q: request.texts, target: request.to.to_s) + .tap { |payload| payload[:source] = request.from.to_s unless request.from.nil? } + end + + # One text comes back as an object rather than a one-element array, so the + # envelope is always coerced to a list before it is mapped. + def parse_translate_response(body, _headers, request) + results = results_from(body) + + TranslationDiff::Translation::Response.build( + request: request, + texts: results.map { |r| r["translation"] }, + detected_source: results.first&.dig("detectedLanguage")&.downcase, + usage: usage_for(request, results) + ) + end + + def detect(text) + request = TranslationDiff::Translation::Request.new(texts: [text], from: nil, to: "en") + translate(request).detected_source + end + + private + + def results_from(body) + data = body["data"] + data.is_a?(Array) ? data : [data].compact + end + + def usage_for(request, results) + billed = results.filter_map { |r| r["billedCharacters"] }.sum + + TranslationDiff::Translation::Usage.new( + characters: request.texts.sum(&:size), + billed_characters: billed.positive? ? billed : nil + ) + end +end + +TranslationDiff::Providers.register(:modernmt, TranslationDiff::Providers::ModernMT) diff --git a/test/translation_diff/providers/modernmt_test.rb b/test/translation_diff/providers/modernmt_test.rb new file mode 100644 index 0000000..62f6447 --- /dev/null +++ b/test/translation_diff/providers/modernmt_test.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +require "test_helper" +require "support/provider_contract" +require "support/http_provider_contract" +require "support/stubbed_provider" +require "faraday" + +class ModernMTProviderTest < Minitest::Test + include ProviderContract + include HTTPProviderContract + include StubbedProvider + + # Shaped from modernmt.com/api's own "Translate" reference (read + # 2026-09-09), not from a live call: no ModernMT key was available for this + # task. + BODY = { "data" => [ + { "translation" => "один", "billedCharacters" => 3, "characters" => 3, "detectedLanguage" => "en" }, + { "translation" => "два", "billedCharacters" => 3, "characters" => 3, "detectedLanguage" => "en" } + ] }.freeze + + attr_reader :config + + def setup + TranslationDiff.reset! + @config = TranslationDiff::Configuration.new + @config.modernmt_api_key = "test-key" + end + + def provider_class = TranslationDiff::Providers::ModernMT + + # When `body:` is left nil, the stub echoes back whatever texts were + # actually sent, so the shared ProviderContract tests -- which call + # `provider` with no knowledge of how many texts they are about to send -- + # get a response the same size as their request instead of tripping + # Response.build's count check. + def provider(body: nil, status: 200, headers: {}) + stub_provider(route: "/translate", body: body || method(:echo_translations), + status: status, headers: headers, name: :modernmt) + end + + def test_it_sends_the_key_in_the_documented_header + assert_equal "test-key", + TranslationDiff::Providers::ModernMT.new(config).headers["MMT-ApiKey"] + end + + def test_it_sends_the_texts_and_the_language_pair_in_the_body + provider.translate(translation_request(%w[one two])) + + assert_equal %w[one two], sent["q"] + assert_equal "en", sent["source"] + assert_equal "ru", sent["target"] + end + + def test_it_asks_for_html_by_mime_type + provider.translate(translation_request(%w[one])) + + assert_equal "text/html", sent["format"] + end + + def test_it_unwraps_the_data_envelope + response = provider(body: BODY).translate(translation_request(%w[one two])) + + assert_equal %w[один два], response.texts + assert_equal "en", response.detected_source + assert_equal 6, response.usage.billed_characters + end + + # One text comes back as an object, not a one-element array. A provider + # that passes that through hands the pipeline a Hash where it expects a + # list, and the count check is what catches it. + def test_a_single_text_comes_back_unwrapped_and_is_still_a_list + single = { "data" => { "translation" => "один", "detectedLanguage" => "en" } } + response = provider(body: single).translate(translation_request(%w[one])) + + assert_equal %w[один], response.texts + end + + def test_its_batch_limit_is_the_documented_maximum + assert_equal 128, TranslationDiff::Providers::ModernMT.capabilities.max_batch_size + end + + private + + def echo_translations(env) + texts = JSON.parse(env.body)["q"] + { "data" => texts.map { |text| { "translation" => text, "detectedLanguage" => "en" } } } + end +end From aa27d96cb3a2fbbdc02b89b35ca6e7a86b4151a1 Mon Sep 17 00:00:00 2001 From: IG Date: Wed, 9 Sep 2026 18:31:50 +0400 Subject: [PATCH 12/26] feat: add a LibreTranslate provider MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Docker was available, so the notranslate capability was probed for real rather than left an unverified guess: against libretranslate/libretranslate --load-only en,ru, Bold Mountain is a good place. came back with the span tag intact but "Bold Mountain" translated to "Смелая гора" anyway. LibreTranslate's HTML format preserves markup; it does not honour the notranslate marker, so the constant is a documented false rather than a safe-default false. --- lib/translation_diff.rb | 1 + .../providers/libretranslate.rb | 72 +++++++++++ .../providers/libretranslate_test.rb | 114 ++++++++++++++++++ 3 files changed, 187 insertions(+) create mode 100644 lib/translation_diff/providers/libretranslate.rb create mode 100644 test/translation_diff/providers/libretranslate_test.rb diff --git a/lib/translation_diff.rb b/lib/translation_diff.rb index 23fd27d..607236f 100644 --- a/lib/translation_diff.rb +++ b/lib/translation_diff.rb @@ -27,6 +27,7 @@ require "translation_diff/providers/google" require "translation_diff/providers/azure" require "translation_diff/providers/modernmt" +require "translation_diff/providers/libretranslate" require "translation_diff/segmenters" require "translation_diff/segmenters/simple" diff --git a/lib/translation_diff/providers/libretranslate.rb b/lib/translation_diff/providers/libretranslate.rb new file mode 100644 index 0000000..2b03b2c --- /dev/null +++ b/lib/translation_diff/providers/libretranslate.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +# LibreTranslate: open source, self-hosted, and the only provider here that +# can be run against for free, which is why it is worth supporting even +# though its translations are not the best of this set. +# +# It inverts the usual configuration: the base URL is required, because +# everyone runs their own instance, and the API key is optional, because most +# instances do not ask for one. +class TranslationDiff::Providers::LibreTranslate < TranslationDiff::HTTPProvider + DEFAULT_FORMAT = "html" + + # The API's own way of asking for detection: `source` is required and + # "auto" is the value that means "work it out". + AUTO = "auto" + + # Observed, not assumed: probed 2026-09-09 against `docker run + # libretranslate/libretranslate --load-only en,ru` (the argos-translate + # en->ru model). `Bold Mountain is a good + # place.` came back with the span tag intact but its content translated + # anyway -- "Bold Mountain" became "Смелая гора". LibreTranslate's HTML + # format preserves markup; it does not honour the notranslate marker. + LIBRETRANSLATE_HONOURS_NOTRANSLATE = false + + # LibreTranslate publishes no per-request limits -- it is whatever the + # instance operator configured. These are this library's own conservative + # numbers, not the vendor's, and a self-hoster with a bigger instance can + # raise them by subclassing. + def self.capabilities + TranslationDiff::Capabilities.new( + max_request_size: 5_000, max_batch_size: 50, max_text_size: nil, + html: :format, notranslate: LIBRETRANSLATE_HONOURS_NOTRANSLATE, + detects_language: true, reports_billing: false + ) + end + + def self.configuration_options = %i[libretranslate_api_key libretranslate_api_base] + def self.configuration_requirements = %i[libretranslate_api_base] + + def api_base = config.libretranslate_api_base + def translate_url = "translate" + + def render_translate_payload(request) + { format: DEFAULT_FORMAT } + .merge(request.options) + .merge(q: request.texts, target: request.to.to_s, + source: request.from.nil? ? AUTO : request.from.to_s) + .tap { |payload| payload[:api_key] = config.libretranslate_api_key if config.libretranslate_api_key } + end + + def parse_translate_response(body, _headers, request) + translated = body["translatedText"] + detected = body["detectedLanguage"] + detected = detected.first if detected.is_a?(Array) + + TranslationDiff::Translation::Response.build( + request: request, + texts: translated.is_a?(Array) ? translated : [translated].compact, + detected_source: detected.is_a?(Hash) ? detected["language"]&.downcase : nil, + usage: TranslationDiff::Translation::Usage.new(characters: request.texts.sum(&:size)) + ) + end + + def detect(text) + payload = { q: text } + payload[:api_key] = config.libretranslate_api_key if config.libretranslate_api_key + + post("detect", payload).body.dig(0, "language")&.downcase + end +end + +TranslationDiff::Providers.register(:libretranslate, TranslationDiff::Providers::LibreTranslate) diff --git a/test/translation_diff/providers/libretranslate_test.rb b/test/translation_diff/providers/libretranslate_test.rb new file mode 100644 index 0000000..61afd54 --- /dev/null +++ b/test/translation_diff/providers/libretranslate_test.rb @@ -0,0 +1,114 @@ +# frozen_string_literal: true + +require "test_helper" +require "support/provider_contract" +require "support/http_provider_contract" +require "support/stubbed_provider" +require "faraday" + +class LibreTranslateProviderTest < Minitest::Test + include ProviderContract + include HTTPProviderContract + include StubbedProvider + + attr_reader :config + + def setup + TranslationDiff.reset! + @config = TranslationDiff::Configuration.new + @config.libretranslate_api_base = "https://libretranslate.test" + end + + def provider_class = TranslationDiff::Providers::LibreTranslate + + # When `body:` is left nil, the stub echoes back whatever texts were + # actually sent, so the shared ProviderContract tests -- which call + # `provider` with no knowledge of how many texts they are about to send -- + # get a response the same size as their request instead of tripping + # Response.build's count check. + def provider(body: nil, status: 200, headers: {}) + stub_provider(route: "/translate", body: body || method(:echo_translations), + status: status, headers: headers, name: :libretranslate) + end + + # Everyone self-hosts this one, so the base URL is the requirement and the + # key is the option -- the reverse of every other provider here. + def test_the_api_base_is_required_and_the_key_is_not + config.libretranslate_api_base = nil + + error = assert_raises(TranslationDiff::ConfigurationError) do + TranslationDiff::Providers::LibreTranslate.new(config) + end + + assert_match(/libretranslate_api_base/, error.message) + end + + def test_the_key_travels_in_the_body_when_set + config.libretranslate_api_key = "test-key" + provider.translate(translation_request(%w[one])) + + assert_equal "test-key", sent["api_key"] + end + + def test_the_key_is_absent_from_the_body_when_unset + provider.translate(translation_request(%w[one])) + + refute sent.key?("api_key") + end + + # source is required by the API, and "auto" is how detection is asked for. + def test_a_missing_source_language_becomes_auto + provider.translate(translation_request(%w[one], from: nil)) + + assert_equal "auto", sent["source"] + end + + def test_it_asks_for_html + provider.translate(translation_request(%w[one])) + + assert_equal "html", sent["format"] + end + + def test_it_reads_an_array_response + body = { "translatedText" => %w[один два], + "detectedLanguage" => [{ "language" => "en", "confidence" => 92.0 }] } + response = provider(body: body).translate(translation_request(%w[one two])) + + assert_equal %w[один два], response.texts + assert_equal "en", response.detected_source + end + + # A single q comes back as a bare string, not a one-element array. + def test_it_reads_a_single_string_response + body = { "translatedText" => "один", "detectedLanguage" => { "language" => "en" } } + response = provider(body: body).translate(translation_request(%w[one])) + + assert_equal %w[один], response.texts + end + + def test_a_short_response_raises_rather_than_shifting_nils_into_the_results + short = { "translatedText" => "один" } + + assert_raises(TranslationDiff::ResponseError) do + provider(body: short).translate(translation_request(%w[one two])) + end + end + + def test_detect_returns_the_language_libretranslate_reports + detector = stub_provider(route: "/detect", body: [{ "language" => "en", "confidence" => 92.0 }], + name: :libretranslate) + + assert_equal "en", detector.detect("something") + end + + def test_its_batch_limit_is_this_librarys_own_conservative_choice + assert_equal 50, TranslationDiff::Providers::LibreTranslate.capabilities.max_batch_size + end + + private + + def echo_translations(env) + texts = JSON.parse(env.body)["q"] + { "translatedText" => texts, "detectedLanguage" => texts.map { { "language" => "en" } } } + end +end From 7886a48cf0a8158b353f6a676af9eb2ebf329724 Mon Sep 17 00:00:00 2001 From: IG Date: Wed, 9 Sep 2026 18:33:47 +0400 Subject: [PATCH 13/26] refactor: migrate DeepL and Google tests onto the shared stub helper Both files carried their own copy of the request-recording Faraday stub, written before StubbedProvider's body: callable existed. Now that the shared helper can echo request-sized responses, both migrate onto it, leaving the local copy in Azure's test as the only style and StubbedProvider as the one place the plumbing lives. Assertions are unchanged; only how each stub is built moved. --- test/translation_diff/providers/deepl_test.rb | 67 ++++++------------- .../translation_diff/providers/google_test.rb | 65 ++++++------------ 2 files changed, 42 insertions(+), 90 deletions(-) diff --git a/test/translation_diff/providers/deepl_test.rb b/test/translation_diff/providers/deepl_test.rb index 24ebb61..e7ede02 100644 --- a/test/translation_diff/providers/deepl_test.rb +++ b/test/translation_diff/providers/deepl_test.rb @@ -3,11 +3,13 @@ require "test_helper" require "support/provider_contract" require "support/http_provider_contract" +require "support/stubbed_provider" require "faraday" class DeepLProviderTest < Minitest::Test include ProviderContract include HTTPProviderContract + include StubbedProvider # A real response body, captured from api-free.deepl.com on 2026-09-09. TRANSLATE_BODY = { @@ -17,35 +19,25 @@ class DeepLProviderTest < Minitest::Test ] }.freeze - attr_reader :config, :requests + attr_reader :config def setup TranslationDiff.reset! @config = TranslationDiff::Configuration.new @config.deepl_api_key = "test-key:fx" - @requests = [] - end - - # Builds a provider whose connection answers from a stub and records what - # was sent, so a test can assert on the payload as well as the parse. - # - # When neither `body:` nor `texts:` is given, the stub echoes back - # whatever texts were actually sent (rather than a fixed pair), so the - # shared ProviderContract tests -- which call `provider` with no - # knowledge of how many texts they are about to send -- get a response - # the same size as their request instead of tripping Response.build's - # count check. - def provider(body: nil, status: 200, texts: nil) - stubs = stub_translate(body: body, status: status, texts: texts) - built = TranslationDiff::Providers::DeepL.new(config) - built.name = :deepl - built.instance_variable_set(:@connection, built.send(:build_connection) do |faraday| - faraday.adapter :test, stubs - end) - built - end - - def sent = JSON.parse(requests.first.body) + end + + def provider_class = TranslationDiff::Providers::DeepL + + # When `body:` is left nil, the stub echoes back whatever texts were + # actually sent (rather than a fixed pair), so the shared ProviderContract + # tests -- which call `provider` with no knowledge of how many texts they + # are about to send -- get a response the same size as their request + # instead of tripping Response.build's count check. + def provider(body: nil, status: 200, headers: {}) + stub_provider(route: "/v2/translate", body: body || method(:echo_translations), + status: status, headers: headers, name: :deepl) + end def test_a_free_key_selects_the_free_host assert_equal "https://api-free.deepl.com", TranslationDiff::Providers::DeepL.new(config).api_base @@ -141,9 +133,10 @@ def test_a_short_response_raises_rather_than_shifting_nils_into_the_results # DeepL has no detection endpoint, so it detects by translating a sample # and reading what it says the source was. #detect sends exactly one - # text, so the stub is given exactly one text to echo back. + # text, and the stub echoes it back, so the count matches without an + # override. def test_detect_returns_the_language_deepl_reports - assert_equal "en", provider(texts: %w[x]).detect("something") + assert_equal "en", provider.detect("something") end def test_its_batch_limit_is_deepls_documented_fifty @@ -161,24 +154,8 @@ def test_it_claims_html_and_notranslate private - def stub_translate(body:, status:, texts:) - recorder = @requests - Faraday::Adapter::Test::Stubs.new do |stub| - stub.post("/v2/translate") do |env| - # Faraday's test adapter reuses this env for the response, mutating - # its body in place once the block returns -- capture a copy now or - # every read after #translate returns sees the reply, not the - # request. - recorder << env.dup - [status, { "Content-Type" => "application/json" }, translate_response(body, texts, env).to_json] - end - end - end - - def translate_response(body, texts, env) - return body if body - - response_texts = texts || JSON.parse(env.body)["text"] - { "translations" => response_texts.map { |t| { "text" => t, "detected_source_language" => "EN" } } } + def echo_translations(env) + texts = JSON.parse(env.body)["text"] + { "translations" => texts.map { |t| { "text" => t, "detected_source_language" => "EN" } } } end end diff --git a/test/translation_diff/providers/google_test.rb b/test/translation_diff/providers/google_test.rb index d142b9c..f3db00a 100644 --- a/test/translation_diff/providers/google_test.rb +++ b/test/translation_diff/providers/google_test.rb @@ -3,12 +3,14 @@ require "test_helper" require "support/provider_contract" require "support/http_provider_contract" +require "support/stubbed_provider" require "faraday" require "cgi" class GoogleProviderTest < Minitest::Test include ProviderContract include HTTPProviderContract + include StubbedProvider # A real response envelope, shaped from the Cloud Translation v2 REST # reference read 2026-09-09: translations live under a nested "data" key, @@ -20,36 +22,26 @@ class GoogleProviderTest < Minitest::Test ] } }.freeze - attr_reader :config, :requests + attr_reader :config def setup TranslationDiff.reset! @config = TranslationDiff::Configuration.new @config.google_api_key = "test-key" - @requests = [] - end - - # Builds a provider whose connection answers from a stub and records what - # was sent, so a test can assert on the payload as well as the parse. - # - # When neither `body:` nor `texts:` is given, the stub echoes back - # whatever texts were actually sent (rather than a fixed pair), so the - # shared ProviderContract tests -- which call `provider` with no - # knowledge of how many texts they are about to send -- get a response - # the same size as their request instead of tripping Response.build's - # count check. - def provider(body: nil, status: 200, texts: nil) - stubs = stub_translate(body: body, status: status, texts: texts) - built = TranslationDiff::Providers::Google.new(config) - built.name = :google - built.instance_variable_set(:@connection, built.send(:build_connection) do |faraday| - faraday.adapter :test, stubs - end) - built - end - - def sent = JSON.parse(requests.first.body) - def query = CGI.parse(requests.first.url.query.to_s) + end + + def provider_class = TranslationDiff::Providers::Google + + # When `body:` is left nil, the stub echoes back whatever texts were + # actually sent (rather than a fixed pair), so the shared ProviderContract + # tests -- which call `provider` with no knowledge of how many texts they + # are about to send -- get a response the same size as their request + # instead of tripping Response.build's count check. The content type + # matches what Cloud Translation v2 actually sends. + def provider(body: nil, status: 200, headers: { "Content-Type" => "application/json; charset=UTF-8" }) + stub_provider(route: "/language/translate/v2", body: body || method(:echo_translations), + status: status, headers: headers, name: :google) + end def test_the_key_travels_in_the_query_string provider.translate(translation_request(%w[one])) @@ -150,26 +142,9 @@ def test_google_reports_no_billing private - def stub_translate(body:, status:, texts:) - recorder = @requests - Faraday::Adapter::Test::Stubs.new do |stub| - stub.post("/language/translate/v2") do |env| - # Faraday's test adapter reuses this env for the response, mutating - # its body in place once the block returns -- capture a copy now or - # every read after #translate returns sees the reply, not the - # request. - recorder << env.dup - [status, { "Content-Type" => "application/json; charset=UTF-8" }, - translate_response(body, texts, env).to_json] - end - end - end - - def translate_response(body, texts, env) - return body if body - - response_texts = texts || JSON.parse(env.body)["q"] - translations = response_texts.map { |t| { "translatedText" => t, "detectedSourceLanguage" => "en" } } + def echo_translations(env) + texts = JSON.parse(env.body)["q"] + translations = texts.map { |t| { "translatedText" => t, "detectedSourceLanguage" => "en" } } { "data" => { "translations" => translations } } end end From 90b0120ca6847a6631a62842addad046f617fadf Mon Sep 17 00:00:00 2001 From: IG Date: Wed, 9 Sep 2026 18:42:25 +0400 Subject: [PATCH 14/26] feat: add an Amazon Translate provider --- Gemfile | 7 + lib/translation_diff.rb | 1 + lib/translation_diff/providers/amazon.rb | 140 ++++++++++++++++++ .../translation_diff/providers/amazon_test.rb | 129 ++++++++++++++++ 4 files changed, 277 insertions(+) create mode 100644 lib/translation_diff/providers/amazon.rb create mode 100644 test/translation_diff/providers/amazon_test.rb diff --git a/Gemfile b/Gemfile index 89c72eb..c57ecc0 100644 --- a/Gemfile +++ b/Gemfile @@ -26,3 +26,10 @@ gem "ratelimit", "~> 1.1", require: false # query string it sent, and Ruby 4.0 removed CGI.parse from the default # load path; "install cgi gem" is Ruby's own suggested fix. gem "cgi", "~> 0.5", require: false + +# Not a runtime dependency of the gem (see the gemspec) -- the Amazon +# provider requires it lazily when it signs its first request, so an +# application using another provider never needs it installed. It is here so +# the test suite, which signs against the real library rather than a +# stand-in, has it available. +gem "aws-sigv4", "~> 1.12", require: false diff --git a/lib/translation_diff.rb b/lib/translation_diff.rb index 607236f..994916c 100644 --- a/lib/translation_diff.rb +++ b/lib/translation_diff.rb @@ -28,6 +28,7 @@ require "translation_diff/providers/azure" require "translation_diff/providers/modernmt" require "translation_diff/providers/libretranslate" +require "translation_diff/providers/amazon" require "translation_diff/segmenters" require "translation_diff/segmenters/simple" diff --git a/lib/translation_diff/providers/amazon.rb b/lib/translation_diff/providers/amazon.rb new file mode 100644 index 0000000..70a2428 --- /dev/null +++ b/lib/translation_diff/providers/amazon.rb @@ -0,0 +1,140 @@ +# frozen_string_literal: true + +# Amazon Translate. The odd one out of this set in three ways, all of which +# the capabilities declare rather than hide: +# +# - It translates one text per call. There is no batch form of TranslateText, +# so a hundred sentences are a hundred requests. `max_batch_size: 1` makes +# Chunker produce one text per chunk, which is correct and slow. +# - It has no HTML mode, so a notranslate span sent to it is translated like +# any other text. `notranslate: false` is what lets the rest of the library +# warn instead of discovering it in production. +# - Its requests are signed rather than merely headed, which is why this +# class overrides #translate instead of filling in the usual seams, and +# why it overrides #build_connection to drop the JSON request middleware: +# the signature covers the body exactly as sent, so nothing may re-encode +# it afterwards. +class TranslationDiff::Providers::Amazon < TranslationDiff::HTTPProvider + SERVICE = "translate" + TARGET = "AWSShineFrontendService_20170701.TranslateText" + CONTENT_TYPE = "application/x-amz-json-1.1" + + # Amazon's own way of asking for detection. It reaches Amazon Comprehend + # under the hood and is only available in regions that have it. + AUTO = "auto" + + def self.capabilities + TranslationDiff::Capabilities.new( + max_request_size: 10_000, max_batch_size: 1, max_text_size: 10_000, + html: :none, notranslate: false, detects_language: true, reports_billing: false + ) + end + + def self.configuration_options + %i[amazon_access_key_id amazon_secret_access_key amazon_session_token + amazon_region amazon_api_base] + end + + def self.configuration_requirements + %i[amazon_access_key_id amazon_secret_access_key amazon_region] + end + + def api_base = config.amazon_api_base || "https://#{SERVICE}.#{config.amazon_region}.amazonaws.com" + + # One request per text, in order. The response's detected language is the + # first one Amazon reported: every text in a chunk comes from the same + # document, so they share a source language. + def translate(request) + detected = nil + texts = request.texts.map do |text| + body = call(text, request) + detected ||= body["SourceLanguageCode"]&.downcase + body["TranslatedText"] + end + + TranslationDiff::Translation::Response.build( + request: request, texts: texts, detected_source: detected, + usage: TranslationDiff::Translation::Usage.new(characters: request.texts.sum(&:size)) + ) + end + + def detect(text) + call(text, TranslationDiff::Translation::Request.new(texts: [text], from: nil, to: "en")) + .fetch("SourceLanguageCode", nil)&.downcase + end + + private + + def call(text, request) + payload = { + "Text" => text, + "SourceLanguageCode" => request.from.nil? ? AUTO : request.from.to_s, + "TargetLanguageCode" => request.to.to_s + }.merge(request.options.transform_keys(&:to_s)) + + post_signed(JSON.generate(payload)).body + end + + def post_signed(body) + raw = connection.post("/", body, signed_headers(body)) + response = decoded_response(raw) + raise_for_status!(response) + response + rescue *TRANSPORT_FAILURES => e + # The message is the transport's, never the payload's: the payload is the + # customer's text. + raise TranslationDiff::TransportError, "#{self.class}: #{e.class}: #{e.message}" + end + + # Reuses the base's own decoding (`decode`, `Decoded`, `json?`) rather than + # a second, Faraday-middleware-based path: that middleware is exactly what + # HTTPProvider's own #post avoids, since it breaks under the `json` 3 gem + # that ships by default on Ruby 4.x. + def decoded_response(raw) = Decoded.new(status: raw.status, headers: raw.headers, body: decode(raw)) + + # aws-sigv4 is Amazon's own signing library and nothing more: no clients, no + # service models, one dependency. It is required here rather than at load + # time so an application using another provider never needs it installed. + def signer + @signer ||= begin + require_sigv4 + Aws::Sigv4::Signer.new( + service: SERVICE, region: config.amazon_region, + access_key_id: config.amazon_access_key_id, + secret_access_key: config.amazon_secret_access_key, + session_token: config.amazon_session_token + ) + end + end + + def require_sigv4 + require "aws-sigv4" + rescue LoadError + raise TranslationDiff::Error, + "provider is :amazon but the `aws-sigv4` gem is not available. " \ + 'Add `gem "aws-sigv4"` to your Gemfile.' + end + + def signed_headers(body) + signature = signer.sign_request( + http_method: "POST", url: "#{api_base}/", body: body, + headers: { "Content-Type" => CONTENT_TYPE, "X-Amz-Target" => TARGET } + ) + + signature.headers.merge("Content-Type" => CONTENT_TYPE, "X-Amz-Target" => TARGET) + end + + # The signature covers the body exactly as sent, so this connection must + # not have a JSON request middleware re-encoding it afterwards -- unlike + # the base class's #build_connection, this one omits `faraday.request + # :json`. + def build_connection(&block) + Faraday.new(url: api_base, headers: headers) do |faraday| + faraday.request :retry, retry_options + adapt(faraday, &block) + apply_timeouts(faraday) + end + end +end + +TranslationDiff::Providers.register(:amazon, TranslationDiff::Providers::Amazon) diff --git a/test/translation_diff/providers/amazon_test.rb b/test/translation_diff/providers/amazon_test.rb new file mode 100644 index 0000000..0498b49 --- /dev/null +++ b/test/translation_diff/providers/amazon_test.rb @@ -0,0 +1,129 @@ +# frozen_string_literal: true + +require "test_helper" +require "support/provider_contract" +require "support/http_provider_contract" +require "faraday" +require "aws-sigv4" + +class AmazonProviderTest < Minitest::Test + include ProviderContract + include HTTPProviderContract + + attr_reader :config, :requests + + def setup + TranslationDiff.reset! + @config = TranslationDiff::Configuration.new + @config.amazon_access_key_id = "AKIAEXAMPLE" + @config.amazon_secret_access_key = "secret" + @config.amazon_region = "eu-central-1" + @requests = [] + end + + # There is no AWS key available for this task, so unlike DeepL's and + # Google's fixtures -- both captured from a live call -- this response + # body is shaped from Amazon's own Translate API reference documentation + # ("TranslateText", read 2026-09-09), not from an observed response. + # Nobody should mistake it for one. + def provider(texts: nil) + built = TranslationDiff::Providers::Amazon.new(config) + built.name = :amazon + built.instance_variable_set(:@connection, built.send(:build_connection) do |faraday| + faraday.adapter :test, build_stubs(texts) + end) + built + end + + def build_stubs(texts) + recorder = @requests + Faraday::Adapter::Test::Stubs.new do |stub| + stub.post("/") { |env| respond_to_translate(env, texts, recorder) } + end + end + + def respond_to_translate(env, texts, recorder) + # Faraday's test adapter reuses this env for the response, mutating its + # body in place once the block returns -- capture a copy now or every + # read after #translate returns sees the reply, not the request (see + # test/support/stubbed_provider.rb). + recorder << env.dup + body = JSON.parse(env.body) + translated = texts&.shift || "#{body['Text']}-ru" + [200, { "Content-Type" => "application/x-amz-json-1.1" }, + { "TranslatedText" => translated, "SourceLanguageCode" => "en", + "TargetLanguageCode" => "ru" }.to_json] + end + + def test_the_endpoint_is_regional + assert_equal "https://translate.eu-central-1.amazonaws.com", + TranslationDiff::Providers::Amazon.new(config).api_base + end + + def test_missing_credentials_are_named_before_any_request + config.amazon_secret_access_key = nil + + error = assert_raises(TranslationDiff::ConfigurationError) do + TranslationDiff::Providers::Amazon.new(config) + end + + assert_match(/amazon_secret_access_key/, error.message) + end + + def test_it_sends_the_json_rpc_target_header + provider.translate(translation_request(%w[one])) + + assert_equal "AWSShineFrontendService_20170701.TranslateText", + requests.first.request_headers["X-Amz-Target"] + end + + # This runs against the real aws-sigv4 library rather than a stand-in, so + # it is real evidence that this provider signs correctly -- not just that + # some string ended up in the Authorization header. + def test_it_signs_the_request + provider.translate(translation_request(%w[one])) + authorization = requests.first.request_headers["Authorization"] + + assert_match(/\AAWS4-HMAC-SHA256 Credential=AKIAEXAMPLE/, authorization) + assert_match(/Signature=[0-9a-f]{64}\z/, authorization) + end + + def test_it_sends_one_text_per_call_because_the_api_has_no_batch + provider(texts: %w[один два три]).translate(translation_request(%w[one two three])) + + assert_equal 3, requests.size + sent = requests.map { |r| JSON.parse(r.body)["Text"] } + + assert_equal %w[one two three], sent + end + + def test_it_returns_the_translations_in_the_order_they_were_asked_for + response = provider(texts: %w[один два три]).translate(translation_request(%w[one two three])) + + assert_equal %w[один два три], response.texts + end + + def test_a_missing_source_language_becomes_auto + provider.translate(translation_request(%w[one], from: nil)) + + assert_equal "auto", JSON.parse(requests.first.body)["SourceLanguageCode"] + end + + def test_it_reports_the_source_language_amazon_resolved + response = provider.translate(translation_request(%w[one], from: nil)) + + assert_equal "en", response.detected_source + end + + # The capability is the warning. Amazon has no HTML mode at all, so a + # notranslate span sent to it WILL be translated, and the only honest thing + # to do is say so where the rest of the library can read it. + def test_it_claims_neither_html_nor_notranslate + capabilities = TranslationDiff::Providers::Amazon.capabilities + + refute_predicate capabilities, :html? + refute_predicate capabilities, :notranslate? + assert_equal 1, capabilities.max_batch_size + assert_equal 10_000, capabilities.max_text_size + end +end From af7c7fdcde485aef9ec623e599d5c6eb32972a63 Mon Sep 17 00:00:00 2001 From: IG Date: Wed, 9 Sep 2026 19:01:31 +0400 Subject: [PATCH 15/26] feat!: read limits and abilities from provider capabilities Wires the pipeline to the Provider/Capabilities/Translation::Request-Response contract every provider already speaks: Request#chunks reads Capabilities instead of two provider methods nothing defines any more, #detect_language checks capabilities.detects_language? instead of a respond_to? check that was true for every provider, and #call_api sends a Translation::Request and returns a Translation::Response, dropping the now-dead manual count check. Without this, TranslationDiff.translate raised NoMethodError for every provider on this branch. Also fixes a real aliasing bug the required chunking test surfaced: a provider that hands back the same array it was given (rather than a fresh one, as Null does) had that array silently drained to empty by Cache#store's destructive #shift, visible to anything else still holding the reference. call_api now dups its return value. Migrates the three test files still carrying pre-Provider doubles (instrumentation_test.rb, context_test.rb, request_test.rb) onto the real contract, preferring the actual :null provider where a double added nothing. --- CHANGELOG.md | 35 ++ README.md | 476 +++++++++--------- lib/translation_diff/request.rb | 32 +- test/translation_diff/context_test.rb | 17 +- test/translation_diff/instrumentation_test.rb | 16 +- test/translation_diff/request_test.rb | 132 +++-- 6 files changed, 377 insertions(+), 331 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3910600..a7cf040 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -75,6 +75,22 @@ described below. Everything here is relative to `deepl_diff` 2.2.0. configured above 600 seconds is enforced over 600 seconds instead -- up to six times more eager than the configuration reads. Keep `rate_interval` within that range, or expect a tighter effective window than configured. +- Providers must inherit `TranslationDiff::Provider`. A duck-typed object is + no longer accepted: the base class supplies the transport, the + configuration check and the capability defaults, and a provider without + them is a provider that fails in the ways this library has already been + bitten by twice. +- `provider.translate(texts, from:, to:, **options)` is now + `provider.translate(request)`, taking a `Translation::Request` and + returning a `Translation::Response`. The response carries the detected + source language and, where the provider reports it, the characters billed. +- `max_request_size` and `max_batch_size` move from provider methods to + `Capabilities`. +- `TranslationDiff::Providers::Naming` is gone; the registry stamps + `cache_key` and `Provider` implements it. +- `deepl-rb` and `google-cloud-translate-v2` are no longer used at all. + `faraday` and `faraday-retry` become runtime dependencies; `aws-sigv4` is + required lazily by the Amazon provider only. ### Removed @@ -181,6 +197,22 @@ described below. Everything here is relative to `deepl_diff` 2.2.0. told apart by an argument's value. A provider with no `detect` makes `from:` required and raises a clear error when it is missing, instead of `NoMethodError`. +- Four new providers: `TranslationDiff::Providers::Azure` (`:azure`), + `TranslationDiff::Providers::ModernMT` (`:modernmt`), + `TranslationDiff::Providers::LibreTranslate` (`:libretranslate`), and + `TranslationDiff::Providers::Amazon` (`:amazon`), Amazon Translate, signed + with `aws-sigv4` rather than headed. Every provider's limits, HTML + support, `notranslate` handling, detection and billing reporting are + declared through `Capabilities` and measured against the vendor rather + than assumed -- see the provider table in the README. +- An error hierarchy for everything a provider's transport can do wrong: + `TranslationDiff::ConfigurationError`, `TranslationDiff::ProviderError` + (and its `AuthenticationError`, `QuotaExceededError`, + `InvalidRequestError`, `ServiceError` and `RateLimitError` subclasses), + `TranslationDiff::TransportError`, `TranslationDiff::ResponseError` and + `TranslationDiff::InvalidProviderError`, all under `TranslationDiff::Error`. +- `config.open_timeout`, `config.timeout` and `config.max_retries`, read by + every HTTP provider's connection and retry policy. ### Changed @@ -218,6 +250,9 @@ described below. Everything here is relative to `deepl_diff` 2.2.0. first check, and raises `TranslationDiff::Error` naming the gem to add when it is missing. Previously the bare constant surfaced a raw `NameError` instead of the message the Redis path already raises. +- DeepL's batch limit was declared as 300 sentences per request; DeepL + documents 50. The request-size limit (1,700 escaped characters) was + already correct and is unchanged. ## [2.2.0] - 2026-09-07 diff --git a/README.md b/README.md index 0f2c84a..36f7c76 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,10 @@ # TranslationDiff A translation cache that helps translate only changes between revisions of -long texts. It ships with DeepL and Google Cloud Translation providers, but -any translation service can be plugged in by implementing a small provider -contract -- this gem has no hard dependency on either of them, or on any -other provider. +long texts. It ships with six providers -- DeepL, Google Cloud Translation, +Azure AI Translator, ModernMT, LibreTranslate and Amazon Translate -- but any +translation service can be plugged in by subclassing a small base class; see +[Providers](#providers). **TranslationDiff** based on [GoogleTranslateDiff](https://github.com/gzigzigzeo/google_translate_diff) @@ -21,21 +21,24 @@ Much better approach is to try to translate every repeated structural element (s ## Dependencies -This gem loads two: [`ox`](https://github.com/ohler55/ox) to walk the HTML, and -[`pragmatic_segmenter`](https://github.com/diasks2/pragmatic_segmenter), which backs -the default sentence segmenter and has zero dependencies of its own. See [Segmenters -and the segmenter contract](#segmenters-and-the-segmenter-contract) below if you want -to avoid the second dependency. - -Everything else is duck typed and supplied by you: `deepl-rb` only if you use -the DeepL provider (the default) and `google-cloud-translate-v2` only if you -use the Google one, each required lazily the first time it is needed, with a -clear error if it is missing. The same is true of `redis` and -`connection_pool` once you configure `redis_url`, and of `ratelimit` on the -first check once you configure `rate_limit`. `redis-namespace` (for the Redis -cache store) goes one step further: this gem never requires it at all, so -your application must `require` it itself before using the Redis-backed -store. See [Getting started](#getting-started) below. +This gem loads four at require time: [`ox`](https://github.com/ohler55/ox) to +walk the HTML; [`pragmatic_segmenter`](https://github.com/diasks2/pragmatic_segmenter), +which backs the default sentence segmenter and has zero dependencies of its +own (see [Segmenters and the segmenter contract](#segmenters-and-the-segmenter-contract) +below if you want to avoid it); and [`faraday`](https://github.com/lostisland/faraday) +with [`faraday-retry`](https://github.com/lostisland/faraday-retry), the HTTP +transport every REST-backed provider (DeepL, Google, Azure, ModernMT, +LibreTranslate) inherits and owns directly -- none of them wraps a +vendor-supplied SDK any more. + +Everything else is duck typed and supplied by you: `aws-sigv4`, required +lazily the first time the Amazon provider signs a request, with a clear +error if it is missing. The same is true of `redis` and `connection_pool` +once you configure `redis_url`, and of `ratelimit` on the first check once +you configure `rate_limit`. `redis-namespace` (for the Redis cache store) +goes one step further: this gem never requires it at all, so your +application must `require` it itself before using the Redis-backed store. +See [Getting started](#getting-started) below. ## Installation @@ -64,8 +67,9 @@ end TranslationDiff.translate("Привет.", from: "ru", to: "en") ``` -Both of those have sensible defaults, so with `DEEPL_AUTH_KEY` and `REDIS_URL` -in the environment there is nothing to configure at all. Without `REDIS_URL` +`deepl_api_key` is required -- the provider checks for it at build time and +raises `TranslationDiff::ConfigurationError` naming what is missing, rather +than failing on the first real request. `redis_url` is optional: without it the cache lives in the process, which means the library runs before any infrastructure does. @@ -74,7 +78,7 @@ of your own**: ```ruby TranslationDiff.configure do |config| - config.provider = :deepl # or any object satisfying the provider contract + config.provider = :deepl # or any TranslationDiff::Provider of your own -- see Providers config.cache = :redis # or any object satisfying the cache store contract config.segmenter = :pragmatic # or any object satisfying the segmenter contract end @@ -89,7 +93,7 @@ at all, so an unset environment variable never has to be special-cased. | Option | Default | Meaning | | --- | --- | --- | -| `provider` | `:deepl` | The translation provider: a registered name or an object satisfying the [provider contract](#the-provider-contract). | +| `provider` | `:deepl` | The translation provider: a registered name or a `TranslationDiff::Provider` of your own. See [Providers](#providers). | | `cache` | `nil` | The cache store: a registered name or an object satisfying the [cache store contract](#the-cache-store-contract). `nil` means "choose for me" -- see below. | | `cache_ttl` | `604_800` (one week) | Seconds a Redis cache entry is kept. Only meaningful for `RedisCacheStore`; `MemoryCacheStore` evicts by size instead. | | `cache_namespace` | `"translation-diff"` | Prefix applied to every Redis key this gem writes -- both cache entries and the rate limiter's own bookkeeping. | @@ -103,24 +107,33 @@ at all, so an unset environment variable never has to be special-cased. | `segmenter` | `:pragmatic` | The sentence segmenter: a registered name or an object satisfying the [segmenter contract](#segmenters-and-the-segmenter-contract). | | `instrumenter` | `nil` | Anything satisfying `ActiveSupport::Notifications`' `#instrument(name, payload) { }` interface. See [Instrumentation and logging](#instrumentation-and-logging). | | `logger` | `nil` | A standard `Logger`. Receives one `debug` line per provider resolution, naming the provider class -- never content and never a credential. See [Instrumentation and logging](#instrumentation-and-logging). | +| `open_timeout` | `5` | Seconds an HTTP-backed provider waits to open a connection before raising `TranslationDiff::TransportError`. | +| `timeout` | `30` | Seconds an HTTP-backed provider waits for a response before raising `TranslationDiff::TransportError`. | +| `max_retries` | `3` | Retries `faraday-retry` attempts on a transport failure or a `429`/`500`/`502`/`503`/`504` response, with exponential backoff. `faraday-retry` honours a `Retry-After` header itself, so a `429` usually exhausts its retries before `TranslationDiff::RateLimitError` is ever raised. | -The `:deepl` provider declares two options of its own, registered the moment +Every provider declares its own configuration options, registered the moment `translation_diff` is required: -| Option | Default | Meaning | -| --- | --- | --- | -| `deepl_api_key` | `nil` | Forwarded to `deepl-rb` as `auth_key`. Left unset, `deepl-rb` reads `DEEPL_AUTH_KEY` from the environment itself. | -| `deepl_host` | `nil` | Overrides `deepl-rb`'s automatic free/paid host selection (from the `:fx` suffix on the key). Rarely needed. | - -The `:google` provider declares two of its own, on the same terms: - -| Option | Default | Meaning | +| Provider | Options | Meaning | | --- | --- | --- | -| `google_api_key` | `nil` | Forwarded to `google-cloud-translate-v2` as `key`. Left unset, that gem reads `TRANSLATE_KEY` or `GOOGLE_CLOUD_KEY` from the environment itself, and falls back to application default credentials when there is no key at all. | -| `google_project_id` | `nil` | Only consulted on the credentials path; an API key needs no project. Left unset, the gem reads `TRANSLATE_PROJECT`. | +| `:deepl` | `deepl_api_key` (required) | Sent as `DeepL-Auth-Key`. | +| | `deepl_api_base` | Overrides the automatic free/paid host selection (from the `:fx` suffix on the key). Rarely needed. | +| `:google` | `google_api_key` (required) | Sent as the `key` query parameter. | +| | `google_project_id` | Declared for a future credentials path; not currently read -- an API key needs no project. | +| | `google_api_base` | Overrides the default `https://translation.googleapis.com`. | +| `:azure` | `azure_api_key` (required) | Sent as `Ocp-Apim-Subscription-Key`. | +| | `azure_region` | Sent as `Ocp-Apim-Subscription-Region`. Required by a multi-service Azure resource; a single-service resource needs no region. | +| | `azure_api_base` | Overrides the default `https://api.cognitive.microsofttranslator.com`. | +| `:modernmt` | `modernmt_api_key` (required) | Sent as `MMT-ApiKey`. | +| | `modernmt_api_base` | Overrides the default `https://api.modernmt.com`. | +| `:libretranslate` | `libretranslate_api_base` (required) | Every instance is self-hosted; there is no default to fall back to. | +| | `libretranslate_api_key` | Sent as `api_key` in the request body. Most instances do not require one. | +| `:amazon` | `amazon_access_key_id`, `amazon_secret_access_key`, `amazon_region` (all required) | Used to sign each request with `aws-sigv4`. | +| | `amazon_session_token` | For temporary credentials. | +| | `amazon_api_base` | Overrides the default `https://translate..amazonaws.com`. | A provider you register yourself can declare its own options the same way -- -see [Registering your own provider](#registering-your-own-provider) below. +see [Writing a provider](#writing-a-provider) below. **Configure once, before the first translation.** `provider`, `cache`, `segmenter` and `rate_limiter` each resolve to a collaborator on first use @@ -147,136 +160,128 @@ source and target language codes, a digest of the provider options that call passed (`formality:`, a glossary id, ...), and a digest of the sentence itself. `RedisCacheStore` prefixes all of that with `cache_namespace`. -**`deepl_host` is deliberately not part of the key.** Two configurations -pointing `deepl_host` at different endpoints share cache entries. For DeepL's -own free and paid hosts that is correct -- they return the same translations --- but a self-hosted or proxied endpoint may not, and it would be served, and -would serve, the real service's entries. Give such a configuration its own -`cache_namespace` (or its own Redis database). The key format is left alone -here on purpose: changing its shape invalidates every entry already cached, -everywhere, at once. - -### The DeepL provider - -`config.provider = :deepl` is the default. It sends `tag_handling: :html` -and `tag_handling_version: "v2"` with every translation, because what -reaches a provider is not plain text: a `notranslate` span arrives whole, -tags included. DeepL honours `class="notranslate"` and `translate="no"` -only under HTML tag handling -- without it, in DeepL's own words, "tags are -treated as regular text". - -That failure was a quiet one, worth knowing about if you translated with an -older version: DeepL leaves the tags themselves alone either way, so the -markup looks untouched and only the protected content comes back changed. - -``` -"Bold Mountain is a good place." -no tag handling -> "Болд-Маунтин — отличное место." -tag_handling -> "Bold Mountain — это хорошее место." -``` - -Both values are overridable per call, as any provider option is. Under HTML -tag handling DeepL defaults `split_sentences` to `nonewlines`; this library -sends one sentence at a time, so that changes nothing. - -### The Google provider - -`config.provider = :google` translates through Cloud Translation v2 (Basic). -It needs the `google-cloud-translate-v2` gem, which this gem requires lazily -the first time the provider is built: - -```ruby -gem "google-cloud-translate-v2", "~> 1.2" -``` - -```ruby -TranslationDiff.configure do |config| - config.provider = :google - config.google_api_key = ENV["GOOGLE_TRANSLATE_KEY"] -end -``` - -An API key is the whole setup -- no project id, no service account. Leave -`google_api_key` unset and the gem reads `TRANSLATE_KEY` or -`GOOGLE_CLOUD_KEY` itself; with no key anywhere it falls back to application -default credentials, which is the path where `google_project_id` matters. - -Two things this provider does on your behalf, both of which would otherwise -be silent problems: - -- **It asks for HTML**, which is Google's own default and what this gem has - always sent. What reaches a provider is not plain text: a `notranslate` - span arrives whole, tags included -- that is how the tokenizer marks - content the provider must leave alone -- and entities such as `&` - stay in the text it emits. Asking for `text` makes Google translate the - protected span and drop its markup: - - ``` - "Bold Mountain is a good place." - format: text -> "Болд Маунтин — хорошее место." - format: html -> "Bold Mountain — хорошее место." - ``` - - The cost is that Google escapes its own output: a literal apostrophe - returns as `'`. Inside the HTML fragment these values usually are, - that renders as an apostrophe and is correct. Inside a value that never - had any markup in it, it is noise -- pass `format: :text` per call when - translating bare strings. -- **It downcases bare language codes.** Google's codes are lowercase, and a - configuration written against DeepL says `"EN"`. Codes carrying a subtag - -- `"zh-Hans"`, `"zh-CN"`, `"pt-BR"` -- are passed through untouched, - because the casing of a script or region subtag is its own. - -Its limits are Google's documented ones: 128 strings per request (a hard -limit -- a larger batch is rejected), and 5,000 characters per request (the -documented recommendation, well under the hard 100 KB ceiling). Chunker -measures the URL-escaped form of each string, which is never smaller than -its UTF-8 byte count, so a chunk within that bound in escaped characters is -within it in bytes as well. - -`google_project_id` is not part of the cache key, on the same reasoning as -`deepl_host` above: it selects an account to bill, not a translation. - -## Registering your own provider +**No provider's `*_api_base` option is part of the key.** Two configurations +pointing `deepl_api_base` (or any other provider's `_api_base`) at different +endpoints share cache entries. For DeepL's own free and paid hosts that is +correct -- they return the same translations -- but a self-hosted or proxied +endpoint may not, and it would be served, and would serve, the real +service's entries. Give such a configuration its own `cache_namespace` (or +its own Redis database). The key format is left alone here on purpose: +changing its shape invalidates every entry already cached, everywhere, at +once. + +## Providers + +Every provider declares what it can do through +`TranslationDiff::Capabilities` -- there is nowhere else these numbers live, +so this table is generated from the same source the library reads at +runtime: + +| Provider | `config.provider` | Auth option(s) | Batch size | Request size (escaped chars) | HTML support | `notranslate` | Detects language | Reports billing | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | +| Null | `:null` | none | 1,000,000 | 1,000,000 | no | no | no | no | +| DeepL | `:deepl` (default) | `deepl_api_key` | 50 | 1,700 | yes (`tag_handling`) | yes | yes | yes | +| Google | `:google` | `google_api_key` | 128 | 5,000 | yes (`format`) | yes | yes | no | +| Azure | `:azure` | `azure_api_key` | 1,000 | 50,000 | yes (`textType`) | yes | yes | yes | +| ModernMT | `:modernmt` | `modernmt_api_key` | 128 | 5,000 | yes (`format`) | no | yes | yes | +| LibreTranslate | `:libretranslate` | `libretranslate_api_base` | 50 | 5,000 | yes (`format`) | no | yes | no | +| Amazon | `:amazon` | `amazon_access_key_id`, `amazon_secret_access_key`, `amazon_region` | 1 | 10,000 | no | no | yes | no | + +"Request size" is what `Chunker` measures: the URL-escaped form of each +string (`CGI.escape(text).size`), which is never smaller than its UTF-8 byte +count. "HTML support" names the provider option that turns HTML handling on +-- every vendor spells it differently, which is exactly what +`Capabilities#html` is for. A provider whose "Detects language" column says +no makes `from:` required; passing it makes every provider's `#detect` call +unnecessary regardless of whether it has one. + +**Amazon translates one text per call and honours no `notranslate`.** There +is no batch form of `TranslateText`, so a hundred sentences are a hundred +requests -- slow, but correct, and `Capabilities#max_batch_size` reflects +it. Amazon also has no HTML mode: a `notranslate` span reaches it as plain +text and is translated like everything else, tags and all. Both facts are +worth weighing before your bill and your brand names arrive, not after. + +**LibreTranslate does not honour `notranslate` either -- measured, not +assumed.** Its HTML format preserves markup, but probing a real instance +(`docker run libretranslate/libretranslate --load-only en,ru`) with +`Bold Mountain is a good place.` came back +with the span tag intact and its content translated anyway -- "Bold +Mountain" became "Смелая гора". The tags survive; what they were meant to +protect does not. + +ModernMT's `notranslate: false` is the conservative default rather than a +measurement: it documents an HTML format but says nothing about +`class="notranslate"`, and no key was available to probe it. A capability +that under-promises costs a warning; one that over-promises costs a +customer's protected content reaching a competitor's brand voice. + +### Writing a provider Any translation service can be a provider -- no change to this gem's own -code is required. Registering a provider also declares the options it needs, -so `config.yandex_api_key` below does not exist until `YandexProvider` -is registered: +code is required. Subclass `TranslationDiff::HTTPProvider` for a REST +service; it owns the Faraday connection, retries, timeouts and turns HTTP +status codes into this library's error hierarchy, and asks only for three +seams per operation: the URL, how to render a request, how to parse a +reply. Subclass `TranslationDiff::Provider` directly for anything that +reaches its service some other way -- signed requests, another gem, an +LLM client -- and implement `#translate` outright, the way +`TranslationDiff::Providers::Amazon` does. + +Registering a provider also declares the options it needs, so +`config.yandex_api_key` below does not exist until `YandexProvider` is +registered: ```ruby -class YandexProvider - # Declares this provider's own configuration options. TranslationDiff::Providers.register - # adds each one to TranslationDiff::Configuration as a side effect. +class YandexProvider < TranslationDiff::HTTPProvider + # Declares this provider's own configuration options. + # TranslationDiff::Providers.register adds each one to + # TranslationDiff::Configuration as a side effect. def self.configuration_options = %i[yandex_api_key] - def self.build(config) = new(config.yandex_api_key) + def self.configuration_requirements = %i[yandex_api_key] + + # What this provider can do, checked once by the pipeline for chunking, + # detection and cache-key safety. + def self.capabilities + TranslationDiff::Capabilities.new( + max_request_size: 10_000, max_batch_size: 100, max_text_size: nil, + html: :format, notranslate: true, detects_language: true, + reports_billing: false + ) + end + + def api_base = "https://translate.api.cloud.yandex.net" + def headers = { "Authorization" => "Api-Key #{config.yandex_api_key}" } + def translate_url = "translate/v2/translate" - def initialize(api_key) - @client = SomeYandexClient.new(key: api_key) + # The three seams: build the request body, decode the reply. + def render_translate_payload(request) + { format: "HTML", texts: request.texts, targetLanguageCode: request.to.to_s } + .tap { |body| body[:sourceLanguageCode] = request.from.to_s unless request.from.nil? } end - # Required: translate an array of strings, return one string per input, in - # the same order. Provider-specific options (formality, glossary, ...) - # arrive through **options untouched. - def translate(texts, from:, to:, **options) - texts.map { |text| @client.translate(text, from: from, to: to)[:text] } + def parse_translate_response(body, _headers, request) + translations = Array(body["translations"]) + + TranslationDiff::Translation::Response.build( + request: request, + texts: translations.map { |t| t["text"] }, + detected_source: translations.first&.dig("detectedLanguageCode")&.downcase + ) end - # Required: this provider's own request- and batch-size limits. - def max_request_size = 30_000 - def max_batch_size = 128 + def detect_url = "translate/v2/detect" - # Optional: omit entirely if the provider has no detection endpoint, or if - # callers of this gem always pass `from:` explicitly. def detect(text) - @client.detect(text)[:language] + response = post(detect_url, { text: text }) + response.body["languageCode"]&.downcase end - # cache_key is optional too: TranslationDiff::Providers.register mixes a - # module into any provider that does not define its own #cache_key, and - # that module stamps every instance built through the registry with its - # registered name -- nothing here needs to supply one by hand. See - # "Provider objects and cache_key" below for what happens without it. + # cache_key is optional: TranslationDiff::Providers.register stamps every + # instance built through the registry with its registered name, and + # Provider#cache_key falls back to that. Define it yourself only if this + # provider will also be instantiated and assigned directly, bypassing the + # registry -- see "Provider objects and cache_key" below. end TranslationDiff::Providers.register(:yandex, YandexProvider) @@ -287,6 +292,12 @@ TranslationDiff.configure do |config| end ``` +`translate_url`/`render_translate_payload`/`parse_translate_response` are +the three seams `HTTPProvider#translate` calls in order; `detect` is +entirely optional -- omit it (and leave `capabilities.detects_language: +false`) if the provider has no detection endpoint, or if callers of this +gem always pass `from:` explicitly. + **Provider names must be unique.** `TranslationDiff::Providers.register` overwrites whatever was previously registered under that name, silently -- there is no error for registering `:deepl` twice. This is deliberate: a @@ -316,64 +327,20 @@ and a Rails reload both re-run registration. `TranslationDiff::Providers.names` lists every registered provider; `TranslationDiff::Providers.registered?(:yandex)` checks one. -## The provider contract - -`config.provider` accepts either a registered name (`:deepl`, `:google`, -`:null`, or anything you registered yourself) or an object of your own that satisfies -this contract directly, bypassing the registry entirely: - -```ruby -# Translates an array of strings and returns an array of strings, one per -# input, in the same order. Provider-specific options (e.g. `formality:`) -# arrive through **options and are passed straight through to the provider. -def translate(texts, from:, to:, **options); end - -# Detects the source language of a single string and returns it. Optional: -# omit this method entirely if the provider has no detection endpoint, or if -# callers of this gem always pass `from:` explicitly. When `detect` is -# missing and `from:` is not given, TranslationDiff raises rather than -# guessing. -def detect(text); end - -# The largest single request the provider accepts, in characters of the -# escaped form -- which is what the chunker measures (CGI.escape(text).size, -# not String#size). For Cyrillic and other non-Latin text this is 6 to 9 -# times the raw character count. Declaring the provider's raw character -# limit here will either waste most of the budget (if you under-report) or -# raise Chunker::Error on text the provider would actually have accepted -# (if you over-report). Used to split long texts into multiple requests. -def max_request_size; end - -# The largest number of strings the provider accepts in one batched request. -# Used to split large arrays into multiple requests. -def max_batch_size; end - -# A short, stable, non-empty string identifying this provider. Used to -# namespace cache keys, so that switching providers does not return one -# provider's cached translations for another. Not required when a provider -# is only ever built through TranslationDiff::Providers.register -- see -# below. -def cache_key; end -``` - -`test/support/provider_contract.rb` is the executable form of this contract: -include `ProviderContract` in a test class that defines `#provider`, and it -verifies `translate`, `max_request_size` and `max_batch_size` behave as -documented above. - -Two providers ship with this gem: `TranslationDiff::Providers::DeepL` (the -default, registered as `:deepl`), wrapping the -[`deepl-rb`](https://github.com/wikiti/deepl-rb) gem (not a dependency of -this one -- required at build time); and `TranslationDiff::Providers::Null` -(`:null`), which hands back exactly what it was given, for tests and for -wiring up a pipeline before a real provider is available. +`test/support/provider_contract.rb` and `test/support/http_provider_contract.rb` +are the executable form of the provider contract: include `ProviderContract` +(and, for an `HTTPProvider` subclass, `HTTPProviderContract`) in a test class +that defines `#provider`, and they verify a provider inherits +`TranslationDiff::Provider`, that `#translate` preserves order and returns +one string per input, and that its declared capabilities are internally +consistent (a provider claiming `notranslate` must also claim an HTML mode). ### Provider objects and cache_key A provider built through `TranslationDiff::Providers.build` (which is what happens when `config.provider` is a symbol) is stamped with its registered -name automatically, and never needs to define `cache_key` itself -- the -registry mixes in a module that supplies it. +name automatically, and never needs to define `cache_key` itself -- +`Provider#cache_key` falls back to that stamped name. A provider object assigned straight to `config.provider` never passes through the registry, so it gets no name and **must define `cache_key` @@ -638,31 +605,17 @@ same guarantee applies to it as to instrumentation payloads: no log line this library writes carries the text being translated, its translation, or a credential. -**deepl-rb has request logging of its own, and this library deliberately -does not enable it.** `config.logger` is never passed to `deepl-rb`. Given a -logger, `deepl-rb` writes a `Request details:` line at DEBUG holding the full -`Authorization: DeepL-Auth-Key ...` header and the request payload -- your API -key and the text being translated. Forwarding this gem's logger into it would -break the guarantee above at the exact moment someone raises the log level to -diagnose a problem, which is why the provider does not. - -If you want that log anyway, ask for it explicitly: build the `DeepL::API` -yourself, wrap it in the provider, and assign the object. - -```ruby -api = DeepL::API.new( - DeepL::Configuration.new(auth_key: ENV["DEEPL_AUTH_KEY"], logger: verbose_logger) -) - -provider = TranslationDiff::Providers::DeepL.new(api) -provider.name = :deepl # the cache key a registry-built provider gets for free - -TranslationDiff.configure { |config| config.provider = provider } -``` - -Everything `verbose_logger` then receives -- source text, translations, and -the auth key -- goes wherever it writes. Point it somewhere disposable, not at -the application log, and do not leave it on. +**No HTTP-backed provider ever receives `config.logger`, and there is no way +to opt one in.** `TranslationDiff::HTTPProvider` installs no logging +middleware on its Faraday connection and never passes a logger to it -- this +is enforced by `test/support/http_provider_contract.rb`, not merely +documented. Earlier versions wrapped `deepl-rb`, which logged a +`Request details:` line at DEBUG holding the full +`Authorization: DeepL-Auth-Key ...` header and the request payload -- your +API key and the text being translated -- if you gave it a logger of its own. +Owning the transport directly closed that door rather than working around +it: nothing this library builds writes source text, a translation, or a +credential anywhere, and no configuration option reopens that. ## Errors @@ -671,22 +624,42 @@ so rescuing the gem's failures in one clause is a single `rescue TranslationDiff ``` TranslationDiff::Error -├── TranslationDiff::Request::Error # e.g. provider returned the wrong number of -│ # translations, from: missing and the -│ # provider cannot detect, cache_key -│ # missing on an assigned provider object -├── TranslationDiff::Cache::Error # provider options have no stable -│ # serialisation for the cache key -├── TranslationDiff::Chunker::Error # a single value is larger than the -│ # provider's max_request_size +├── TranslationDiff::ConfigurationError # a provider is missing a required option +├── TranslationDiff::ProviderError # the service answered and said no +│ ├── AuthenticationError # 401/403 +│ ├── RateLimitError # 429, once faraday-retry's own retries +│ │ # are exhausted -- carries #retry_after +│ │ # when the service sent one +│ ├── QuotaExceededError # 456 +│ ├── InvalidRequestError # any other 4xx +│ └── ServiceError # 5xx, or anything else +├── TranslationDiff::TransportError # nobody answered: connection failed, +│ # timed out, or TLS failed +├── TranslationDiff::ResponseError # the answer was well-formed HTTP but broke +│ # this library's contract -- a body that +│ # is not JSON, or a provider that returned +│ # the wrong number of translations +├── TranslationDiff::InvalidProviderError # a class registered without inheriting +│ # TranslationDiff::Provider +├── TranslationDiff::Request::Error # from: missing and the provider cannot +│ # detect, cache_key missing on an +│ # assigned provider object +├── TranslationDiff::Cache::Error # provider options have no stable +│ # serialisation for the cache key +├── TranslationDiff::Chunker::Error # a single value is larger than the +│ # provider's declared max_request_size ├── TranslationDiff::Segmenters::Pragmatic::Error -│ # Pragmatic computed offsets that -│ # violate its own postcondition -- -│ # not raised by ordinary use +│ # Pragmatic computed offsets that +│ # violate its own postcondition -- +│ # not raised by ordinary use └── TranslationDiff::RedisRateLimiter::RateLimitExceeded - # the configured rate_limit was exceeded + # the configured rate_limit was exceeded ``` +`ProviderError` and its subclasses carry `#provider` (the registered name) +and `#status` (the HTTP status code), so a caller can log or branch on which +service and which response caused the failure without parsing the message. + `TranslationDiff::Registry` -- which backs the provider, cache store and segmenter registries -- also raises `TranslationDiff::Error` directly (not a dedicated subclass) for an unknown name, listing what is actually @@ -728,11 +701,13 @@ TranslationDiff.translate("Black", from: "en", to: "es") ## Very long texts -Every provider limits how large a single request or a single batch can be. -Providers state their own limits through `#max_request_size` and -`#max_batch_size`; if your text is longer than that, TranslationDiff splits it -into multiple requests automatically. The DeepL provider, for example, caps -requests at 1700 characters and batches at 300 sentences. +Every provider limits how large a single request or a single batch can be, +declared through `TranslationDiff::Capabilities#max_request_size` and +`#max_batch_size`; if your text is longer than that, TranslationDiff splits +it into multiple requests automatically. See the [provider +table](#providers) for each built-in provider's actual numbers -- DeepL, for +example, caps requests at 1,700 escaped characters and batches at 50 +sentences. ## Former name and upgrading @@ -752,6 +727,15 @@ configured, also read the upgrading note in never actually enforcing your threshold before 3.1.0, and it starts doing so now. See [CHANGELOG.md](CHANGELOG.md) for the full list of breaking changes. +**If you registered a custom provider,** it must now subclass +`TranslationDiff::Provider` (or `TranslationDiff::HTTPProvider`), declare +`self.capabilities`, and implement `#translate(request)` taking a +`TranslationDiff::Translation::Request` and returning a +`TranslationDiff::Translation::Response` -- the duck-typed +`#translate(texts, from:, to:, **options)` plus `#max_request_size` and +`#max_batch_size` methods are no longer read at all. See [Writing a +provider](#writing-a-provider). + ## Development After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. diff --git a/lib/translation_diff/request.rb b/lib/translation_diff/request.rb index d967e80..1fe09fe 100644 --- a/lib/translation_diff/request.rb +++ b/lib/translation_diff/request.rb @@ -60,8 +60,12 @@ def nothing_to_translate? text_tokens_texts.all?(&:empty?) end + def capabilities = api.class.capabilities + def detect_language - raise Error, "Pass from: -- #{api.class} cannot detect the source language" unless api.respond_to?(:detect) + unless capabilities.detects_language? + raise Error, "Pass from: -- provider #{provider_cache_key} cannot detect the source language" + end api.detect(text_tokens_texts.join(" ")[0..100]) end @@ -118,8 +122,8 @@ def text_tokens_texts def chunks @chunks ||= TranslationDiff::Chunker.new( text_tokens_texts, - limit: api.max_request_size, - count_limit: api.max_batch_size + limit: capabilities.max_request_size, + count_limit: capabilities.max_batch_size ).call end @@ -181,17 +185,19 @@ def translation def call_api(values) check_rate_limit(values) - translations = instrument("request", provider: provider_cache_key, - batch: values.size, - characters: values.sum(&:size)) do - api.translate(values, from: from, to: to, **options) + request = TranslationDiff::Translation::Request.new( + texts: values, from: from, to: to, options: options + ) + response = instrument("request", provider: provider_cache_key, batch: values.size, + characters: values.sum(&:size)) do + api.translate(request) end - return translations if translations.size == values.size - - # Letting a short response through means shifting nils into the results, - # which surfaces much later as a NoMethodError far from the cause. - raise Error, - "Provider returned #{translations.size} translations for #{values.size} values" + # Dup'd because Cache#store consumes this array destructively (#shift). + # A provider is free to hand back the very array it was given -- Null + # does with a fresh one, but nothing requires that -- and without the + # dup here, a provider or caller holding onto that reference would watch + # it drain to empty out from under them. + response.texts.dup end def cache diff --git a/test/translation_diff/context_test.rb b/test/translation_diff/context_test.rb index 52268b8..86b5c24 100644 --- a/test/translation_diff/context_test.rb +++ b/test/translation_diff/context_test.rb @@ -3,19 +3,6 @@ require "test_helper" class ContextTest < Minitest::Test - # TranslationDiff::Providers::Null now speaks Translation::Request/Response - # (provider-transport work); request.rb still calls a provider the old way - # and is migrated onto the new contract in a later task. This double keeps - # that old shape so this file can keep exercising Context#translate without - # touching request.rb. - class NullDouble - # rubocop:disable-next Lint/UnusedMethodArgument - def translate(texts, from:, to:, **_options) = texts - def max_request_size = 1_000_000 - def max_batch_size = 1_000_000 - def cache_key = "null" - end - def setup TranslationDiff.reset! TranslationDiff.configure do |c| @@ -55,7 +42,9 @@ def test_two_contexts_build_separate_collaborators end def test_a_context_translates_through_its_own_configuration - context = TranslationDiff.context { |c| c.provider = NullDouble.new } + context = TranslationDiff.context do |c| + c.provider = TranslationDiff::Providers::Null.new(TranslationDiff::Configuration.new) + end assert_equal "Hello.", context.translate("Hello.", from: "en", to: "ru") end diff --git a/test/translation_diff/instrumentation_test.rb b/test/translation_diff/instrumentation_test.rb index 488f315..b31b047 100644 --- a/test/translation_diff/instrumentation_test.rb +++ b/test/translation_diff/instrumentation_test.rb @@ -23,25 +23,11 @@ class FakeRateLimiter def check(_size) = nil end - # TranslationDiff::Providers::Null now speaks Translation::Request/Response - # (provider-transport work); request.rb still calls a provider the old way - # and is migrated onto the new contract in a later task. This double keeps - # that old shape -- and the "null" cache key the assertions below check -- - # so this file can keep exercising the instrumentation pipeline without - # touching request.rb. - class NullDouble - # rubocop:disable-next Lint/UnusedMethodArgument - def translate(texts, from:, to:, **_options) = texts - def max_request_size = 1_000_000 - def max_batch_size = 1_000_000 - def cache_key = "null" - end - def setup super @recorder = Recorder.new TranslationDiff.configure do |c| - c.provider = NullDouble.new + c.provider = :null # Pinned so a developer with REDIS_URL set does not have these tests # resolve the Redis store and open a real socket -- the same reason # context_test.rb pins it. It weakens no assertion here. diff --git a/test/translation_diff/request_test.rb b/test/translation_diff/request_test.rb index 9405f08..005b87c 100644 --- a/test/translation_diff/request_test.rb +++ b/test/translation_diff/request_test.rb @@ -3,22 +3,32 @@ require "test_helper" class RequestTest < ConfiguredTest - # A minimal adapter. Records what it was asked to translate so the call - # can be asserted on, and answers with a canned response. - class FakeApi - attr_reader :calls, :max_request_size, :max_batch_size - - def initialize(response, detected: nil, max_request_size: 1_000_000, max_batch_size: 1_000_000) + # A minimal provider. Records what it was asked to translate so the call + # can be asserted on, and answers with a canned response. Detects a + # language when told to, so it can stand in for both a detecting and a + # non-detecting provider depending on which test needs which. + class FakeApi < TranslationDiff::Provider + CAPABILITIES = TranslationDiff::Capabilities.new( + max_request_size: 1_000_000, max_batch_size: 1_000_000, max_text_size: nil, + html: :none, notranslate: false, detects_language: true, reports_billing: false + ).freeze + + def self.capabilities = CAPABILITIES + + attr_reader :calls + + def initialize(response, detected: nil) + super(TranslationDiff::Configuration.new) @response = response @detected = detected - @max_request_size = max_request_size - @max_batch_size = max_batch_size @calls = [] end - def translate(texts, from:, to:, **options) - @calls << [texts, from, to, options] - @response.shift(texts.size) + def translate(request) + @calls << [request.texts, request.from, request.to, request.options] + TranslationDiff::Translation::Response.build( + request: request, texts: @response.shift(request.texts.size) + ) end def detect(text) @@ -29,34 +39,31 @@ def detect(text) def cache_key = "fake" end - # request.rb still asks `respond_to?(:detect)` to learn whether a provider - # can detect a source language -- the capability this library now expresses - # through Capabilities#detects_language? for anything built as a - # TranslationDiff::Provider. TranslationDiff::Providers::Null moved onto - # that base class (provider-transport work) and, like every Provider, - # always defines #detect (raising NotImplementedError), so it no longer - # answers `respond_to?(:detect)` honestly for this still-old pipeline. This - # double keeps the pre-migration shape -- no #detect at all -- so this file - # can keep testing request.rb's own "cannot detect" guard without touching - # request.rb itself. - class NonDetectingApi - # rubocop:disable-next Lint/UnusedMethodArgument - def translate(texts, from:, to:, **_options) = texts - def max_request_size = 1_000_000 - def max_batch_size = 1_000_000 - def cache_key = "null" + # Proves the generalisation took effect: a provider whose declared + # capabilities cap the batch at one text per request must change the + # chunking, not merely be asked to. + class NarrowBatchApi < FakeApi + def self.capabilities + TranslationDiff::Capabilities.new( + max_request_size: 1_000_000, max_batch_size: 1, max_text_size: nil, + html: :none, notranslate: false, detects_language: true, reports_billing: false + ) + end end - # Same rationale as NonDetectingApi: a Provider built through the registry - # now speaks Translation::Request/Response, which request.rb does not call - # yet. This double is registered under :echo so + # Registered under :echo so # test_the_provider_keyword_overrides_the_configured_provider_for_one_call - # can still exercise resolving a provider by name through the registry. + # can exercise resolving a provider by name through the registry. class EchoProvider < TranslationDiff::Provider - # rubocop:disable-next Lint/UnusedMethodArgument - def translate(texts, from:, to:, **_options) = texts - def max_request_size = 1_000_000 - def max_batch_size = 1_000_000 + def self.capabilities + TranslationDiff::Capabilities.new( + max_request_size: 1_000_000, max_batch_size: 1_000_000, max_text_size: nil, + html: :none, notranslate: false, detects_language: false, reports_billing: false + ) + end + + def translate(request) = TranslationDiff::Translation::Response.build(request: request, texts: request.texts) + def cache_key = "echo" end TranslationDiff::Providers.register(:echo, EchoProvider) unless TranslationDiff::Providers.registered?(:echo) @@ -80,8 +87,7 @@ def write(key, value) # A provider object assigned straight to `config.provider` never passed # through the registry, so nothing stamped it with a name. This one defines - # its own #cache_key and returns an empty segment from it, which is the case - # TranslationDiff::Providers::Naming cannot catch. + # its own #cache_key and returns an empty segment from it. class NamelessApi < FakeApi def cache_key = "" end @@ -194,20 +200,60 @@ def test_skips_the_translation_when_the_detected_language_is_the_target assert_equal [[:detect, "привет"]], api.calls end - def test_raises_when_from_is_missing_and_the_adapter_cannot_detect - configure_with(NonDetectingApi.new) + # Chunker's limits used to come from two provider methods. They now come + # from the declared capabilities, which is the only place a new provider + # states them. + # rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength + def test_chunking_uses_the_providers_declared_capabilities + narrow = Class.new(TranslationDiff::Provider) do + def self.capabilities + TranslationDiff::Capabilities.new( + max_request_size: 20, max_batch_size: 1, max_text_size: nil, + html: :none, notranslate: false, detects_language: false, reports_billing: false + ) + end + + attr_reader :batches + + def initialize(config) + super + @batches = [] + end + + def translate(request) + @batches << request.texts + TranslationDiff::Translation::Response.build(request: request, texts: request.texts) + end + + def cache_key = "narrow" + end + + provider = narrow.new(TranslationDiff::Configuration.new) + TranslationDiff.translate("One. Two. Three.", from: "en", to: "ru", provider: provider) + assert(provider.batches.all? { |batch| batch.size == 1 }, + "expected one text per request, got #{provider.batches.inspect}") + end + + # The old check was `respond_to?(:detect)`, which a provider could satisfy + # by inheriting the base class's raising stub. + def test_a_provider_that_cannot_detect_says_so_before_it_is_called error = assert_raises(TranslationDiff::Request::Error) do - TranslationDiff::Request.new("text", to: :ru).call + TranslationDiff.translate("Some text.", to: "ru", provider: :null) end assert_match(/cannot detect/, error.message) + assert_match(/null/, error.message) end + # The count check used to live in Request#call_api. It now lives in + # Translation::Response.build, which is why the error is a ResponseError + # rather than a Request::Error: it holds for every provider, including one + # that overrides #translate outright instead of using the HTTP seams. def test_raises_when_the_api_returns_fewer_translations_than_asked_for configure_with(FakeApi.new(%w[Один])) - error = assert_raises(TranslationDiff::Request::Error) do + error = assert_raises(TranslationDiff::ResponseError) do TranslationDiff::Request.new({ a: "One", b: "Two" }, from: :en, to: :ru).call end @@ -246,9 +292,9 @@ def test_passes_nested_scalars_through_and_still_blanks_out_nils end # Proves the generalisation took effect rather than merely being - # described: an adapter declaring tiny limits must change the batching. + # described: a provider declaring tiny limits must change the batching. def test_batches_according_to_the_limits_the_adapter_declares - api = FakeApi.new(%w[Один Два], max_batch_size: 1) + api = NarrowBatchApi.new(%w[Один Два]) configure_with(api) TranslationDiff::Request.new({ a: "One", b: "Two" }, from: :en, to: :ru).call From bff01a73c1468f826237724c0eccf0c9b5ff5d13 Mon Sep 17 00:00:00 2001 From: IG Date: Wed, 9 Sep 2026 19:37:36 +0400 Subject: [PATCH 16/26] Compress multi-line comments to a single line Every multi-line comment block in lib/ and test/ is reduced to at most one line, keeping the load-bearing fact (a decision, a vendor quirk, a measured observation) and dropping restated prose and usage examples. --- lib/translation_diff.rb | 13 +- lib/translation_diff/cache.rb | 21 +-- lib/translation_diff/capabilities.rb | 10 +- lib/translation_diff/chunker.rb | 8 +- lib/translation_diff/configuration.rb | 61 +------- .../configuration/provider_option_owners.rb | 26 +--- lib/translation_diff/context.rb | 4 +- lib/translation_diff/error.rb | 4 +- lib/translation_diff/errors.rb | 22 +-- lib/translation_diff/http_provider.rb | 38 +---- lib/translation_diff/instrumentation.rb | 19 +-- lib/translation_diff/memory_cache_store.rb | 11 +- lib/translation_diff/provider.rb | 37 +---- lib/translation_diff/providers.rb | 18 +-- lib/translation_diff/providers/amazon.rb | 39 +----- lib/translation_diff/providers/azure.rb | 15 +- lib/translation_diff/providers/deepl.rb | 24 +--- lib/translation_diff/providers/google.rb | 20 +-- .../providers/libretranslate.rb | 23 +-- lib/translation_diff/providers/modernmt.rb | 15 +- lib/translation_diff/providers/null.rb | 6 +- lib/translation_diff/redis_cache_store.rb | 3 +- lib/translation_diff/redis_rate_limiter.rb | 13 +- lib/translation_diff/registry.rb | 11 +- lib/translation_diff/request.rb | 53 +------ lib/translation_diff/segmenters/pragmatic.rb | 132 ++---------------- lib/translation_diff/segmenters/simple.rb | 45 ++---- lib/translation_diff/stores.rb | 3 +- lib/translation_diff/tokenizer.rb | 14 +- lib/translation_diff/translation/request.rb | 4 +- lib/translation_diff/translation/response.rb | 9 +- lib/translation_diff/translation/usage.rb | 6 +- test/support/cache_store_contract.rb | 3 +- test/support/http_provider_contract.rb | 4 +- test/support/provider_contract.rb | 8 +- test/support/stubbed_provider.rb | 16 +-- test/test_helper.rb | 6 +- test/translation_diff/cache_test.rb | 27 ++-- test/translation_diff/capabilities_test.rb | 7 +- test/translation_diff/chunker_test.rb | 7 +- test/translation_diff/configuration_test.rb | 62 ++------ test/translation_diff/context_test.rb | 3 +- test/translation_diff/errors_test.rb | 3 +- test/translation_diff/golden_rules_test.rb | 25 +--- test/translation_diff/http_provider_test.rb | 15 +- test/translation_diff/instrumentation_test.rb | 15 +- test/translation_diff/provider_test.rb | 8 +- .../translation_diff/providers/amazon_test.rb | 19 +-- test/translation_diff/providers/azure_test.rb | 18 +-- test/translation_diff/providers/deepl_test.rb | 16 +-- .../translation_diff/providers/google_test.rb | 14 +- .../providers/libretranslate_test.rb | 9 +- .../providers/modernmt_test.rb | 14 +- test/translation_diff/providers_test.rb | 49 ++----- .../redis_cache_store_test.rb | 18 +-- .../redis_rate_limiter_test.rb | 25 +--- test/translation_diff/request_test.rb | 74 +++------- .../segmenters/pragmatic_test.rb | 81 ++--------- .../segmenters/simple_test.rb | 7 +- test/translation_diff/tokenizer_test.rb | 13 +- .../translation/response_test.rb | 5 +- 61 files changed, 237 insertions(+), 1061 deletions(-) diff --git a/lib/translation_diff.rb b/lib/translation_diff.rb index 994916c..95b4142 100644 --- a/lib/translation_diff.rb +++ b/lib/translation_diff.rb @@ -52,20 +52,13 @@ def config = @config ||= Configuration.new def configure = yield(config) - # Tests need this, and without it one test's configuration leaks into - # every test that runs after it. + # Without this, one test's configuration leaks into every test that runs after it. def reset! = @config = nil - # An isolated copy of the configuration with the same entry point, for - # per-tenant or per-request settings. The global configuration is left - # alone. - # - # tenant = TranslationDiff.context { |c| c.deepl_api_key = key } - # tenant.translate("Hello.", from: "en", to: "ru") + # An isolated copy of the configuration with the same entry point, for per-tenant settings. def context(&) = Context.new(config.copy.tap(&)) - # `provider:` and `config:` are reserved; every other keyword is - # forwarded to the provider untouched. + # `provider:` and `config:` are reserved; every other keyword is forwarded to the provider. def translate(values, from: nil, to: nil, provider: nil, **) Request.new(values, from: from, to: to, provider: provider, config: config, **).call end diff --git a/lib/translation_diff/cache.rb b/lib/translation_diff/cache.rb index 1f00872..15107fd 100644 --- a/lib/translation_diff/cache.rb +++ b/lib/translation_diff/cache.rb @@ -3,14 +3,10 @@ class TranslationDiff::Cache class Error < TranslationDiff::Error; end - # An application uses a handful of distinct option sets, so 32 bits of - # digest is ample to keep them apart; the full 128-bit MD5 would just - # bloat every key in a cache that may hold millions of them. + # 32 bits of digest is ample to keep option sets apart without bloating every key in the cache. DIGEST_LENGTH = 8 - # `store` is the cache store this instance reads and writes through. It has - # no reader: #store is already the public method that writes a chunk of - # translations back, and an attr_reader would silently replace it. + # No attr_reader for `store`: #store is already the public method that writes translations back. def initialize(from, to, provider:, store:, options: {}) @from = from @to = to @@ -48,26 +44,19 @@ def key(value) [provider, language(from), language(to), options_digest, hash].compact.join(":") end - # "EN" and :en are the same language; without this they are two entries - # for identical work. The collision argument for #key depends on none of - # its segments containing a colon: from/to are caller-supplied, so this - # normalisation must not introduce one. + # "EN" and :en are the same language; also must never introduce a colon, the key-join separator. def language(code) code.to_s.downcase end - # Two calls differing only in formality or glossary are two different - # translations and must not share a key. + # Two calls differing only in formality or glossary must not share a key. def options_digest return @options_digest if defined?(@options_digest) @options_digest = options.empty? ? nil : Digest::MD5.hexdigest(canonical(options))[0, DIGEST_LENGTH] end - # Object#inspect is not a stable serialisation: Ruby 3.4 changed how - # symbol-keyed hashes render, and an object without its own #inspect embeds - # a memory address. Either would silently change every cache key and make - # the application pay for every translation a second time. + # Object#inspect isn't stable: Ruby 3.4 changed hash rendering, and default #inspect embeds an address. def canonical(value) case value when Hash then value.sort_by { |key, _| key.to_s }.map { |key, item| "#{key}=#{canonical(item)}" }.join(",") diff --git a/lib/translation_diff/capabilities.rb b/lib/translation_diff/capabilities.rb index e4401e9..d2a0a23 100644 --- a/lib/translation_diff/capabilities.rb +++ b/lib/translation_diff/capabilities.rb @@ -1,14 +1,6 @@ # frozen_string_literal: true -# What one provider can do and how much it will accept, declared rather than -# discovered. Before this existed, `max_batch_size` was a method, "can it -# detect a language" was `respond_to?(:detect)`, and "does it honour -# notranslate" was not expressed anywhere -- which is how two providers -# shipped with notranslate silently broken. -# -# `html` holds the name of the provider option that turns HTML handling on, -# because every vendor spells it differently (`tag_handling`, `format`, -# `textType`), or :none when the provider has no HTML mode at all. +# Declared, not discovered -- duck-typing left notranslate silently broken on two providers. TranslationDiff::Capabilities = Data.define(:max_request_size, :max_batch_size, :max_text_size, :html, :notranslate, :detects_language, :reports_billing) do diff --git a/lib/translation_diff/chunker.rb b/lib/translation_diff/chunker.rb index 0a12a73..155b506 100644 --- a/lib/translation_diff/chunker.rb +++ b/lib/translation_diff/chunker.rb @@ -5,9 +5,7 @@ class Error < TranslationDiff::Error; end Chunk = Struct.new(:texts, :escaped_size) - # No defaults. With one provider a default looks meaningful; with two it - # silently lies about the second, which is the class of bug this file - # already had once. + # No defaults: a default silently lies about the second provider, a bug this file already had once. def initialize(values, limit:, count_limit:) @values = values @limit = limit @@ -43,9 +41,7 @@ def next_chunk?(tail, value) tail.texts.size >= count_limit end - # What the limit is about is the size of the request that goes over the - # wire, so every measurement here is of the escaped form. Mixing it with - # String#size lets a chunk of non-ASCII text run several times over. + # The limit is on wire size, so measure the escaped form -- String#size lets non-ASCII text run over. def escaped_size(text) CGI.escape(text).size end diff --git a/lib/translation_diff/configuration.rb b/lib/translation_diff/configuration.rb index 20adb36..cecde2e 100644 --- a/lib/translation_diff/configuration.rb +++ b/lib/translation_diff/configuration.rb @@ -1,24 +1,6 @@ # frozen_string_literal: true -# Every setting this library has, declared in one place with its default. -# -# TranslationDiff.configure do |config| -# config.deepl_api_key = ENV["DEEPL_API_KEY"] -# config.redis_url = ENV["REDIS_URL"] -# end -# -# Options are declared with `option`, which generates a reader that falls back -# to the default and a writer that normalises a blank string to nil -- so an -# unset environment variable behaves as though the option was never touched. -# -# A default may be a literal or a callable. A callable is invoked on read, not -# at load time, so `-> { ENV["REDIS_URL"] }` reflects the environment when the -# value is needed rather than when this file was required. -# -# Provider-specific options such as `deepl_api_key` are NOT declared here. -# A provider declares its own and registers them through -# TranslationDiff::Providers.register, which keeps the core ignorant of any -# particular translation service. +# Every declared setting in one place; callable defaults are invoked on read, not at load time. class TranslationDiff::Configuration class << self def option(key, default = nil) @@ -35,10 +17,7 @@ def option(key, default = nil) options << key end - # Declares the options a provider needs, remembering which provider - # declared each one. See ProviderOptionOwners (configuration/ - # provider_option_owners.rb) for the conflict rules and the - # all-or-nothing guarantee. + # See ProviderOptionOwners for the conflict rules and the all-or-nothing guarantee. def register_provider_options(keys, provider) keys = Array(keys).map(&:to_sym) provider_option_owners.claim(keys, provider) @@ -71,18 +50,7 @@ def provider_option_owners = @provider_option_owners ||= ProviderOptionOwners.ne option :timeout, 30 option :max_retries, 3 - # Values are copied; memoised collaborators (`provider_instance`, - # `cache_store`, `segmenter_instance`, `rate_limiter_instance` and - # `redis_pool`) are deliberately not -- `copy` walks `self.class.options` - # only, which never includes those readers' instance variables, so a copy - # builds its own provider, store, rate limiter and connection pool from - # its own values instead of inheriting the original's. This matters for - # `rate_limiter_instance` in particular: a tenant context that sets its - # own `cache_namespace` must not be rate-limited against its parent's - # namespace just because the parent had already resolved a limiter before - # the copy was made. An object the caller assigned is an option value and - # is therefore shared -- which is correct: someone who hands us one - # connection pool means one connection pool. + # Memoised collaborators aren't copied, or a tenant's own cache_namespace leaks its parent's rate limiter. def copy self.class.new.tap do |other| self.class.options.each do |key| @@ -91,15 +59,11 @@ def copy end end - # The provider actually used. `provider` holds what the caller set -- a - # symbol or an object -- and this turns it into an instance once. def provider_instance @provider_instance ||= resolve(provider, TranslationDiff::Providers) end - # `cache` unset means "choose for me": Redis when a URL is configured, - # otherwise the in-process store, so the library works before anything is - # running. + # Unset `cache` means Redis when a URL is configured, otherwise in-process -- works before anything runs. def cache_store @cache_store ||= resolve(cache || (redis_url ? :redis : :memory), TranslationDiff::Stores) end @@ -108,18 +72,7 @@ def segmenter_instance @segmenter_instance ||= resolve(segmenter, TranslationDiff::Segmenters.registry) end - # `rate_limiter` holds what the caller set -- an object, or nil -- exactly - # like `provider`, `cache` and `segmenter` hold theirs. nil, not a null - # object: Request checks for nil and skips the whole rate-limiting path, - # which is the common case and should cost nothing. - # - # The assigned object when there is one; nil when no `rate_limit` - # threshold was ever configured; a RedisRateLimiter built from this - # config's own values otherwise. Memoised under its own instance - # variable, like `provider_instance`, `cache_store` and - # `segmenter_instance`, so a copy builds its own limiter from its own - # settings instead of inheriting one built for a different config's - # namespace or connection pool. + # nil, not a null object: Request checks for nil and skips rate-limiting entirely -- costs nothing normally. def rate_limiter_instance return rate_limiter unless rate_limiter.nil? return nil if rate_limit.nil? @@ -127,9 +80,7 @@ def rate_limiter_instance @rate_limiter_instance ||= TranslationDiff::RedisRateLimiter.build(self) end - # One pool for the cache store and the rate limiter both. Callers used to - # build this themselves and pass it to each, keeping the namespaces in step - # by hand. + # One pool shared by the cache store and the rate limiter; callers used to build and pass it by hand. def redis_pool @redis_pool ||= build_redis_pool end diff --git a/lib/translation_diff/configuration/provider_option_owners.rb b/lib/translation_diff/configuration/provider_option_owners.rb index df4f0f5..d0286e7 100644 --- a/lib/translation_diff/configuration/provider_option_owners.rb +++ b/lib/translation_diff/configuration/provider_option_owners.rb @@ -1,29 +1,12 @@ # frozen_string_literal: true -# Tracks which provider declared each provider-specific configuration option -# name, so `Configuration.option` -- which returns early on a name it -# already knows -- never lets two providers silently share one accessor. If -# it did, a credential set for one provider would be handed to the other; -# that is a packaging conflict a human has to resolve, so #claim raises and -# names both. -# -# The same provider re-declaring its own options is not a conflict: a -# defensive double `require` and a Rails development reload both re-run -# registration, and a reload yields a *new* class object for the same -# constant, which is why identity is not the only test. Nor is a subclass of -# the declaring provider a conflict -- a provider subclassed to point it at -# a different host or account is meant to share its parent's options. +# Tracks which provider declared each option; two silently sharing one accessor would leak a credential. class TranslationDiff::Configuration::ProviderOptionOwners def initialize @owners = {} end - # All-or-nothing: every key is checked for a conflict before any of them - # is recorded as owned. Checking and recording key by key would leave the - # options before a conflicting one already attributed to a provider that - # then failed to register -- so a later, legitimate registration of one - # of those names would be refused, blaming a provider that was never - # registered. + # All-or-nothing: recording key by key would attribute earlier keys to a provider that then failed. def claim(keys, provider) validate!(keys, provider) keys.each { |key| @owners[key] = provider } @@ -40,10 +23,7 @@ def available?(key, provider) owner.nil? || same_provider?(owner, provider) end - # `<=>` returns non-nil (0 for equal, -1/1 for either direction) exactly - # when `owner` and `provider` sit on one inheritance chain, and nil for - # two unrelated classes -- which is also how it answers "or is not a - # Module at all", since only Modules define `<=>` this way. + # `<=>` is non-nil exactly when both sit on one inheritance chain -- also true if `provider` isn't a Module. def same_provider?(owner, provider) owner.equal?(provider) || (!owner.name.nil? && owner.name == provider.name) || diff --git a/lib/translation_diff/context.rb b/lib/translation_diff/context.rb index 5440e6c..21fd9ce 100644 --- a/lib/translation_diff/context.rb +++ b/lib/translation_diff/context.rb @@ -1,8 +1,6 @@ # frozen_string_literal: true -# An isolated configuration scope offering the same entry point as the -# TranslationDiff module itself, for multi-tenant applications and -# per-request overrides. Created with TranslationDiff.context. +# An isolated configuration scope with the same entry point as TranslationDiff itself. class TranslationDiff::Context attr_reader :config diff --git a/lib/translation_diff/error.rb b/lib/translation_diff/error.rb index c231e62..7a49a39 100644 --- a/lib/translation_diff/error.rb +++ b/lib/translation_diff/error.rb @@ -1,6 +1,4 @@ # frozen_string_literal: true -# Common ancestor for every error this gem raises. Without it, a caller who -# wants to rescue anything TranslationDiff can throw has to list four -# unrelated classes; with it, `rescue TranslationDiff::Error` is enough. +# Common ancestor for every error this gem raises, so `rescue TranslationDiff::Error` is enough. class TranslationDiff::Error < StandardError; end diff --git a/lib/translation_diff/errors.rb b/lib/translation_diff/errors.rb index 4b48e52..a0fdf6a 100644 --- a/lib/translation_diff/errors.rb +++ b/lib/translation_diff/errors.rb @@ -1,16 +1,6 @@ # frozen_string_literal: true -# One hierarchy for everything that can go wrong, so a caller handles a rate -# limit the same way whichever provider raised it. -# -# The three branches answer three different questions. ConfigurationError -# means the caller set something up wrong and no request was made. -# ProviderError means the service answered and said no. TransportError means -# nobody answered. ResponseError means the answer was well-formed HTTP but -# broke this library's contract. -# -# No error carries the text being translated. Errors are logged, and this -# library handles other people's content. +# No error carries the text being translated -- errors are logged, and this library handles other people's content. module TranslationDiff class ConfigurationError < Error; end @@ -30,10 +20,7 @@ class InvalidRequestError < ProviderError; end class ServiceError < ProviderError; end class RateLimitError < ProviderError - # Seconds the provider asked us to wait, when it said so at all. Faraday's - # retry middleware honours the header itself; this is for a caller who - # rescues the error after the retries are exhausted and wants to schedule - # its own attempt. + # Faraday's retry middleware already honours this header; it's here for a caller scheduling its own retry. attr_reader :retry_after def initialize(message, provider: nil, status: nil, retry_after: nil) @@ -45,9 +32,6 @@ def initialize(message, provider: nil, status: nil, retry_after: nil) class TransportError < Error; end class ResponseError < Error; end - # Raised when a class is offered to a registry that requires a particular - # ancestor. Its own class, rather than the generic Error, so that a caller - # rescuing "this class is the wrong shape" cannot also swallow an unrelated - # failure such as an option-name collision. + # Its own class, not the generic Error, so rescuing "wrong shape" can't also swallow an option-name collision. class InvalidProviderError < Error; end end diff --git a/lib/translation_diff/http_provider.rb b/lib/translation_diff/http_provider.rb index 3fb4a6e..db884a2 100644 --- a/lib/translation_diff/http_provider.rb +++ b/lib/translation_diff/http_provider.rb @@ -4,19 +4,7 @@ require "faraday/retry" require "json" -# Every provider reached over HTTP inherits this. It owns one Faraday -# connection and turns HTTP status codes into this library's errors, so a -# caller handles a rate limit the same way whichever service produced it. -# -# A subclass supplies where to talk (#api_base, #headers) and three seams per -# operation: the URL, how to render a request, how to parse a reply. The -# seams are a convenience of this class, not a requirement of Provider -- -# Amazon signs its requests instead and overrides #translate outright. -# -# No logging middleware is installed, ever, and no logger is passed to -# Faraday. This library's log lines carry no source text, no translation and -# no credential; a request logger would carry all three, and it would do so -# at exactly the moment someone turns DEBUG on to diagnose a problem. +# Every HTTP provider inherits this; no logging middleware, ever -- lines must carry no source text or credential. class TranslationDiff::HTTPProvider < TranslationDiff::Provider RETRY_STATUSES = [429, 500, 502, 503, 504].freeze @@ -45,10 +33,7 @@ def connection = @connection ||= build_connection private - # What #post hands back: a decoded body next to the headers it arrived - # with, so #raise_for_status! and a subclass's #parse_translate_response - # both see the same shape a Faraday::Response would have given them had - # its own JSON middleware still been in the stack. + # Mirrors the shape a Faraday::Response would give if its own JSON middleware were still in the stack. Decoded = Data.define(:status, :headers, :body) private_constant :Decoded @@ -58,15 +43,11 @@ def post(url, payload) raise_for_status!(response) response rescue *TRANSPORT_FAILURES => e - # The message is the transport's, never the payload's: the payload is the - # customer's text. + # The message is the transport's, never the payload's: the payload is the customer's text. raise TranslationDiff::TransportError, "#{self.class}: #{e.class}: #{e.message}" end - # Faraday's response-JSON middleware passes parser options positionally, - # which json 3 removed -- and json 3 is the default gem on Ruby 4.x, so - # relying on that middleware would break this library for most modern - # applications. Decoding here costs one call and depends on nothing. + # Faraday's JSON middleware passes parser options positionally, which json 3 (default on Ruby 4.x) removed. def decode(response) body = response.body return body unless body.is_a?(String) @@ -81,9 +62,7 @@ def decode(response) def json?(response) = response.headers["content-type"].to_s.match?(/\bjson\b/) - # The block is how a test swaps in Faraday's test adapter. Amazon overrides - # this with the same signature, because its signature covers the body - # exactly as sent and a JSON request middleware would re-encode it. + # The block is how a test swaps in Faraday's test adapter; Amazon overrides it too, to sign the body as sent. def build_connection(&block) Faraday.new(url: api_base, headers: headers) do |faraday| faraday.request :json @@ -102,9 +81,7 @@ def apply_timeouts(faraday) faraday.options.timeout = config.timeout end - # faraday-retry reads Retry-After itself, which is why a 429 usually never - # reaches #raise_for_status!. What is left when it does is a service that - # kept saying no for every attempt. + # faraday-retry reads Retry-After itself, which is why a 429 usually never reaches #raise_for_status!. def retry_options { max: config.max_retries, interval: 0.5, backoff_factor: 2, interval_randomness: 0.5, retry_statuses: RETRY_STATUSES, methods: %i[post get], @@ -135,8 +112,7 @@ def error_options(response) options.merge(retry_after: response.headers["Retry-After"]&.to_i) end - # The service's own words, truncated. A provider's error body is a - # diagnostic, and an untruncated one can be a whole HTML error page. + # Truncated: an untruncated provider error body can be a whole HTML error page. def error_message(response) body = response.body text = body.is_a?(Hash) ? (body["message"] || body["error"] || body.to_s) : body.to_s diff --git a/lib/translation_diff/instrumentation.rb b/lib/translation_diff/instrumentation.rb index c4522c7..85cc722 100644 --- a/lib/translation_diff/instrumentation.rb +++ b/lib/translation_diff/instrumentation.rb @@ -1,26 +1,13 @@ # frozen_string_literal: true -# Emits events to whatever the application configured as its instrumenter -- -# ActiveSupport::Notifications satisfies the interface as-is. When nothing is -# configured, #instrument yields (if given a block) and returns, so call -# sites never branch on whether instrumentation is on. -# -# Payloads carry counts, language codes and provider names. They never carry -# the text being translated, its translation, or a credential: this library -# handles other people's content, and an instrumenter usually writes -# somewhere that content must not go. +# Payloads carry counts, language codes and provider names -- never the text, its translation, or a credential. module TranslationDiff::Instrumentation SUFFIX = ".translation_diff" - # Both methods are private: they are internal plumbing for the class that - # includes this module, not part of its public surface. `include` ignores - # the includer's own `private` keyword, so the visibility has to be - # declared here. + # `include` ignores the includer's own `private` keyword, so visibility has to be declared here. private - # A point event (no block) reports a fact that already happened -- a cache - # hit/miss tally, say -- rather than wrapping work, so it only yields when - # a block was actually given. + # A point event (no block) reports a fact that already happened, e.g. a cache hit/miss tally. def instrument(name, payload = {}) instrumenter = config.instrumenter return yield if instrumenter.nil? && block_given? diff --git a/lib/translation_diff/memory_cache_store.rb b/lib/translation_diff/memory_cache_store.rb index 69844a9..93bd3aa 100644 --- a/lib/translation_diff/memory_cache_store.rb +++ b/lib/translation_diff/memory_cache_store.rb @@ -1,15 +1,6 @@ # frozen_string_literal: true -# The default cache: a bounded LRU in the current process, so the library -# works the moment it is required and without Redis running. -# -# Ruby hashes keep insertion order, so "least recently used" is delete and -# reinsert on every touch, and eviction is a shift of the first pair. -# -# NOT thread-safe, and deliberately so -- a lock here would be a tax on the -# single-threaded case to make the multi-threaded one merely less wrong. A -# process that needs a cache shared between threads or machines sets -# `redis_url` and gets TranslationDiff::RedisCacheStore instead. +# The default cache, a bounded in-process LRU. NOT thread-safe, deliberately -- set `redis_url` for that. class TranslationDiff::MemoryCacheStore def self.build(config) = new(max_size: config.cache_max_size) diff --git a/lib/translation_diff/provider.rb b/lib/translation_diff/provider.rb index 4ea093c..204b9f0 100644 --- a/lib/translation_diff/provider.rb +++ b/lib/translation_diff/provider.rb @@ -1,28 +1,8 @@ # frozen_string_literal: true -# A Provider connects this library to one translation service. It knows where -# to talk, who it is, and what it can do. It knows nothing about HTTP -- that -# is HTTPProvider, which most providers inherit instead. Inheriting Provider -# directly is for services reached some other way: Amazon, whose requests are -# signed rather than merely headed, and an LLM-backed provider that delegates -# to another gem. -# -# Subclass, declare the options you need, then register: -# -# class Acme < TranslationDiff::Provider -# def self.configuration_options = %i[acme_api_key] -# def self.configuration_requirements = %i[acme_api_key] -# def self.capabilities = TranslationDiff::Capabilities.new(...) -# -# def translate(request) = TranslationDiff::Translation::Response.build(...) -# end -# -# TranslationDiff::Providers.register(:acme, Acme) +# Connects this library to one translation service; knows nothing about HTTP itself -- that's HTTPProvider. class TranslationDiff::Provider - # What a provider can do when it says nothing: the least capable thing that - # can still translate. A subclass that forgets to declare its capabilities - # therefore under-promises rather than over-promises -- the failure mode is - # smaller batches, not a rejected request or silently unprotected content. + # A subclass that forgets to declare capabilities under-promises, not over-promises: smaller batches, not silent risk. DEFAULT_CAPABILITIES = TranslationDiff::Capabilities.new( max_request_size: 1_000, max_batch_size: 1, max_text_size: nil, html: :none, notranslate: false, detects_language: false, reports_billing: false @@ -38,17 +18,12 @@ def initialize(config) ensure_configured! end - # Translate a Translation::Request, return a Translation::Response. def translate(_request) = raise NotImplementedError, "#{self.class} must implement #translate" - # Return the source language of a sample of text, lowercased. Only called - # when `capabilities.detects_language?`. + # Only called when `capabilities.detects_language?`. def detect(_text) = raise NotImplementedError, "#{self.class} must implement #detect" - # The segment of every cache key that keeps one provider's translations from - # being served for another. Raising when the provider was never stamped -- - # rather than falling back to "" -- is deliberate: an empty segment would - # merge two providers' namespaces silently. + # Raising when never stamped, rather than falling back to "", is deliberate: "" would merge namespaces silently. def cache_key return name.to_s unless name.nil? @@ -61,9 +36,7 @@ def cache_key class << self def configuration_options = [] - # The subset of configuration_options without which this provider cannot - # work. Checked once, at build time, so a caller learns what to set before - # any request is attempted rather than from a vendor's own exception. + # Checked once, at build time, so a caller learns what to set before a vendor's own exception does. def configuration_requirements = [] def capabilities = DEFAULT_CAPABILITIES diff --git a/lib/translation_diff/providers.rb b/lib/translation_diff/providers.rb index 9826d16..207ace5 100644 --- a/lib/translation_diff/providers.rb +++ b/lib/translation_diff/providers.rb @@ -1,23 +1,9 @@ # frozen_string_literal: true -# Translation providers, by name. This is the one registry that does more -# than look a class up: registering a provider also declares that provider's -# configuration options, which is what keeps names like `deepl_api_key` out -# of TranslationDiff::Configuration and out of this library's core. -# -# TranslationDiff::Providers.register(:acme, AcmeProvider) -# -# TranslationDiff.configure do |config| -# config.provider = :acme -# config.acme_api_key = ENV["ACME_API_KEY"] -# end -# -# Nothing in lib/ changes to make that work. +# Translation providers, by name; registering one also declares its configuration options. module TranslationDiff::Providers class << self - # Options are declared before the registry entry is written, so a - # provider whose option names collide with another's raises without - # having replaced anything under `name`. + # Options are declared before the registry entry is written, so a name collision raises without replacing. def register(name, klass) unless klass < TranslationDiff::Provider raise TranslationDiff::InvalidProviderError, diff --git a/lib/translation_diff/providers/amazon.rb b/lib/translation_diff/providers/amazon.rb index 70a2428..c57f1f2 100644 --- a/lib/translation_diff/providers/amazon.rb +++ b/lib/translation_diff/providers/amazon.rb @@ -1,26 +1,12 @@ # frozen_string_literal: true -# Amazon Translate. The odd one out of this set in three ways, all of which -# the capabilities declare rather than hide: -# -# - It translates one text per call. There is no batch form of TranslateText, -# so a hundred sentences are a hundred requests. `max_batch_size: 1` makes -# Chunker produce one text per chunk, which is correct and slow. -# - It has no HTML mode, so a notranslate span sent to it is translated like -# any other text. `notranslate: false` is what lets the rest of the library -# warn instead of discovering it in production. -# - Its requests are signed rather than merely headed, which is why this -# class overrides #translate instead of filling in the usual seams, and -# why it overrides #build_connection to drop the JSON request middleware: -# the signature covers the body exactly as sent, so nothing may re-encode -# it afterwards. +# Amazon Translate: no batch API (one text per call), no HTML mode, and requests are signed, not just headed. class TranslationDiff::Providers::Amazon < TranslationDiff::HTTPProvider SERVICE = "translate" TARGET = "AWSShineFrontendService_20170701.TranslateText" CONTENT_TYPE = "application/x-amz-json-1.1" - # Amazon's own way of asking for detection. It reaches Amazon Comprehend - # under the hood and is only available in regions that have it. + # Amazon's own way of asking for detection; reaches Comprehend under the hood, in regions that have it. AUTO = "auto" def self.capabilities @@ -41,9 +27,7 @@ def self.configuration_requirements def api_base = config.amazon_api_base || "https://#{SERVICE}.#{config.amazon_region}.amazonaws.com" - # One request per text, in order. The response's detected language is the - # first one Amazon reported: every text in a chunk comes from the same - # document, so they share a source language. + # Detected language is the first one Amazon reported: every text in a chunk shares a source language. def translate(request) detected = nil texts = request.texts.map do |text| @@ -81,20 +65,14 @@ def post_signed(body) raise_for_status!(response) response rescue *TRANSPORT_FAILURES => e - # The message is the transport's, never the payload's: the payload is the - # customer's text. + # The message is the transport's, never the payload's: the payload is the customer's text. raise TranslationDiff::TransportError, "#{self.class}: #{e.class}: #{e.message}" end - # Reuses the base's own decoding (`decode`, `Decoded`, `json?`) rather than - # a second, Faraday-middleware-based path: that middleware is exactly what - # HTTPProvider's own #post avoids, since it breaks under the `json` 3 gem - # that ships by default on Ruby 4.x. + # Reuses the base's own decoding rather than a second, Faraday-middleware-based path (see HTTPProvider#decode). def decoded_response(raw) = Decoded.new(status: raw.status, headers: raw.headers, body: decode(raw)) - # aws-sigv4 is Amazon's own signing library and nothing more: no clients, no - # service models, one dependency. It is required here rather than at load - # time so an application using another provider never needs it installed. + # Required here, not at load time, so an application using another provider never needs it installed. def signer @signer ||= begin require_sigv4 @@ -124,10 +102,7 @@ def signed_headers(body) signature.headers.merge("Content-Type" => CONTENT_TYPE, "X-Amz-Target" => TARGET) end - # The signature covers the body exactly as sent, so this connection must - # not have a JSON request middleware re-encoding it afterwards -- unlike - # the base class's #build_connection, this one omits `faraday.request - # :json`. + # The signature covers the body exactly as sent, so this omits `faraday.request :json` unlike the base class. def build_connection(&block) Faraday.new(url: api_base, headers: headers) do |faraday| faraday.request :retry, retry_options diff --git a/lib/translation_diff/providers/azure.rb b/lib/translation_diff/providers/azure.rb index ecc6f3a..b6dfc57 100644 --- a/lib/translation_diff/providers/azure.rb +++ b/lib/translation_diff/providers/azure.rb @@ -1,15 +1,11 @@ # frozen_string_literal: true -# Azure AI Translator, REST v3.0. The cheapest of the paid services per -# character and the most generous per request: a thousand strings and fifty -# thousand characters at a time. +# Azure AI Translator, REST v3.0: cheapest per character, most generous per request (1,000 strings/50,000 chars). class TranslationDiff::Providers::Azure < TranslationDiff::HTTPProvider HOST = "https://api.cognitive.microsofttranslator.com" API_VERSION = "3.0" - # Azure spells HTML handling `textType`, and under it honours - # `class=notranslate` -- the same marker the tokenizer emits and the same - # one Google and DeepL honour under their own spellings. + # Azure spells HTML handling `textType`, and under it honours `class=notranslate` like DeepL and Google do. DEFAULT_TEXT_TYPE = "html" def self.capabilities @@ -24,16 +20,13 @@ def self.configuration_requirements = %i[azure_api_key] def api_base = config.azure_api_base || HOST - # A multi-service resource needs the region header and a single-service one - # rejects nothing without it, so it is sent only when configured. + # A multi-service resource needs the region header; a single-service one rejects nothing without it. def headers { "Ocp-Apim-Subscription-Key" => config.azure_api_key.to_s } .tap { |h| h["Ocp-Apim-Subscription-Region"] = config.azure_region if config.azure_region } end - # Azure takes the language pair in the query string and the texts in the - # body, which is why this provider builds its URL per request rather than - # answering a constant. + # Azure takes the language pair in the query string, so the URL is built per request, not a constant. def translate_url = "translate" def translate(request) diff --git a/lib/translation_diff/providers/deepl.rb b/lib/translation_diff/providers/deepl.rb index 2b7b3dd..d46b8c6 100644 --- a/lib/translation_diff/providers/deepl.rb +++ b/lib/translation_diff/providers/deepl.rb @@ -1,27 +1,17 @@ # frozen_string_literal: true -# Talks to DeepL's REST API directly. This used to wrap deepl-rb; owning the -# request removed a dependency and, more to the point, removed a layer whose -# defaults were not ours -- deepl-rb logs the auth key and the payload at -# DEBUG, and its tag handling default silently disabled notranslate. +# Talks to DeepL's REST API directly, not deepl-rb: it logged the auth key at DEBUG and defaulted notranslate off. class TranslationDiff::Providers::DeepL < TranslationDiff::HTTPProvider PAID_HOST = "https://api.deepl.com" FREE_HOST = "https://api-free.deepl.com" - # A key ending in :fx is a free-plan key, and the free plan lives on its - # own host. DeepL's own libraries do this; so do we, now. + # A key ending in :fx is a free-plan key, and the free plan lives on its own host. FREE_KEY_SUFFIX = ":fx" - # What arrives here is not plain text: the tokenizer hands over a - # notranslate span with its tags. DeepL honours class="notranslate" only - # under HTML tag handling; without it, in DeepL's words, "tags are treated - # as regular text", and the protected content is translated while the tags - # survive -- a failure nothing about the output reveals. + # DeepL honours class="notranslate" only under HTML tag handling -- otherwise content translates, tags survive. DEFAULT_OPTIONS = { tag_handling: :html, tag_handling_version: "v2" }.freeze - # 50 texts and a 128 KiB body are DeepL's documented per-request limits. - # The request size stays at the 1700 escaped characters this library has - # always used; the batch count is the number that was wrong (it said 300). + # 50 texts / 128 KiB are DeepL's documented per-request limits; max_batch_size was wrong before (it said 300). def self.capabilities TranslationDiff::Capabilities.new( max_request_size: 1_700, max_batch_size: 50, max_text_size: nil, @@ -32,8 +22,7 @@ def self.capabilities def self.configuration_options = %i[deepl_api_key deepl_api_base] def self.configuration_requirements = %i[deepl_api_key] - # DeepL requires a target language even when only the detection is wanted, - # so the provider picks one rather than making the caller do it. + # DeepL requires a target language even when only detection is wanted, so the provider picks one. DETECTION_TARGET = "EN" def api_base @@ -62,8 +51,7 @@ def parse_translate_response(body, _headers, request) ) end - # DeepL has no detection endpoint. Translating a sample and reading what it - # says the source was is the only way, and is what this has always done. + # DeepL has no detection endpoint; translating a sample and reading the source it reports is the only way. def detect(text) request = TranslationDiff::Translation::Request.new(texts: [text], from: nil, to: DETECTION_TARGET) diff --git a/lib/translation_diff/providers/google.rb b/lib/translation_diff/providers/google.rb index 34a2225..a8be8b5 100644 --- a/lib/translation_diff/providers/google.rb +++ b/lib/translation_diff/providers/google.rb @@ -1,22 +1,13 @@ # frozen_string_literal: true -# Talks to Cloud Translation v2 (Basic) directly. This used to wrap -# google-cloud-translate-v2, which pulled googleauth, signet, os, -# google-protobuf and grpc in order to send one POST with a key in the query -# string. +# Talks to Cloud Translation v2 directly, not google-cloud-translate-v2, which pulled in grpc for one POST. class TranslationDiff::Providers::Google < TranslationDiff::HTTPProvider HOST = "https://translation.googleapis.com" - # Google's own default, and what the tokenizer's output requires: a - # notranslate span arrives with its tags, and entities such as & stay - # in the text. Asking for `text` makes Google translate the protected span - # and drop its markup -- verified against the live API. + # Verified against the live API: `text` format translates the protected span and drops its markup. DEFAULT_FORMAT = :html - # Google's documented limits: 128 strings per request, and a recommended - # 5,000 characters (the hard ceiling is 100 KB). Chunker measures the - # URL-escaped form, never smaller than the UTF-8 byte count, so a chunk - # inside 5,000 escaped characters is inside it in bytes too. + # Google's documented limits: 128 strings/request, 5,000 chars recommended (hard ceiling 100 KB). def self.capabilities TranslationDiff::Capabilities.new( max_request_size: 5_000, max_batch_size: 128, max_text_size: nil, @@ -27,10 +18,7 @@ def self.capabilities def self.configuration_options = %i[google_api_key google_project_id google_api_base] def self.configuration_requirements = %i[google_api_key] - # A bare alphabetic code is downcased, so a configuration written for DeepL - # ("EN") keeps working. Anything carrying a subtag ("zh-Hans", "pt-BR") is - # passed through untouched: the casing of a script or region subtag is its - # own, and a blanket downcase would corrupt it. + # A bare code is downcased for DeepL-style configs ("EN"); a subtag ("zh-Hans") is passed through untouched. BARE_LANGUAGE_CODE = /\A[A-Za-z]{2,3}\z/ def api_base = config.google_api_base || HOST diff --git a/lib/translation_diff/providers/libretranslate.rb b/lib/translation_diff/providers/libretranslate.rb index 2b03b2c..01d59f1 100644 --- a/lib/translation_diff/providers/libretranslate.rb +++ b/lib/translation_diff/providers/libretranslate.rb @@ -1,31 +1,16 @@ # frozen_string_literal: true -# LibreTranslate: open source, self-hosted, and the only provider here that -# can be run against for free, which is why it is worth supporting even -# though its translations are not the best of this set. -# -# It inverts the usual configuration: the base URL is required, because -# everyone runs their own instance, and the API key is optional, because most -# instances do not ask for one. +# The only free, self-hosted provider here; base URL is required (everyone runs their own), API key is optional. class TranslationDiff::Providers::LibreTranslate < TranslationDiff::HTTPProvider DEFAULT_FORMAT = "html" - # The API's own way of asking for detection: `source` is required and - # "auto" is the value that means "work it out". + # The API's own way of asking for detection: `source` is required, and "auto" means "work it out". AUTO = "auto" - # Observed, not assumed: probed 2026-09-09 against `docker run - # libretranslate/libretranslate --load-only en,ru` (the argos-translate - # en->ru model). `Bold Mountain is a good - # place.` came back with the span tag intact but its content translated - # anyway -- "Bold Mountain" became "Смелая гора". LibreTranslate's HTML - # format preserves markup; it does not honour the notranslate marker. + # Observed 2026-09-09 via Docker: LibreTranslate's HTML format preserves markup but translates content anyway. LIBRETRANSLATE_HONOURS_NOTRANSLATE = false - # LibreTranslate publishes no per-request limits -- it is whatever the - # instance operator configured. These are this library's own conservative - # numbers, not the vendor's, and a self-hoster with a bigger instance can - # raise them by subclassing. + # LibreTranslate publishes no per-request limits; these are this library's own conservative numbers. def self.capabilities TranslationDiff::Capabilities.new( max_request_size: 5_000, max_batch_size: 50, max_text_size: nil, diff --git a/lib/translation_diff/providers/modernmt.rb b/lib/translation_diff/providers/modernmt.rb index 2c5d055..55b675d 100644 --- a/lib/translation_diff/providers/modernmt.rb +++ b/lib/translation_diff/providers/modernmt.rb @@ -1,22 +1,16 @@ # frozen_string_literal: true -# ModernMT. Adaptive translation with translation memories, which is -# thematically the closest of these services to what this library does. +# ModernMT: adaptive translation with translation memories. class TranslationDiff::Providers::ModernMT < TranslationDiff::HTTPProvider HOST = "https://api.modernmt.com" # ModernMT spells its formats as MIME types. DEFAULT_FORMAT = "text/html" - # Unverified. ModernMT documents an HTML format but says nothing about - # class="notranslate", and no key was available to probe it. Declaring - # false is the safe direction: a capability that under-promises costs a - # warning, one that over-promises costs a customer's protected content. + # Unverified, not observed: no key was available to probe it; false is the safe assumption either way. MODERNMT_HONOURS_NOTRANSLATE = false - # 128 texts is documented. The per-request character limit is not, so the - # conservative 5,000 Google recommends is used rather than a number nobody - # published. + # 128 texts is documented; the character limit is not, so Google's 5,000 recommendation is borrowed. def self.capabilities TranslationDiff::Capabilities.new( max_request_size: 5_000, max_batch_size: 128, max_text_size: nil, @@ -39,8 +33,7 @@ def render_translate_payload(request) .tap { |payload| payload[:source] = request.from.to_s unless request.from.nil? } end - # One text comes back as an object rather than a one-element array, so the - # envelope is always coerced to a list before it is mapped. + # One text comes back as an object rather than a one-element array, so the envelope is always coerced. def parse_translate_response(body, _headers, request) results = results_from(body) diff --git a/lib/translation_diff/providers/null.rb b/lib/translation_diff/providers/null.rb index fd6cd80..f6e6fe4 100644 --- a/lib/translation_diff/providers/null.rb +++ b/lib/translation_diff/providers/null.rb @@ -1,10 +1,8 @@ # frozen_string_literal: true -# Hands back what it was given. For tests, and for wiring a pipeline up -# before a real provider is available. +# Hands back what it was given -- for tests, and for wiring a pipeline up before a real provider is available. class TranslationDiff::Providers::Null < TranslationDiff::Provider - # Deliberately not detecting: detection is optional in the contract, and - # this is the provider that proves the optional branch works. + # Deliberately not detecting: this is the provider that proves the optional branch works. def self.capabilities TranslationDiff::Capabilities.new( max_request_size: 1_000_000, max_batch_size: 1_000_000, max_text_size: nil, diff --git a/lib/translation_diff/redis_cache_store.rb b/lib/translation_diff/redis_cache_store.rb index 576a744..b9f89aa 100644 --- a/lib/translation_diff/redis_cache_store.rb +++ b/lib/translation_diff/redis_cache_store.rb @@ -8,8 +8,7 @@ def self.build(config) new(config.redis_pool, timeout: config.cache_ttl, namespace: config.cache_namespace) end - # `connection_pool` is anything answering to #with, and what it yields is - # anything Redis::Namespace accepts. Neither gem is a dependency of this one. + # `connection_pool` is duck-typed to #with; neither connection_pool nor redis-namespace is a hard dependency. def initialize(connection_pool, timeout: ONE_WEEK, namespace: DEFAULT_NAMESPACE) @connection_pool = connection_pool @timeout = timeout diff --git a/lib/translation_diff/redis_rate_limiter.rb b/lib/translation_diff/redis_rate_limiter.rb index c70f64c..8004ac6 100644 --- a/lib/translation_diff/redis_rate_limiter.rb +++ b/lib/translation_diff/redis_rate_limiter.rb @@ -7,9 +7,7 @@ class RateLimitExceeded < TranslationDiff::Error; end DEFAULT_INTERVAL = 60 DEFAULT_NAMESPACE = "translation-diff" - # Ratelimit counts per subject. This library limits the provider as a - # whole rather than per caller, so there is exactly one subject and it - # only has to be stable. + # This library limits the provider as a whole rather than per caller, so there is exactly one subject. SUBJECT = "call" def self.build(config) @@ -19,8 +17,7 @@ def self.build(config) namespace: config.cache_namespace) end - # `connection_pool` is anything answering to #with, and what it yields is - # anything Ratelimit accepts. Neither gem is a dependency of this one. + # `connection_pool` is duck-typed to #with; neither connection_pool nor ratelimit is a hard dependency. def initialize(connection_pool, threshold: DEFAULT_THRESHOLD, interval: DEFAULT_INTERVAL, @@ -46,11 +43,7 @@ def check(size) attr_reader :connection_pool, :threshold, :interval, :namespace - # `ratelimit` is not a dependency of this gem, so it is required here, at - # the first check, rather than at load time -- an application that - # configures no `rate_limit` never needs it installed. Naming the bare - # constant instead surfaced its absence as a raw NameError; this raises the - # same "add this gem" TranslationDiff::Error the Redis path already does. + # Required at first check, not load time; naming the bare constant instead would raise a raw NameError. def ratelimit_class require "ratelimit" ::Ratelimit diff --git a/lib/translation_diff/registry.rb b/lib/translation_diff/registry.rb index 9d9d90b..2e2740d 100644 --- a/lib/translation_diff/registry.rb +++ b/lib/translation_diff/registry.rb @@ -1,15 +1,8 @@ # frozen_string_literal: true -# Maps a short symbol to a class that knows how to build itself from a -# Configuration. Three of these exist -- providers, cache stores and -# segmenters -- so that every extension option in this library can accept -# either a symbol naming a built-in or an object the caller supplies. -# -# The only thing the three kinds have in common is that a registered class -# answers `build(config)`. That is deliberately the whole contract. +# Maps a symbol to a class that builds itself from a Configuration; the whole contract is answering `build(config)`. class TranslationDiff::Registry - # `kind` appears in the error message for an unknown name, so it should be - # the singular noun a reader would use: "provider", "cache store". + # `kind` appears in the unknown-name error message, so it should be a singular noun: "provider". def initialize(kind) @kind = kind @entries = {} diff --git a/lib/translation_diff/request.rb b/lib/translation_diff/request.rb index 1fe09fe..89e2990 100644 --- a/lib/translation_diff/request.rb +++ b/lib/translation_diff/request.rb @@ -30,8 +30,7 @@ def call attr_reader :values, :options, :to, :config - # The provider for this call: the `provider:` keyword when the caller gave - # one, otherwise whatever the configuration resolves to. + # The `provider:` keyword when the caller gave one, otherwise whatever the configuration resolves to. def api @api ||= (@provider.nil? ? config.provider_instance : resolve_provider(@provider)) .tap { |provider| log("provider #{provider.class}") } @@ -47,15 +46,12 @@ def from @from ||= detect_language end - # A detected language arrives as a String while :to is usually a Symbol, so - # the two have to be compared on equal footing or the short circuit never - # fires and the text gets translated into its own language. + # A detected language is a String while :to is usually a Symbol -- without casecmp? this never short-circuits. def same_language? !to.nil? && from.to_s.casecmp?(to.to_s) end - # Covers values holding no translatable text at all: "", nil, an empty - # collection, or a scalar the tokenizer has nothing to say about. + # Covers "", nil, an empty collection, or a scalar the tokenizer has nothing to say about. def nothing_to_translate? text_tokens_texts.all?(&:empty?) end @@ -70,34 +66,21 @@ def detect_language api.detect(text_tokens_texts.join(" ")[0..100]) end - # Extracts flat text array - # => "Name", "Good boy" - # - # #values might be something like { name: "Name", bio: "Good boy" } def texts @texts ||= linearize(values) end - # Converts each array item to token list - # => [..., [["", :markup], ["Good", :text], ...]] def tokens @tokens ||= texts.map do |value| TranslationDiff::Tokenizer.tokenize(value, segmenter: config.segmenter_instance, language: source_language) end end - # The segmenter's language, not the resolved one: `from` triggers - # auto-detection the first time it is called, and detection builds its - # sample from the segmented text, so asking `from` here would be circular. - # Only a language the caller actually passed is usable at this point -- - # everything else genuinely doesn't know yet, and nil is the honest - # answer. + # Not the resolved `from`: detection builds its sample from the segmented text, so asking `from` here is circular. def source_language @from&.to_s end - # Extracts text tokens from token list - # => { ..., "1_1" => "Good", 1_3 => "Boy", ... } def text_tokens @text_tokens ||= extract_text_tokens.to_h end @@ -110,15 +93,10 @@ def extract_text_tokens end end - # Extracts values from text tokens - # => [ ..., "Good", "Boy", ... ] def text_tokens_texts @text_tokens_texts ||= linearize(text_tokens).map(&:to_s).map(&:strip) end - # Splits things requires translations to per-request chunks - # (groups less 2k sym) - # => [[ ..., "Good", "Boy", ... ]] def chunks @chunks ||= TranslationDiff::Chunker.new( text_tokens_texts, @@ -127,8 +105,6 @@ def chunks ).call end - # Translates/loads from cache values from each chunk - # => [[ ..., "Horoshiy", "Malchik", ... ]] def chunks_translated @chunks_translated ||= chunks.map do |chunk| cached, missing = cache.cached_and_missing(chunk) @@ -141,15 +117,11 @@ def chunks_translated end end - # Restores indexes for translated tokens - # => { ..., "1_1" => "Horoshiy", 1_3 => "Malchik", ... } def text_tokens_translated @text_tokens_translated ||= restore(text_tokens, chunks_translated.flatten) end - # Restores tokens translated + adds same spacing as in source token - # => [[..., [ "Horoshiy", :text ], ...]] # rubocop:disable-next Metrics/AbcSize def tokens_translated @tokens_translated ||= tokens.dup.tap do |tokens| @@ -165,13 +137,10 @@ def restore_spacing(source_value, value) TranslationDiff::Spacing.restore(source_value, value) end - # Restores texts from tokens - # [..., "Horoshiy Malchik", ...] def texts_translated @texts_translated ||= tokens_translated.map.with_index do |group, index| source = texts[index] - # Only strings are rebuilt from tokens. Anything else has no tokens to - # rebuild from; nil keeps collapsing to "" the way it always has. + # Only strings are rebuilt from tokens; nil keeps collapsing to "" the way it always has. next source unless source.nil? || source.is_a?(String) group.map { |value, type| type == :text ? value : fix_ascii(value) }.join @@ -192,11 +161,7 @@ def call_api(values) characters: values.sum(&:size)) do api.translate(request) end - # Dup'd because Cache#store consumes this array destructively (#shift). - # A provider is free to hand back the very array it was given -- Null - # does with a fresh one, but nothing requires that -- and without the - # dup here, a provider or caller holding onto that reference would watch - # it drain to empty out from under them. + # Dup'd: Cache#store consumes this array destructively (#shift), and a provider may hand back its own array. response.texts.dup end @@ -206,11 +171,7 @@ def cache ) end - # A provider built through the registry is stamped with its name. An object - # assigned straight to `config.provider` never passed through the registry, - # so it has to supply this itself -- without it two providers' translations - # would share cache entries and a caller would be served the wrong service's - # answer. + # An object assigned straight to `config.provider` never passed through the registry's stamping. def provider_cache_key key = api.cache_key if api.respond_to?(:cache_key) return key unless key.nil? || key.to_s.strip.empty? diff --git a/lib/translation_diff/segmenters/pragmatic.rb b/lib/translation_diff/segmenters/pragmatic.rb index 6314509..c5608ad 100644 --- a/lib/translation_diff/segmenters/pragmatic.rb +++ b/lib/translation_diff/segmenters/pragmatic.rb @@ -2,62 +2,15 @@ require "pragmatic_segmenter" -# Splits a string into sentence-sized cache units using the -# `pragmatic_segmenter` gem's per-language rule sets. This is the default -# segmenter: measured against the Golden Rules corpus -- the -# `context "Golden Rules" do` block of each of the 10 per-language spec -# files on diasks2/pragmatic_segmenter, 80 exemplars in total; a sample of -# the same corpus is in test/translation_diff/golden_rules_test.rb -- this -# class scores 76/80 against TranslationDiff::Segmenters::Simple's 47/80 on -# the same corpus -- and the gap is worst on exactly the languages Simple -# cannot reason about at all: Arabic, Hindi, Armenian, Greek, because they -# have no letter case for Simple's central "does the next letter look -# lowercase" rule to use. -# -# `pragmatic_segmenter` returns sentence strings, not offsets, and drops the -# whitespace between them. This gem reassembles the source document from -# offsets (see TranslationDiff::Segmenters::Simple), so the strings have to -# be turned back into split points by finding each one in the source, in -# order, starting the search where the previous one left off. -# -# That recovery is only as good as the assumption that each returned -# sentence still appears verbatim in the text pragmatic_segmenter was given -# -- and pinned version 0.3.24's cleaner does not preserve that in every -# case. It collapses runs of three or more spaces inside a sentence, it -# respaces "Ph.D." into "Ph. D.", and (language-specific) its Japanese rules -# delete a "\n" that follows "の". None of these are rare: an English -# sentence mentioning a degree, or HTML indented with more than two spaces, -# hits one of them routinely. #recover_offsets does not raise when this -# happens. It recovers every offset it can verify, in order, and stops at -# the first sentence it cannot -- but the boundary at the end of the last -# sentence it did verify is not thrown away with the rest: it was matched -# character for character, so it is still emitted, and only the genuinely -# unverifiable remainder becomes one final unit. This is a coarsening, not -# a fallback: every offset this class ever emits has been proved to exist -# at that position in the source, so the document reassembles exactly -# either way -- the last cache unit is just bigger when recovery stops -# early, not the whole node. +# Default segmenter: scores 76/80 on the Golden Rules corpus vs Simple's 47/80 (golden_rules_test.rb). class TranslationDiff::Segmenters::Pragmatic - # Raised only if #split_offsets itself computed offsets that violate its - # own postcondition (start at 0, strictly increasing, all within the - # text) -- not by ordinary use of pragmatic_segmenter, however it rewrites - # a sentence. Kept as public API: it is documented in the README's error - # tree, and a caller may already rescue it. + # Raised only if computed offsets violate their own postcondition, never by ordinary sentence rewriting. class Error < TranslationDiff::Error; end - # pragmatic_segmenter defaults to English rules when no language is given. - # Without this, Russian (among others) mis-segments: it treats "Проф." as - # a full sentence and stops there instead of reading through to the next - # real terminator. Passing the caller's language avoids that; see - # TranslationDiff::Request for where the language comes from and why it is - # sometimes nil despite this. + # Without a language, Russian mis-segments: it treats "Проф." as a full sentence and stops there. DEFAULT_LANGUAGE = "en" - # A single newline -- one with neither a preceding nor a following "\n" -- - # is incidental source formatting almost everywhere this gem is used (HTML - # indentation, hand-wrapped prose), not a paragraph break. A run of two or - # more newlines is left alone: that is a real paragraph break, and - # pragmatic_segmenter already handles it correctly. See #shadow_newlines. + # A lone "\n" is incidental source formatting, not a paragraph break; a run of two or more is left alone. SINGLE_NEWLINE = /(? offsets.last offsets end - # Walks the sentences in order, returning the starts of every one located - # (see #locate), the cursor left after the last one located, and whether - # the walk stopped before exhausting every sentence. + # Returns the starts of every sentence located, the cursor after the last one, and whether the walk stopped early. def walk(shadow, sentences) cursor = 0 starts = [] @@ -175,16 +80,7 @@ def walk(shadow, sentences) [starts, cursor, false] end - # Returns [start, next_cursor] for a sentence found at or after cursor; - # :skip for an empty sentence, which #recover_offsets passes over without - # ending the walk (an empty pattern would "match" at cursor itself and - # never advance past it -- distinct from not being found, which should - # end the walk, not stall it); or nil if a non-empty sentence cannot be - # found there at all, which does end the walk. A located non-empty - # sentence always advances past cursor by construction (String#index - # never returns a position before where the search started), so there is - # no further case to guard here -- #assert_valid_offsets is the actual - # backstop if that ever stops holding. + # :skip for empty (an empty match would never advance the cursor); nil if a non-empty sentence isn't found. def locate(shadow, sentence, cursor) return :skip if sentence.empty? @@ -194,11 +90,7 @@ def locate(shadow, sentence, cursor) [start, start + sentence.length] end - # The postcondition every caller of #split_offsets depends on: offsets - # start at 0, strictly increase, and every one is a valid index into the - # text. #recover_offsets is built to guarantee this by construction: this - # asserts it rather than trusting that construction forever, since a wrong - # offset here would silently corrupt the document it feeds back into. + # Asserts the postcondition rather than trusting construction forever: a wrong offset would silently corrupt. def assert_valid_offsets(text, offsets) return if offsets.first.zero? && offsets.each_cons(2).all? { |a, b| a < b } && diff --git a/lib/translation_diff/segmenters/simple.rb b/lib/translation_diff/segmenters/simple.rb index 3288cde..73a48f1 100644 --- a/lib/translation_diff/segmenters/simple.rb +++ b/lib/translation_diff/segmenters/simple.rb @@ -1,48 +1,24 @@ # frozen_string_literal: true -# Splits a string into sentence-sized cache units, with no runtime -# dependency beyond Ruby's own Unicode data. -# -# A wrong boundary never corrupts the document -- the tokenizer slices by -# offset and joins the pieces back together -- but the two kinds of error are -# not equal. A missed boundary just makes the cache unit bigger: still a -# coherent, correctly translated piece of text. A false boundary sends half a -# sentence to the provider on its own, and it comes back wrong. So this class -# is deliberately conservative: it only splits on strong signals, and every -# guard below exists to turn a would-be split back off, never to add one. -# -# Its central rule -- "the next visible character is lowercase, so do not -# split" -- has no meaning in scripts without case, such as Arabic, Hindi or -# Hebrew, so it fares worse than TranslationDiff::Segmenters::Pragmatic (the -# default) on those languages. It exists for callers who want zero extra -# dependencies and translate only from cased scripts. +# No runtime dependency, but deliberately conservative: every guard exists to turn a split off, never on. class TranslationDiff::Segmenters::Simple - # A period, question mark, exclamation mark or ellipsis, run together - # (`?!`, `!!!`) so a whole run is treated as a single terminator; or a - # run of CJK terminators, which need no trailing whitespace and no guards - # -- those scripts have no case ambiguity and no abbreviation periods. + # CJK terminators need no trailing whitespace and no guards -- those scripts have no case or abbreviations. TERMINATOR = /[.!?…]+|[。!?]+/ CJK_TERMINATOR = /\A[。!?]/ - # A short list of words that end in "." without ending a sentence. - # Matched case-insensitively against the word immediately before the - # period. A caller can extend this constant to cover their own domain. + # Words that end in "." without ending a sentence, matched case-insensitively; extend for your own domain. ABBREVIATIONS = %w[ т.е. т.д. т.п. см. рис. стр. гр. ул. г. руб. проф. акад. тыс. млн. млрд. им. Mr. Mrs. Ms. Dr. Prof. St. etc. e.g. i.e. vs. approx. No. im. fig. Fig. vol. p. pp. ].map(&:downcase).freeze - # The trailing run of letters, digits and periods in a preceding word -- - # what is actually compared against ABBREVIATIONS and checked for being a - # single initial. Strips leading punctuation such as an opening quote so - # `"Dr. Smith` still guards on "Dr.". + # Strips leading punctuation like an opening quote so `"Dr. Smith` still guards on "Dr.". WORD_TAIL = /[\p{L}\p{N}.]+\z/ def self.build(_config) = new - # language: is part of the shared segmenter contract but is ignored here -- - # this segmenter's rules (case, digits, punctuation) are language-neutral. + # language: is part of the shared segmenter contract but ignored here -- these rules are language-neutral. # rubocop:disable-next Lint/UnusedMethodArgument def split_offsets(text, language: nil) offsets = [0] @@ -67,17 +43,13 @@ def boundary_for(text, match) end end - # CJK terminators need no trailing whitespace to split, but if whitespace - # does follow, it is attached to the sentence that just ended rather than - # left as a leading gap on the next one. + # If whitespace follows, it's attached to the sentence that just ended, not left as a leading gap. def cjk_boundary(text, run_end) boundary = skip_whitespace(text, run_end) boundary if boundary < text.length end - # Latin terminators only count as a candidate when followed by whitespace; - # a bare "." with nothing after it is not a sentence break, it is a string - # that stops mid-thought. + # A bare "." with nothing after it is not a sentence break, it is a string that stops mid-thought. def latin_boundary(text, match) run_end = match.end(0) return unless whitespace?(text[run_end]) @@ -128,8 +100,7 @@ def url_or_email?(word_before, run) token.include?("://") || token.include?("@") end - # The maximal run of non-whitespace characters immediately before the - # terminator, i.e. the "word" the terminator is attached to. + # The maximal run of non-whitespace characters immediately before the terminator. def word_before(text, run_start) start = run_start start -= 1 while start.positive? && !whitespace?(text[start - 1]) diff --git a/lib/translation_diff/stores.rb b/lib/translation_diff/stores.rb index 1fcbd07..170da16 100644 --- a/lib/translation_diff/stores.rb +++ b/lib/translation_diff/stores.rb @@ -1,5 +1,4 @@ # frozen_string_literal: true -# Cache stores, by name. `config.cache = :redis` resolves through here; -# assigning an object bypasses it entirely. +# Cache stores, by name; assigning an object to `config.cache` bypasses this entirely. TranslationDiff::Stores = TranslationDiff::Registry.new("cache store") diff --git a/lib/translation_diff/tokenizer.rb b/lib/translation_diff/tokenizer.rb index f3199e7..e77f3be 100644 --- a/lib/translation_diff/tokenizer.rb +++ b/lib/translation_diff/tokenizer.rb @@ -30,15 +30,7 @@ def start_element(name) start_markup(name) end - # Ox reports a comment, a doctype and a CDATA section as events of their - # own, each with its own byte position, and each needs a handler here. - # Without one the bytes it covers belong to no token: a leading comment - # disappears from the rebuilt string outright, and one in the middle of a - # sentence leaks its " "application/json" }.merge(headers), diff --git a/test/test_helper.rb b/test/test_helper.rb index deb188f..8c0936f 100644 --- a/test/test_helper.rb +++ b/test/test_helper.rb @@ -8,8 +8,7 @@ require "minitest/autorun" -# Stands in for a connection_pool. The gem only ever calls #with on whatever -# it is handed, so this is the whole contract. +# Stands in for a connection_pool: the gem only ever calls #with on whatever it is handed. class FakeConnectionPool def initialize(connection) @connection = connection @@ -20,8 +19,7 @@ def with end end -# Any test that configures anything must reset afterwards, or its settings -# leak into every test that runs after it. +# Any test that configures anything must reset afterwards, or its settings leak into later tests. class ConfiguredTest < Minitest::Test def setup = TranslationDiff.reset! def teardown = TranslationDiff.reset! diff --git a/test/translation_diff/cache_test.rb b/test/translation_diff/cache_test.rb index 6203d03..5b961fb 100644 --- a/test/translation_diff/cache_test.rb +++ b/test/translation_diff/cache_test.rb @@ -20,9 +20,7 @@ def write(_key, value) end end - # A store that answers with fixed, caller-chosen results regardless of the - # keys it is asked about, to pin the positional contract between the keys - # sent and the results read back. + # Answers with fixed results regardless of keys asked, to pin the positional contract. class PositionalStore def initialize(responses) @responses = responses @@ -37,8 +35,7 @@ def setup @store = RecordingStore.new end - # Two providers writing to one store used to collide: switching DeepL for - # another provider silently returned DeepL's translations. + # Two providers writing to one store used to collide, silently returning DeepL's translations for another. def test_the_provider_is_part_of_the_key key_for(provider: "deepl") key_for(provider: "google") @@ -46,8 +43,7 @@ def test_the_provider_is_part_of_the_key refute_equal @store.keys[0], @store.keys[1] end - # formality: :less used to share a key with the default, so whichever - # translated first won. + # formality: :less used to share a key with the default, so whichever translated first won. def test_the_provider_options_are_part_of_the_key key_for(options: {}) key_for(options: { formality: :less }) @@ -62,8 +58,7 @@ def test_the_options_digest_is_order_independent assert_equal @store.keys[0], @store.keys[1] end - # "EN" and :en are the same language and used to produce two entries for - # identical work. + # "EN" and :en are the same language and used to produce two entries for identical work. def test_the_language_codes_are_normalised key_for(from: "EN", to: "RU") key_for(from: :en, to: :ru) @@ -78,8 +73,7 @@ def test_leading_and_trailing_space_does_not_change_the_key assert_equal @store.keys[0], @store.keys[1] end - # nil and "" are different values; #to_s would collide them, so the digest - # must be built from a serialisation that keeps them apart. + # nil and "" are different values; #to_s would collide them. def test_nil_and_empty_string_option_values_produce_different_keys key_for(options: { a: nil }) key_for(options: { a: "" }) @@ -87,8 +81,7 @@ def test_nil_and_empty_string_option_values_produce_different_keys refute_equal @store.keys[0], @store.keys[1] end - # An Array-valued option (e.g. glossary_ids: %w[a b]) exercises the Array - # branch of #canonical, which no other test in this file reaches. + # Exercises the Array branch of #canonical, which no other test in this file reaches. def test_an_array_option_value_is_part_of_the_digest key_for(options: { glossary_ids: %w[a b] }) key_for(options: { glossary_ids: %w[a c] }) @@ -96,16 +89,12 @@ def test_an_array_option_value_is_part_of_the_digest refute_equal @store.keys[0], @store.keys[1] end - # An option value with no stable serialisation (no #inspect of its own, - # or one that embeds a memory address) must not be allowed to silently - # produce an unreproducible cache key. + # An option value with no stable serialisation must not silently produce an unreproducible cache key. def test_an_unsupported_option_value_raises assert_raises(TranslationDiff::Cache::Error) { key_for(options: { a: Object.new }) } end - # cached_and_missing pairs the store's response with the requested values - # by position, trusting the store to return results in key order. A - # database-backed store answering `WHERE key IN (...)` will not. + # Trusts the store to return results in key order; a `WHERE key IN (...)` store will not. def test_cached_and_missing_pairs_results_positionally store = PositionalStore.new(["cached one", nil, "cached three"]) diff --git a/test/translation_diff/capabilities_test.rb b/test/translation_diff/capabilities_test.rb index 52572fd..38cdab5 100644 --- a/test/translation_diff/capabilities_test.rb +++ b/test/translation_diff/capabilities_test.rb @@ -9,9 +9,7 @@ def capabilities(**overrides) reports_billing: false, **overrides) end - # Every reader asks a yes-or-no question, and Data.define generates plain - # readers. Declaring the predicates once stops the codebase from asking - # `detects_language` in one place and `detects_language?` in another. + # Declaring the predicates once stops the codebase asking `detects_language` in one place, `?` in another. def test_it_answers_in_predicates assert_predicate capabilities, :html? assert_predicate capabilities, :notranslate? @@ -19,8 +17,7 @@ def test_it_answers_in_predicates refute_predicate capabilities, :reports_billing? end - # `html` holds the name of the provider option that turns HTML on, which is - # different for every vendor, so :none is the only way to say "cannot". + # `html` holds the option name that turns HTML on, different per vendor, so :none says "cannot". def test_html_is_false_only_when_the_provider_has_no_html_mode refute_predicate capabilities(html: :none), :html? assert_predicate capabilities(html: :tag_handling), :html? diff --git a/test/translation_diff/chunker_test.rb b/test/translation_diff/chunker_test.rb index 70e2542..3affa24 100644 --- a/test/translation_diff/chunker_test.rb +++ b/test/translation_diff/chunker_test.rb @@ -42,14 +42,11 @@ def test_raises_when_a_single_value_exceeds_the_limit assert_match(/Too long part/, error.message) end - # The limit is about the size of the request that goes over the wire, and - # CGI.escape inflates Cyrillic sixfold. Measuring the raw String#size - # anywhere here let chunks of non-ASCII text run several times over. + # CGI.escape inflates Cyrillic sixfold; measuring raw String#size let chunks of non-ASCII text run over. def test_measures_non_ascii_values_by_their_escaped_size value = "я" * 3 - # Three characters raw, eighteen escaped. Measured raw, both values fit - # in one chunk of 20; measured as sent, they cannot. + # Measured raw, both values fit in one chunk of 20; measured as sent, they cannot. assert_equal 3, value.size assert_equal 18, CGI.escape(value).size assert_equal [[value], [value]], chunk([value, value]) diff --git a/test/translation_diff/configuration_test.rb b/test/translation_diff/configuration_test.rb index 1cf1cce..b7006e1 100644 --- a/test/translation_diff/configuration_test.rb +++ b/test/translation_diff/configuration_test.rb @@ -3,8 +3,7 @@ require "test_helper" class ConfigurationTest < Minitest::Test - # A hand-written double for TranslationDiff::Registry: this project's - # Minitest (6.0) dropped minitest/mock, so there is no Minitest::Mock here. + # A hand-written double for TranslationDiff::Registry: Minitest 6.0 dropped minitest/mock. ResolvingRegistry = Struct.new(:answer) do attr_reader :asked @@ -46,8 +45,7 @@ def test_assigning_a_blank_string_stores_nil_so_an_unset_env_var_behaves_as_unse end def test_assigning_false_is_kept_and_not_treated_as_unset - # test_flag is registered here purely to exercise `option`; it is not a - # production option and mutating class-level state with it is harmless. + # test_flag is registered here purely to exercise `option`; it is not a production option. TranslationDiff::Configuration.option(:test_flag, true) @config.test_flag = false @@ -64,9 +62,7 @@ def test_copy_carries_values_and_leaves_the_original_alone assert_equal 120, copy.cache_ttl end - # Stand-ins for two provider classes. register_provider_options is handed - # the class so it can tell "the same provider declaring its options again" - # from "another provider claiming a name that is already taken". + # Stand-ins for two provider classes, to distinguish redeclaring from a name collision. class AcmeOptionOwner def self.configuration_options = %i[acme_api_key] end @@ -76,9 +72,7 @@ def self.configuration_options = %i[contested_key] end def test_register_provider_options_adds_readers_and_writers - # acme_api_key is registered here purely to exercise - # register_provider_options; it is not a production option and mutating - # class-level state with it is harmless. + # acme_api_key is registered here purely to exercise register_provider_options; not a production option. TranslationDiff::Configuration.register_provider_options(%i[acme_api_key], AcmeOptionOwner) @config.acme_api_key = "secret" @@ -86,8 +80,7 @@ def test_register_provider_options_adds_readers_and_writers assert_includes TranslationDiff::Configuration.options, :acme_api_key end - # A double `require` and a Rails development reload both re-run - # registration; neither is a conflict. + # A double `require` and a Rails development reload both re-run registration; neither is a conflict. def test_the_same_provider_may_redeclare_its_own_options TranslationDiff::Configuration.register_provider_options(%i[acme_repeat_key], AcmeOptionOwner) TranslationDiff::Configuration.register_provider_options(%i[acme_repeat_key], AcmeOptionOwner) @@ -95,9 +88,7 @@ def test_the_same_provider_may_redeclare_its_own_options assert_includes TranslationDiff::Configuration.options, :acme_repeat_key end - # Without this, `option` returns early on the already-declared name and the - # two providers silently share one accessor -- so the credential set for - # one is handed to the other. + # Without this, `option` returns early and two providers silently share one accessor. def test_a_different_provider_declaring_a_declared_option_raises TranslationDiff::Configuration.register_provider_options(%i[contested_key], AcmeOptionOwner) @@ -110,9 +101,7 @@ def test_a_different_provider_declaring_a_declared_option_raises assert_match(/RivalOptionOwner/, error.message) end - # A subclass wanting its parent's options is the one case where sharing - # the accessor is correct -- subclassing a provider to point it at a - # different host or account is an obvious thing to want. + # A subclass wanting its parent's options is the one case where sharing the accessor is correct. class AcmeSubclassOwner < AcmeOptionOwner; end def test_a_subclass_of_the_declaring_provider_may_redeclare_its_options @@ -122,8 +111,7 @@ def test_a_subclass_of_the_declaring_provider_may_redeclare_its_options assert_includes TranslationDiff::Configuration.options, :acme_subclass_key end - # An unrelated class is still refused, even though a subclass is now - # allowed -- the guard only relaxes for an actual inheritance relationship. + # The guard only relaxes for an actual inheritance relationship; an unrelated class is still refused. def test_an_unrelated_class_claiming_a_subclassable_providers_option_still_raises TranslationDiff::Configuration.register_provider_options(%i[acme_unrelated_key], AcmeOptionOwner) @@ -134,16 +122,12 @@ def test_an_unrelated_class_claiming_a_subclassable_providers_option_still_raise assert_match(/acme_unrelated_key/, error.message) end - # A provider that conflicts on its *second* option, once its first - # (fresh_option) has already been checked. + # Conflicts on its *second* option, once its first (fresh_option) has already been checked. class IntruderOptionOwner def self.configuration_options = %i[fresh_option owned_by_acme] end - # register_provider_options used to declare and record ownership key by - # key, so a provider whose *second* option conflicted still left its first - # option declared and owned by the class that failed to register. Fixed to - # check every key before mutating any of them. + # Used to declare ownership key by key, leaving the first option owned by a class that failed to register. def test_a_conflict_on_a_later_option_leaves_no_partial_state TranslationDiff::Configuration.register_provider_options(%i[owned_by_acme], AcmeOptionOwner) @@ -154,16 +138,14 @@ def test_a_conflict_on_a_later_option_leaves_no_partial_state refute_includes TranslationDiff::Configuration.options, :fresh_option - # If the failed attempt had already declared or claimed :fresh_option, - # this would raise, blaming a provider that was never registered. + # If the failed attempt had already claimed :fresh_option, this would raise, blaming the wrong provider. TranslationDiff::Configuration.register_provider_options(%i[fresh_option], RivalOptionOwner) assert_includes TranslationDiff::Configuration.options, :fresh_option end def test_registering_an_option_twice_does_not_clobber_the_first_default - # shared_option is not a production option; mutating class-level state - # with it is harmless. + # shared_option is not a production option; mutating class-level state with it is harmless. TranslationDiff::Configuration.option(:shared_option, "first") TranslationDiff::Configuration.option(:shared_option, "second") @@ -244,14 +226,7 @@ def test_the_default_rate_limiter_is_memoised assert_same @config.rate_limiter_instance, @config.rate_limiter_instance end - # `rate_limiter_instance` follows the same rule as `provider_instance`, - # `cache_store` and `segmenter_instance`: a config that never had an - # object assigned starts from nothing on `copy` and builds its own - # limiter. This is the case that used to be broken -- a copy inherited an - # already-built limiter, which would silently rate-limit a tenant against - # its parent's namespace. See - # `test_a_copy_that_changes_its_namespace_gets_a_rate_limiter_using_that_namespace` - # below for the scenario that actually surfaces the bleed. + # Used to be broken: a copy inherited an already-built limiter and rate-limited a tenant against its parent's. def test_copy_does_not_share_a_built_default_rate_limiter @config.rate_limit = 100 original_limiter = @config.rate_limiter_instance @@ -259,12 +234,7 @@ def test_copy_does_not_share_a_built_default_rate_limiter refute_same original_limiter, @config.copy.rate_limiter_instance end - # An explicitly assigned object is different: it is the option's own - # value, exactly like an assigned `cache` or `provider`, and `copy` - # carries option values over on purpose -- "someone who hands us one - # object means one object." Only the *default build* must not survive a - # copy; an object the caller supplied is never rebuilt in the first - # place, so there is nothing for a copy to get wrong. + # An assigned object is an option value, and `copy` carries option values over on purpose. def test_copy_shares_an_assigned_rate_limiter_object limiter = Object.new @config.rate_limiter = limiter @@ -272,9 +242,7 @@ def test_copy_shares_an_assigned_rate_limiter_object assert_same limiter, @config.copy.rate_limiter_instance end - # The regression this whole area was fixed for: a tenant context that - # sets its own `cache_namespace` must get a rate limiter scoped to that - # namespace, not one built for -- and still carrying -- its parent's. + # The regression this whole area was fixed for. def test_a_copy_that_changes_its_namespace_gets_a_rate_limiter_using_that_namespace @config.rate_limit = 100 @config.redis_url = "redis://localhost:6379" diff --git a/test/translation_diff/context_test.rb b/test/translation_diff/context_test.rb index 86b5c24..0cd675e 100644 --- a/test/translation_diff/context_test.rb +++ b/test/translation_diff/context_test.rb @@ -7,8 +7,7 @@ def setup TranslationDiff.reset! TranslationDiff.configure do |c| c.provider = :null - # Pinned rather than left to the default so a developer with REDIS_URL - # set in their environment does not have these tests reach for a socket. + # Pinned so a developer with REDIS_URL set doesn't have these tests reach for a socket. c.cache = :memory end end diff --git a/test/translation_diff/errors_test.rb b/test/translation_diff/errors_test.rb index 0893a77..2cd09a9 100644 --- a/test/translation_diff/errors_test.rb +++ b/test/translation_diff/errors_test.rb @@ -14,8 +14,7 @@ def test_every_error_descends_from_the_common_ancestor end end - # Rescuing "the provider said no" must not also catch a local - # configuration mistake or a socket timeout. + # Rescuing "the provider said no" must not also catch a config mistake or a socket timeout. def test_provider_errors_are_a_family_of_their_own [TranslationDiff::AuthenticationError, TranslationDiff::RateLimitError, TranslationDiff::QuotaExceededError, TranslationDiff::InvalidRequestError, diff --git a/test/translation_diff/golden_rules_test.rb b/test/translation_diff/golden_rules_test.rb index ae2315b..278059a 100644 --- a/test/translation_diff/golden_rules_test.rb +++ b/test/translation_diff/golden_rules_test.rb @@ -2,21 +2,7 @@ require "test_helper" -# Eleven exemplars from the "Golden Rules", the de-facto benchmark for -# sentence segmentation, adapted from the `context "Golden Rules" do` block -# of each per-language spec file under spec/pragmatic_segmenter/languages/ -# on https://github.com/diasks2/pragmatic_segmenter (MIT licence, Copyright -# (c) 2015 Kevin S. Dias). The full corpus (80 exemplars across 10 -# languages) lives outside this repo; this sample exists so a future change -# to the default segmenter cannot quietly regress segmentation quality -# without a test noticing here first. -# -# It deliberately weights the languages that have no letter case -- Arabic -# (two exemplars), Greek, Hindi -- and includes one Japanese exemplar, -# because those are exactly the languages TranslationDiff::Segmenters::Simple -# cannot reason about (its central rule, "does the next letter look -# lowercase", has no meaning for them) and where Pragmatic earns its place -# as the default. +# Adapted from diasks2/pragmatic_segmenter's Golden Rules (MIT, Copyright (c) 2015 Kevin S. Dias). class GoldenRulesTest < Minitest::Test EXEMPLARS = [ { language: "en", text: "Hello World. My name is Jonas.", @@ -43,9 +29,7 @@ class GoldenRulesTest < Minitest::Test "يقول معارضو الرئيس الإيراني إن الطريقة التي اعلنت بها النتائج كانت مثيرة للاستغراب." ] }, { language: "ar", - # Contains U+202A/U+202C (left-to-right embedding) around the - # abbreviation's period -- a real bidi-formatting shape, and a good - # stress test for offset recovery finding a sentence verbatim. + # Contains U+202A/U+202C (left-to-right embedding) around the abbreviation's period, a real bidi shape. text: "وقال د‪.‬ ديفيد ريدي و الأطباء الذين كانوا يعالجونها في مستشفى برمنجهام إنها كانت " \ "تعاني من أمراض أخرى. وليس معروفا ما اذا كانت قد توفيت بسبب اصابتها بأنفلونزا الخنازير.", expected: [ @@ -81,10 +65,7 @@ def test_pragmatic_the_default_segmenter_matches_the_golden_rules_sample end end - # Simple is not held to the Golden Rules' exact boundaries -- it cannot be, - # for languages without letter case -- but it must never corrupt the - # document while trying. This is the structural guarantee that still has - # to hold when a caller opts into the zero-dependency segmenter. + # Simple isn't held to exact boundaries, but must never corrupt the document while trying. def test_simple_still_reconstructs_every_exemplar_even_where_it_under_or_over_splits segmenter = TranslationDiff::Segmenters::Simple.new diff --git a/test/translation_diff/http_provider_test.rb b/test/translation_diff/http_provider_test.rb index 92f211f..f36c12c 100644 --- a/test/translation_diff/http_provider_test.rb +++ b/test/translation_diff/http_provider_test.rb @@ -4,8 +4,7 @@ require "faraday" class HTTPProviderTest < Minitest::Test - # A provider that exists only to exercise the base class. Its seams are the - # smallest thing that can round-trip. + # A provider that exists only to exercise the base class. class Echo < TranslationDiff::HTTPProvider def api_base = "https://echo.test" def headers = { "X-Echo" => "1" } @@ -25,10 +24,7 @@ def request(texts = %w[one]) TranslationDiff::Translation::Request.new(texts: texts, from: "en", to: "ru") end - # Builds an Echo whose connection uses Faraday's test adapter. Minitest 6 - # dropped minitest/mock, and stubbing HTTP is exactly what the test adapter - # is for -- no webmock, no network, and the same middleware stack the real - # connection has. + # Faraday's test adapter: no webmock, no network, same middleware stack the real connection has. def provider_for(status:, body:, headers: {}) stubs = Faraday::Adapter::Test::Stubs.new do |stub| stub.post("/v1/translate") { [status, headers, body] } @@ -75,8 +71,7 @@ def test_a_500_becomes_a_service_error assert_raises(TranslationDiff::ServiceError) { provider.translate(request) } end - # 429 survives the retries only when they are exhausted, so the test turns - # them off; what is asserted here is the mapping, not the retrying. + # Retries are turned off; what is asserted here is the mapping, not the retrying. def test_a_429_becomes_a_rate_limit_error_carrying_retry_after @config.max_retries = 0 provider = provider_for(status: 429, body: "slow down", headers: { "Retry-After" => "17" }) @@ -99,9 +94,7 @@ def test_a_connection_failure_becomes_a_transport_error assert_raises(TranslationDiff::TransportError) { provider.translate(request) } end - # The guarantee that no line this library writes carries source text or a - # credential now has a mechanism: we own the connection, and nothing - # installs a logging middleware on it. + # No line this library writes may carry source text or a credential. def test_no_logging_middleware_is_installed_even_when_a_logger_is_configured @config.logger = Logger.new(StringIO.new) handlers = Echo.new(@config).connection.builder.handlers diff --git a/test/translation_diff/instrumentation_test.rb b/test/translation_diff/instrumentation_test.rb index b31b047..eb80b29 100644 --- a/test/translation_diff/instrumentation_test.rb +++ b/test/translation_diff/instrumentation_test.rb @@ -14,11 +14,7 @@ def instrument(name, payload) end end - # Assignable via `config.rate_limiter =`, same as `config.cache =`. Always - # lets the call through, so `check_rate_limit` has something to call - # without needing a real Redis connection -- and so the `rate_limit` event - # fires on every translation in this file, alongside `translate`, `cache` - # and `request`. + # Always lets the call through, so the `rate_limit` event fires without a real Redis connection. class FakeRateLimiter def check(_size) = nil end @@ -28,9 +24,7 @@ def setup @recorder = Recorder.new TranslationDiff.configure do |c| c.provider = :null - # Pinned so a developer with REDIS_URL set does not have these tests - # resolve the Redis store and open a real socket -- the same reason - # context_test.rb pins it. It weakens no assertion here. + # Pinned so a developer with REDIS_URL set doesn't have these tests open a real socket. c.cache = :memory c.instrumenter = @recorder c.rate_limiter = FakeRateLimiter.new @@ -80,10 +74,7 @@ def test_the_rate_limit_event_carries_the_provider_and_a_character_count ALL_EVENT_NAMES = %w[translate.translation_diff cache.translation_diff request.translation_diff rate_limit.translation_diff].sort.freeze - # A guard that only checked payload content would pass even if an event - # quietly stopped firing -- asserting the full set of names first makes - # sure every event this library emits is actually present and inspected, - # not just whichever ones happened to show up. + # A guard that only checked payload content would pass even if an event quietly stopped firing. def test_no_payload_ever_contains_the_text_being_translated secret = "Zaphod Beeblebrox is president." TranslationDiff.translate(secret, from: "en", to: "ru") diff --git a/test/translation_diff/provider_test.rb b/test/translation_diff/provider_test.rb index fdffb05..1a52271 100644 --- a/test/translation_diff/provider_test.rb +++ b/test/translation_diff/provider_test.rb @@ -22,9 +22,7 @@ def test_a_provider_without_requirements_builds assert_instance_of Bare, Bare.new(@config) end - # The old behaviour was a DeepL:: error raised from inside a vendor SDK, or - # `ArgumentError, "project_id is missing"` from another. Neither named the - # option a caller of THIS library has to set. + # The old behaviour raised a vendor SDK's own error, which never named the option this library needs set. def test_it_names_every_missing_option_at_once error = assert_raises(TranslationDiff::ConfigurationError) { Demanding.new(@config) } @@ -51,9 +49,7 @@ def test_detect_raises_until_a_subclass_implements_it assert_raises(NotImplementedError) { Bare.new(@config).detect("etwas") } end - # cache_key is a segment of every cache key this provider reads or writes. - # A quietly empty one would let two providers share a namespace and serve - # one service's translations for another. + # A quietly empty cache_key would let two providers share a namespace and serve the wrong translations. def test_cache_key_is_the_registered_name provider = Bare.new(@config) provider.name = :bare diff --git a/test/translation_diff/providers/amazon_test.rb b/test/translation_diff/providers/amazon_test.rb index 0498b49..cf3bf82 100644 --- a/test/translation_diff/providers/amazon_test.rb +++ b/test/translation_diff/providers/amazon_test.rb @@ -21,11 +21,7 @@ def setup @requests = [] end - # There is no AWS key available for this task, so unlike DeepL's and - # Google's fixtures -- both captured from a live call -- this response - # body is shaped from Amazon's own Translate API reference documentation - # ("TranslateText", read 2026-09-09), not from an observed response. - # Nobody should mistake it for one. + # No AWS key was available: shaped from Amazon's "TranslateText" reference (read 2026-09-09), not observed. def provider(texts: nil) built = TranslationDiff::Providers::Amazon.new(config) built.name = :amazon @@ -43,10 +39,7 @@ def build_stubs(texts) end def respond_to_translate(env, texts, recorder) - # Faraday's test adapter reuses this env for the response, mutating its - # body in place once the block returns -- capture a copy now or every - # read after #translate returns sees the reply, not the request (see - # test/support/stubbed_provider.rb). + # Faraday's test adapter mutates this env's body in place for the response -- dup it now, or lose the request. recorder << env.dup body = JSON.parse(env.body) translated = texts&.shift || "#{body['Text']}-ru" @@ -77,9 +70,7 @@ def test_it_sends_the_json_rpc_target_header requests.first.request_headers["X-Amz-Target"] end - # This runs against the real aws-sigv4 library rather than a stand-in, so - # it is real evidence that this provider signs correctly -- not just that - # some string ended up in the Authorization header. + # Runs against the real aws-sigv4 library, not a stand-in, so this is real evidence signing works. def test_it_signs_the_request provider.translate(translation_request(%w[one])) authorization = requests.first.request_headers["Authorization"] @@ -115,9 +106,7 @@ def test_it_reports_the_source_language_amazon_resolved assert_equal "en", response.detected_source end - # The capability is the warning. Amazon has no HTML mode at all, so a - # notranslate span sent to it WILL be translated, and the only honest thing - # to do is say so where the rest of the library can read it. + # The capability is the warning: Amazon has no HTML mode, so a notranslate span sent to it WILL be translated. def test_it_claims_neither_html_nor_notranslate capabilities = TranslationDiff::Providers::Amazon.capabilities diff --git a/test/translation_diff/providers/azure_test.rb b/test/translation_diff/providers/azure_test.rb index 0e39c5f..6dd2f91 100644 --- a/test/translation_diff/providers/azure_test.rb +++ b/test/translation_diff/providers/azure_test.rb @@ -12,11 +12,7 @@ class AzureProviderTest < Minitest::Test include HTTPProviderContract include StubbedProvider - # There is no Azure key available for this task, so unlike DeepL's and - # Google's fixtures -- both captured from a live call -- this body is - # shaped from Microsoft's own Azure AI Translator v3 "Translate" reference - # documentation (read 2026-09-09), not from an observed response. Nobody - # should mistake it for one. + # No Azure key was available: shaped from Microsoft's v3 "Translate" reference (read 2026-09-09), not observed. BODY = [ { "detectedLanguage" => { "language" => "en", "score" => 1.0 }, "translations" => [{ "text" => "один", "to" => "ru" }] }, @@ -34,11 +30,7 @@ def setup def provider_class = TranslationDiff::Providers::Azure - # When `body:` is left nil, the stub echoes back whatever texts were - # actually sent (rather than a fixed pair), so the shared ProviderContract - # tests -- which call `provider` with no knowledge of how many texts they - # are about to send -- get a response the same size as their request - # instead of tripping Response.build's count check. + # Left nil, `body:` echoes back whatever texts were sent, so ProviderContract's count check never trips. def provider(body: nil, status: 200, headers: {}) stub_provider(route: "/translate", body: body || method(:echo_translations), status: status, headers: headers, name: :azure) @@ -60,8 +52,7 @@ def test_it_sends_the_key_in_the_documented_header TranslationDiff::Providers::Azure.new(config).headers["Ocp-Apim-Subscription-Key"] end - # A single-service key needs no region and a multi-service one does, so the - # header appears only when the option is set. + # A single-service key needs no region and a multi-service one does. def test_the_region_header_appears_only_when_configured refute TranslationDiff::Providers::Azure.new(config).headers.key?("Ocp-Apim-Subscription-Region") @@ -122,8 +113,7 @@ def test_it_reads_the_billed_characters_from_the_metered_usage_header assert_equal 6, response.usage.billed_characters end - # Absent, the header must yield nil rather than 0 -- 0 is a false claim - # about billing, not "unknown". + # Absent, the header must yield nil rather than 0 -- 0 is a false claim about billing, not "unknown". def test_billed_characters_is_nil_when_the_header_is_absent response = provider(body: BODY).translate(translation_request(%w[one two])) diff --git a/test/translation_diff/providers/deepl_test.rb b/test/translation_diff/providers/deepl_test.rb index e7ede02..b9b0c18 100644 --- a/test/translation_diff/providers/deepl_test.rb +++ b/test/translation_diff/providers/deepl_test.rb @@ -29,11 +29,7 @@ def setup def provider_class = TranslationDiff::Providers::DeepL - # When `body:` is left nil, the stub echoes back whatever texts were - # actually sent (rather than a fixed pair), so the shared ProviderContract - # tests -- which call `provider` with no knowledge of how many texts they - # are about to send -- get a response the same size as their request - # instead of tripping Response.build's count check. + # Left nil, `body:` echoes back whatever texts were sent, so ProviderContract's count check never trips. def provider(body: nil, status: 200, headers: {}) stub_provider(route: "/v2/translate", body: body || method(:echo_translations), status: status, headers: headers, name: :deepl) @@ -92,10 +88,7 @@ def test_it_omits_the_source_language_when_none_was_given refute sent.key?("source_lang") end - # Regression: notranslate spans reach the provider with their tags, and - # DeepL honours class="notranslate" only under HTML tag handling. Without - # this the protected content is translated while the tags survive, which is - # invisible in review. + # DeepL honours class="notranslate" only under HTML tag handling; otherwise content translates, tags survive. def test_it_asks_for_html_tag_handling provider.translate(translation_request(%w[one two])) @@ -131,10 +124,7 @@ def test_a_short_response_raises_rather_than_shifting_nils_into_the_results end end - # DeepL has no detection endpoint, so it detects by translating a sample - # and reading what it says the source was. #detect sends exactly one - # text, and the stub echoes it back, so the count matches without an - # override. + # DeepL has no detection endpoint; it detects by translating a sample and reading the reported source. def test_detect_returns_the_language_deepl_reports assert_equal "en", provider.detect("something") end diff --git a/test/translation_diff/providers/google_test.rb b/test/translation_diff/providers/google_test.rb index f3db00a..4b58e19 100644 --- a/test/translation_diff/providers/google_test.rb +++ b/test/translation_diff/providers/google_test.rb @@ -12,9 +12,7 @@ class GoogleProviderTest < Minitest::Test include HTTPProviderContract include StubbedProvider - # A real response envelope, shaped from the Cloud Translation v2 REST - # reference read 2026-09-09: translations live under a nested "data" key, - # not at the top level the way DeepL's do. + # Shaped from the Cloud Translation v2 REST reference read 2026-09-09: translations nest under "data". TRANSLATE_BODY = { "data" => { "translations" => [ { "translatedText" => "один", "detectedSourceLanguage" => "en" }, @@ -32,12 +30,7 @@ def setup def provider_class = TranslationDiff::Providers::Google - # When `body:` is left nil, the stub echoes back whatever texts were - # actually sent (rather than a fixed pair), so the shared ProviderContract - # tests -- which call `provider` with no knowledge of how many texts they - # are about to send -- get a response the same size as their request - # instead of tripping Response.build's count check. The content type - # matches what Cloud Translation v2 actually sends. + # Left nil, `body:` echoes back whatever texts were sent, so ProviderContract's count check never trips. def provider(body: nil, status: 200, headers: { "Content-Type" => "application/json; charset=UTF-8" }) stub_provider(route: "/language/translate/v2", body: body || method(:echo_translations), status: status, headers: headers, name: :google) @@ -69,8 +62,7 @@ def test_a_caller_may_ask_for_plain_text assert_equal "text", sent["format"] end - # Google's codes are lower case and a config written for DeepL says "EN"; - # but "zh-Hans" and "pt-BR" carry subtags whose casing is their own. + # Google's codes are lower case, but "zh-Hans" carries a subtag whose casing is its own. def test_it_downcases_bare_codes_and_leaves_subtagged_ones_alone provider.translate(translation_request(%w[one], from: "EN", to: "zh-Hans")) diff --git a/test/translation_diff/providers/libretranslate_test.rb b/test/translation_diff/providers/libretranslate_test.rb index 61afd54..fe5d5da 100644 --- a/test/translation_diff/providers/libretranslate_test.rb +++ b/test/translation_diff/providers/libretranslate_test.rb @@ -21,18 +21,13 @@ def setup def provider_class = TranslationDiff::Providers::LibreTranslate - # When `body:` is left nil, the stub echoes back whatever texts were - # actually sent, so the shared ProviderContract tests -- which call - # `provider` with no knowledge of how many texts they are about to send -- - # get a response the same size as their request instead of tripping - # Response.build's count check. + # Left nil, `body:` echoes back whatever texts were sent, so ProviderContract's count check never trips. def provider(body: nil, status: 200, headers: {}) stub_provider(route: "/translate", body: body || method(:echo_translations), status: status, headers: headers, name: :libretranslate) end - # Everyone self-hosts this one, so the base URL is the requirement and the - # key is the option -- the reverse of every other provider here. + # Everyone self-hosts this one, so base URL is the requirement and key is the option -- the reverse of the rest. def test_the_api_base_is_required_and_the_key_is_not config.libretranslate_api_base = nil diff --git a/test/translation_diff/providers/modernmt_test.rb b/test/translation_diff/providers/modernmt_test.rb index 62f6447..56d2a57 100644 --- a/test/translation_diff/providers/modernmt_test.rb +++ b/test/translation_diff/providers/modernmt_test.rb @@ -11,9 +11,7 @@ class ModernMTProviderTest < Minitest::Test include HTTPProviderContract include StubbedProvider - # Shaped from modernmt.com/api's own "Translate" reference (read - # 2026-09-09), not from a live call: no ModernMT key was available for this - # task. + # Shaped from modernmt.com/api's reference (read 2026-09-09), not a live call: no key was available. BODY = { "data" => [ { "translation" => "один", "billedCharacters" => 3, "characters" => 3, "detectedLanguage" => "en" }, { "translation" => "два", "billedCharacters" => 3, "characters" => 3, "detectedLanguage" => "en" } @@ -29,11 +27,7 @@ def setup def provider_class = TranslationDiff::Providers::ModernMT - # When `body:` is left nil, the stub echoes back whatever texts were - # actually sent, so the shared ProviderContract tests -- which call - # `provider` with no knowledge of how many texts they are about to send -- - # get a response the same size as their request instead of tripping - # Response.build's count check. + # Left nil, `body:` echoes back whatever texts were sent, so ProviderContract's count check never trips. def provider(body: nil, status: 200, headers: {}) stub_provider(route: "/translate", body: body || method(:echo_translations), status: status, headers: headers, name: :modernmt) @@ -66,9 +60,7 @@ def test_it_unwraps_the_data_envelope assert_equal 6, response.usage.billed_characters end - # One text comes back as an object, not a one-element array. A provider - # that passes that through hands the pipeline a Hash where it expects a - # list, and the count check is what catches it. + # One text comes back as an object, not a one-element array, which would hand the pipeline a bare Hash. def test_a_single_text_comes_back_unwrapped_and_is_still_a_list single = { "data" => { "translation" => "один", "detectedLanguage" => "en" } } response = provider(body: single).translate(translation_request(%w[one])) diff --git a/test/translation_diff/providers_test.rb b/test/translation_diff/providers_test.rb index 162210a..e832791 100644 --- a/test/translation_diff/providers_test.rb +++ b/test/translation_diff/providers_test.rb @@ -3,8 +3,7 @@ require "test_helper" class ProvidersTest < Minitest::Test - # A provider defined entirely outside this library, to prove that adding a - # translation service requires no change to lib/. + # Defined entirely outside this library, to prove adding a translation service requires no change to lib/. class AcmeProvider < TranslationDiff::Provider def self.configuration_options = %i[acme_token] @@ -23,8 +22,7 @@ def translate(request) end end - # Two providers that both want the same option name. Registering the - # second must raise rather than hand it the first one's accessor. + # Registering the second must raise rather than hand it the first one's accessor. class ConflictingProviderA < TranslationDiff::Provider def self.configuration_options = %i[shared_provider_token] end @@ -33,13 +31,10 @@ class ConflictingProviderB < TranslationDiff::Provider def self.configuration_options = %i[shared_provider_token] end - # A subclass wanting AcmeProvider's own option -- subclassing a provider - # to point it at a different host or account is an obvious thing to want. + # A subclass wanting AcmeProvider's own option. class SubclassOfAcmeProvider < AcmeProvider; end - # The second-key-conflicts shape: PartialB would declare :partial_own_key - # successfully if checked eagerly, but conflicts with PartialA's - # :partial_shared_key on its second option. + # PartialB would declare :partial_own_key successfully if checked eagerly, but conflicts on its second. class PartialProviderA < TranslationDiff::Provider def self.configuration_options = %i[partial_shared_key] end @@ -83,17 +78,14 @@ def test_null_keeps_its_own_cache_key assert_equal "null", TranslationDiff::Providers.build(:null, @config).cache_key end - # A defensive double `require` and a Rails development reload both re-run - # registration, so the same provider redeclaring its own options must stay - # silent. #setup has already registered AcmeProvider once. + # A defensive double `require` and a Rails reload both re-run registration; redeclaring must stay silent. def test_registering_the_same_provider_twice_is_not_a_conflict TranslationDiff::Providers.register(:acme, AcmeProvider) assert_includes TranslationDiff::Configuration.options, :acme_token end - # Rails reloading yields a *new* class object under the same constant, so - # identity alone would make an ordinary development reload raise. + # Rails reloading yields a *new* class object under the same constant, so identity alone would raise. def test_a_reloaded_class_of_the_same_name_is_not_a_conflict Object.const_set(:ReloadedProvider, reloadable_provider_class) TranslationDiff::Providers.register(:reloaded, ReloadedProvider) @@ -109,9 +101,7 @@ def test_a_reloaded_class_of_the_same_name_is_not_a_conflict Object.send(:remove_const, :ReloadedProvider) if Object.const_defined?(:ReloadedProvider) end - # Reproduces the credential crossing: `Configuration.option` returns early - # on a name it already knows, so without this guard ConflictingProviderB - # would be handed the accessor -- and the value -- ConflictingProviderA set. + # Without this guard, ConflictingProviderB would be handed the accessor and value ConflictingProviderA set. def test_a_second_provider_claiming_a_declared_option_name_raises TranslationDiff::Providers.register(:conflict_a, ConflictingProviderA) @@ -125,10 +115,7 @@ def test_a_second_provider_claiming_a_declared_option_name_raises refute TranslationDiff::Providers.registered?(:conflict_b) end - # AcmeProvider (registered as :acme in #setup) owns :acme_token. A - # subclass inherits that option and must still be registerable under its - # own name -- this used to raise, since a subclass claiming its inherited - # option looked identical to an unrelated class claiming a taken one. + # Used to raise: a subclass claiming its inherited option looked identical to an unrelated class claiming it. def test_a_subclass_of_a_registered_provider_may_be_registered TranslationDiff::Providers.register(:acme_subclass, SubclassOfAcmeProvider) @@ -136,8 +123,7 @@ def test_a_subclass_of_a_registered_provider_may_be_registered assert_includes TranslationDiff::Configuration.options, :acme_token end - # An unrelated class is still refused for the very option a subclass may - # now share -- the relaxation is specific to an inheritance relationship. + # The relaxation is specific to an inheritance relationship; an unrelated class is still refused. def test_an_unrelated_class_claiming_a_subclassable_option_still_raises unrelated = Class.new(TranslationDiff::Provider) do def self.configuration_options = %i[acme_token] @@ -151,10 +137,7 @@ def self.configuration_options = %i[acme_token] refute TranslationDiff::Providers.registered?(:acme_unrelated) end - # Registration must be all-or-nothing: PartialProviderB conflicts on its - # second option, so it must not leave its first option declared, owned by - # PartialProviderB, or itself registered -- and a later, legitimate - # provider claiming that first option name must succeed. + # Registration must be all-or-nothing: a conflict on the second option must not leave the first declared. def test_a_failed_registration_leaves_no_partial_option_state TranslationDiff::Providers.register(:partial_a, PartialProviderA) @@ -171,10 +154,7 @@ def test_a_failed_registration_leaves_no_partial_option_state assert_includes TranslationDiff::Configuration.options, :partial_own_key end - # ruby_llm requires a Provider subclass and so do we now. A duck-typed - # object cannot be given the transport, the requirement check or the - # capability defaults, and every one of those is a place this library has - # already been bitten. + # A duck-typed object cannot be given the transport, the requirement check, or the capability defaults. def test_registering_a_class_that_is_not_a_provider_raises not_a_provider = Class.new do def self.configuration_options = [] @@ -189,12 +169,7 @@ def self.build(_config) = new refute TranslationDiff::Providers.registered?(:impostor) end - # InvalidProviderError is specific to a provider of the wrong shape (see - # its definition in errors.rb): a caller rescuing "this class cannot be a - # provider" must not also, by accident, swallow an unrelated failure. An - # option-name collision is that unrelated failure -- two well-shaped - # providers fighting over one option name -- so it must raise the generic - # TranslationDiff::Error, not the specific one. + # A caller rescuing "this class cannot be a provider" must not also accidentally swallow an option collision. def test_an_option_collision_raises_the_generic_error_not_the_invalid_provider_one TranslationDiff::Providers.register(:collision_a, ConflictingProviderA) diff --git a/test/translation_diff/redis_cache_store_test.rb b/test/translation_diff/redis_cache_store_test.rb index ce76cc9..ffc8cfa 100644 --- a/test/translation_diff/redis_cache_store_test.rb +++ b/test/translation_diff/redis_cache_store_test.rb @@ -3,17 +3,7 @@ require "test_helper" require "support/cache_store_contract" -# The gem depends on neither redis nor redis-namespace at runtime -- it just -# calls into whatever the application supplies. This stand-in applies the -# namespace the way redis-namespace does, so the keys reaching Redis can be -# asserted on. -# -# TranslationDiff::Configuration#redis_pool requires the real "redis" gem -# lazily, at call time, so whether ::Redis is already defined when this file -# runs depends on test order -- Minitest randomises it. Requiring it -# explicitly here, and nesting this stand-in inside the real class instead of -# declaring a fake top-level `Redis` module, means this file never collides -# with -- or races -- that real constant. +# Nested inside the real Redis class, requiring it explicitly, so this never races Configuration's lazy require. require "redis" class Redis::Namespace @@ -34,11 +24,7 @@ def setex(key, timeout, value) class RedisCacheStoreTest < Minitest::Test include CacheStoreContract - # `values`, when given, forces every #mget to return it regardless of the - # keys asked for -- what the namespacing tests below use to inspect the - # keys reaching Redis without needing real storage behind them. Without - # it, #mget and #setex behave like a real key/value store, which is what - # the shared CacheStoreContract needs. + # `values`, when given, forces #mget to return it regardless of keys asked, to inspect keys without real storage. class FakeRedis attr_reader :calls diff --git a/test/translation_diff/redis_rate_limiter_test.rb b/test/translation_diff/redis_rate_limiter_test.rb index 17e2f44..199248d 100644 --- a/test/translation_diff/redis_rate_limiter_test.rb +++ b/test/translation_diff/redis_rate_limiter_test.rb @@ -2,18 +2,11 @@ require "test_helper" -# RedisRateLimiter requires "ratelimit" lazily, at the first check, so this -# file exercises the production integration against the real Ratelimit class -# rather than a stand-in. The stand-in this file used to define hid a real -# defect: the gem's signature is `add(subject, count)`, so `add(size)` was -# counting under a subject named after the character count while `exceeded?` -# read a subject nothing ever incremented -- the limit never fired. +# A prior stand-in here hid a real defect: `add(size)` counted under the wrong subject and the limit never fired. require "ratelimit" class RedisRateLimiterTest < Minitest::Test - # An in-memory Redis server implementing exactly the commands ratelimit 1.1 - # issues, plus the two Lua scripts it loads (interpreted here rather than - # run). No socket is opened; nothing here is a stub of the gem under test. + # An in-memory Redis server implementing exactly the commands ratelimit 1.1 issues; no socket is opened. class FakeRedisServer attr_reader :hashes, :expiries, :count_spans @@ -86,8 +79,7 @@ def test_check_counts_under_a_custom_namespace assert_equal({ "ratelimit:tenant-42:call" => 7 }, server.totals) end - # Ratelimit's own rule is `count >= threshold`, so the check that carries - # the count over the line still passes and the next one raises. + # Ratelimit's own rule is `count >= threshold`, so the check that carries the count over the line still passes. def test_check_raises_once_the_threshold_is_passed server = FakeRedisServer.new @@ -107,10 +99,7 @@ def test_check_uses_the_default_threshold assert_raises(TranslationDiff::RedisRateLimiter::RateLimitExceeded) { limiter(server).check(1) } end - # Ratelimit buckets five seconds at a time, so the number of buckets its - # count script sweeps is the interval divided by five -- exactly, since - # both intervals here are multiples of five. That is the only observable - # the interval has. + # Ratelimit buckets five seconds at a time, so buckets swept is the interval divided by five. def test_check_looks_back_over_the_default_interval server = FakeRedisServer.new @@ -127,11 +116,7 @@ def test_check_looks_back_over_a_custom_interval assert_equal [120], server.count_spans end - # The gem used to name the bare `Ratelimit` constant, so an application - # that had not installed it got a raw NameError rather than the "add this - # gem" message the Redis path raises. The singleton `require` here is the - # narrowest way to simulate the gem being absent: it shadows Kernel#require - # for this one object only. + # Naming the bare `Ratelimit` constant used to raise a raw NameError instead of this gem's own message. def test_a_missing_ratelimit_gem_raises_a_translation_diff_error limiter = limiter(FakeRedisServer.new) limiter.define_singleton_method(:require) { |_name| raise LoadError } diff --git a/test/translation_diff/request_test.rb b/test/translation_diff/request_test.rb index 005b87c..de2cc25 100644 --- a/test/translation_diff/request_test.rb +++ b/test/translation_diff/request_test.rb @@ -3,10 +3,7 @@ require "test_helper" class RequestTest < ConfiguredTest - # A minimal provider. Records what it was asked to translate so the call - # can be asserted on, and answers with a canned response. Detects a - # language when told to, so it can stand in for both a detecting and a - # non-detecting provider depending on which test needs which. + # Records what it was asked to translate, and answers with a canned response. class FakeApi < TranslationDiff::Provider CAPABILITIES = TranslationDiff::Capabilities.new( max_request_size: 1_000_000, max_batch_size: 1_000_000, max_text_size: nil, @@ -39,9 +36,7 @@ def detect(text) def cache_key = "fake" end - # Proves the generalisation took effect: a provider whose declared - # capabilities cap the batch at one text per request must change the - # chunking, not merely be asked to. + # Proves the generalisation took effect: capabilities capping the batch at one must change the chunking. class NarrowBatchApi < FakeApi def self.capabilities TranslationDiff::Capabilities.new( @@ -51,9 +46,7 @@ def self.capabilities end end - # Registered under :echo so - # test_the_provider_keyword_overrides_the_configured_provider_for_one_call - # can exercise resolving a provider by name through the registry. + # Registered under :echo to exercise resolving a provider by name through the registry. class EchoProvider < TranslationDiff::Provider def self.capabilities TranslationDiff::Capabilities.new( @@ -85,24 +78,17 @@ def write(key, value) end end - # A provider object assigned straight to `config.provider` never passed - # through the registry, so nothing stamped it with a name. This one defines - # its own #cache_key and returns an empty segment from it. + # An object assigned straight to `config.provider` never passed through the registry's stamping. class NamelessApi < FakeApi def cache_key = "" end - # A whitespace-only key is just as blank as an empty one -- it must not - # slip past the guard and cache translations under a segment that looks - # empty to anyone reading the store. + # A whitespace-only key is just as blank as an empty one; it must not slip past the guard. class WhitespaceNamedApi < FakeApi def cache_key = " " end - # Already has every key cached, regardless of what it is asked for. Proves - # the all-cached short circuit in Request#chunks_translated: the gem's - # headline behaviour is serving a translation from cache without calling - # the adapter at all. + # Proves the all-cached short circuit: serving a translation from cache without calling the adapter at all. class AllCachedStore def initialize(responses) @responses = responses @@ -163,10 +149,7 @@ def test_translates_text_around_markup_and_leaves_the_markup_alone assert_equal [[%w[One Black So Red that], :en, :ru, {}]], api.calls end - # True of the public interface, but not new: 2.1.0 already protected the - # caller's hash from mutation by dup-ing it in the initializer. Keyword - # arguments keep that guarantee for a different reason (a fresh hash per - # call), but this test alone cannot tell the two implementations apart. + # True of the public interface, but not new: 2.1.0 already protected the caller's hash by dup-ing it. def test_repeated_calls_leave_the_callers_options_hash_alone options = { from: :en, to: :ru } @@ -178,18 +161,14 @@ def test_repeated_calls_leave_the_callers_options_hash_alone assert_equal({ from: :en, to: :ru }, options) end - # This is what actually changed: the initializer declares one positional - # parameter now, so a single positional options hash -- which is what every - # pre-task caller passed -- is no longer accepted. + # The initializer now declares one positional parameter, so a positional options hash is no longer accepted. def test_the_positional_options_hash_is_no_longer_accepted assert_raises(ArgumentError) do TranslationDiff::Request.new("text", { from: :en, to: :ru }) end end - # A detected language comes back as a String while :to is usually a Symbol, - # so the source == target short circuit never fired and the text was paid - # for and translated into its own language. + # A detected language is a String while :to is usually a Symbol -- without casecmp? this never short-circuits. def test_skips_the_translation_when_the_detected_language_is_the_target api = FakeApi.new([], detected: "RU") configure_with(api) @@ -200,9 +179,7 @@ def test_skips_the_translation_when_the_detected_language_is_the_target assert_equal [[:detect, "привет"]], api.calls end - # Chunker's limits used to come from two provider methods. They now come - # from the declared capabilities, which is the only place a new provider - # states them. + # Chunker's limits used to come from two provider methods; they now come from the declared capabilities. # rubocop:disable-next Metrics/AbcSize, Metrics/MethodLength def test_chunking_uses_the_providers_declared_capabilities narrow = Class.new(TranslationDiff::Provider) do @@ -235,8 +212,7 @@ def cache_key = "narrow" "expected one text per request, got #{provider.batches.inspect}") end - # The old check was `respond_to?(:detect)`, which a provider could satisfy - # by inheriting the base class's raising stub. + # The old check was `respond_to?(:detect)`, satisfied by inheriting the base class's raising stub. def test_a_provider_that_cannot_detect_says_so_before_it_is_called error = assert_raises(TranslationDiff::Request::Error) do TranslationDiff.translate("Some text.", to: "ru", provider: :null) @@ -246,10 +222,7 @@ def test_a_provider_that_cannot_detect_says_so_before_it_is_called assert_match(/null/, error.message) end - # The count check used to live in Request#call_api. It now lives in - # Translation::Response.build, which is why the error is a ResponseError - # rather than a Request::Error: it holds for every provider, including one - # that overrides #translate outright instead of using the HTTP seams. + # The count check now lives in Translation::Response.build, so it holds for every provider, hence ResponseError. def test_raises_when_the_api_returns_fewer_translations_than_asked_for configure_with(FakeApi.new(%w[Один])) @@ -281,8 +254,7 @@ def test_passes_through_nil_untouched assert_empty api.calls end - # Scalars nested in a structure are passed through too, while nil keeps - # collapsing to "" the way it always has. + # Scalars nested in a structure are passed through too, while nil keeps collapsing to "". def test_passes_nested_scalars_through_and_still_blanks_out_nils configure_with(FakeApi.new(%w[Один])) @@ -291,8 +263,7 @@ def test_passes_nested_scalars_through_and_still_blanks_out_nils assert_equal({ a: "Один", n: 42, skip: "" }, result) end - # Proves the generalisation took effect rather than merely being - # described: a provider declaring tiny limits must change the batching. + # Proves the generalisation took effect: a provider declaring tiny limits must change the batching. def test_batches_according_to_the_limits_the_adapter_declares api = NarrowBatchApi.new(%w[Один Два]) configure_with(api) @@ -302,9 +273,7 @@ def test_batches_according_to_the_limits_the_adapter_declares assert_equal 2, api.calls.size, "one call per text at a batch size of 1" end - # Every fake cache store elsewhere in this file always misses, so this is - # the only Request-level test exercising a cache hit: a translation served - # from the store without ever reaching the adapter. + # The only Request-level test exercising a cache hit; every other fake store in this file always misses. def test_serves_a_translation_from_cache_without_calling_the_adapter api = FakeApi.new([]) configure_with(api, AllCachedStore.new(["Какая-то строка"])) @@ -315,9 +284,7 @@ def test_serves_a_translation_from_cache_without_calling_the_adapter assert_empty api.calls end - # `provider:` picks the provider for one call. A name is built through the - # registry against this call's configuration; the configured provider is - # left untouched and unused. + # `provider:` picks the provider for one call; the configured provider is left untouched and unused. def test_the_provider_keyword_overrides_the_configured_provider_for_one_call api = FakeApi.new(%w[Один]) configure_with(api) @@ -328,9 +295,7 @@ def test_the_provider_keyword_overrides_the_configured_provider_for_one_call assert_empty api.calls end - # An empty cache-key segment would put this provider's translations in the - # same namespace as every other provider's, and a caller would be served - # another service's answer. Refusing is the only safe response. + # An empty cache-key segment would put this provider's translations in every other provider's namespace. def test_a_provider_whose_cache_key_is_empty_is_refused_rather_than_sharing_a_namespace configure_with(NamelessApi.new(%w[Один])) @@ -353,7 +318,6 @@ def test_a_provider_whose_cache_key_is_whitespace_is_refused_rather_than_sharing private - # Translates `values` from :en to :ru against fakes. # Returns the translation and the API fake, so the call can be asserted on. def translate(values, response) api = FakeApi.new(response) @@ -362,9 +326,7 @@ def translate(values, response) [TranslationDiff::Request.new(values, from: :en, to: :ru).call, api] end - # The two collaborators every test here needs, assigned as configuration - # options. An object assigned to `provider` or `cache` is used as-is, so a - # fake goes in exactly where a registered name would. + # An object assigned to `provider` or `cache` is used as-is, so a fake goes in exactly where a name would. def configure_with(api, store = FakeCacheStore.new) TranslationDiff.configure do |config| config.provider = api diff --git a/test/translation_diff/segmenters/pragmatic_test.rb b/test/translation_diff/segmenters/pragmatic_test.rb index d2e4a02..61ba024 100644 --- a/test/translation_diff/segmenters/pragmatic_test.rb +++ b/test/translation_diff/segmenters/pragmatic_test.rb @@ -11,16 +11,13 @@ class PragmaticSegmenterTest < Minitest::Test ["Смеркалось. Ворчало. Кричало.", "ru"], ["Набор «Солнечная механика» от 4М — это 6 экспериментов.\n\n" \ "Юному изобретателю предстоит воочию посмотреть на чудеса.", "ru"], - # Multiple blank lines: more than one blank line in a single gap, and - # more than one such gap in the same text. Untouched by shadowing -- - # SINGLE_NEWLINE only matches a "\n" with no adjoining "\n". + # Multiple blank lines, untouched by shadowing: SINGLE_NEWLINE only matches "\n" with no adjoining "\n". ["First paragraph.\n\n\nSecond paragraph.\n\n\n\nThird paragraph.", "en"], ["見て。すごい!次はどうなる?", "ja"], ["Проф. Иванов пришёл домой. Было поздно.", "ru"], ["سؤال وجواب: ماذا حدث؟ طرح الكثير من التساؤلات.", "ar"], ["Ի՞նչ ես մտածում: Ոչինչ:", "hy"], - # Single newlines (shadowed) and a blank-line paragraph break (not - # shadowed), together, so the invariant is exercised against both paths. + # Single newlines (shadowed) and a blank-line paragraph break (not), together, exercising both paths. ["The cat sat on the mat\nand looked at the moon. It was content.\n\n" \ "A new paragraph starts here.", "en"], ["これは父の\n家です。それはペンです。", "ja"] @@ -60,9 +57,7 @@ def test_cjk_terminators_split_without_a_language_hint assert_equal [0, "見て。".length], @segmenter.split_offsets(text) end - # This is the trap the brief calls out by name: without a language, - # pragmatic_segmenter falls back to English rules, and English rules read - # "Проф." as a complete sentence on its own. + # Without a language, pragmatic_segmenter falls back to English rules, which read "Проф." as a full sentence. def test_without_a_language_russian_abbreviations_are_mis_segmented text = "Проф. Иванов пришёл домой. Было поздно." offsets = @segmenter.split_offsets(text) @@ -70,9 +65,7 @@ def test_without_a_language_russian_abbreviations_are_mis_segmented assert_equal [0, "Проф. ".length, "Проф. Иванов пришёл домой. ".length], offsets end - # The same text, with the language supplied, segments correctly -- proving - # the language argument is actually threaded through to pragmatic_segmenter - # rather than merely accepted and ignored. + # Proves the language argument is actually threaded through to pragmatic_segmenter, not merely accepted. def test_with_the_language_russian_abbreviations_are_respected text = "Проф. Иванов пришёл домой. Было поздно." offsets = @segmenter.split_offsets(text, language: "ru") @@ -88,15 +81,7 @@ def test_blank_lines_between_sentences_are_attached_to_the_first_sentence assert_equal [0, "Набор «Солнечная механика» от 4М — это 6 экспериментов.\n\n".length], offsets end - # pragmatic_segmenter treats essentially any single newline as a sentence - # boundary candidate, independent of punctuation -- confirmed on ordinary, - # punctuation-free, line-wrapped prose, including with whitespace on both - # sides of the newline (not just a newline glued to non-whitespace). A - # false split is the harmful kind of error this whole gem exists to avoid, - # and it is common: HTML text nodes routinely carry incidental newlines - # from source formatting. Shadowing single newlines before segmenting - # fixes this while leaving the original text -- newline included -- in - # the output. + # pragmatic_segmenter treats any single newline as a sentence boundary, confirmed false on wrapped prose. def test_a_single_newline_with_no_punctuation_does_not_split_the_sentence text = "Some text \n continues here without any punctuation at the break" assert_equal [0], @segmenter.split_offsets(text) @@ -109,10 +94,7 @@ def test_a_single_newline_between_two_real_sentences_is_preserved_as_a_boundary_ assert_equal [0, "This is a sentence\ncut off by a line wrap. ".length], offsets end - # A blank-line run is a real paragraph break, not incidental formatting, - # and shadowing deliberately leaves it alone -- pragmatic_segmenter already - # handles it correctly (also covered by the reconstruction invariant above, - # and by TokenizerTest's own blank-line case). + # A blank-line run is a real paragraph break; shadowing deliberately leaves it alone. def test_a_blank_line_paragraph_break_still_splits text = "Первое предложение.\n\nВторое предложение." offsets = @segmenter.split_offsets(text, language: "ru") @@ -120,9 +102,7 @@ def test_a_blank_line_paragraph_break_still_splits assert_equal [0, "Первое предложение.\n\n".length], offsets end - # Shadowing fixes the real trigger reported in the previous round: this no - # longer raises, and the newline survives in the output exactly as it - # appeared in the source. + # Shadowing fixes the real trigger reported in the previous round: this no longer raises. def test_the_japanese_newline_after_a_common_particle_now_segments_instead_of_raising text = "これは父の\n家です。それはペンです。" offsets = @segmenter.split_offsets(text, language: "ja") @@ -130,15 +110,7 @@ def test_the_japanese_newline_after_a_common_particle_now_segments_instead_of_ra assert_equal [0, "これは父の\n家です。".length], offsets end - # H1 (fix round 2): pragmatic_segmenter's cleaner rewrites the sentence it - # hands back in ways shadowing does not touch -- collapsing runs of three - # or more spaces, respacing "Ph.D." into "Ph. D.", deleting a formatting - # artefact outright. None of these are rare (an English sentence naming a - # degree, or HTML indented with more than two spaces, hits one of them - # routinely), and none of them may abort translation any more: recovery - # stops at the first sentence it cannot verify and the remainder of the - # text stands as one final unit -- a coarsening, not a failure. Each case - # below is a real, reproduced trigger, not a hypothetical. + # H1 (fix round 2): pragmatic_segmenter's cleaner respaces "Ph.D." into "Ph. D.", a real reproduced trigger. def test_ph_d_no_longer_aborts_and_the_original_text_is_untouched text = "He has a Ph.D. in physics. It took years." assert_equal [0], @segmenter.split_offsets(text, language: "en") @@ -159,22 +131,13 @@ def test_a_newline_plus_a_two_space_indent_no_longer_aborts assert_equal [0], @segmenter.split_offsets(text, language: "en") end - # The inline-formatting artefact pragmatic_segmenter deletes outright - # (lib/pragmatic_segmenter/cleaner/rules.rb, InlineFormattingRule) is what - # the previous round used to prove the (now-removed) raise fired on real - # behaviour. It now proves the opposite: recovery still stops cleanly - # instead of guessing, and the whole node survives as one unit. + # pragmatic_segmenter's InlineFormattingRule deletes this artefact outright; recovery now stops cleanly instead. def test_a_deleted_formatting_artefact_no_longer_aborts text = "This is a sentence{b^>3 [ "! Киловольт. Смеркалось. Ворчало. Кричало.", [ - # Pragmatic does not treat a lone terminator with no preceding - # content as a sentence of its own, so "!" stays merged with - # "Киловольт." -- a missed boundary, which only makes the cache - # unit bigger, not a false one. + # A lone terminator with no preceding content stays merged: a missed boundary, not a false one. ["! Киловольт. ", :text], ["", :markup], ["Смеркалось. ", :text], @@ -106,9 +103,7 @@ class TokenizerTest < Minitest::Test ["", :markup] ] ], - # Ox reports a comment, a doctype and a CDATA section as their own SAX - # events. Each one needs a handler: without it the bytes it covers are - # attributed to no token at all and vanish from the rebuilt string. + # Without a handler, the bytes a comment/doctype/CDATA event covers vanish from the rebuilt string. "an_html_comment" => [ " Visible text.", [ @@ -158,8 +153,6 @@ class TokenizerTest < Minitest::Test private - # The segmenter these expectations were written against, and the one the - # configuration still defaults to. Passed explicitly now that the tokenizer - # takes its segmenter as a collaborator rather than reaching for a global. + # Passed explicitly now that the tokenizer takes its segmenter as a collaborator, not a global. def segmenter = TranslationDiff::Segmenters::Pragmatic.new end diff --git a/test/translation_diff/translation/response_test.rb b/test/translation_diff/translation/response_test.rb index a5e5589..a87967f 100644 --- a/test/translation_diff/translation/response_test.rb +++ b/test/translation_diff/translation/response_test.rb @@ -17,10 +17,7 @@ def test_build_returns_the_texts_it_was_given assert_equal %w[один два], response.texts end - # A short response means nils get shifted into the results and surface much - # later as a NoMethodError far from the cause. The check lives in the - # constructor rather than in a base-class method so that it still holds for - # a provider that overrides #translate outright. + # A short response would shift nils into the results, surfacing much later as a distant NoMethodError. def test_build_raises_when_the_provider_returned_the_wrong_number_of_texts error = assert_raises(TranslationDiff::ResponseError) do TranslationDiff::Translation::Response.build(request: request, texts: %w[один]) From 4a3ebeb09782aa4a45023843580c368125202261 Mon Sep 17 00:00:00 2001 From: IG Date: Wed, 9 Sep 2026 19:41:20 +0400 Subject: [PATCH 17/26] Drop frozen_string_literal magic comments, require Ruby 3.4 Dedup savings from the magic comment are traded for plain mutable string literals; explicit .freeze keeps the constants that need to stay frozen. Ruby 3.4 chills string literals without the comment, so it becomes the supported floor. --- .rubocop.yml | 7 ++++++- Gemfile | 2 -- Rakefile | 2 -- lib/translation_diff.rb | 2 -- lib/translation_diff/cache.rb | 2 -- lib/translation_diff/capabilities.rb | 2 -- lib/translation_diff/chunker.rb | 2 -- lib/translation_diff/configuration.rb | 2 -- .../configuration/provider_option_owners.rb | 2 -- lib/translation_diff/context.rb | 2 -- lib/translation_diff/error.rb | 2 -- lib/translation_diff/errors.rb | 2 -- lib/translation_diff/http_provider.rb | 6 ++---- lib/translation_diff/instrumentation.rb | 4 +--- lib/translation_diff/linearizer.rb | 2 -- lib/translation_diff/memory_cache_store.rb | 2 -- lib/translation_diff/provider.rb | 2 -- lib/translation_diff/providers.rb | 2 -- lib/translation_diff/providers/amazon.rb | 14 ++++++-------- lib/translation_diff/providers/azure.rb | 8 +++----- lib/translation_diff/providers/deepl.rb | 10 ++++------ lib/translation_diff/providers/google.rb | 4 +--- lib/translation_diff/providers/libretranslate.rb | 6 ++---- lib/translation_diff/providers/modernmt.rb | 6 ++---- lib/translation_diff/providers/null.rb | 2 -- lib/translation_diff/redis_cache_store.rb | 4 +--- lib/translation_diff/redis_rate_limiter.rb | 6 ++---- lib/translation_diff/registry.rb | 2 -- lib/translation_diff/request.rb | 2 -- lib/translation_diff/segmenters.rb | 2 -- lib/translation_diff/segmenters/pragmatic.rb | 4 +--- lib/translation_diff/segmenters/simple.rb | 2 -- lib/translation_diff/spacing.rb | 2 -- lib/translation_diff/stores.rb | 2 -- lib/translation_diff/tokenizer.rb | 2 -- lib/translation_diff/translation/request.rb | 2 -- lib/translation_diff/translation/response.rb | 2 -- lib/translation_diff/translation/usage.rb | 2 -- lib/translation_diff/version.rb | 4 +--- test/support/cache_store_contract.rb | 2 -- test/support/http_provider_contract.rb | 2 -- test/support/provider_contract.rb | 2 -- test/support/stubbed_provider.rb | 2 -- test/test_helper.rb | 2 -- test/translation_diff/cache_test.rb | 2 -- test/translation_diff/capabilities_test.rb | 2 -- test/translation_diff/chunker_test.rb | 4 +--- test/translation_diff/configuration_test.rb | 2 -- test/translation_diff/context_test.rb | 2 -- test/translation_diff/errors_test.rb | 2 -- test/translation_diff/golden_rules_test.rb | 2 -- test/translation_diff/http_provider_test.rb | 2 -- test/translation_diff/instrumentation_test.rb | 2 -- test/translation_diff/linearizer_test.rb | 2 -- test/translation_diff/memory_cache_store_test.rb | 2 -- test/translation_diff/provider_test.rb | 2 -- test/translation_diff/providers/amazon_test.rb | 2 -- test/translation_diff/providers/azure_test.rb | 2 -- test/translation_diff/providers/deepl_test.rb | 2 -- test/translation_diff/providers/google_test.rb | 2 -- .../providers/libretranslate_test.rb | 2 -- test/translation_diff/providers/modernmt_test.rb | 2 -- test/translation_diff/providers/null_test.rb | 2 -- test/translation_diff/providers_test.rb | 2 -- test/translation_diff/redis_cache_store_test.rb | 2 -- test/translation_diff/redis_rate_limiter_test.rb | 2 -- test/translation_diff/registry_test.rb | 2 -- test/translation_diff/request_test.rb | 2 -- test/translation_diff/segmenters/pragmatic_test.rb | 2 -- test/translation_diff/segmenters/simple_test.rb | 2 -- test/translation_diff/spacing_test.rb | 2 -- test/translation_diff/tokenizer_test.rb | 6 ++---- test/translation_diff/translation/response_test.rb | 2 -- test/translation_diff_test.rb | 2 -- translation_diff.gemspec | 4 +--- 75 files changed, 36 insertions(+), 179 deletions(-) diff --git a/.rubocop.yml b/.rubocop.yml index 4127e85..dea43ba 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,5 +1,5 @@ AllCops: - TargetRubyVersion: 3.2 + TargetRubyVersion: 3.4 NewCops: enable Style/Documentation: @@ -21,6 +21,11 @@ Layout/LineLength: Gemspec/DevelopmentDependencies: EnforcedStyle: gemspec +# Ruby 3.4 chills string literals in files without this comment, and the +# supported floor is now 3.4. +Style/FrozenStringLiteralComment: + Enabled: false + # Request takes its collaborators as keyword arguments -- values, from:, to:, # provider:, config: and the provider's own options. The confusion this cop # guards against is positional: six named arguments at a call site read diff --git a/Gemfile b/Gemfile index c57ecc0..281e85e 100644 --- a/Gemfile +++ b/Gemfile @@ -1,5 +1,3 @@ -# frozen_string_literal: true - source "https://rubygems.org" gemspec diff --git a/Rakefile b/Rakefile index 34f1378..d4ffd43 100644 --- a/Rakefile +++ b/Rakefile @@ -1,5 +1,3 @@ -# frozen_string_literal: true - require "bundler/gem_tasks" require "rake/testtask" diff --git a/lib/translation_diff.rb b/lib/translation_diff.rb index 95b4142..2011962 100644 --- a/lib/translation_diff.rb +++ b/lib/translation_diff.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - require "cgi/escape" require "digest/md5" require "forwardable" diff --git a/lib/translation_diff/cache.rb b/lib/translation_diff/cache.rb index 15107fd..fcee513 100644 --- a/lib/translation_diff/cache.rb +++ b/lib/translation_diff/cache.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - class TranslationDiff::Cache class Error < TranslationDiff::Error; end diff --git a/lib/translation_diff/capabilities.rb b/lib/translation_diff/capabilities.rb index d2a0a23..e76f27c 100644 --- a/lib/translation_diff/capabilities.rb +++ b/lib/translation_diff/capabilities.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - # Declared, not discovered -- duck-typing left notranslate silently broken on two providers. TranslationDiff::Capabilities = Data.define(:max_request_size, :max_batch_size, :max_text_size, :html, :notranslate, diff --git a/lib/translation_diff/chunker.rb b/lib/translation_diff/chunker.rb index 155b506..60268dd 100644 --- a/lib/translation_diff/chunker.rb +++ b/lib/translation_diff/chunker.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - class TranslationDiff::Chunker class Error < TranslationDiff::Error; end diff --git a/lib/translation_diff/configuration.rb b/lib/translation_diff/configuration.rb index cecde2e..c813844 100644 --- a/lib/translation_diff/configuration.rb +++ b/lib/translation_diff/configuration.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - # Every declared setting in one place; callable defaults are invoked on read, not at load time. class TranslationDiff::Configuration class << self diff --git a/lib/translation_diff/configuration/provider_option_owners.rb b/lib/translation_diff/configuration/provider_option_owners.rb index d0286e7..55e7759 100644 --- a/lib/translation_diff/configuration/provider_option_owners.rb +++ b/lib/translation_diff/configuration/provider_option_owners.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - # Tracks which provider declared each option; two silently sharing one accessor would leak a credential. class TranslationDiff::Configuration::ProviderOptionOwners def initialize diff --git a/lib/translation_diff/context.rb b/lib/translation_diff/context.rb index 21fd9ce..e0c94b0 100644 --- a/lib/translation_diff/context.rb +++ b/lib/translation_diff/context.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - # An isolated configuration scope with the same entry point as TranslationDiff itself. class TranslationDiff::Context attr_reader :config diff --git a/lib/translation_diff/error.rb b/lib/translation_diff/error.rb index 7a49a39..f733c1e 100644 --- a/lib/translation_diff/error.rb +++ b/lib/translation_diff/error.rb @@ -1,4 +1,2 @@ -# frozen_string_literal: true - # Common ancestor for every error this gem raises, so `rescue TranslationDiff::Error` is enough. class TranslationDiff::Error < StandardError; end diff --git a/lib/translation_diff/errors.rb b/lib/translation_diff/errors.rb index a0fdf6a..974d48d 100644 --- a/lib/translation_diff/errors.rb +++ b/lib/translation_diff/errors.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - # No error carries the text being translated -- errors are logged, and this library handles other people's content. module TranslationDiff class ConfigurationError < Error; end diff --git a/lib/translation_diff/http_provider.rb b/lib/translation_diff/http_provider.rb index db884a2..b591578 100644 --- a/lib/translation_diff/http_provider.rb +++ b/lib/translation_diff/http_provider.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - require "faraday" require "faraday/retry" require "json" @@ -63,11 +61,11 @@ def decode(response) def json?(response) = response.headers["content-type"].to_s.match?(/\bjson\b/) # The block is how a test swaps in Faraday's test adapter; Amazon overrides it too, to sign the body as sent. - def build_connection(&block) + def build_connection(&) Faraday.new(url: api_base, headers: headers) do |faraday| faraday.request :json faraday.request :retry, retry_options - adapt(faraday, &block) + adapt(faraday, &) apply_timeouts(faraday) end end diff --git a/lib/translation_diff/instrumentation.rb b/lib/translation_diff/instrumentation.rb index 85cc722..b251e3e 100644 --- a/lib/translation_diff/instrumentation.rb +++ b/lib/translation_diff/instrumentation.rb @@ -1,8 +1,6 @@ -# frozen_string_literal: true - # Payloads carry counts, language codes and provider names -- never the text, its translation, or a credential. module TranslationDiff::Instrumentation - SUFFIX = ".translation_diff" + SUFFIX = ".translation_diff".freeze # `include` ignores the includer's own `private` keyword, so visibility has to be declared here. private diff --git a/lib/translation_diff/linearizer.rb b/lib/translation_diff/linearizer.rb index b99e6e8..3e2f913 100644 --- a/lib/translation_diff/linearizer.rb +++ b/lib/translation_diff/linearizer.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - class TranslationDiff::Linearizer class << self def linearize(struct, array = []) diff --git a/lib/translation_diff/memory_cache_store.rb b/lib/translation_diff/memory_cache_store.rb index 93bd3aa..d7871aa 100644 --- a/lib/translation_diff/memory_cache_store.rb +++ b/lib/translation_diff/memory_cache_store.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - # The default cache, a bounded in-process LRU. NOT thread-safe, deliberately -- set `redis_url` for that. class TranslationDiff::MemoryCacheStore def self.build(config) = new(max_size: config.cache_max_size) diff --git a/lib/translation_diff/provider.rb b/lib/translation_diff/provider.rb index 204b9f0..f817af4 100644 --- a/lib/translation_diff/provider.rb +++ b/lib/translation_diff/provider.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - # Connects this library to one translation service; knows nothing about HTTP itself -- that's HTTPProvider. class TranslationDiff::Provider # A subclass that forgets to declare capabilities under-promises, not over-promises: smaller batches, not silent risk. diff --git a/lib/translation_diff/providers.rb b/lib/translation_diff/providers.rb index 207ace5..a16e1e4 100644 --- a/lib/translation_diff/providers.rb +++ b/lib/translation_diff/providers.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - # Translation providers, by name; registering one also declares its configuration options. module TranslationDiff::Providers class << self diff --git a/lib/translation_diff/providers/amazon.rb b/lib/translation_diff/providers/amazon.rb index c57f1f2..ba91603 100644 --- a/lib/translation_diff/providers/amazon.rb +++ b/lib/translation_diff/providers/amazon.rb @@ -1,13 +1,11 @@ -# frozen_string_literal: true - # Amazon Translate: no batch API (one text per call), no HTML mode, and requests are signed, not just headed. class TranslationDiff::Providers::Amazon < TranslationDiff::HTTPProvider - SERVICE = "translate" - TARGET = "AWSShineFrontendService_20170701.TranslateText" - CONTENT_TYPE = "application/x-amz-json-1.1" + SERVICE = "translate".freeze + TARGET = "AWSShineFrontendService_20170701.TranslateText".freeze + CONTENT_TYPE = "application/x-amz-json-1.1".freeze # Amazon's own way of asking for detection; reaches Comprehend under the hood, in regions that have it. - AUTO = "auto" + AUTO = "auto".freeze def self.capabilities TranslationDiff::Capabilities.new( @@ -103,10 +101,10 @@ def signed_headers(body) end # The signature covers the body exactly as sent, so this omits `faraday.request :json` unlike the base class. - def build_connection(&block) + def build_connection(&) Faraday.new(url: api_base, headers: headers) do |faraday| faraday.request :retry, retry_options - adapt(faraday, &block) + adapt(faraday, &) apply_timeouts(faraday) end end diff --git a/lib/translation_diff/providers/azure.rb b/lib/translation_diff/providers/azure.rb index b6dfc57..e2e972c 100644 --- a/lib/translation_diff/providers/azure.rb +++ b/lib/translation_diff/providers/azure.rb @@ -1,12 +1,10 @@ -# frozen_string_literal: true - # Azure AI Translator, REST v3.0: cheapest per character, most generous per request (1,000 strings/50,000 chars). class TranslationDiff::Providers::Azure < TranslationDiff::HTTPProvider - HOST = "https://api.cognitive.microsofttranslator.com" - API_VERSION = "3.0" + HOST = "https://api.cognitive.microsofttranslator.com".freeze + API_VERSION = "3.0".freeze # Azure spells HTML handling `textType`, and under it honours `class=notranslate` like DeepL and Google do. - DEFAULT_TEXT_TYPE = "html" + DEFAULT_TEXT_TYPE = "html".freeze def self.capabilities TranslationDiff::Capabilities.new( diff --git a/lib/translation_diff/providers/deepl.rb b/lib/translation_diff/providers/deepl.rb index d46b8c6..4ecb109 100644 --- a/lib/translation_diff/providers/deepl.rb +++ b/lib/translation_diff/providers/deepl.rb @@ -1,12 +1,10 @@ -# frozen_string_literal: true - # Talks to DeepL's REST API directly, not deepl-rb: it logged the auth key at DEBUG and defaulted notranslate off. class TranslationDiff::Providers::DeepL < TranslationDiff::HTTPProvider - PAID_HOST = "https://api.deepl.com" - FREE_HOST = "https://api-free.deepl.com" + PAID_HOST = "https://api.deepl.com".freeze + FREE_HOST = "https://api-free.deepl.com".freeze # A key ending in :fx is a free-plan key, and the free plan lives on its own host. - FREE_KEY_SUFFIX = ":fx" + FREE_KEY_SUFFIX = ":fx".freeze # DeepL honours class="notranslate" only under HTML tag handling -- otherwise content translates, tags survive. DEFAULT_OPTIONS = { tag_handling: :html, tag_handling_version: "v2" }.freeze @@ -23,7 +21,7 @@ def self.configuration_options = %i[deepl_api_key deepl_api_base] def self.configuration_requirements = %i[deepl_api_key] # DeepL requires a target language even when only detection is wanted, so the provider picks one. - DETECTION_TARGET = "EN" + DETECTION_TARGET = "EN".freeze def api_base config.deepl_api_base || (free_key? ? FREE_HOST : PAID_HOST) diff --git a/lib/translation_diff/providers/google.rb b/lib/translation_diff/providers/google.rb index a8be8b5..f3fd543 100644 --- a/lib/translation_diff/providers/google.rb +++ b/lib/translation_diff/providers/google.rb @@ -1,8 +1,6 @@ -# frozen_string_literal: true - # Talks to Cloud Translation v2 directly, not google-cloud-translate-v2, which pulled in grpc for one POST. class TranslationDiff::Providers::Google < TranslationDiff::HTTPProvider - HOST = "https://translation.googleapis.com" + HOST = "https://translation.googleapis.com".freeze # Verified against the live API: `text` format translates the protected span and drops its markup. DEFAULT_FORMAT = :html diff --git a/lib/translation_diff/providers/libretranslate.rb b/lib/translation_diff/providers/libretranslate.rb index 01d59f1..a3fe5f4 100644 --- a/lib/translation_diff/providers/libretranslate.rb +++ b/lib/translation_diff/providers/libretranslate.rb @@ -1,11 +1,9 @@ -# frozen_string_literal: true - # The only free, self-hosted provider here; base URL is required (everyone runs their own), API key is optional. class TranslationDiff::Providers::LibreTranslate < TranslationDiff::HTTPProvider - DEFAULT_FORMAT = "html" + DEFAULT_FORMAT = "html".freeze # The API's own way of asking for detection: `source` is required, and "auto" means "work it out". - AUTO = "auto" + AUTO = "auto".freeze # Observed 2026-09-09 via Docker: LibreTranslate's HTML format preserves markup but translates content anyway. LIBRETRANSLATE_HONOURS_NOTRANSLATE = false diff --git a/lib/translation_diff/providers/modernmt.rb b/lib/translation_diff/providers/modernmt.rb index 55b675d..f150465 100644 --- a/lib/translation_diff/providers/modernmt.rb +++ b/lib/translation_diff/providers/modernmt.rb @@ -1,11 +1,9 @@ -# frozen_string_literal: true - # ModernMT: adaptive translation with translation memories. class TranslationDiff::Providers::ModernMT < TranslationDiff::HTTPProvider - HOST = "https://api.modernmt.com" + HOST = "https://api.modernmt.com".freeze # ModernMT spells its formats as MIME types. - DEFAULT_FORMAT = "text/html" + DEFAULT_FORMAT = "text/html".freeze # Unverified, not observed: no key was available to probe it; false is the safe assumption either way. MODERNMT_HONOURS_NOTRANSLATE = false diff --git a/lib/translation_diff/providers/null.rb b/lib/translation_diff/providers/null.rb index f6e6fe4..2882676 100644 --- a/lib/translation_diff/providers/null.rb +++ b/lib/translation_diff/providers/null.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - # Hands back what it was given -- for tests, and for wiring a pipeline up before a real provider is available. class TranslationDiff::Providers::Null < TranslationDiff::Provider # Deliberately not detecting: this is the provider that proves the optional branch works. diff --git a/lib/translation_diff/redis_cache_store.rb b/lib/translation_diff/redis_cache_store.rb index b9f89aa..9dde87d 100644 --- a/lib/translation_diff/redis_cache_store.rb +++ b/lib/translation_diff/redis_cache_store.rb @@ -1,8 +1,6 @@ -# frozen_string_literal: true - class TranslationDiff::RedisCacheStore ONE_WEEK = 60 * 60 * 24 * 7 - DEFAULT_NAMESPACE = "translation-diff" + DEFAULT_NAMESPACE = "translation-diff".freeze def self.build(config) new(config.redis_pool, timeout: config.cache_ttl, namespace: config.cache_namespace) diff --git a/lib/translation_diff/redis_rate_limiter.rb b/lib/translation_diff/redis_rate_limiter.rb index 8004ac6..fd42ed1 100644 --- a/lib/translation_diff/redis_rate_limiter.rb +++ b/lib/translation_diff/redis_rate_limiter.rb @@ -1,14 +1,12 @@ -# frozen_string_literal: true - class TranslationDiff::RedisRateLimiter class RateLimitExceeded < TranslationDiff::Error; end DEFAULT_THRESHOLD = 8000 DEFAULT_INTERVAL = 60 - DEFAULT_NAMESPACE = "translation-diff" + DEFAULT_NAMESPACE = "translation-diff".freeze # This library limits the provider as a whole rather than per caller, so there is exactly one subject. - SUBJECT = "call" + SUBJECT = "call".freeze def self.build(config) new(config.redis_pool, diff --git a/lib/translation_diff/registry.rb b/lib/translation_diff/registry.rb index 2e2740d..9d98fcc 100644 --- a/lib/translation_diff/registry.rb +++ b/lib/translation_diff/registry.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - # Maps a symbol to a class that builds itself from a Configuration; the whole contract is answering `build(config)`. class TranslationDiff::Registry # `kind` appears in the unknown-name error message, so it should be a singular noun: "provider". diff --git a/lib/translation_diff/request.rb b/lib/translation_diff/request.rb index 89e2990..b097d12 100644 --- a/lib/translation_diff/request.rb +++ b/lib/translation_diff/request.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - class TranslationDiff::Request extend Forwardable include TranslationDiff::Instrumentation diff --git a/lib/translation_diff/segmenters.rb b/lib/translation_diff/segmenters.rb index 7e178ac..5d2a5f2 100644 --- a/lib/translation_diff/segmenters.rb +++ b/lib/translation_diff/segmenters.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - module TranslationDiff::Segmenters # Segmenters, by name. `config.segmenter = :simple` resolves through here. def self.registry = @registry ||= TranslationDiff::Registry.new("segmenter") diff --git a/lib/translation_diff/segmenters/pragmatic.rb b/lib/translation_diff/segmenters/pragmatic.rb index c5608ad..a147d0e 100644 --- a/lib/translation_diff/segmenters/pragmatic.rb +++ b/lib/translation_diff/segmenters/pragmatic.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - require "pragmatic_segmenter" # Default segmenter: scores 76/80 on the Golden Rules corpus vs Simple's 47/80 (golden_rules_test.rb). @@ -8,7 +6,7 @@ class TranslationDiff::Segmenters::Pragmatic class Error < TranslationDiff::Error; end # Without a language, Russian mis-segments: it treats "Проф." as a full sentence and stops there. - DEFAULT_LANGUAGE = "en" + DEFAULT_LANGUAGE = "en".freeze # A lone "\n" is incidental source formatting, not a paragraph break; a run of two or more is left alone. SINGLE_NEWLINE = /(?foo" \ "barbaz" \ - "" + "".freeze # source => expected tokens CASES = { diff --git a/test/translation_diff/translation/response_test.rb b/test/translation_diff/translation/response_test.rb index a87967f..8b2a258 100644 --- a/test/translation_diff/translation/response_test.rb +++ b/test/translation_diff/translation/response_test.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - require "test_helper" class TranslationResponseTest < Minitest::Test diff --git a/test/translation_diff_test.rb b/test/translation_diff_test.rb index 52bd8ed..0b4f449 100644 --- a/test/translation_diff_test.rb +++ b/test/translation_diff_test.rb @@ -1,5 +1,3 @@ -# frozen_string_literal: true - require "test_helper" class TranslationDiffTest < ConfiguredTest diff --git a/translation_diff.gemspec b/translation_diff.gemspec index ebbf482..bf8b2a6 100644 --- a/translation_diff.gemspec +++ b/translation_diff.gemspec @@ -1,5 +1,3 @@ -# frozen_string_literal: true - lib = File.expand_path("lib", __dir__) $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) require "translation_diff/version" @@ -21,7 +19,7 @@ small edit costs the price of the edit, not the whole text. ) spec.homepage = "https://github.com/Halvanhelv/translation_diff" spec.license = "MIT" - spec.required_ruby_version = ">= 3.2" + spec.required_ruby_version = ">= 3.4" if spec.respond_to?(:metadata) spec.metadata["allowed_push_host"] = "https://rubygems.org" From fe3d9282884e633e97747c1efa085f2a1992fb4a Mon Sep 17 00:00:00 2001 From: IG Date: Wed, 9 Sep 2026 19:45:49 +0400 Subject: [PATCH 18/26] fix: reject a non-String translation in Response.build A provider returning a well-formed response that carries nil for one input passed Response.build -- it checked only the count. The nil was written into the cache under a real key and the caller got NoMethodError: undefined method 'strip' for nil out of Spacing.restore, naming neither the provider nor the position. Azure documents exactly this shape: 200 for a batch where one element carries `error` instead of `translations`. Five of six providers can produce it since the branch replaced SDK objects with raw JSON lookups. Response.build now names the class and the position of the first offender, never the value, which is the customer's text or the provider's error object. --- lib/translation_diff/translation/response.rb | 26 +++++++++--- test/translation_diff/providers/azure_test.rb | 15 +++++++ .../translation/response_test.rb | 42 +++++++++++++++++++ 3 files changed, 78 insertions(+), 5 deletions(-) diff --git a/lib/translation_diff/translation/response.rb b/lib/translation_diff/translation/response.rb index 53cf2b0..3829c2f 100644 --- a/lib/translation_diff/translation/response.rb +++ b/lib/translation_diff/translation/response.rb @@ -1,13 +1,29 @@ -# ::build checks the count: fewer texts than requested would shift nils in and surface as a distant NoMethodError. +# ::build is the one guard over every provider's parse step: a wrong count or a non-String would surface far away. module TranslationDiff::Translation Response = Data.define(:texts, :detected_source, :usage) do def self.build(request:, texts:, detected_source: nil, usage: nil) - if texts.size != request.texts.size - raise TranslationDiff::ResponseError, - "Provider returned #{texts.size} translations for #{request.texts.size} values" - end + ensure_count!(request, texts) + ensure_strings!(texts) new(texts: texts, detected_source: detected_source, usage: usage) end + + def self.ensure_count!(request, texts) + return if texts.size == request.texts.size + + raise TranslationDiff::ResponseError, + "Provider returned #{texts.size} translations for #{request.texts.size} values" + end + + # The class, never the value: the value is either the customer's text or the provider's own error object. + def self.ensure_strings!(texts) + index = texts.index { |text| !text.is_a?(String) } + return if index.nil? + + raise TranslationDiff::ResponseError, + "Provider returned #{texts[index].class} rather than a translation at position " \ + "#{index} of #{texts.size}. A response can be well-formed and still carry no " \ + "translation for one input -- a per-string failure inside a batch that answered 200." + end end end diff --git a/test/translation_diff/providers/azure_test.rb b/test/translation_diff/providers/azure_test.rb index b7ab08d..4ff3172 100644 --- a/test/translation_diff/providers/azure_test.rb +++ b/test/translation_diff/providers/azure_test.rb @@ -133,6 +133,21 @@ def test_detect_returns_the_language_azure_reports assert_equal "en", detector.detect("something") end + # Azure answers 200 for a batch where one string failed, carrying `error` in place of `translations`. + def test_a_per_string_failure_in_the_middle_of_a_batch_raises_rather_than_caching_nil + body = [ + { "translations" => [{ "text" => "один", "to" => "ru" }] }, + { "error" => { "code" => 400_050, "message" => "The input is too long." } }, + { "translations" => [{ "text" => "три", "to" => "ru" }] } + ] + + error = assert_raises(TranslationDiff::ResponseError) do + provider(body: body).translate(translation_request(%w[one two three])) + end + + assert_match(/position 1/, error.message) + end + def test_its_limits_are_azures_documented_ones capabilities = TranslationDiff::Providers::Azure.capabilities diff --git a/test/translation_diff/translation/response_test.rb b/test/translation_diff/translation/response_test.rb index 8b2a258..1ce7f7e 100644 --- a/test/translation_diff/translation/response_test.rb +++ b/test/translation_diff/translation/response_test.rb @@ -25,6 +25,48 @@ def test_build_raises_when_the_provider_returned_the_wrong_number_of_texts assert_match(/2/, error.message) end + # A nil translation used to reach Spacing.restore and die there as NoMethodError, naming nothing. + def test_build_raises_when_a_translation_is_not_a_string + error = assert_raises(TranslationDiff::ResponseError) do + TranslationDiff::Translation::Response.build(request: request, texts: ["один", nil]) + end + + assert_match(/position 1/, error.message) + assert_match(/NilClass/, error.message) + end + + # Azure returns 200 for a batch where one element carries `error` instead of `translations`. + def test_build_raises_for_a_nil_in_the_middle_of_a_batch + batch = request(%w[one two three]) + + error = assert_raises(TranslationDiff::ResponseError) do + TranslationDiff::Translation::Response.build(request: batch, texts: ["один", nil, "три"]) + end + + assert_match(/position 1/, error.message) + end + + def test_build_names_only_the_first_offending_position + batch = request(%w[one two three]) + + error = assert_raises(TranslationDiff::ResponseError) do + TranslationDiff::Translation::Response.build(request: batch, texts: [nil, nil, 42]) + end + + assert_match(/position 0/, error.message) + refute_match(/position 2/, error.message) + end + + # The offending value is the customer's text or a provider's error object; neither belongs in a message. + def test_build_names_the_class_but_never_the_offending_value + error = assert_raises(TranslationDiff::ResponseError) do + TranslationDiff::Translation::Response.build(request: request, texts: ["один", { "error" => "s3cret" }]) + end + + refute_match(/s3cret/, error.message) + assert_match(/Hash/, error.message) + end + def test_detected_source_and_usage_default_to_nil response = TranslationDiff::Translation::Response.build(request: request, texts: %w[один два]) From 9d717afec6df02d08565f523a0e00dbf143c2e04 Mon Sep 17 00:00:00 2001 From: IG Date: Wed, 9 Sep 2026 19:46:49 +0400 Subject: [PATCH 19/26] fix: stop Cache#store consuming the array it is handed #store shifted its `updates` argument empty. Task 10 dup'd at the one call site, which fixes that caller and nothing else: #store is a public method on a public class taking an outside array, so the next call site anyone adds reintroduces the bug in the same silent form -- the caller's array comes back empty with no error. It now indexes instead of shifting. The dup at the call site stays: a provider's array is the provider's, and Request should not hand it to a collaborator either. Pins the exact cache keys so this change, and the option-declaration change that follows, are shown not to move one. --- lib/translation_diff/cache.rb | 5 ++++- lib/translation_diff/request.rb | 2 +- test/translation_diff/cache_test.rb | 26 ++++++++++++++++++++++++++ 3 files changed, 31 insertions(+), 2 deletions(-) diff --git a/lib/translation_diff/cache.rb b/lib/translation_diff/cache.rb index fcee513..15afd92 100644 --- a/lib/translation_diff/cache.rb +++ b/lib/translation_diff/cache.rb @@ -21,9 +21,12 @@ def cached_and_missing(values) [cached, missing] end + # Indexes rather than shifting: #store is public and takes an outside array, which is not ours to consume. def store(values, cached, updates) + update = -1 + cached.map.with_index do |value, index| - value || store_value(values[index], updates.shift) + value || store_value(values[index], updates[update += 1]) end end diff --git a/lib/translation_diff/request.rb b/lib/translation_diff/request.rb index b097d12..9ffe92a 100644 --- a/lib/translation_diff/request.rb +++ b/lib/translation_diff/request.rb @@ -159,7 +159,7 @@ def call_api(values) characters: values.sum(&:size)) do api.translate(request) end - # Dup'd: Cache#store consumes this array destructively (#shift), and a provider may hand back its own array. + # Dup'd: the array is the provider's own, and handing it to a collaborator makes it the collaborator's too. response.texts.dup end diff --git a/test/translation_diff/cache_test.rb b/test/translation_diff/cache_test.rb index 7a8bc30..5dcb33d 100644 --- a/test/translation_diff/cache_test.rb +++ b/test/translation_diff/cache_test.rb @@ -103,6 +103,32 @@ def test_cached_and_missing_pairs_results_positionally assert_equal ["two"], missing end + # #store is public and takes an outside array; shifting it emptied the caller's own array. + def test_store_does_not_consume_the_updates_it_is_given + updates = %w[один три] + cache = TranslationDiff::Cache.new(:en, :ru, provider: "deepl", store: @store) + + cache.store(%w[one two three], [nil, "два", nil], updates) + + assert_equal %w[один три], updates + end + + def test_store_fills_the_gaps_in_key_order + cache = TranslationDiff::Cache.new(:en, :ru, provider: "deepl", store: @store) + + assert_equal %w[один два три], + cache.store(%w[one two three], [nil, "два", nil], %w[один три]) + end + + # The exact key a translation is stored under. Change it and every user re-translates their whole corpus. + def test_the_key_is_the_one_users_already_have_in_their_caches + key_for(value: "text", from: :en, to: :ru, provider: "deepl") + key_for(value: "text", from: :en, to: :ru, provider: "deepl", options: { formality: :less }) + + assert_equal "deepl:en:ru:1cb251ec0d568de6a929b520c4aed8d1", @store.keys[0] + assert_equal "deepl:en:ru:80df90b8:1cb251ec0d568de6a929b520c4aed8d1", @store.keys[1] + end + private def key_for(value: "text", from: :en, to: :ru, provider: "deepl", options: {}) From 41cd8b9d4418a666f8f489a396008d15d49f6e81 Mon Sep 17 00:00:00 2001 From: IG Date: Wed, 9 Sep 2026 19:51:11 +0400 Subject: [PATCH 20/26] fix: restore the environment-variable credential fallbacks The design promised that DEEPL_AUTH_KEY, TRANSLATE_KEY/GOOGLE_CLOUD_KEY and TRANSLATE_PROJECT would be read by us now that the SDKs that read them are gone. Nothing implemented it and nothing could: configuration_options was a bare symbol array with nowhere to put a default. An application that set DEEPL_AUTH_KEY and never assigned config.deepl_api_key worked on main and raised ConfigurationError on the first translate here -- at boot, for a Rails app. configuration_options now takes either a bare symbol or `key => default`, and a default routes through Configuration.option's existing callable support, so it is resolved on read rather than at load: the variable may be exported after this gem is required. A blank default reads as unset, the rule assignment already followed, so DEEPL_AUTH_KEY= is a missing key rather than an empty credential. Amazon declares no fallback on purpose and now says so. CHANGELOG: the sentence claiming Google reads those variables and falls back to application default credentials was false twice over -- corrected, ADC recorded as dropped, and the two missing Breaking entries added (deepl_host -> deepl_api_base, and a count mismatch raising ResponseError rather than Request::Error). --- CHANGELOG.md | 46 +++++++++--- README.md | 32 +++++++-- lib/translation_diff/configuration.rb | 22 ++++-- lib/translation_diff/providers/amazon.rb | 1 + lib/translation_diff/providers/deepl.rb | 6 +- lib/translation_diff/providers/google.rb | 8 ++- test/support/env_stub.rb | 16 +++++ test/translation_diff/configuration_test.rb | 71 +++++++++++++++++++ .../translation_diff/providers/amazon_test.rb | 14 ++++ test/translation_diff/providers/deepl_test.rb | 38 ++++++++++ .../translation_diff/providers/google_test.rb | 33 +++++++++ 11 files changed, 264 insertions(+), 23 deletions(-) create mode 100644 test/support/env_stub.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index a7cf040..9f4af0d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -91,6 +91,20 @@ described below. Everything here is relative to `deepl_diff` 2.2.0. - `deepl-rb` and `google-cloud-translate-v2` are no longer used at all. `faraday` and `faraday-retry` become runtime dependencies; `aws-sigv4` is required lazily by the Amazon provider only. +- `config.deepl_host` is renamed `config.deepl_api_base`, matching the + `_api_base` name every other provider uses. There is no alias: a + configuration still setting `deepl_host` raises `NoMethodError` on + `TranslationDiff.configure`. Rename it. +- A provider returning the wrong number of translations now raises + `TranslationDiff::ResponseError`, not `TranslationDiff::Request::Error`. + `Request::Error` still exists, and still means "`from:` is missing and the + provider cannot detect"; a `rescue TranslationDiff::Request::Error` written + to catch a short response no longer catches one. Both are + `TranslationDiff::Error`, so a rescue of the base class is unaffected. +- A provider returning a well-formed response that carries no translation for + one input -- Azure answers 200 for a batch where a single string failed -- + also raises `TranslationDiff::ResponseError`, naming the position. It + previously reached `Spacing.restore` and died there as `NoMethodError`. ### Removed @@ -102,15 +116,29 @@ described below. Everything here is relative to `deepl_diff` 2.2.0. ### Added - A Google provider: `config.provider = :google` translates through Cloud - Translation v2 (Basic), on the `google-cloud-translate-v2` gem, required - lazily so an application using DeepL never needs it installed. It declares - `google_api_key` and `google_project_id`; an API key alone is enough, and - with none configured the gem reads `TRANSLATE_KEY`/`GOOGLE_CLOUD_KEY` or - falls back to application default credentials. The provider asks for - `format: :html`, which the tokenizer's output requires -- a `notranslate` - span is handed over with its tags -- and downcases bare language codes so - a configuration written for DeepL (`"EN"`) keeps working, leaving - subtagged codes such as `"zh-Hans"` alone. + Translation v2 (Basic) over HTTP directly, with no Google gem installed. It + declares `google_api_key`, `google_project_id` and `google_api_base`; an API + key alone is enough. With no key configured it reads `TRANSLATE_KEY` and + then `GOOGLE_CLOUD_KEY`, and `google_project_id` falls back to + `TRANSLATE_PROJECT` -- the variables `google-cloud-translate-v2` used to + read on your behalf. Application default credentials are **not** supported: + that path lived in the gem that is gone, and an application relying on ADC + must now configure an API key. The provider asks for `format: :html`, which + the tokenizer's output requires -- a `notranslate` span is handed over with + its tags -- and downcases bare language codes so a configuration written for + DeepL (`"EN"`) keeps working, leaving subtagged codes such as `"zh-Hans"` + alone. +- Environment-variable credential fallbacks, read by this library now that the + vendor SDKs that read them are gone: `DEEPL_AUTH_KEY` for `deepl_api_key`, + `TRANSLATE_KEY` then `GOOGLE_CLOUD_KEY` for `google_api_key`, and + `TRANSLATE_PROJECT` for `google_project_id`. Each is read on use rather than + at load, so setting one after requiring the gem still works, and an + explicitly configured value always wins. The Amazon provider deliberately + has no environment fallback: `aws-sigv4` is handed explicit credentials and + this library does not implement the AWS credential chain. +- A provider declares a default for one of its options by writing + `key => default` in `configuration_options` instead of a bare symbol; a + callable default is evaluated on every read. - `TranslationDiff::Configuration`, a declarative settings object built through the `option(key, default)` macro. Options fall back to their default until assigned, treat a blank string as unset, and support a diff --git a/README.md b/README.md index 36f7c76..342e095 100644 --- a/README.md +++ b/README.md @@ -69,7 +69,9 @@ TranslationDiff.translate("Привет.", from: "ru", to: "en") `deepl_api_key` is required -- the provider checks for it at build time and raises `TranslationDiff::ConfigurationError` naming what is missing, rather -than failing on the first real request. `redis_url` is optional: without it +than failing on the first real request. Leave it unset and `DEEPL_AUTH_KEY` +is read instead, on use rather than at load, so the variable may be exported +after this gem is required. `redis_url` is optional: without it the cache lives in the process, which means the library runs before any infrastructure does. @@ -116,10 +118,10 @@ Every provider declares its own configuration options, registered the moment | Provider | Options | Meaning | | --- | --- | --- | -| `:deepl` | `deepl_api_key` (required) | Sent as `DeepL-Auth-Key`. | +| `:deepl` | `deepl_api_key` (required) | Sent as `DeepL-Auth-Key`. Falls back to `ENV["DEEPL_AUTH_KEY"]`. | | | `deepl_api_base` | Overrides the automatic free/paid host selection (from the `:fx` suffix on the key). Rarely needed. | -| `:google` | `google_api_key` (required) | Sent as the `key` query parameter. | -| | `google_project_id` | Declared for a future credentials path; not currently read -- an API key needs no project. | +| `:google` | `google_api_key` (required) | Sent as the `key` query parameter. Falls back to `ENV["TRANSLATE_KEY"]`, then `ENV["GOOGLE_CLOUD_KEY"]`. | +| | `google_project_id` | Declared for a future credentials path; not currently read -- an API key needs no project. Falls back to `ENV["TRANSLATE_PROJECT"]`. | | | `google_api_base` | Overrides the default `https://translation.googleapis.com`. | | `:azure` | `azure_api_key` (required) | Sent as `Ocp-Apim-Subscription-Key`. | | | `azure_region` | Sent as `Ocp-Apim-Subscription-Region`. Required by a multi-service Azure resource; a single-service resource needs no region. | @@ -128,7 +130,7 @@ Every provider declares its own configuration options, registered the moment | | `modernmt_api_base` | Overrides the default `https://api.modernmt.com`. | | `:libretranslate` | `libretranslate_api_base` (required) | Every instance is self-hosted; there is no default to fall back to. | | | `libretranslate_api_key` | Sent as `api_key` in the request body. Most instances do not require one. | -| `:amazon` | `amazon_access_key_id`, `amazon_secret_access_key`, `amazon_region` (all required) | Used to sign each request with `aws-sigv4`. | +| `:amazon` | `amazon_access_key_id`, `amazon_secret_access_key`, `amazon_region` (all required) | Used to sign each request with `aws-sigv4`. No environment fallback: this library does not implement the AWS credential chain, so `AWS_ACCESS_KEY_ID` and friends are not read. | | | `amazon_session_token` | For temporary credentials. | | | `amazon_api_base` | Overrides the default `https://translate..amazonaws.com`. | @@ -315,6 +317,21 @@ the log to say so. Registering over an existing name is not a way to substitute a service -- give the replacement its own name, or clear the cache (`cache_namespace` is the cheapest way to do that). +**An option can declare a default.** A bare symbol in +`configuration_options` declares an option with no default. Writing +`key => default` instead declares one, and a callable default is evaluated on +every read rather than at load time -- which is what lets an environment +variable work when the application exports it after requiring this gem: + +```ruby +def self.configuration_options + [:yandex_api_base, { yandex_api_key: -> { ENV.fetch("YANDEX_API_KEY", nil) } }] +end +``` + +An explicitly configured value always wins over a default, and a default that +resolves to a blank string reads as unset -- the same rule assignment follows. + **Option names are unique too, and enforced.** Two providers declaring the same `configuration_options` name would share one accessor on `TranslationDiff::Configuration`, which would hand one service's credential @@ -637,8 +654,9 @@ TranslationDiff::Error │ # timed out, or TLS failed ├── TranslationDiff::ResponseError # the answer was well-formed HTTP but broke │ # this library's contract -- a body that -│ # is not JSON, or a provider that returned -│ # the wrong number of translations +│ # is not JSON, a provider that returned +│ # the wrong number of translations, or one +│ # that returned no translation for an input ├── TranslationDiff::InvalidProviderError # a class registered without inheriting │ # TranslationDiff::Provider ├── TranslationDiff::Request::Error # from: missing and the provider cannot diff --git a/lib/translation_diff/configuration.rb b/lib/translation_diff/configuration.rb index c813844..8b67edf 100644 --- a/lib/translation_diff/configuration.rb +++ b/lib/translation_diff/configuration.rb @@ -16,10 +16,10 @@ def option(key, default = nil) end # See ProviderOptionOwners for the conflict rules and the all-or-nothing guarantee. - def register_provider_options(keys, provider) - keys = Array(keys).map(&:to_sym) - provider_option_owners.claim(keys, provider) - keys.each { |key| option(key) } + def register_provider_options(declared, provider) + declared = normalise_declarations(declared) + provider_option_owners.claim(declared.keys, provider) + declared.each { |key, default| option(key, default) } end def options = @options ||= [] @@ -27,6 +27,15 @@ def defaults = @defaults ||= {} private + # `:key` declares an option with no default; `{ key => default }` declares one, and a callable is read lazily. + def normalise_declarations(declared) + entries = declared.is_a?(Hash) ? [declared] : Array(declared) + + entries.each_with_object({}) do |entry, result| + entry.is_a?(Hash) ? result.merge!(entry.transform_keys(&:to_sym)) : result[entry.to_sym] = nil + end + end + def provider_option_owners = @provider_option_owners ||= ProviderOptionOwners.new end @@ -98,14 +107,17 @@ def build_redis_pool '`gem "redis-namespace"` to your Gemfile.' end + # A default is resolved on every read, and a blank one is unset -- the rule the writer already applies. def read(key) value = instance_variable_get(:"@#{key}") return value unless value.nil? default = self.class.defaults[key] - default.respond_to?(:call) ? default.call : default + blank_to_nil(default.respond_to?(:call) ? default.call : default) end + def blank_to_nil(value) = value.is_a?(String) && value.strip.empty? ? nil : value + # The symbol-or-object rule, implemented once for all three extension points. def resolve(value, registry) value.is_a?(Symbol) || value.is_a?(String) ? registry.build(value, self) : value diff --git a/lib/translation_diff/providers/amazon.rb b/lib/translation_diff/providers/amazon.rb index ba91603..d51d9d1 100644 --- a/lib/translation_diff/providers/amazon.rb +++ b/lib/translation_diff/providers/amazon.rb @@ -14,6 +14,7 @@ def self.capabilities ) end + # Deliberately no environment fallback: this library does not implement the AWS credential chain. def self.configuration_options %i[amazon_access_key_id amazon_secret_access_key amazon_session_token amazon_region amazon_api_base] diff --git a/lib/translation_diff/providers/deepl.rb b/lib/translation_diff/providers/deepl.rb index 4ecb109..8999b09 100644 --- a/lib/translation_diff/providers/deepl.rb +++ b/lib/translation_diff/providers/deepl.rb @@ -17,7 +17,11 @@ def self.capabilities ) end - def self.configuration_options = %i[deepl_api_key deepl_api_base] + # DEEPL_AUTH_KEY is what deepl-rb read on our behalf; the callable keeps it read on use, not at load. + def self.configuration_options + [:deepl_api_base, { deepl_api_key: -> { ENV.fetch("DEEPL_AUTH_KEY", nil) } }] + end + def self.configuration_requirements = %i[deepl_api_key] # DeepL requires a target language even when only detection is wanted, so the provider picks one. diff --git a/lib/translation_diff/providers/google.rb b/lib/translation_diff/providers/google.rb index f3fd543..5d3ec23 100644 --- a/lib/translation_diff/providers/google.rb +++ b/lib/translation_diff/providers/google.rb @@ -13,7 +13,13 @@ def self.capabilities ) end - def self.configuration_options = %i[google_api_key google_project_id google_api_base] + # TRANSLATE_KEY then GOOGLE_CLOUD_KEY, the order google-cloud-translate-v2 read them in. + def self.configuration_options + [:google_api_base, + { google_api_key: -> { ENV.fetch("TRANSLATE_KEY", nil) || ENV.fetch("GOOGLE_CLOUD_KEY", nil) }, + google_project_id: -> { ENV.fetch("TRANSLATE_PROJECT", nil) } }] + end + def self.configuration_requirements = %i[google_api_key] # A bare code is downcased for DeepL-style configs ("EN"); a subtag ("zh-Hans") is passed through untouched. diff --git a/test/support/env_stub.rb b/test/support/env_stub.rb new file mode 100644 index 0000000..62ee2cc --- /dev/null +++ b/test/support/env_stub.rb @@ -0,0 +1,16 @@ +# Sets environment variables around a block and restores exactly what was there, including "was not set". +module EnvStub + def with_env(values) + original = values.keys.to_h { |key| [key, ENV.fetch(key, nil)] } + apply_env(values) + yield + ensure + apply_env(original) + end + + private + + def apply_env(values) + values.each { |key, value| value.nil? ? ENV.delete(key) : ENV[key] = value } + end +end diff --git a/test/translation_diff/configuration_test.rb b/test/translation_diff/configuration_test.rb index 30ddff5..601eb5d 100644 --- a/test/translation_diff/configuration_test.rb +++ b/test/translation_diff/configuration_test.rb @@ -1,6 +1,9 @@ require "test_helper" +require "support/env_stub" class ConfigurationTest < Minitest::Test + include EnvStub + # A hand-written double for TranslationDiff::Registry: Minitest 6.0 dropped minitest/mock. ResolvingRegistry = Struct.new(:answer) do attr_reader :asked @@ -142,6 +145,74 @@ def test_a_conflict_on_a_later_option_leaves_no_partial_state assert_includes TranslationDiff::Configuration.options, :fresh_option end + # A provider needing no default keeps the bare-symbol form; one needing a default declares `key => default`. + class DefaultingOptionOwner + def self.configuration_options + [:defaulting_bare, { defaulting_keyed: -> { ENV.fetch("DEFAULTING_TEST_VAR", nil) } }] + end + end + + def test_a_provider_declares_bare_symbols_and_defaults_in_one_list + TranslationDiff::Configuration.register_provider_options( + DefaultingOptionOwner.configuration_options, DefaultingOptionOwner + ) + + assert_includes TranslationDiff::Configuration.options, :defaulting_bare + assert_includes TranslationDiff::Configuration.options, :defaulting_keyed + assert_nil TranslationDiff::Configuration.new.defaulting_bare + end + + # Evaluated on read, not at load: an application that sets the variable after requiring us still gets it. + def test_a_declared_callable_default_is_evaluated_on_every_read + TranslationDiff::Configuration.register_provider_options( + DefaultingOptionOwner.configuration_options, DefaultingOptionOwner + ) + + with_env("DEFAULTING_TEST_VAR" => "from-the-environment") do + assert_equal "from-the-environment", TranslationDiff::Configuration.new.defaulting_keyed + end + end + + def test_an_assigned_value_wins_over_a_declared_default + TranslationDiff::Configuration.register_provider_options( + DefaultingOptionOwner.configuration_options, DefaultingOptionOwner + ) + config = TranslationDiff::Configuration.new + config.defaulting_keyed = "assigned" + + with_env("DEFAULTING_TEST_VAR" => "from-the-environment") do + assert_equal "assigned", config.defaulting_keyed + end + end + + # The blank rule the writer already applies: a variable exported empty means unset, not an empty credential. + def test_a_blank_default_reads_as_unset + TranslationDiff::Configuration.register_provider_options( + DefaultingOptionOwner.configuration_options, DefaultingOptionOwner + ) + + with_env("DEFAULTING_TEST_VAR" => " ") do + assert_nil TranslationDiff::Configuration.new.defaulting_keyed + end + end + + class KeyedConflictOwner + def self.configuration_options = [:keyed_fresh_option, { owned_by_acme: -> { "x" } }] + end + + # Ownership is claimed for every key however it was written, so a keyed clash still leaves no partial state. + def test_a_conflict_on_a_keyed_option_leaves_no_partial_state + TranslationDiff::Configuration.register_provider_options(%i[owned_by_acme], AcmeOptionOwner) + + assert_raises(TranslationDiff::Error) do + TranslationDiff::Configuration.register_provider_options( + KeyedConflictOwner.configuration_options, KeyedConflictOwner + ) + end + + refute_includes TranslationDiff::Configuration.options, :keyed_fresh_option + end + def test_registering_an_option_twice_does_not_clobber_the_first_default # shared_option is not a production option; mutating class-level state with it is harmless. TranslationDiff::Configuration.option(:shared_option, "first") diff --git a/test/translation_diff/providers/amazon_test.rb b/test/translation_diff/providers/amazon_test.rb index e51657d..6b17863 100644 --- a/test/translation_diff/providers/amazon_test.rb +++ b/test/translation_diff/providers/amazon_test.rb @@ -1,12 +1,14 @@ require "test_helper" require "support/provider_contract" require "support/http_provider_contract" +require "support/env_stub" require "faraday" require "aws-sigv4" class AmazonProviderTest < Minitest::Test include ProviderContract include HTTPProviderContract + include EnvStub attr_reader :config, :requests @@ -46,6 +48,18 @@ def respond_to_translate(env, texts, recorder) "TargetLanguageCode" => "ru" }.to_json] end + # Deliberate: aws-sigv4 takes explicit credentials and this library does not implement the credential chain. + def test_it_reads_no_aws_environment_variables + with_env("AWS_ACCESS_KEY_ID" => "AKIAENV", "AWS_SECRET_ACCESS_KEY" => "secret", + "AWS_REGION" => "eu-west-1") do + fresh = TranslationDiff::Configuration.new + + assert_nil fresh.amazon_access_key_id + assert_nil fresh.amazon_secret_access_key + assert_nil fresh.amazon_region + end + end + def test_the_endpoint_is_regional assert_equal "https://translate.eu-central-1.amazonaws.com", TranslationDiff::Providers::Amazon.new(config).api_base diff --git a/test/translation_diff/providers/deepl_test.rb b/test/translation_diff/providers/deepl_test.rb index c94f289..b12d1b7 100644 --- a/test/translation_diff/providers/deepl_test.rb +++ b/test/translation_diff/providers/deepl_test.rb @@ -2,12 +2,14 @@ require "support/provider_contract" require "support/http_provider_contract" require "support/stubbed_provider" +require "support/env_stub" require "faraday" class DeepLProviderTest < Minitest::Test include ProviderContract include HTTPProviderContract include StubbedProvider + include EnvStub # A real response body, captured from api-free.deepl.com on 2026-09-09. TRANSLATE_BODY = { @@ -33,6 +35,42 @@ def provider(body: nil, status: 200, headers: {}) status: status, headers: headers, name: :deepl) end + # deepl-rb read DEEPL_AUTH_KEY on our behalf; an app that set it and configured nothing kept working. + def test_the_deepl_auth_key_environment_variable_supplies_an_unset_key + with_env("DEEPL_AUTH_KEY" => "env-key:fx") do + assert_equal "env-key:fx", TranslationDiff::Configuration.new.deepl_api_key + end + end + + # The default is a callable, so setting the variable after this file was required still works. + def test_the_environment_is_read_on_use_not_at_load_time + built = TranslationDiff::Configuration.new + + with_env("DEEPL_AUTH_KEY" => "set-after-the-fact") do + assert_equal "set-after-the-fact", built.deepl_api_key + end + end + + def test_an_explicitly_configured_key_wins_over_the_environment + with_env("DEEPL_AUTH_KEY" => "env-key") do + assert_equal "test-key:fx", config.deepl_api_key + end + end + + def test_a_blank_environment_variable_is_not_a_key + with_env("DEEPL_AUTH_KEY" => " ") do + assert_nil TranslationDiff::Configuration.new.deepl_api_key + end + end + + # The scenario: ensure_configured! raised ConfigurationError at boot for an app that only set the variable. + def test_the_environment_variable_satisfies_the_configuration_requirement + with_env("DEEPL_AUTH_KEY" => "env-key") do + assert_instance_of TranslationDiff::Providers::DeepL, + TranslationDiff::Providers::DeepL.new(TranslationDiff::Configuration.new) + end + end + def test_a_free_key_selects_the_free_host assert_equal "https://api-free.deepl.com", TranslationDiff::Providers::DeepL.new(config).api_base end diff --git a/test/translation_diff/providers/google_test.rb b/test/translation_diff/providers/google_test.rb index 084efd0..c71b5c3 100644 --- a/test/translation_diff/providers/google_test.rb +++ b/test/translation_diff/providers/google_test.rb @@ -2,6 +2,7 @@ require "support/provider_contract" require "support/http_provider_contract" require "support/stubbed_provider" +require "support/env_stub" require "faraday" require "cgi" @@ -9,6 +10,7 @@ class GoogleProviderTest < Minitest::Test include ProviderContract include HTTPProviderContract include StubbedProvider + include EnvStub # Shaped from the Cloud Translation v2 REST reference read 2026-09-09: translations nest under "data". TRANSLATE_BODY = { @@ -34,6 +36,37 @@ def provider(body: nil, status: 200, headers: { "Content-Type" => "application/j status: status, headers: headers, name: :google) end + # google-cloud-translate-v2 read these on our behalf, TRANSLATE_KEY first. + def test_translate_key_supplies_an_unset_api_key + with_env("TRANSLATE_KEY" => "from-translate-key", "GOOGLE_CLOUD_KEY" => nil) do + assert_equal "from-translate-key", TranslationDiff::Configuration.new.google_api_key + end + end + + def test_google_cloud_key_is_the_second_fallback + with_env("TRANSLATE_KEY" => nil, "GOOGLE_CLOUD_KEY" => "from-google-cloud-key") do + assert_equal "from-google-cloud-key", TranslationDiff::Configuration.new.google_api_key + end + end + + def test_translate_key_wins_over_google_cloud_key + with_env("TRANSLATE_KEY" => "first", "GOOGLE_CLOUD_KEY" => "second") do + assert_equal "first", TranslationDiff::Configuration.new.google_api_key + end + end + + def test_an_explicitly_configured_key_wins_over_both_variables + with_env("TRANSLATE_KEY" => "first", "GOOGLE_CLOUD_KEY" => "second") do + assert_equal "test-key", config.google_api_key + end + end + + def test_translate_project_supplies_an_unset_project_id + with_env("TRANSLATE_PROJECT" => "a-project") do + assert_equal "a-project", TranslationDiff::Configuration.new.google_project_id + end + end + def test_the_key_travels_in_the_query_string provider.translate(translation_request(%w[one])) From 984fbfc77a7149aef72a9f81f8e0dd54abf05ad0 Mon Sep 17 00:00:00 2001 From: IG Date: Wed, 9 Sep 2026 19:53:39 +0400 Subject: [PATCH 21/26] fix: refuse a non-Provider however it was supplied Providers.register raised InvalidProviderError for a class that skipped the base class, but config.provider = -- a documented extension point -- returned any non-Symbol untouched, so an object was never checked. An app that upgraded with its own duck-typed provider object heard nothing at configure time and got NoMethodError: undefined method 'capabilities' from request.rb on the first translate, with no message naming the contract change. The `provider:` keyword had the same hole. All three paths now raise InvalidProviderError in the same words. cache, segmenter and rate_limiter are still genuinely duck-typed and stay untouched, which a test now pins. Also fixes the registry guard itself (M8): `klass < Provider` raised NoMethodError for an instance, nil or a symbol, and ArgumentError for a non-Module -- the first thing someone writing their own provider hits. The message names the class, never the object, whose #to_s would render its own contents. --- lib/translation_diff/configuration.rb | 4 +- lib/translation_diff/providers.rb | 36 ++++++++++++++---- lib/translation_diff/request.rb | 4 +- test/translation_diff/configuration_test.rb | 42 +++++++++++++++++++++ test/translation_diff/providers_test.rb | 34 +++++++++++++++++ test/translation_diff/request_test.rb | 9 +++++ 6 files changed, 119 insertions(+), 10 deletions(-) diff --git a/lib/translation_diff/configuration.rb b/lib/translation_diff/configuration.rb index 8b67edf..944b5bd 100644 --- a/lib/translation_diff/configuration.rb +++ b/lib/translation_diff/configuration.rb @@ -66,8 +66,10 @@ def copy end end + # Guarded, unlike cache/segmenter/rate_limiter: only the provider gained a base class to check against. def provider_instance - @provider_instance ||= resolve(provider, TranslationDiff::Providers) + @provider_instance ||= + TranslationDiff::Providers.ensure_provider!(resolve(provider, TranslationDiff::Providers)) end # Unset `cache` means Redis when a URL is configured, otherwise in-process -- works before anything runs. diff --git a/lib/translation_diff/providers.rb b/lib/translation_diff/providers.rb index a16e1e4..4d5c917 100644 --- a/lib/translation_diff/providers.rb +++ b/lib/translation_diff/providers.rb @@ -1,20 +1,35 @@ # Translation providers, by name; registering one also declares its configuration options. module TranslationDiff::Providers + # Said once, so a provider refused by name, by class or as an object is refused in the same words. + CONTRACT = "The base class supplies the transport, the configuration check and the " \ + "capability defaults, so a provider that skips it has none of them.".freeze + class << self # Options are declared before the registry entry is written, so a name collision raises without replacing. def register(name, klass) - unless klass < TranslationDiff::Provider - raise TranslationDiff::InvalidProviderError, - "#{klass} cannot be registered as a provider: it does not inherit " \ - "TranslationDiff::Provider. The base class supplies the transport, the " \ - "configuration check and the capability defaults, so a provider that " \ - "skips it has none of them." - end - + ensure_provider_class!(klass) TranslationDiff::Configuration.register_provider_options(klass.configuration_options, klass) registry.register(name, klass) end + # Guards registration. `klass < Provider` alone raises NoMethodError for an instance or a non-Module. + def ensure_provider_class!(klass) + return klass if klass.is_a?(Class) && klass < TranslationDiff::Provider + + raise TranslationDiff::InvalidProviderError, + "#{describe(klass)} cannot be registered as a provider: the registry takes a " \ + "class inheriting TranslationDiff::Provider. #{CONTRACT}" + end + + # Guards the other two ways a provider arrives: assigned to config.provider, or passed as `provider:`. + def ensure_provider!(instance) + return instance if instance.is_a?(TranslationDiff::Provider) + + raise TranslationDiff::InvalidProviderError, + "#{describe(instance)} cannot be used as a provider: it does not inherit " \ + "TranslationDiff::Provider. #{CONTRACT}" + end + def build(name, config) registry.build(name, config).tap { |provider| provider.name = name.to_sym } end @@ -22,5 +37,10 @@ def build(name, config) def registered?(name) = registry.registered?(name) def names = registry.names def registry = @registry ||= TranslationDiff::Registry.new("provider") + + private + + # Never #to_s on a non-Module: an arbitrary object renders its own content, or an address. + def describe(value) = value.is_a?(Module) ? value.to_s : "an instance of #{value.class}" end end diff --git a/lib/translation_diff/request.rb b/lib/translation_diff/request.rb index 9ffe92a..b193084 100644 --- a/lib/translation_diff/request.rb +++ b/lib/translation_diff/request.rb @@ -35,7 +35,9 @@ def api end def resolve_provider(value) - value.is_a?(Symbol) || value.is_a?(String) ? TranslationDiff::Providers.build(value, config) : value + return TranslationDiff::Providers.build(value, config) if value.is_a?(Symbol) || value.is_a?(String) + + TranslationDiff::Providers.ensure_provider!(value) end def rate_limiter = config.rate_limiter_instance diff --git a/test/translation_diff/configuration_test.rb b/test/translation_diff/configuration_test.rb index 601eb5d..d6bb527 100644 --- a/test/translation_diff/configuration_test.rb +++ b/test/translation_diff/configuration_test.rb @@ -5,6 +5,21 @@ class ConfigurationTest < Minitest::Test include EnvStub # A hand-written double for TranslationDiff::Registry: Minitest 6.0 dropped minitest/mock. + # Duck-typed collaborators, deliberately inheriting nothing: the provider is the only tightened one. + class FakeStore + def read_multi(keys) = [nil] * keys.size + def write(_key, value) = value + end + + class FakeSegmenter + def split_offsets(_text, **) = [] + end + + class FakeRateLimiter + # The real limiter raises when the threshold is passed and returns nothing useful otherwise. + def check(_size) = nil + end + ResolvingRegistry = Struct.new(:answer) do attr_reader :asked @@ -221,6 +236,33 @@ def test_registering_an_option_twice_does_not_clobber_the_first_default assert_equal "first", TranslationDiff::Configuration.new.shared_option end + # Providers.register refuses a class that skips the base class; assigning an object bypassed that entirely. + def test_an_assigned_provider_object_that_is_not_a_provider_is_refused + @config.provider = Object.new + + error = assert_raises(TranslationDiff::InvalidProviderError) { @config.provider_instance } + + assert_match(/TranslationDiff::Provider/, error.message) + end + + def test_an_assigned_provider_object_that_is_a_provider_is_used_as_is + provider = TranslationDiff::Providers::Null.new(TranslationDiff::Configuration.new) + @config.provider = provider + + assert_same provider, @config.provider_instance + end + + # Only the provider gained a base class; these three are still genuinely duck-typed. + def test_the_other_extension_points_stay_duck_typed + @config.cache = FakeStore.new + @config.segmenter = FakeSegmenter.new + @config.rate_limiter = FakeRateLimiter.new + + assert_instance_of FakeStore, @config.cache_store + assert_instance_of FakeSegmenter, @config.segmenter_instance + assert_instance_of FakeRateLimiter, @config.rate_limiter_instance + end + def test_resolve_builds_from_a_registry_for_a_symbol built = Object.new registry = ResolvingRegistry.new(built) diff --git a/test/translation_diff/providers_test.rb b/test/translation_diff/providers_test.rb index 16747d1..64e2c27 100644 --- a/test/translation_diff/providers_test.rb +++ b/test/translation_diff/providers_test.rb @@ -167,6 +167,40 @@ def self.build(_config) = new refute TranslationDiff::Providers.registered?(:impostor) end + # `klass < Provider` raised NoMethodError for an instance -- the first thing someone writing a provider hits. + def test_registering_an_instance_rather_than_a_class_raises_the_registry_error + instance = TranslationDiff::Providers::Null.new(TranslationDiff::Configuration.new) + + error = assert_raises(TranslationDiff::InvalidProviderError) do + TranslationDiff::Providers.register(:impostor_instance, instance) + end + + assert_match(/TranslationDiff::Provider/, error.message) + refute TranslationDiff::Providers.registered?(:impostor_instance) + end + + # `klass < Provider` raised ArgumentError for a non-Module, and NoMethodError for nil or a symbol. + def test_registering_a_non_module_raises_the_registry_error + [nil, :deepl, 42, Object.new].each do |value| + assert_raises(TranslationDiff::InvalidProviderError) do + TranslationDiff::Providers.register(:impostor_value, value) + end + end + + refute TranslationDiff::Providers.registered?(:impostor_value) + end + + # The message names the class, never the object: an arbitrary #to_s renders its own content or an address. + def test_the_registry_error_names_the_class_and_not_the_object + secret = Struct.new(:token).new("s3cret") + + error = assert_raises(TranslationDiff::InvalidProviderError) do + TranslationDiff::Providers.register(:impostor_secret, secret) + end + + refute_match(/s3cret/, error.message) + end + # A caller rescuing "this class cannot be a provider" must not also accidentally swallow an option collision. def test_an_option_collision_raises_the_generic_error_not_the_invalid_provider_one TranslationDiff::Providers.register(:collision_a, ConflictingProviderA) diff --git a/test/translation_diff/request_test.rb b/test/translation_diff/request_test.rb index 0394b56..ebce31c 100644 --- a/test/translation_diff/request_test.rb +++ b/test/translation_diff/request_test.rb @@ -293,6 +293,15 @@ def test_the_provider_keyword_overrides_the_configured_provider_for_one_call assert_empty api.calls end + # The `provider:` keyword is the third way to supply one; it must fail in the same words as the other two. + def test_a_provider_object_passed_for_one_call_that_is_not_a_provider_is_refused + error = assert_raises(TranslationDiff::InvalidProviderError) do + TranslationDiff::Request.new("One", from: :en, to: :ru, provider: Object.new).call + end + + assert_match(/TranslationDiff::Provider/, error.message) + end + # An empty cache-key segment would put this provider's translations in every other provider's namespace. def test_a_provider_whose_cache_key_is_empty_is_refused_rather_than_sharing_a_namespace configure_with(NamelessApi.new(%w[Один])) From 8c7571859fd3e119bf26ec29c6b049f1af5ecd21 Mon Sep 17 00:00:00 2001 From: IG Date: Wed, 9 Sep 2026 19:56:28 +0400 Subject: [PATCH 22/26] fix: normalise language codes in every provider, once Only Google and DeepL normalised. Measured with from: "EN", to: "RU", Amazon sent "SourceLanguageCode" => "EN", LibreTranslate and ModernMT sent "EN"/"RU", and Azure sent &to=RU&from=EN. Amazon Translate rejects those outright and LibreTranslate answers 400 -- on every call, for anyone who followed the README's "switch provider by changing config.provider" while keeping DeepL-style upper-case codes. The rule Google had is now Provider#language, reachable by the base class and by every provider, with self.language_case choosing the casing: DeepL up-cases, the other five down-case. The subtag exception is kept and now applies to DeepL too, which up-cased "zh-Hans" into "ZH-HANS" -- the casing of a script or region subtag is its own and a blanket transform corrupts it. Tested per provider, both the bare code and the subtag, not only Google. --- CHANGELOG.md | 10 +++++ README.md | 9 ++++ lib/translation_diff/provider.rb | 15 +++++++ lib/translation_diff/providers/amazon.rb | 4 +- lib/translation_diff/providers/azure.rb | 4 +- lib/translation_diff/providers/deepl.rb | 6 +-- lib/translation_diff/providers/google.rb | 12 ------ .../providers/libretranslate.rb | 4 +- lib/translation_diff/providers/modernmt.rb | 4 +- test/translation_diff/provider_test.rb | 41 +++++++++++++++++++ .../translation_diff/providers/amazon_test.rb | 17 ++++++++ test/translation_diff/providers/azure_test.rb | 15 +++++++ test/translation_diff/providers/deepl_test.rb | 16 ++++++++ .../translation_diff/providers/google_test.rb | 14 +++++++ .../providers/libretranslate_test.rb | 15 +++++++ .../providers/modernmt_test.rb | 14 +++++++ 16 files changed, 177 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f4af0d..68320c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -88,6 +88,16 @@ described below. Everything here is relative to `deepl_diff` 2.2.0. `Capabilities`. - `TranslationDiff::Providers::Naming` is gone; the registry stamps `cache_key` and `Provider` implements it. +- Every provider normalises language codes to the casing its own vendor + documents and accepts either casing from the caller: DeepL upper-cases, the + other five lower-case. A code carrying a script or region subtag + (`"zh-Hans"`, `"pt-BR"`) is passed through untouched. `Provider#language` + is the shared rule and `self.language_case` selects the casing, so a + provider of your own gets it by inheriting. Previously only Google and + DeepL normalised at all -- Amazon Translate rejected `"EN"`/`"RU"` and + LibreTranslate answered 400, on every call, for anyone who followed the + README's "switch provider by changing `config.provider`" with DeepL-style + codes -- and DeepL upper-cased subtags too, corrupting `"zh-Hans"`. - `deepl-rb` and `google-cloud-translate-v2` are no longer used at all. `faraday` and `faraday-retry` become runtime dependencies; `aws-sigv4` is required lazily by the Amazon provider only. diff --git a/README.md b/README.md index 342e095..507d696 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,15 @@ runtime: | LibreTranslate | `:libretranslate` | `libretranslate_api_base` | 50 | 5,000 | yes (`format`) | no | yes | no | | Amazon | `:amazon` | `amazon_access_key_id`, `amazon_secret_access_key`, `amazon_region` | 1 | 10,000 | no | no | yes | no | +**Language codes are normalised per vendor, so switching provider needs no +other change.** A bare code (`"EN"`, `:ru`) is cased the way the vendor +documents it -- DeepL takes upper case, every other provider here takes lower +case -- whichever casing you wrote. A code carrying a script or region subtag +(`"zh-Hans"`, `"pt-BR"`) is passed through untouched, because the casing of a +subtag is its own. A provider of your own gets the same rule from +`TranslationDiff::Provider#language`; declare `def self.language_case = +:upcase` if your vendor wants upper case. + "Request size" is what `Chunker` measures: the URL-escaped form of each string (`CGI.escape(text).size`), which is never smaller than its UTF-8 byte count. "HTML support" names the provider option that turns HTML handling on diff --git a/lib/translation_diff/provider.rb b/lib/translation_diff/provider.rb index f817af4..4aad074 100644 --- a/lib/translation_diff/provider.rb +++ b/lib/translation_diff/provider.rb @@ -6,6 +6,9 @@ class TranslationDiff::Provider html: :none, notranslate: false, detects_language: false, reports_billing: false ).freeze + # A bare alphabetic code is cased the way the vendor documents; anything with a subtag is left alone. + BARE_LANGUAGE_CODE = /\A[A-Za-z]{2,3}\z/ + # Stamped by the registry at build time. See #cache_key. attr_accessor :name @@ -16,6 +19,15 @@ def initialize(config) ensure_configured! end + # Callers write whichever casing their old configuration used; the vendor gets the one it documents. + def language(value) + code = value.to_s + return nil if code.empty? + return code unless code.match?(BARE_LANGUAGE_CODE) + + self.class.language_case == :upcase ? code.upcase : code.downcase + end + def translate(_request) = raise NotImplementedError, "#{self.class} must implement #translate" # Only called when `capabilities.detects_language?`. @@ -32,6 +44,9 @@ def cache_key end class << self + # The casing this vendor documents for a bare code. DeepL upcases; everyone else takes lower case. + def language_case = :downcase + def configuration_options = [] # Checked once, at build time, so a caller learns what to set before a vendor's own exception does. diff --git a/lib/translation_diff/providers/amazon.rb b/lib/translation_diff/providers/amazon.rb index d51d9d1..e910f44 100644 --- a/lib/translation_diff/providers/amazon.rb +++ b/lib/translation_diff/providers/amazon.rb @@ -51,8 +51,8 @@ def detect(text) def call(text, request) payload = { "Text" => text, - "SourceLanguageCode" => request.from.nil? ? AUTO : request.from.to_s, - "TargetLanguageCode" => request.to.to_s + "SourceLanguageCode" => request.from.nil? ? AUTO : language(request.from), + "TargetLanguageCode" => language(request.to) }.merge(request.options.transform_keys(&:to_s)) post_signed(JSON.generate(payload)).body diff --git a/lib/translation_diff/providers/azure.rb b/lib/translation_diff/providers/azure.rb index e2e972c..7501565 100644 --- a/lib/translation_diff/providers/azure.rb +++ b/lib/translation_diff/providers/azure.rb @@ -56,9 +56,9 @@ def detect(text) private def url_for(request) - params = { "api-version" => API_VERSION, "to" => request.to.to_s, + params = { "api-version" => API_VERSION, "to" => language(request.to), "textType" => DEFAULT_TEXT_TYPE } - params["from"] = request.from.to_s unless request.from.nil? + params["from"] = language(request.from) unless request.from.nil? params.merge!(request.options.transform_keys(&:to_s)) "#{translate_url}?#{URI.encode_www_form(params)}" diff --git a/lib/translation_diff/providers/deepl.rb b/lib/translation_diff/providers/deepl.rb index 8999b09..674eda3 100644 --- a/lib/translation_diff/providers/deepl.rb +++ b/lib/translation_diff/providers/deepl.rb @@ -24,6 +24,9 @@ def self.configuration_options def self.configuration_requirements = %i[deepl_api_key] + # DeepL is the one vendor documenting upper-case codes. + def self.language_case = :upcase + # DeepL requires a target language even when only detection is wanted, so the provider picks one. DETECTION_TARGET = "EN".freeze @@ -64,9 +67,6 @@ def detect(text) def free_key? = config.deepl_api_key.to_s.end_with?(FREE_KEY_SUFFIX) - # DeepL's language codes are upper case. - def language(value) = value.to_s.upcase - def usage_for(request, translations) billed = translations.filter_map { |t| t["billed_characters"] }.sum diff --git a/lib/translation_diff/providers/google.rb b/lib/translation_diff/providers/google.rb index 5d3ec23..020e4a8 100644 --- a/lib/translation_diff/providers/google.rb +++ b/lib/translation_diff/providers/google.rb @@ -22,9 +22,6 @@ def self.configuration_options def self.configuration_requirements = %i[google_api_key] - # A bare code is downcased for DeepL-style configs ("EN"); a subtag ("zh-Hans") is passed through untouched. - BARE_LANGUAGE_CODE = /\A[A-Za-z]{2,3}\z/ - def api_base = config.google_api_base || HOST def translate_url = "language/translate/v2?key=#{CGI.escape(config.google_api_key.to_s)}" def detect_url = "language/translate/v2/detect?key=#{CGI.escape(config.google_api_key.to_s)}" @@ -51,15 +48,6 @@ def detect(text) response = post(detect_url, { q: [text] }) response.body.dig("data", "detections", 0, 0, "language")&.downcase end - - private - - def language(value) - code = value.to_s - return nil if code.empty? - - code.match?(BARE_LANGUAGE_CODE) ? code.downcase : code - end end TranslationDiff::Providers.register(:google, TranslationDiff::Providers::Google) diff --git a/lib/translation_diff/providers/libretranslate.rb b/lib/translation_diff/providers/libretranslate.rb index a3fe5f4..5a2728a 100644 --- a/lib/translation_diff/providers/libretranslate.rb +++ b/lib/translation_diff/providers/libretranslate.rb @@ -26,8 +26,8 @@ def translate_url = "translate" def render_translate_payload(request) { format: DEFAULT_FORMAT } .merge(request.options) - .merge(q: request.texts, target: request.to.to_s, - source: request.from.nil? ? AUTO : request.from.to_s) + .merge(q: request.texts, target: language(request.to), + source: request.from.nil? ? AUTO : language(request.from)) .tap { |payload| payload[:api_key] = config.libretranslate_api_key if config.libretranslate_api_key } end diff --git a/lib/translation_diff/providers/modernmt.rb b/lib/translation_diff/providers/modernmt.rb index f150465..2170aba 100644 --- a/lib/translation_diff/providers/modernmt.rb +++ b/lib/translation_diff/providers/modernmt.rb @@ -27,8 +27,8 @@ def translate_url = "translate" def render_translate_payload(request) { format: DEFAULT_FORMAT } .merge(request.options) - .merge(q: request.texts, target: request.to.to_s) - .tap { |payload| payload[:source] = request.from.to_s unless request.from.nil? } + .merge(q: request.texts, target: language(request.to)) + .tap { |payload| payload[:source] = language(request.from) unless request.from.nil? } end # One text comes back as an object rather than a one-element array, so the envelope is always coerced. diff --git a/test/translation_diff/provider_test.rb b/test/translation_diff/provider_test.rb index 343a9f2..124b9b1 100644 --- a/test/translation_diff/provider_test.rb +++ b/test/translation_diff/provider_test.rb @@ -16,6 +16,47 @@ def setup @config = TranslationDiff::Configuration.new end + class Upcasing < TranslationDiff::Provider + def self.language_case = :upcase + end + + # The rule lived in google.rb and nowhere else; four providers shipped without it. + def test_a_bare_code_is_cased_the_way_the_vendor_documents_it + downcasing = Bare.new(@config) + upcasing = Upcasing.new(@config) + + assert_equal "ru", downcasing.language("RU") + assert_equal "ru", downcasing.language(:ru) + assert_equal "RU", upcasing.language("ru") + assert_equal "RU", upcasing.language(:RU) + end + + # A script or region subtag has its own casing; a blanket transform corrupts it. + def test_a_subtagged_code_passes_through_untouched + downcasing = Bare.new(@config) + upcasing = Upcasing.new(@config) + + %w[zh-Hans pt-BR EN-GB sr-Latn-RS].each do |code| + assert_equal code, downcasing.language(code) + assert_equal code, upcasing.language(code) + end + end + + # nil so a provider can leave the field out of the payload entirely rather than sending "". + def test_an_absent_code_is_nil + assert_nil Bare.new(@config).language(nil) + assert_nil Bare.new(@config).language("") + end + + def test_providers_downcase_unless_they_say_otherwise + assert_equal :downcase, TranslationDiff::Provider.language_case + assert_equal :upcase, TranslationDiff::Providers::DeepL.language_case + + %w[Google Azure ModernMT LibreTranslate Amazon].each do |name| + assert_equal :downcase, TranslationDiff::Providers.const_get(name).language_case + end + end + def test_a_provider_without_requirements_builds assert_instance_of Bare, Bare.new(@config) end diff --git a/test/translation_diff/providers/amazon_test.rb b/test/translation_diff/providers/amazon_test.rb index 6b17863..647c5bd 100644 --- a/test/translation_diff/providers/amazon_test.rb +++ b/test/translation_diff/providers/amazon_test.rb @@ -60,6 +60,23 @@ def test_it_reads_no_aws_environment_variables end end + # Amazon Translate rejected "EN"/"RU" outright, on every call. + def test_it_downcases_a_bare_language_code_whichever_casing_the_caller_used + provider.translate(translation_request(%w[one], from: "EN", to: "RU")) + body = JSON.parse(requests.first.body) + + assert_equal "en", body["SourceLanguageCode"] + assert_equal "ru", body["TargetLanguageCode"] + end + + def test_it_leaves_a_subtagged_code_untouched + provider.translate(translation_request(%w[one], from: "zh-Hans", to: "pt-BR")) + body = JSON.parse(requests.first.body) + + assert_equal "zh-Hans", body["SourceLanguageCode"] + assert_equal "pt-BR", body["TargetLanguageCode"] + end + def test_the_endpoint_is_regional assert_equal "https://translate.eu-central-1.amazonaws.com", TranslationDiff::Providers::Amazon.new(config).api_base diff --git a/test/translation_diff/providers/azure_test.rb b/test/translation_diff/providers/azure_test.rb index 4ff3172..cba68a2 100644 --- a/test/translation_diff/providers/azure_test.rb +++ b/test/translation_diff/providers/azure_test.rb @@ -78,6 +78,21 @@ def test_the_languages_travel_in_the_query_string_not_the_body assert_equal ["ru"], query["to"] end + # Azure rejected "&to=RU"; a caller keeping DeepL-style codes hit it on every call. + def test_it_downcases_a_bare_language_code_whichever_casing_the_caller_used + provider.translate(translation_request(%w[one], from: "EN", to: "RU")) + + assert_equal ["en"], query["from"] + assert_equal ["ru"], query["to"] + end + + def test_it_leaves_a_subtagged_code_untouched + provider.translate(translation_request(%w[one], from: "zh-Hans", to: "pt-BR")) + + assert_equal ["zh-Hans"], query["from"] + assert_equal ["pt-BR"], query["to"] + end + def test_it_omits_from_when_none_was_given provider.translate(translation_request(%w[one], from: nil)) diff --git a/test/translation_diff/providers/deepl_test.rb b/test/translation_diff/providers/deepl_test.rb index b12d1b7..6389178 100644 --- a/test/translation_diff/providers/deepl_test.rb +++ b/test/translation_diff/providers/deepl_test.rb @@ -71,6 +71,22 @@ def test_the_environment_variable_satisfies_the_configuration_requirement end end + # DeepL documents upper-case codes; a caller writing Google-style lower case must still work. + def test_it_upcases_a_bare_language_code_whichever_casing_the_caller_used + provider.translate(translation_request(%w[one], from: "en", to: "ru")) + + assert_equal "EN", sent["source_lang"] + assert_equal "RU", sent["target_lang"] + end + + # The casing of a script or region subtag is its own; upcasing "pt-BR" corrupts it. + def test_it_leaves_a_subtagged_code_untouched + provider.translate(translation_request(%w[one], from: "zh-Hans", to: "pt-BR")) + + assert_equal "zh-Hans", sent["source_lang"] + assert_equal "pt-BR", sent["target_lang"] + end + def test_a_free_key_selects_the_free_host assert_equal "https://api-free.deepl.com", TranslationDiff::Providers::DeepL.new(config).api_base end diff --git a/test/translation_diff/providers/google_test.rb b/test/translation_diff/providers/google_test.rb index c71b5c3..d434729 100644 --- a/test/translation_diff/providers/google_test.rb +++ b/test/translation_diff/providers/google_test.rb @@ -67,6 +67,20 @@ def test_translate_project_supplies_an_unset_project_id end end + def test_it_downcases_a_bare_language_code_whichever_casing_the_caller_used + provider.translate(translation_request(%w[one], from: "EN", to: "RU")) + + assert_equal "en", sent["source"] + assert_equal "ru", sent["target"] + end + + def test_it_leaves_a_subtagged_code_untouched + provider.translate(translation_request(%w[one], from: "zh-Hans", to: "pt-BR")) + + assert_equal "zh-Hans", sent["source"] + assert_equal "pt-BR", sent["target"] + end + def test_the_key_travels_in_the_query_string provider.translate(translation_request(%w[one])) diff --git a/test/translation_diff/providers/libretranslate_test.rb b/test/translation_diff/providers/libretranslate_test.rb index afa2b8e..f576f3b 100644 --- a/test/translation_diff/providers/libretranslate_test.rb +++ b/test/translation_diff/providers/libretranslate_test.rb @@ -56,6 +56,21 @@ def test_a_missing_source_language_becomes_auto assert_equal "auto", sent["source"] end + # LibreTranslate answered 400 for "EN"/"RU" on every call. + def test_it_downcases_a_bare_language_code_whichever_casing_the_caller_used + provider.translate(translation_request(%w[one], from: "EN", to: "RU")) + + assert_equal "en", sent["source"] + assert_equal "ru", sent["target"] + end + + def test_it_leaves_a_subtagged_code_untouched + provider.translate(translation_request(%w[one], from: "zh-Hans", to: "pt-BR")) + + assert_equal "zh-Hans", sent["source"] + assert_equal "pt-BR", sent["target"] + end + def test_it_asks_for_html provider.translate(translation_request(%w[one])) diff --git a/test/translation_diff/providers/modernmt_test.rb b/test/translation_diff/providers/modernmt_test.rb index acc9fba..8de5a39 100644 --- a/test/translation_diff/providers/modernmt_test.rb +++ b/test/translation_diff/providers/modernmt_test.rb @@ -44,6 +44,20 @@ def test_it_sends_the_texts_and_the_language_pair_in_the_body assert_equal "ru", sent["target"] end + def test_it_downcases_a_bare_language_code_whichever_casing_the_caller_used + provider.translate(translation_request(%w[one], from: "EN", to: "RU")) + + assert_equal "en", sent["source"] + assert_equal "ru", sent["target"] + end + + def test_it_leaves_a_subtagged_code_untouched + provider.translate(translation_request(%w[one], from: "zh-Hans", to: "pt-BR")) + + assert_equal "zh-Hans", sent["source"] + assert_equal "pt-BR", sent["target"] + end + def test_it_asks_for_html_by_mime_type provider.translate(translation_request(%w[one])) From 1ba0ead12738aff7a311c66951995da44b95036d Mon Sep 17 00:00:00 2001 From: IG Date: Wed, 9 Sep 2026 19:57:44 +0400 Subject: [PATCH 23/26] fix: one convention for billed characters across the three that report it deepl.rb and modernmt.rb mapped a summed 0 to nil while azure.rb returned 0 when its header said 0, so the same fact reached a caller two ways depending on the provider. nil now means the provider reported nothing and a number, 0 included, means it reported that number -- Azure's behaviour, and the one Translation::Usage's own comment already described. A provider that bills nothing for a call (a translation memory hit) no longer has that reported as "unknown". The rule lives once, in Provider#billed_characters, rather than twice. --- README.md | 5 +++++ lib/translation_diff/provider.rb | 6 ++++++ lib/translation_diff/providers/deepl.rb | 4 +--- lib/translation_diff/providers/modernmt.rb | 4 +--- test/translation_diff/providers/azure_test.rb | 8 ++++++++ test/translation_diff/providers/deepl_test.rb | 16 ++++++++++++++++ test/translation_diff/providers/modernmt_test.rb | 16 ++++++++++++++++ 7 files changed, 53 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 507d696..de40ca5 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,11 @@ runtime: | LibreTranslate | `:libretranslate` | `libretranslate_api_base` | 50 | 5,000 | yes (`format`) | no | yes | no | | Amazon | `:amazon` | `amazon_access_key_id`, `amazon_secret_access_key`, `amazon_region` | 1 | 10,000 | no | no | yes | no | +**`usage.billed_characters` is `nil` when the provider said nothing about +billing and a number -- `0` included -- when it said something.** All three +providers that report billing follow that rule; the other four always answer +`nil`. + **Language codes are normalised per vendor, so switching provider needs no other change.** A bare code (`"EN"`, `:ru`) is cased the way the vendor documents it -- DeepL takes upper case, every other provider here takes lower diff --git a/lib/translation_diff/provider.rb b/lib/translation_diff/provider.rb index 4aad074..413977b 100644 --- a/lib/translation_diff/provider.rb +++ b/lib/translation_diff/provider.rb @@ -28,6 +28,12 @@ def language(value) self.class.language_case == :upcase ? code.upcase : code.downcase end + # nil means the provider reported no billing at all; 0 means it reported zero. Both are claims. + def billed_characters(reported) + values = reported.compact + values.empty? ? nil : values.sum + end + def translate(_request) = raise NotImplementedError, "#{self.class} must implement #translate" # Only called when `capabilities.detects_language?`. diff --git a/lib/translation_diff/providers/deepl.rb b/lib/translation_diff/providers/deepl.rb index 674eda3..6283a3e 100644 --- a/lib/translation_diff/providers/deepl.rb +++ b/lib/translation_diff/providers/deepl.rb @@ -68,11 +68,9 @@ def detect(text) def free_key? = config.deepl_api_key.to_s.end_with?(FREE_KEY_SUFFIX) def usage_for(request, translations) - billed = translations.filter_map { |t| t["billed_characters"] }.sum - TranslationDiff::Translation::Usage.new( characters: request.texts.sum(&:size), - billed_characters: billed.positive? ? billed : nil + billed_characters: billed_characters(translations.map { |t| t["billed_characters"] }) ) end end diff --git a/lib/translation_diff/providers/modernmt.rb b/lib/translation_diff/providers/modernmt.rb index 2170aba..b3bc68d 100644 --- a/lib/translation_diff/providers/modernmt.rb +++ b/lib/translation_diff/providers/modernmt.rb @@ -56,11 +56,9 @@ def results_from(body) end def usage_for(request, results) - billed = results.filter_map { |r| r["billedCharacters"] }.sum - TranslationDiff::Translation::Usage.new( characters: request.texts.sum(&:size), - billed_characters: billed.positive? ? billed : nil + billed_characters: billed_characters(results.map { |r| r["billedCharacters"] }) ) end end diff --git a/test/translation_diff/providers/azure_test.rb b/test/translation_diff/providers/azure_test.rb index cba68a2..5acf528 100644 --- a/test/translation_diff/providers/azure_test.rb +++ b/test/translation_diff/providers/azure_test.rb @@ -133,6 +133,14 @@ def test_billed_characters_is_nil_when_the_header_is_absent assert_nil response.usage.billed_characters end + # The same convention the other two billing providers now follow. + def test_a_reported_zero_is_zero_not_unknown + response = provider(body: BODY, headers: { "X-metered-usage" => "0" }) + .translate(translation_request(%w[one two])) + + assert_equal 0, response.usage.billed_characters + end + def test_a_short_response_raises_rather_than_shifting_nils_into_the_results short = [BODY.first] diff --git a/test/translation_diff/providers/deepl_test.rb b/test/translation_diff/providers/deepl_test.rb index 6389178..a0a778b 100644 --- a/test/translation_diff/providers/deepl_test.rb +++ b/test/translation_diff/providers/deepl_test.rb @@ -168,6 +168,22 @@ def test_it_reports_the_detected_source_and_the_billed_characters assert_equal 6, response.usage.billed_characters end + # nil means "the provider did not say"; a reported 0 is a claim, and mapping it to nil erased one. + def test_a_reported_zero_is_zero_not_unknown + body = { "translations" => [{ "text" => "один", "billed_characters" => 0 }, + { "text" => "два", "billed_characters" => 0 }] } + response = provider(body: body).translate(translation_request(%w[one two])) + + assert_equal 0, response.usage.billed_characters + end + + def test_billed_characters_is_nil_when_no_translation_reported_it + body = { "translations" => [{ "text" => "один" }, { "text" => "два" }] } + response = provider(body: body).translate(translation_request(%w[one two])) + + assert_nil response.usage.billed_characters + end + def test_a_short_response_raises_rather_than_shifting_nils_into_the_results short = { "translations" => [{ "text" => "один" }] } diff --git a/test/translation_diff/providers/modernmt_test.rb b/test/translation_diff/providers/modernmt_test.rb index 8de5a39..b501cb7 100644 --- a/test/translation_diff/providers/modernmt_test.rb +++ b/test/translation_diff/providers/modernmt_test.rb @@ -72,6 +72,22 @@ def test_it_unwraps_the_data_envelope assert_equal 6, response.usage.billed_characters end + # Same convention as DeepL and Azure: a reported 0 is a claim, not "unknown". + def test_a_reported_zero_is_zero_not_unknown + body = { "data" => [{ "translation" => "один", "billedCharacters" => 0 }, + { "translation" => "два", "billedCharacters" => 0 }] } + response = provider(body: body).translate(translation_request(%w[one two])) + + assert_equal 0, response.usage.billed_characters + end + + def test_billed_characters_is_nil_when_no_result_reported_it + body = { "data" => [{ "translation" => "один" }, { "translation" => "два" }] } + response = provider(body: body).translate(translation_request(%w[one two])) + + assert_nil response.usage.billed_characters + end + # One text comes back as an object, not a one-element array, which would hand the pipeline a bare Hash. def test_a_single_text_comes_back_unwrapped_and_is_still_a_list single = { "data" => { "translation" => "один", "detectedLanguage" => "en" } } From b0ec2a12ae581379501476628b0faeba961d11c4 Mon Sep 17 00:00:00 2001 From: IG Date: Wed, 9 Sep 2026 19:59:02 +0400 Subject: [PATCH 24/26] fix: one option-merge order for all six providers amazon.rb applied its mandatory fields first and the caller's options second, so TranslationDiff.translate(text, to: :ru, TargetLanguageCode: "de") sent German, and Text: could substitute somebody else's text for the caller's. Azure had the same shape, reachable by anyone building a Translation::Request directly. All six now apply defaults, then caller options, then mandatory fields, which is what the other four already did: a caller can still override a default such as Azure's textType, and can no longer displace the language pair or the texts. --- README.md | 7 ++++++ lib/translation_diff/providers/amazon.rb | 4 ++-- lib/translation_diff/providers/azure.rb | 7 +++--- .../translation_diff/providers/amazon_test.rb | 22 +++++++++++++++++++ test/translation_diff/providers/azure_test.rb | 13 +++++++++++ 5 files changed, 48 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index de40ca5..e8c2177 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,13 @@ runtime: | LibreTranslate | `:libretranslate` | `libretranslate_api_base` | 50 | 5,000 | yes (`format`) | no | yes | no | | Amazon | `:amazon` | `amazon_access_key_id`, `amazon_secret_access_key`, `amazon_region` | 1 | 10,000 | no | no | yes | no | +**Every keyword other than `from:`, `to:`, `provider:` and `config:` is +forwarded to the provider, and every provider applies them the same way: +its own defaults first, then your options, then the fields the request cannot +do without.** So `formality: :less` overrides a default, and a keyword +colliding with the language pair or the texts themselves is overridden rather +than obeyed. + **`usage.billed_characters` is `nil` when the provider said nothing about billing and a number -- `0` included -- when it said something.** All three providers that report billing follow that rule; the other four always answer diff --git a/lib/translation_diff/providers/amazon.rb b/lib/translation_diff/providers/amazon.rb index e910f44..8278655 100644 --- a/lib/translation_diff/providers/amazon.rb +++ b/lib/translation_diff/providers/amazon.rb @@ -49,11 +49,11 @@ def detect(text) private def call(text, request) - payload = { + payload = request.options.transform_keys(&:to_s).merge( "Text" => text, "SourceLanguageCode" => request.from.nil? ? AUTO : language(request.from), "TargetLanguageCode" => language(request.to) - }.merge(request.options.transform_keys(&:to_s)) + ) post_signed(JSON.generate(payload)).body end diff --git a/lib/translation_diff/providers/azure.rb b/lib/translation_diff/providers/azure.rb index 7501565..9cc4d04 100644 --- a/lib/translation_diff/providers/azure.rb +++ b/lib/translation_diff/providers/azure.rb @@ -55,11 +55,12 @@ def detect(text) private + # Defaults, then caller options, then mandatory fields: a caller must not displace the language pair. def url_for(request) - params = { "api-version" => API_VERSION, "to" => language(request.to), - "textType" => DEFAULT_TEXT_TYPE } + params = { "textType" => DEFAULT_TEXT_TYPE } + .merge(request.options.transform_keys(&:to_s)) + .merge("api-version" => API_VERSION, "to" => language(request.to)) params["from"] = language(request.from) unless request.from.nil? - params.merge!(request.options.transform_keys(&:to_s)) "#{translate_url}?#{URI.encode_www_form(params)}" end diff --git a/test/translation_diff/providers/amazon_test.rb b/test/translation_diff/providers/amazon_test.rb index 647c5bd..5671961 100644 --- a/test/translation_diff/providers/amazon_test.rb +++ b/test/translation_diff/providers/amazon_test.rb @@ -77,6 +77,28 @@ def test_it_leaves_a_subtagged_code_untouched assert_equal "pt-BR", body["TargetLanguageCode"] end + # Every provider applies defaults, then caller options, then mandatory fields. Amazon had the last two swapped. + def test_a_caller_option_cannot_displace_a_mandatory_field + request = TranslationDiff::Translation::Request.new( + texts: %w[one], from: :en, to: :ru, + options: { TargetLanguageCode: "de", Text: "somebody else's text" } + ) + provider.translate(request) + body = JSON.parse(requests.first.body) + + assert_equal "ru", body["TargetLanguageCode"] + assert_equal "one", body["Text"] + end + + def test_a_caller_option_that_displaces_nothing_still_reaches_amazon + request = TranslationDiff::Translation::Request.new( + texts: %w[one], from: :en, to: :ru, options: { Settings: { "Formality" => "FORMAL" } } + ) + provider.translate(request) + + assert_equal({ "Formality" => "FORMAL" }, JSON.parse(requests.first.body)["Settings"]) + end + def test_the_endpoint_is_regional assert_equal "https://translate.eu-central-1.amazonaws.com", TranslationDiff::Providers::Amazon.new(config).api_base diff --git a/test/translation_diff/providers/azure_test.rb b/test/translation_diff/providers/azure_test.rb index 5acf528..d0597cf 100644 --- a/test/translation_diff/providers/azure_test.rb +++ b/test/translation_diff/providers/azure_test.rb @@ -93,6 +93,19 @@ def test_it_leaves_a_subtagged_code_untouched assert_equal ["pt-BR"], query["to"] end + # Defaults, then caller options, then mandatory fields -- the order the other five already used. + def test_a_caller_option_cannot_displace_a_mandatory_query_parameter + request = TranslationDiff::Translation::Request.new( + texts: %w[one], from: :en, to: :ru, + options: { "to" => "de", "api-version" => "1.0", "textType" => "plain" } + ) + provider.translate(request) + + assert_equal ["ru"], query["to"] + assert_equal ["3.0"], query["api-version"] + assert_equal ["plain"], query["textType"] + end + def test_it_omits_from_when_none_was_given provider.translate(translation_request(%w[one], from: nil)) From a8f282f324fb8ae9b0d2b05f61f1b16ee659007e Mon Sep 17 00:00:00 2001 From: IG Date: Wed, 9 Sep 2026 20:02:32 +0400 Subject: [PATCH 25/26] test: pin every part of the cache key against main M1 changed Cache#store and I1 changed how provider options are declared; either moving a key would make every user re-translate their whole corpus on upgrade. This pins five keys covering the provider segment, the normalised language codes including a subtag, the options digest over a nested hash and an array, and the sentence digest. The five values were read off `main` and are byte-identical there. --- test/translation_diff/cache_test.rb | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/test/translation_diff/cache_test.rb b/test/translation_diff/cache_test.rb index 5dcb33d..2c345db 100644 --- a/test/translation_diff/cache_test.rb +++ b/test/translation_diff/cache_test.rb @@ -120,13 +120,23 @@ def test_store_fills_the_gaps_in_key_order cache.store(%w[one two three], [nil, "два", nil], %w[один три]) end - # The exact key a translation is stored under. Change it and every user re-translates their whole corpus. - def test_the_key_is_the_one_users_already_have_in_their_caches + # The exact keys a translation is stored under, verified equal to main's -- move one and every user + # re-translates their whole corpus, so the digest of the options and of the sentence are pinned too. + # rubocop:disable-next Metrics/MethodLength + def test_the_keys_are_the_ones_users_already_have_in_their_caches key_for(value: "text", from: :en, to: :ru, provider: "deepl") + key_for(value: " text ", from: "EN", to: "RU", provider: "deepl") key_for(value: "text", from: :en, to: :ru, provider: "deepl", options: { formality: :less }) - - assert_equal "deepl:en:ru:1cb251ec0d568de6a929b520c4aed8d1", @store.keys[0] - assert_equal "deepl:en:ru:80df90b8:1cb251ec0d568de6a929b520c4aed8d1", @store.keys[1] + key_for(value: "Hello there.", from: :en, to: :"pt-BR", provider: "google", + options: { glossary_ids: %w[a b], nested: { z: 1, a: nil } }) + key_for(value: "Ein Satz.", from: :de, to: :en, provider: "azure", + options: { a: "", b: 2, c: true }) + + assert_equal ["deepl:en:ru:1cb251ec0d568de6a929b520c4aed8d1", + "deepl:en:ru:1cb251ec0d568de6a929b520c4aed8d1", + "deepl:en:ru:80df90b8:1cb251ec0d568de6a929b520c4aed8d1", + "google:en:pt-br:2744ebab:9d6a2963872077db674a27a39c492e61", + "azure:de:en:da1fef03:84f8c5b939a540fa8da132c838a8d61d"], @store.keys end private From 2fac01106637866673e86c41b383f9e6abc747b0 Mon Sep 17 00:00:00 2001 From: IG Date: Wed, 9 Sep 2026 20:11:59 +0400 Subject: [PATCH 26/26] ci: test only the Ruby versions the gemspec supports The matrix still listed 3.2 and 3.3, which this branch dropped when it raised `required_ruby_version` to 3.4. Both jobs failed at `bundle install`, because bundler correctly refuses to install a gem that declares a floor above the running Ruby -- so the failure was the gemspec being obeyed, not the code being wrong. A matrix that outlives the version it tests turns CI red for a reason unrelated to any change, which is the fastest way to teach a team to merge over a red build. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 18d74c3..46837bb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -11,7 +11,7 @@ jobs: strategy: fail-fast: false matrix: - ruby: ["3.2", "3.3", "3.4", "4.0"] + ruby: ["3.4", "4.0"] steps: - uses: actions/checkout@v4