From 2c34635c62b95e72bc5455a86a243904468e1466 Mon Sep 17 00:00:00 2001 From: Jake Huneau Date: Thu, 29 Oct 2026 10:54:15 +0000 Subject: [PATCH 1/2] Don't re-lock the sender mutex when #add can't restart the thread #add calls #start while holding @mx. When creating the thread raised ThreadError, the rescue in #start locked @mx again and Ruby raised "deadlock; recursive locking". Set @done directly when the current thread already owns the mutex, and stop #add from queuing a message that no thread will send. Fixes #333 --- lib/datadog/statsd/sender.rb | 11 ++++++++++- spec/statsd/sender_spec.rb | 24 ++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/lib/datadog/statsd/sender.rb b/lib/datadog/statsd/sender.rb index 9f50cc8f..c4a47982 100644 --- a/lib/datadog/statsd/sender.rb +++ b/lib/datadog/statsd/sender.rb @@ -81,6 +81,9 @@ def add(message) @message_queue = nil message_buffer.reset start + # the companion thread could not be re-created, don't queue + # messages that nothing will ever send + return if @done @flush_timer.start if @flush_timer && @flush_timer.stop? } end @@ -106,7 +109,13 @@ def start @sender_thread.name = "Statsd Sender" unless Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.3') rescue ThreadError => e @logger.debug { "Statsd: Failed to start sender thread: #{e.message}" } if @logger - @mx.synchronize { @done = true } + # #add calls #start while already holding @mx, and Ruby mutexes + # are not re-entrant + if @mx.owned? + @done = true + else + @mx.synchronize { @done = true } + end end @flush_timer.start if @flush_timer diff --git a/spec/statsd/sender_spec.rb b/spec/statsd/sender_spec.rb index 95c1f015..765ccec5 100644 --- a/spec/statsd/sender_spec.rb +++ b/spec/statsd/sender_spec.rb @@ -239,6 +239,30 @@ end end end + + context 'when the sender thread died and a new one cannot be started' do + let(:thread_class) { class_double(Thread) } + let(:dead_thread) { Thread.new {}.tap(&:join) } + + before do + allow(thread_class).to receive(:new).and_return(dead_thread) + subject.start + allow(thread_class).to receive(:new).and_raise(ThreadError, "can't create Thread") + allow(message_buffer).to receive(:reset) + end + + after do + subject.stop + end + + it 'drops the messages without raising' do + expect do + subject.add('message') + subject.add('message') + end.not_to raise_error + expect(subject.send(:message_queue).length).to eq 0 + end + end end describe '#flush' do From a9df8e14e26bc8d0a33107cdbee1bf9fe433cffe Mon Sep 17 00:00:00 2001 From: Jake Huneau Date: Thu, 29 Oct 2026 11:49:00 +0000 Subject: [PATCH 2/2] Don't wait in #rendez_vous when there is no sender thread After a failed restart the message queue exists but no thread reads it, so #rendez_vous (and Statsd#sync_with_outbound_io) blocked forever on the reply queue. Return early unless the sender thread is alive, as #flush already does. --- lib/datadog/statsd/sender.rb | 2 ++ spec/statsd/sender_spec.rb | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/lib/datadog/statsd/sender.rb b/lib/datadog/statsd/sender.rb index c4a47982..8ff7f914 100644 --- a/lib/datadog/statsd/sender.rb +++ b/lib/datadog/statsd/sender.rb @@ -50,6 +50,8 @@ def flush(sync: false) def rendez_vous # could happen if #start hasn't be called return unless message_queue + # no companion thread to answer, e.g. it could not be re-created + return unless sender_thread && sender_thread.alive? # Initialize and get the thread's sync queue queue = (@thread_class.current[:statsd_sync_queue] ||= @queue_class.new) diff --git a/spec/statsd/sender_spec.rb b/spec/statsd/sender_spec.rb index 765ccec5..f8302698 100644 --- a/spec/statsd/sender_spec.rb +++ b/spec/statsd/sender_spec.rb @@ -262,6 +262,11 @@ end.not_to raise_error expect(subject.send(:message_queue).length).to eq 0 end + + it 'does not wait for the missing companion thread in rendez_vous' do + subject.add('message') + Timeout.timeout(1) { subject.rendez_vous } + end end end