From fd48fac2cd61c410795627b785f356b5b52414bb Mon Sep 17 00:00:00 2001 From: Irina Dominte Date: Tue, 2 Jun 2026 16:08:34 +0300 Subject: [PATCH 01/37] Starting working branch for Otel --- .../When_processing_incoming_message.cs | 32 ++++++++++ .../Traces/When_publishing_messages.cs | 63 +++++++++++++++++++ .../Traces/When_sending_messages.cs | 32 ++++++++++ .../Traces/When_sending_replies.cs | 34 ++++++++++ ...IApprovals.ApproveNServiceBus.approved.txt | 14 +++++ .../OpenTelemetry/ActivityFactoryTests.cs | 4 +- .../Pipeline/MainPipelineExecutorTests.cs | 2 +- .../Pipeline/TestableMessageOperations.cs | 2 +- .../RoutingToDispatchConnectorTests.cs | 20 +++--- .../Hosting/HostingComponent.Configuration.cs | 2 +- .../Hosting/HostingComponent.Settings.cs | 2 + .../OpenTelemetry/InstrumentationOptions.cs | 42 +++++++++++++ .../OpenTelemetry/OpenTelemetryExtensions.cs | 13 ++++ .../OpenTelemetry/OpenTelemetryFeature.cs | 15 +++++ .../PromoteMessagePropertiesToTagsBehavior.cs | 51 +++++++++++++++ .../Tracing/ActivityDisplayNames.cs | 5 ++ .../OpenTelemetry/Tracing/ActivityFactory.cs | 11 +++- .../OpenTelemetry/Tracing/IActivityFactory.cs | 1 + .../Tracing/NoOpActivityFactory.cs | 2 + .../Outgoing/RoutingToDispatchConnector.cs | 18 ++++++ .../Pipeline/Outgoing/SendComponent.cs | 2 +- .../Unicast/MessageOperations.cs | 6 +- 22 files changed, 355 insertions(+), 18 deletions(-) create mode 100644 src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs create mode 100644 src/NServiceBus.Core/OpenTelemetry/PromoteMessagePropertiesToTagsBehavior.cs diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_incoming_message.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_incoming_message.cs index 99fecbaf11c..d386a9906d2 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_incoming_message.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_incoming_message.cs @@ -80,5 +80,37 @@ public Task Handle(IncomingMessage message, IMessageHandlerContext context) } } + [Test] + public async Task Should_use_receive_address_in_span_name_when_opted_in() + { + await Scenario.Define() + .WithEndpoint(e => e + .When(s => s.SendLocal(new IncomingMessage()))) + .Run(); + + var incomingMessageActivities = NServiceBusActivityListener.CompletedActivities.GetReceiveMessageActivities(); + Assert.That(incomingMessageActivities, Has.Count.EqualTo(1)); + + var incomingActivity = incomingMessageActivities.Single(); + Assert.That(incomingActivity.DisplayName, Does.StartWith("process ")); + Assert.That(incomingActivity.DisplayName, Is.Not.EqualTo("process message")); + } + + public class ReceivingEndpointWithDestinationNaming : EndpointConfigurationBuilder + { + public ReceivingEndpointWithDestinationNaming() => + EndpointSetup(b => b.Tracing().UseMessageDestinationInSpanNames = true); + + [Handler] + public class MessageHandler(Context testContext) : IHandleMessages + { + public Task Handle(IncomingMessage message, IMessageHandlerContext context) + { + testContext.MarkAsCompleted(); + return Task.CompletedTask; + } + } + } + public class IncomingMessage : IMessage; } \ No newline at end of file diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs index 8c402afb504..0595803fbef 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs @@ -179,5 +179,68 @@ public Task Handle(ThisIsAnEvent @event, IMessageHandlerContext context) } } + [Test] + public async Task Should_use_event_type_in_span_name_when_opted_in() + { + await Scenario.Define() + .WithEndpoint(b => b + .When(ctx => ctx.SomeEventSubscribed, s => s.Publish())) + .WithEndpoint(b => b.When((session, ctx) => + { + if (ctx.HasNativePubSubSupport) + { + ctx.SomeEventSubscribed = true; + } + + return Task.CompletedTask; + })) + .Run(); + + var outgoingEventActivities = NServiceBusActivityListener.CompletedActivities.GetPublishEventActivities(); + Assert.That(outgoingEventActivities, Has.Count.EqualTo(1)); + + var publishedMessage = outgoingEventActivities.Single(); + Assert.That(publishedMessage.DisplayName, Is.EqualTo("publish ThisIsAnEvent")); + } + + class PublisherWithDestinationNaming : EndpointConfigurationBuilder + { + public PublisherWithDestinationNaming() => + EndpointSetup(b => + { + b.Tracing().UseMessageDestinationInSpanNames = true; + b.OnEndpointSubscribed((s, context) => + { + if (s.SubscriberEndpoint.Contains(Conventions.EndpointNamingConvention(typeof(SubscriberForPublisherWithDestinationNaming)))) + { + if (s.MessageType == typeof(ThisIsAnEvent).AssemblyQualifiedName) + { + context.SomeEventSubscribed = true; + } + } + }); + }); + } + + class SubscriberForPublisherWithDestinationNaming : EndpointConfigurationBuilder + { + public SubscriberForPublisherWithDestinationNaming() => + EndpointSetup(c => { }, + metadata => + { + metadata.RegisterPublisherFor(); + }); + + [Handler] + public class ThisHandlesSomethingHandler(Context testContext) : IHandleMessages + { + public Task Handle(ThisIsAnEvent @event, IMessageHandlerContext context) + { + testContext.MarkAsCompleted(); + return Task.CompletedTask; + } + } + } + public class ThisIsAnEvent : IEvent; } \ No newline at end of file diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_messages.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_messages.cs index af0751530d7..e14df1d05e6 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_messages.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_messages.cs @@ -131,5 +131,37 @@ public Task Handle(OutgoingMessage message, IMessageHandlerContext context) } } + [Test] + public async Task Should_use_destination_in_send_span_name_when_opted_in() + { + await Scenario.Define() + .WithEndpoint(b => b + .When(s => s.SendLocal(new OutgoingMessage()))) + .Run(); + + var outgoingMessageActivities = NServiceBusActivityListener.CompletedActivities.GetSendMessageActivities(); + Assert.That(outgoingMessageActivities, Has.Count.EqualTo(1)); + + var sentMessage = outgoingMessageActivities.Single(); + Assert.That(sentMessage.DisplayName, Does.StartWith("send ")); + Assert.That(sentMessage.DisplayName, Is.Not.EqualTo("send message")); + } + + public class TestEndpointWithDestinationNaming : EndpointConfigurationBuilder + { + public TestEndpointWithDestinationNaming() => + EndpointSetup(b => b.Tracing().UseMessageDestinationInSpanNames = true); + + [Handler] + public class MessageHandler(Context testContext) : IHandleMessages + { + public Task Handle(OutgoingMessage message, IMessageHandlerContext context) + { + testContext.MarkAsCompleted(); + return Task.CompletedTask; + } + } + } + public class OutgoingMessage : IMessage; } \ No newline at end of file diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_replies.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_replies.cs index e6299389c51..c6c505d9236 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_replies.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_replies.cs @@ -55,6 +55,40 @@ public Task Handle(OutgoingReply message, IMessageHandlerContext context) } } + [Test] + public async Task Should_use_destination_in_reply_span_name_when_opted_in() + { + await Scenario.Define() + .WithEndpoint(b => b + .When(s => s.SendLocal(new IncomingMessage()))) + .Run(); + + var outgoingMessageActivities = NServiceBusActivityListener.CompletedActivities.GetSendMessageActivities(); + Assert.That(outgoingMessageActivities, Has.Count.EqualTo(2), "2 messages are being sent"); + var replyMessage = outgoingMessageActivities[1]; + + Assert.That(replyMessage.DisplayName, Does.StartWith("reply ")); + } + + public class TestEndpointWithDestinationNaming : EndpointConfigurationBuilder + { + public TestEndpointWithDestinationNaming() => + EndpointSetup(b => b.Tracing().UseMessageDestinationInSpanNames = true); + + [Handler] + public class MessageHandler(Context testContext) : IHandleMessages, + IHandleMessages + { + public Task Handle(IncomingMessage message, IMessageHandlerContext context) => context.Reply(new OutgoingReply()); + + public Task Handle(OutgoingReply message, IMessageHandlerContext context) + { + testContext.MarkAsCompleted(); + return Task.CompletedTask; + } + } + } + public class IncomingMessage : IMessage; public class OutgoingReply : IMessage; diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt index acbacb5d5ae..e5b50f8da1b 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt @@ -632,6 +632,12 @@ namespace NServiceBus StopApplication = 0, Continue = 1, } + public class InstrumentationOptions + { + public InstrumentationOptions() { } + public NServiceBus.MessagePayloadAsTag MessagePayloadAsTag { get; set; } + public bool UseMessageDestinationInSpanNames { get; set; } + } public sealed class KeyedServiceKey { public const string Any = "______________"; @@ -718,6 +724,13 @@ namespace NServiceBus Unsubscribe = 4, Reply = 5, } + public enum MessagePayloadAsTag + { + None = 0, + IncomingMessage = 1, + OutgoingMessage = 2, + All = 3, + } public static class MessageProcessingContextExtensions { public static System.Threading.Tasks.Task Reply(this NServiceBus.IMessageProcessingContext context, object message) { } @@ -772,6 +785,7 @@ namespace NServiceBus { public static void ContinueExistingTraceOnReceive(this NServiceBus.PublishOptions publishOptions) { } public static void StartNewTraceOnReceive(this NServiceBus.SendOptions sendOptions) { } + public static NServiceBus.InstrumentationOptions Tracing(this NServiceBus.EndpointConfiguration config) { } } public static class OutboxConfigExtensions { diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs index 6c22b89bb7b..500740cc6ba 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs @@ -17,7 +17,7 @@ namespace NServiceBus.Core.Tests.OpenTelemetry; [TestFixture] public class ActivityFactoryTests { - readonly ActivityFactory activityFactory = new(); + readonly ActivityFactory activityFactory = new(new InstrumentationOptions()); TestingActivityListener nsbActivityListener; @@ -29,7 +29,7 @@ public class ActivityFactoryTests class NoDiagnosticListeners { - readonly ActivityFactory activityFactory = new(); + readonly ActivityFactory activityFactory = new(new InstrumentationOptions()); [Test] public void Should_return_null_incoming_activity_when_no_listeners() diff --git a/src/NServiceBus.Core.Tests/Pipeline/MainPipelineExecutorTests.cs b/src/NServiceBus.Core.Tests/Pipeline/MainPipelineExecutorTests.cs index f6223a25eac..ee6b76d9e3b 100644 --- a/src/NServiceBus.Core.Tests/Pipeline/MainPipelineExecutorTests.cs +++ b/src/NServiceBus.Core.Tests/Pipeline/MainPipelineExecutorTests.cs @@ -130,7 +130,7 @@ static MainPipelineExecutor CreateMainPipelineExecutor(ServiceProvider servicePr new TestableMessageOperations(), new Notification(), receivePipeline, - new ActivityFactory(), + new ActivityFactory(new InstrumentationOptions()), incomingPipelineMetrics, new EnvelopeUnwrapper([], incomingPipelineMetrics)); diff --git a/src/NServiceBus.Core.Tests/Pipeline/TestableMessageOperations.cs b/src/NServiceBus.Core.Tests/Pipeline/TestableMessageOperations.cs index ddb58d3a66e..909b9fb796a 100644 --- a/src/NServiceBus.Core.Tests/Pipeline/TestableMessageOperations.cs +++ b/src/NServiceBus.Core.Tests/Pipeline/TestableMessageOperations.cs @@ -13,7 +13,7 @@ class TestableMessageOperations : MessageOperations public Pipeline SubscribePipeline => (Pipeline)subscribePipeline; public Pipeline UnsubscribePipeline => (Pipeline)unsubscribePipeline; - public TestableMessageOperations() : base(new MessageMapper(), new Pipeline(), new Pipeline(), new Pipeline(), new Pipeline(), new Pipeline(), new ActivityFactory()) + public TestableMessageOperations() : base(new MessageMapper(), new Pipeline(), new Pipeline(), new Pipeline(), new Pipeline(), new Pipeline(), new ActivityFactory(new InstrumentationOptions())) { } diff --git a/src/NServiceBus.Core.Tests/Routing/RoutingToDispatchConnectorTests.cs b/src/NServiceBus.Core.Tests/Routing/RoutingToDispatchConnectorTests.cs index ad4d34e8037..da97e86e33f 100644 --- a/src/NServiceBus.Core.Tests/Routing/RoutingToDispatchConnectorTests.cs +++ b/src/NServiceBus.Core.Tests/Routing/RoutingToDispatchConnectorTests.cs @@ -17,7 +17,7 @@ public class RoutingToDispatchConnectorTests [Test] public async Task Should_preserve_message_state_for_one_routing_strategy_for_allocation_reasons() { - var behavior = new RoutingToDispatchConnector(); + var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); IEnumerable operations = null; var testableRoutingContext = new TestableRoutingContext { @@ -59,7 +59,7 @@ await behavior.Invoke(testableRoutingContext, context => [Test] public async Task Should_copy_message_state_for_multiple_routing_strategies() { - var behavior = new RoutingToDispatchConnector(); + var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); List operations = null; var testableRoutingContext = new TestableRoutingContext { @@ -135,7 +135,7 @@ await behavior.Invoke(testableRoutingContext, context => [Test] public async Task Should_preserve_headers_generated_by_custom_routing_strategy() { - var behavior = new RoutingToDispatchConnector(); + var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); Dictionary headers = null; await behavior.Invoke(new TestableRoutingContext { RoutingStrategies = [new HeaderModifyingRoutingStrategy()] }, context => { @@ -153,7 +153,7 @@ public async Task Should_dispatch_immediately_if_user_requested() options.RequireImmediateDispatch(); var dispatched = false; - var behavior = new RoutingToDispatchConnector(); + var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); var message = new OutgoingMessage("ID", [], Array.Empty()); await behavior.Invoke(new RoutingContext(message, @@ -170,7 +170,7 @@ await behavior.Invoke(new RoutingContext(message, public async Task Should_dispatch_immediately_if_not_sending_from_a_handler() { var dispatched = false; - var behavior = new RoutingToDispatchConnector(); + var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); var message = new OutgoingMessage("ID", [], Array.Empty()); await behavior.Invoke(new RoutingContext(message, @@ -187,7 +187,7 @@ await behavior.Invoke(new RoutingContext(message, public async Task Should_not_dispatch_by_default() { var dispatched = false; - var behavior = new RoutingToDispatchConnector(); + var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); var message = new OutgoingMessage("ID", [], Array.Empty()); await behavior.Invoke(new RoutingContext(message, @@ -203,7 +203,7 @@ await behavior.Invoke(new RoutingContext(message, [Test] public async Task Should_promote_message_headers_to_pipeline_activity() { - var behavior = new RoutingToDispatchConnector(); + var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); var routingContext = new TestableRoutingContext(); routingContext.Message.Headers[Headers.ContentType] = "test content type"; // one of the headers that will be mapped to tags @@ -257,7 +257,7 @@ class MyMessage : IMessage; [Test] public async Task Should_merge_receive_properties_when_declared_by_transport() { - var behavior = new RoutingToDispatchConnector(); + var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); var receiveProperties = new ReceiveProperties(new Dictionary { @@ -290,7 +290,7 @@ await behavior.Invoke(routingContext, context => [Test] public async Task Should_not_override_user_set_dispatch_property() { - var behavior = new RoutingToDispatchConnector(); + var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); var receiveProperties = new ReceiveProperties(new Dictionary { @@ -324,7 +324,7 @@ await behavior.Invoke(routingContext, context => [Test] public async Task Should_preserve_user_dispatch_properties_even_with_receive_properties() { - var behavior = new RoutingToDispatchConnector(); + var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); var receiveProperties = new ReceiveProperties(new Dictionary { diff --git a/src/NServiceBus.Core/Hosting/HostingComponent.Configuration.cs b/src/NServiceBus.Core/Hosting/HostingComponent.Configuration.cs index 4ed38527c71..6326b1a5006 100644 --- a/src/NServiceBus.Core/Hosting/HostingComponent.Configuration.cs +++ b/src/NServiceBus.Core/Hosting/HostingComponent.Configuration.cs @@ -26,7 +26,7 @@ public static Configuration PrepareConfiguration(Settings settings, List a serviceCollection, settings.ShouldRunInstallers, settings.UserRegistrations, - new ActivityFactory(), + new ActivityFactory(settings.InstrumentationOptions), persistenceConfiguration, installerComponent); diff --git a/src/NServiceBus.Core/Hosting/HostingComponent.Settings.cs b/src/NServiceBus.Core/Hosting/HostingComponent.Settings.cs index f46a6101f05..8e85864c2be 100644 --- a/src/NServiceBus.Core/Hosting/HostingComponent.Settings.cs +++ b/src/NServiceBus.Core/Hosting/HostingComponent.Settings.cs @@ -95,6 +95,8 @@ public bool WriteDiagnosticsToLog get; set; } + public InstrumentationOptions InstrumentationOptions => settings.GetOrDefault() ?? new InstrumentationOptions(); + internal void ConfigureHostLogging(object? endpointIdentifier) { EndpointIdentifier = endpointIdentifier; diff --git a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs new file mode 100644 index 00000000000..aeb480769e1 --- /dev/null +++ b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs @@ -0,0 +1,42 @@ +#nullable enable + +namespace NServiceBus; + +/// +/// Controls opt-in OpenTelemetry instrumentation behaviors. +/// Accessed via endpointConfiguration.Tracing(). +/// +public class InstrumentationOptions +{ + /// + /// Appends the destination to span names following the OTel messaging convention + /// {messaging.operation.name} {destination}, e.g. "process orders" or "send payments". + /// Disabled by default for backward compatibility. + /// + public bool UseMessageDestinationInSpanNames { get; set; } + + /// + /// Promotes public properties of message instances to span attributes + /// as nservicebus.message.{PropertyName}. + /// Defaults to . May expose sensitive data and incurs reflection cost. + /// + public MessagePayloadAsTag MessagePayloadAsTag { get; set; } +} + +/// +/// Controls which message payloads are promoted to span attributes. +/// +public enum MessagePayloadAsTag +{ + /// No message properties are promoted to span tags. + None, + + /// Public properties of the incoming message instance are promoted to span tags. + IncomingMessage, + + /// Public properties of outgoing message instances are promoted to span tags. + OutgoingMessage, + + /// Public properties of both incoming and outgoing message instances are promoted to span tags. + All +} diff --git a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryExtensions.cs b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryExtensions.cs index bc263ffe423..9c3ad6e63e8 100644 --- a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryExtensions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryExtensions.cs @@ -2,11 +2,24 @@ namespace NServiceBus; +using System; + /// /// Gives users control over the depth of an OpenTelemetry trace. /// public static class OpenTelemetryExtensions { + /// + /// Provides access to instrumentation options for OpenTelemetry tracing. + /// + /// The endpoint configuration. + /// The instance for this endpoint. + public static InstrumentationOptions Tracing(this EndpointConfiguration config) + { + ArgumentNullException.ThrowIfNull(config); + return config.Settings.GetOrCreate(); + } + /// /// Start a new OpenTelemetry trace conversation. /// diff --git a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs index c485129eee0..e4804d76a0c 100644 --- a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs +++ b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs @@ -22,5 +22,20 @@ protected override void Setup(FeatureConfigurationContext context) new PopulateRecoverabilityTraceMetadataBehavior(), "Populates the recoverability metadata" ); + + var options = context.Settings.GetOrDefault(); + if (options?.MessagePayloadAsTag is MessagePayloadAsTag.IncomingMessage or MessagePayloadAsTag.All) + { + context.Pipeline.Register( + new IncomingMessagePayloadToTagsBehavior(), + "Promotes incoming message properties to span tags"); + } + + if (options?.MessagePayloadAsTag is MessagePayloadAsTag.OutgoingMessage or MessagePayloadAsTag.All) + { + context.Pipeline.Register( + new OutgoingMessagePayloadToTagsBehavior(), + "Promotes outgoing message properties to span tags"); + } } } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/PromoteMessagePropertiesToTagsBehavior.cs b/src/NServiceBus.Core/OpenTelemetry/PromoteMessagePropertiesToTagsBehavior.cs new file mode 100644 index 00000000000..4435190901a --- /dev/null +++ b/src/NServiceBus.Core/OpenTelemetry/PromoteMessagePropertiesToTagsBehavior.cs @@ -0,0 +1,51 @@ +#nullable enable + +namespace NServiceBus; + +using System; +using System.Diagnostics; +using System.Reflection; +using System.Threading.Tasks; +using Pipeline; + +class IncomingMessagePayloadToTagsBehavior : IBehavior +{ + public Task Invoke(IIncomingLogicalMessageContext context, Func next) + { + var activity = Activity.Current; + if (activity?.IsAllDataRequested == true) + { + PromoteProperties(activity, context.Message.Instance); + } + + return next(context); + } + + // this needs to be changed to be base64 encoding of the entire body not just the properties + + internal static void PromoteProperties(Activity activity, object instance) + { + foreach (var property in instance.GetType().GetProperties(BindingFlags.Public | BindingFlags.Instance)) + { + var value = property.GetValue(instance); + if (value is not null) + { + activity.SetTag($"nservicebus.message.{property.Name}", value.ToString()); + } + } + } +} + +class OutgoingMessagePayloadToTagsBehavior : IBehavior +{ + public Task Invoke(IOutgoingLogicalMessageContext context, Func next) + { + var activity = Activity.Current; + if (activity?.IsAllDataRequested == true) + { + IncomingMessagePayloadToTagsBehavior.PromoteProperties(activity, context.Message.Instance); + } + + return next(context); + } +} diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs index ca6d81aff76..cf56402f4dd 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs @@ -10,4 +10,9 @@ static class ActivityDisplayNames public const string UnsubscribeEvent = "unsubscribe event"; public const string SendMessage = "send message"; public const string ReplyMessage = "reply"; + + // Operation-only prefixes used when UseMessageDestinationInSpanNames is enabled + internal const string ProcessOperation = "process"; + internal const string PublishOperation = "publish"; + internal const string SendOperation = "send"; } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs index 6c0202a51d7..b6548eef2f0 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs @@ -8,6 +8,13 @@ namespace NServiceBus; sealed class ActivityFactory : IActivityFactory { + public ActivityFactory(InstrumentationOptions options) + { + Options = options; + } + + public InstrumentationOptions Options { get; } + public Activity? StartIncomingPipelineActivity(MessageContext context) { // CreateActivity is a no-op if there are no listeners but we are doing a fast path check @@ -66,7 +73,9 @@ sealed class ActivityFactory : IActivityFactory ContextPropagation.PropagateContextFromHeaders(activity, context.Headers); - activity.DisplayName = ActivityDisplayNames.ProcessMessage; + activity.DisplayName = Options.UseMessageDestinationInSpanNames + ? $"{ActivityDisplayNames.ProcessOperation} {context.ReceiveAddress}" + : ActivityDisplayNames.ProcessMessage; activity.SetIdFormat(ActivityIdFormat.W3C); activity.AddTag(ActivityTags.NativeMessageId, context.NativeMessageId); diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs index 8a47acb7fdd..daa6553a2eb 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs @@ -8,6 +8,7 @@ namespace NServiceBus; interface IActivityFactory { + InstrumentationOptions Options { get; } Activity? StartIncomingPipelineActivity(MessageContext context); Activity? StartOutgoingPipelineActivity(string activityName, string displayName, IBehaviorContext outgoingContext); Activity? StartHandlerActivity(MessageHandler messageHandler); diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs index a36d268a3bd..fe1f2f82c12 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs @@ -8,6 +8,8 @@ namespace NServiceBus; sealed class NoOpActivityFactory : IActivityFactory { + public InstrumentationOptions Options { get; } = new InstrumentationOptions(); + public Activity? StartIncomingPipelineActivity(MessageContext context) => null; public Activity? StartOutgoingPipelineActivity(string activityName, string displayName, IBehaviorContext outgoingContext) => null; diff --git a/src/NServiceBus.Core/Pipeline/Outgoing/RoutingToDispatchConnector.cs b/src/NServiceBus.Core/Pipeline/Outgoing/RoutingToDispatchConnector.cs index bc4be17db8b..55f62023b4b 100644 --- a/src/NServiceBus.Core/Pipeline/Outgoing/RoutingToDispatchConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Outgoing/RoutingToDispatchConnector.cs @@ -14,6 +14,13 @@ namespace NServiceBus; class RoutingToDispatchConnector : StageConnector { + readonly IActivityFactory activityFactory; + + public RoutingToDispatchConnector(IActivityFactory activityFactory) + { + this.activityFactory = activityFactory; + } + public override Task Invoke(IRoutingContext context, Func stage) { var dispatchConsistency = DispatchConsistency.Default; @@ -60,6 +67,17 @@ public override Task Invoke(IRoutingContext context, Func 0 + && operations[0].AddressTag is UnicastAddressTag unicastTag + && outgoingMessage.Headers.TryGetValue(Headers.MessageIntent, out var intentStr) + && intentStr is "Send" or "Reply") + { + activity.DisplayName = $"{activity.DisplayName} {unicastTag.Destination}"; + } } if (dispatchConsistency == DispatchConsistency.Default && context.Extensions.TryGet(out var pendingOperations)) diff --git a/src/NServiceBus.Core/Pipeline/Outgoing/SendComponent.cs b/src/NServiceBus.Core/Pipeline/Outgoing/SendComponent.cs index c00d0880656..5899d352981 100644 --- a/src/NServiceBus.Core/Pipeline/Outgoing/SendComponent.cs +++ b/src/NServiceBus.Core/Pipeline/Outgoing/SendComponent.cs @@ -27,7 +27,7 @@ public static SendComponent Initialize(PipelineSettings pipelineSettings, Hostin pipelineSettings.Register(new OutgoingPhysicalToRoutingConnector(), "Starts the message dispatch pipeline"); - pipelineSettings.Register(new RoutingToDispatchConnector(), + pipelineSettings.Register(new RoutingToDispatchConnector(hostingConfiguration.ActivityFactory), "Decides if the current message should be batched or immediately be dispatched to the transport"); pipelineSettings.Register(new BatchToDispatchConnector(), "Passes batched messages over to the immediate dispatch part of the pipeline"); pipelineSettings.Register(b => new ImmediateDispatchTerminator(b.GetRequiredService()), "Hands the outgoing messages over to the transport for immediate delivery"); diff --git a/src/NServiceBus.Core/Unicast/MessageOperations.cs b/src/NServiceBus.Core/Unicast/MessageOperations.cs index 34c60f3cda1..7f3af927a2a 100644 --- a/src/NServiceBus.Core/Unicast/MessageOperations.cs +++ b/src/NServiceBus.Core/Unicast/MessageOperations.cs @@ -65,7 +65,11 @@ async Task Publish(IBehaviorContext context, Type messageType, object message, P MergeDispatchProperties(publishContext, options.DispatchProperties); - using var activity = activityFactory.StartOutgoingPipelineActivity(ActivityNames.OutgoingEventActivityName, ActivityDisplayNames.PublishEvent, publishContext); + var publishDisplayName = activityFactory.Options.UseMessageDestinationInSpanNames + ? $"{ActivityDisplayNames.PublishOperation} {messageType.Name}" + : ActivityDisplayNames.PublishEvent; + + using var activity = activityFactory.StartOutgoingPipelineActivity(ActivityNames.OutgoingEventActivityName, publishDisplayName, publishContext); await publishPipeline.Invoke(publishContext, activity).ConfigureAwait(false); } From dd3a9dbe1e2fbb6b997cacc42a9460f95f96a16c Mon Sep 17 00:00:00 2001 From: Irina Dominte Date: Tue, 2 Jun 2026 16:59:57 +0300 Subject: [PATCH 02/37] Added base64 encoded body --- .../MessagePayloadToTagsBehaviorTests.cs | 102 ++++++++++++++++++ ...or.cs => MessagePayloadToTagsBehaviors.cs} | 22 ++-- 2 files changed, 110 insertions(+), 14 deletions(-) create mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/MessagePayloadToTagsBehaviorTests.cs rename src/NServiceBus.Core/OpenTelemetry/{PromoteMessagePropertiesToTagsBehavior.cs => MessagePayloadToTagsBehaviors.cs} (57%) diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/MessagePayloadToTagsBehaviorTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/MessagePayloadToTagsBehaviorTests.cs new file mode 100644 index 00000000000..4a2cfd6ce2a --- /dev/null +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/MessagePayloadToTagsBehaviorTests.cs @@ -0,0 +1,102 @@ +#nullable enable + +namespace NServiceBus.Core.Tests.OpenTelemetry; + +using System; +using System.Collections.Immutable; +using System.Text; +using System.Text.Json; +using System.Threading.Tasks; +using Helpers; +using NServiceBus.Pipeline; +using NUnit.Framework; +using Testing; +using Unicast.Messages; + +[TestFixture] +public class MessagePayloadToTagsBehaviorTests +{ + TestingActivityListener activityListener; + + [SetUp] + public void SetUp() => activityListener = TestingActivityListener.SetupNServiceBusDiagnosticListener(); + + [TearDown] + public void TearDown() => activityListener.Dispose(); + + [Test] + public async Task Incoming_should_set_base64_encoded_json_body_tag() + { + using var activity = ActivitySources.Main.StartActivity("test"); + + var message = new TestMessage { Name = "Hello", Value = 42 }; + var context = new TestableIncomingLogicalMessageContext + { + Message = new LogicalMessage(new MessageMetadata(typeof(TestMessage)), message) + }; + + await new IncomingMessagePayloadToTagsBehavior().Invoke(context, _ => Task.CompletedTask); + + var tags = activity!.Tags.ToImmutableDictionary(); + Assert.That(tags.ContainsKey("nservicebus.message.body"), Is.True); + + var decoded = Encoding.UTF8.GetString(Convert.FromBase64String(tags["nservicebus.message.body"]!)); + var deserialized = JsonSerializer.Deserialize(decoded); + + using (Assert.EnterMultipleScope()) + { + Assert.That(deserialized!.Name, Is.EqualTo("Hello")); + Assert.That(deserialized.Value, Is.EqualTo(42)); + } + } + + [Test] + public async Task Incoming_should_not_set_tag_when_no_active_activity() + { + var context = new TestableIncomingLogicalMessageContext + { + Message = new LogicalMessage(new MessageMetadata(typeof(TestMessage)), new TestMessage()) + }; + + Assert.DoesNotThrowAsync(() => new IncomingMessagePayloadToTagsBehavior().Invoke(context, _ => Task.CompletedTask)); + } + + [Test] + public async Task Outgoing_should_set_base64_encoded_json_body_tag() + { + using var activity = ActivitySources.Main.StartActivity("test"); + + var message = new TestMessage { Name = "World", Value = 99 }; + var context = new TestableOutgoingLogicalMessageContext(); + context.UpdateMessage(message); + + await new OutgoingMessagePayloadToTagsBehavior().Invoke(context, _ => Task.CompletedTask); + + var tags = activity!.Tags.ToImmutableDictionary(); + Assert.That(tags.ContainsKey("nservicebus.message.body"), Is.True); + + var decoded = Encoding.UTF8.GetString(Convert.FromBase64String(tags["nservicebus.message.body"]!)); + var deserialized = JsonSerializer.Deserialize(decoded); + + using (Assert.EnterMultipleScope()) + { + Assert.That(deserialized!.Name, Is.EqualTo("World")); + Assert.That(deserialized.Value, Is.EqualTo(99)); + } + } + + [Test] + public async Task Outgoing_should_not_set_tag_when_no_active_activity() + { + var context = new TestableOutgoingLogicalMessageContext(); + context.UpdateMessage(new TestMessage()); + + Assert.DoesNotThrowAsync(() => new OutgoingMessagePayloadToTagsBehavior().Invoke(context, _ => Task.CompletedTask)); + } + + class TestMessage + { + public string? Name { get; set; } + public int Value { get; set; } + } +} diff --git a/src/NServiceBus.Core/OpenTelemetry/PromoteMessagePropertiesToTagsBehavior.cs b/src/NServiceBus.Core/OpenTelemetry/MessagePayloadToTagsBehaviors.cs similarity index 57% rename from src/NServiceBus.Core/OpenTelemetry/PromoteMessagePropertiesToTagsBehavior.cs rename to src/NServiceBus.Core/OpenTelemetry/MessagePayloadToTagsBehaviors.cs index 4435190901a..7868307458f 100644 --- a/src/NServiceBus.Core/OpenTelemetry/PromoteMessagePropertiesToTagsBehavior.cs +++ b/src/NServiceBus.Core/OpenTelemetry/MessagePayloadToTagsBehaviors.cs @@ -4,7 +4,8 @@ namespace NServiceBus; using System; using System.Diagnostics; -using System.Reflection; +using System.Text; +using System.Text.Json; using System.Threading.Tasks; using Pipeline; @@ -15,24 +16,17 @@ public Task Invoke(IIncomingLogicalMessageContext context, Func Date: Wed, 3 Jun 2026 11:01:48 +0200 Subject: [PATCH 03/37] Apply suggestion from @ramonsmits --- .../OpenTelemetry/MessagePayloadToTagsBehaviors.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/NServiceBus.Core/OpenTelemetry/MessagePayloadToTagsBehaviors.cs b/src/NServiceBus.Core/OpenTelemetry/MessagePayloadToTagsBehaviors.cs index 7868307458f..7f4129c4705 100644 --- a/src/NServiceBus.Core/OpenTelemetry/MessagePayloadToTagsBehaviors.cs +++ b/src/NServiceBus.Core/OpenTelemetry/MessagePayloadToTagsBehaviors.cs @@ -35,7 +35,7 @@ class OutgoingMessagePayloadToTagsBehavior : IBehavior next) { var activity = Activity.Current; - if (activity?.IsAllDataRequested == true) + if (activity?.IsAllDataRequested) { IncomingMessagePayloadToTagsBehavior.SetMessageBodyTag(activity, context.Message.Instance); } From b083629851bab4a72c9b622fd2aa53f2f9a068e5 Mon Sep 17 00:00:00 2001 From: Irina Dominte Date: Wed, 10 Jun 2026 14:15:16 +0300 Subject: [PATCH 04/37] removed the body serialization --- ...IApprovals.ApproveNServiceBus.approved.txt | 8 -- .../MessagePayloadToTagsBehaviorTests.cs | 102 ------------------ .../OpenTelemetry/InstrumentationOptions.cs | 25 ----- .../MessagePayloadToTagsBehaviors.cs | 45 -------- .../OpenTelemetry/OpenTelemetryFeature.cs | 15 --- 5 files changed, 195 deletions(-) delete mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/MessagePayloadToTagsBehaviorTests.cs delete mode 100644 src/NServiceBus.Core/OpenTelemetry/MessagePayloadToTagsBehaviors.cs diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt index e5b50f8da1b..a5bdc1d3893 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt @@ -635,7 +635,6 @@ namespace NServiceBus public class InstrumentationOptions { public InstrumentationOptions() { } - public NServiceBus.MessagePayloadAsTag MessagePayloadAsTag { get; set; } public bool UseMessageDestinationInSpanNames { get; set; } } public sealed class KeyedServiceKey @@ -724,13 +723,6 @@ namespace NServiceBus Unsubscribe = 4, Reply = 5, } - public enum MessagePayloadAsTag - { - None = 0, - IncomingMessage = 1, - OutgoingMessage = 2, - All = 3, - } public static class MessageProcessingContextExtensions { public static System.Threading.Tasks.Task Reply(this NServiceBus.IMessageProcessingContext context, object message) { } diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/MessagePayloadToTagsBehaviorTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/MessagePayloadToTagsBehaviorTests.cs deleted file mode 100644 index 4a2cfd6ce2a..00000000000 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/MessagePayloadToTagsBehaviorTests.cs +++ /dev/null @@ -1,102 +0,0 @@ -#nullable enable - -namespace NServiceBus.Core.Tests.OpenTelemetry; - -using System; -using System.Collections.Immutable; -using System.Text; -using System.Text.Json; -using System.Threading.Tasks; -using Helpers; -using NServiceBus.Pipeline; -using NUnit.Framework; -using Testing; -using Unicast.Messages; - -[TestFixture] -public class MessagePayloadToTagsBehaviorTests -{ - TestingActivityListener activityListener; - - [SetUp] - public void SetUp() => activityListener = TestingActivityListener.SetupNServiceBusDiagnosticListener(); - - [TearDown] - public void TearDown() => activityListener.Dispose(); - - [Test] - public async Task Incoming_should_set_base64_encoded_json_body_tag() - { - using var activity = ActivitySources.Main.StartActivity("test"); - - var message = new TestMessage { Name = "Hello", Value = 42 }; - var context = new TestableIncomingLogicalMessageContext - { - Message = new LogicalMessage(new MessageMetadata(typeof(TestMessage)), message) - }; - - await new IncomingMessagePayloadToTagsBehavior().Invoke(context, _ => Task.CompletedTask); - - var tags = activity!.Tags.ToImmutableDictionary(); - Assert.That(tags.ContainsKey("nservicebus.message.body"), Is.True); - - var decoded = Encoding.UTF8.GetString(Convert.FromBase64String(tags["nservicebus.message.body"]!)); - var deserialized = JsonSerializer.Deserialize(decoded); - - using (Assert.EnterMultipleScope()) - { - Assert.That(deserialized!.Name, Is.EqualTo("Hello")); - Assert.That(deserialized.Value, Is.EqualTo(42)); - } - } - - [Test] - public async Task Incoming_should_not_set_tag_when_no_active_activity() - { - var context = new TestableIncomingLogicalMessageContext - { - Message = new LogicalMessage(new MessageMetadata(typeof(TestMessage)), new TestMessage()) - }; - - Assert.DoesNotThrowAsync(() => new IncomingMessagePayloadToTagsBehavior().Invoke(context, _ => Task.CompletedTask)); - } - - [Test] - public async Task Outgoing_should_set_base64_encoded_json_body_tag() - { - using var activity = ActivitySources.Main.StartActivity("test"); - - var message = new TestMessage { Name = "World", Value = 99 }; - var context = new TestableOutgoingLogicalMessageContext(); - context.UpdateMessage(message); - - await new OutgoingMessagePayloadToTagsBehavior().Invoke(context, _ => Task.CompletedTask); - - var tags = activity!.Tags.ToImmutableDictionary(); - Assert.That(tags.ContainsKey("nservicebus.message.body"), Is.True); - - var decoded = Encoding.UTF8.GetString(Convert.FromBase64String(tags["nservicebus.message.body"]!)); - var deserialized = JsonSerializer.Deserialize(decoded); - - using (Assert.EnterMultipleScope()) - { - Assert.That(deserialized!.Name, Is.EqualTo("World")); - Assert.That(deserialized.Value, Is.EqualTo(99)); - } - } - - [Test] - public async Task Outgoing_should_not_set_tag_when_no_active_activity() - { - var context = new TestableOutgoingLogicalMessageContext(); - context.UpdateMessage(new TestMessage()); - - Assert.DoesNotThrowAsync(() => new OutgoingMessagePayloadToTagsBehavior().Invoke(context, _ => Task.CompletedTask)); - } - - class TestMessage - { - public string? Name { get; set; } - public int Value { get; set; } - } -} diff --git a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs index aeb480769e1..f920a28b93a 100644 --- a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs @@ -14,29 +14,4 @@ public class InstrumentationOptions /// Disabled by default for backward compatibility. /// public bool UseMessageDestinationInSpanNames { get; set; } - - /// - /// Promotes public properties of message instances to span attributes - /// as nservicebus.message.{PropertyName}. - /// Defaults to . May expose sensitive data and incurs reflection cost. - /// - public MessagePayloadAsTag MessagePayloadAsTag { get; set; } -} - -/// -/// Controls which message payloads are promoted to span attributes. -/// -public enum MessagePayloadAsTag -{ - /// No message properties are promoted to span tags. - None, - - /// Public properties of the incoming message instance are promoted to span tags. - IncomingMessage, - - /// Public properties of outgoing message instances are promoted to span tags. - OutgoingMessage, - - /// Public properties of both incoming and outgoing message instances are promoted to span tags. - All } diff --git a/src/NServiceBus.Core/OpenTelemetry/MessagePayloadToTagsBehaviors.cs b/src/NServiceBus.Core/OpenTelemetry/MessagePayloadToTagsBehaviors.cs deleted file mode 100644 index 7f4129c4705..00000000000 --- a/src/NServiceBus.Core/OpenTelemetry/MessagePayloadToTagsBehaviors.cs +++ /dev/null @@ -1,45 +0,0 @@ -#nullable enable - -namespace NServiceBus; - -using System; -using System.Diagnostics; -using System.Text; -using System.Text.Json; -using System.Threading.Tasks; -using Pipeline; - -class IncomingMessagePayloadToTagsBehavior : IBehavior -{ - public Task Invoke(IIncomingLogicalMessageContext context, Func next) - { - var activity = Activity.Current; - if (activity?.IsAllDataRequested == true) - { - SetMessageBodyTag(activity, context.Message.Instance); - } - - return next(context); - } - - internal static void SetMessageBodyTag(Activity activity, object instance) - { - var json = JsonSerializer.Serialize(instance, instance.GetType()); - var base64 = Convert.ToBase64String(Encoding.UTF8.GetBytes(json)); - activity.SetTag("nservicebus.message.body", base64); - } -} - -class OutgoingMessagePayloadToTagsBehavior : IBehavior -{ - public Task Invoke(IOutgoingLogicalMessageContext context, Func next) - { - var activity = Activity.Current; - if (activity?.IsAllDataRequested) - { - IncomingMessagePayloadToTagsBehavior.SetMessageBodyTag(activity, context.Message.Instance); - } - - return next(context); - } -} diff --git a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs index e4804d76a0c..c485129eee0 100644 --- a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs +++ b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs @@ -22,20 +22,5 @@ protected override void Setup(FeatureConfigurationContext context) new PopulateRecoverabilityTraceMetadataBehavior(), "Populates the recoverability metadata" ); - - var options = context.Settings.GetOrDefault(); - if (options?.MessagePayloadAsTag is MessagePayloadAsTag.IncomingMessage or MessagePayloadAsTag.All) - { - context.Pipeline.Register( - new IncomingMessagePayloadToTagsBehavior(), - "Promotes incoming message properties to span tags"); - } - - if (options?.MessagePayloadAsTag is MessagePayloadAsTag.OutgoingMessage or MessagePayloadAsTag.All) - { - context.Pipeline.Register( - new OutgoingMessagePayloadToTagsBehavior(), - "Promotes outgoing message properties to span tags"); - } } } \ No newline at end of file From af64227212fae3bbe38e38272cd1a7182b692125 Mon Sep 17 00:00:00 2001 From: Tomasz Masternak Date: Tue, 16 Jun 2026 13:36:27 +0200 Subject: [PATCH 05/37] access level fixes for acceptance tests --- .../Core/OpenTelemetry/Traces/When_publishing_messages.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs index 0595803fbef..4281ddc170a 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs @@ -203,7 +203,7 @@ await Scenario.Define() Assert.That(publishedMessage.DisplayName, Is.EqualTo("publish ThisIsAnEvent")); } - class PublisherWithDestinationNaming : EndpointConfigurationBuilder + public class PublisherWithDestinationNaming : EndpointConfigurationBuilder { public PublisherWithDestinationNaming() => EndpointSetup(b => @@ -222,7 +222,7 @@ public PublisherWithDestinationNaming() => }); } - class SubscriberForPublisherWithDestinationNaming : EndpointConfigurationBuilder + public class SubscriberForPublisherWithDestinationNaming : EndpointConfigurationBuilder { public SubscriberForPublisherWithDestinationNaming() => EndpointSetup(c => { }, From bba80f7bd2e6ef25bd942bb85032c0bf5b14c88d Mon Sep 17 00:00:00 2001 From: Tomasz Masternak Date: Tue, 16 Jun 2026 15:12:22 +0200 Subject: [PATCH 06/37] Using DistributedContextPropagator instead of hand-written baggage propagator (#7820) * initial migration to the DistributedContextPropagator for W3C compatibility * new propagator with backwards compatiblity tests * fixing the startnewtrace header progagation * refactor ContextPropagation.cs * small tweak * warning fixes * att tests reflect supported baggage and tracestate formats --- .../Metrics/When_envelope_handler_succeeds.cs | 1 + .../When_ambient_trace_in_message_session.cs | 2 +- ...hen_incoming_message_has_baggage_header.cs | 4 +- .../When_outgoing_activity_has_baggage.cs | 2 +- .../ContextPropagationIncompatibilityTests.cs | 84 +++++++++++++++++ .../OpenTelemetry/ContextPropagationTests.cs | 91 +++++++++++++------ .../OpenTelemetry/LegacyContextPropagator.cs | 87 ++++++++++++++++++ .../Tracing/ActivityExtensions.cs | 2 + .../Tracing/ContextPropagation.cs | 74 ++++++--------- 9 files changed, 269 insertions(+), 78 deletions(-) create mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationIncompatibilityTests.cs create mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagator.cs diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_envelope_handler_succeeds.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_envelope_handler_succeeds.cs index 16c7d1da43b..faca6eed9bb 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_envelope_handler_succeeds.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_envelope_handler_succeeds.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Linq; using System.Threading.Tasks; +using Microsoft.ApplicationInsights.Extensibility; using NServiceBus; using NServiceBus.AcceptanceTesting; using NServiceBus.AcceptanceTests.Core.OpenTelemetry; diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_ambient_trace_in_message_session.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_ambient_trace_in_message_session.cs index 51ae62b9a41..6c3f58c65e5 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_ambient_trace_in_message_session.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_ambient_trace_in_message_session.cs @@ -15,7 +15,7 @@ public async Task Should_attach_to_ambient_trace() using var externalActivitySource = new ActivitySource("external trace source"); using var _ = TestingActivityListener.SetupDiagnosticListener(externalActivitySource.Name); // need to have a registered listener for activities to be created - const string wrapperActivityTraceState = "test trace state"; + const string wrapperActivityTraceState = "tracekey=traceValue"; var context = await Scenario.Define() .WithEndpoint(b => b diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_incoming_message_has_baggage_header.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_incoming_message_has_baggage_header.cs index 29acf036898..ffcc39fd5a1 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_incoming_message_has_baggage_header.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_incoming_message_has_baggage_header.cs @@ -17,7 +17,7 @@ public async Task Should_propagate_baggage_to_activity() { var sendOptions = new SendOptions(); sendOptions.RouteToThisEndpoint(); - sendOptions.SetHeader(Headers.DiagnosticsBaggage, "key1=value1,key2=value2,key3="); + sendOptions.SetHeader(Headers.DiagnosticsBaggage, "key1=value1,key2=value2,key3=value3"); await session.Send(new SomeMessage(), sendOptions); }) ) @@ -29,7 +29,7 @@ public async Task Should_propagate_baggage_to_activity() VerifyBaggageItem("key1", "value1"); VerifyBaggageItem("key2", "value2"); - VerifyBaggageItem("key3", ""); + VerifyBaggageItem("key3", "value3"); return; void VerifyBaggageItem(string key, string expectedValue) diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_outgoing_activity_has_baggage.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_outgoing_activity_has_baggage.cs index 93ff05bcfbf..e8bed02db26 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_outgoing_activity_has_baggage.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_outgoing_activity_has_baggage.cs @@ -33,7 +33,7 @@ public async Task Should_propagate_baggage_to_headers() ) .Run(); - Assert.That(context.BaggageHeader, Is.EqualTo("key3=,key2=value2,key1=value1")); + Assert.That(context.BaggageHeader, Is.EqualTo("key3 = , key2 = value2, key1 = value1")); } public class TestEndpoint : EndpointConfigurationBuilder diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationIncompatibilityTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationIncompatibilityTests.cs new file mode 100644 index 00000000000..4aa93147137 --- /dev/null +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationIncompatibilityTests.cs @@ -0,0 +1,84 @@ +namespace NServiceBus.Core.Tests.OpenTelemetry; + +using System.Collections.Generic; +using System.Diagnostics; +using Extensibility; +using NUnit.Framework; + +[TestFixture] +public class ContextPropagationIncompatibilityTests +{ + delegate void Writer(Activity activity, Dictionary headers, ContextBag context); + delegate void Reader(Activity activity, IDictionary headers); + + static readonly Writer LegacyWrite = LegacyContextPropagator.PropagateContextToHeaders; + static readonly Reader LegacyRead = LegacyContextPropagator.PropagateContextFromHeaders; + static readonly Writer NewWrite = ContextPropagation.PropagateContextToHeaders; + static readonly Reader NewRead = ContextPropagation.PropagateContextFromHeaders; + + // A value exercising every class of special character: structural baggage delimiters + // (',' ';' '='), the escape char '%', quotes, brackets, slashes, ampersand, Unicode and + // an emoji, plus interior spaces. Deliberately has NO leading/trailing whitespace, so this + // value isolates "what happens to special characters" from the separate edge-whitespace + // issue covered by New_propagation_loses_leading_whitespace_in_a_value. + // This already includes property-like syntax (the ';' and '=' delimiters), so a value such as + // "zone=eu;sensitive" is just a subset and needs no separate case here. + const string AllSpecialCharacters = "a b,c;d=e&f'g\"h\\i(j)k{l}m[n]o%p/q?r:s@t~u|vx é ü 😀 z"; + + static Dictionary Send(string value, Writer write) + { + using var sender = new Activity(ActivityNames.OutgoingMessageActivityName); + sender.SetIdFormat(ActivityIdFormat.W3C); + sender.Start(); + sender.AddBaggage("key", value); + + var headers = new Dictionary(); + write(sender, headers, new ContextBag()); + sender.Stop(); + return headers; + } + + static string Receive(Dictionary headers, Reader read) + { + using var receiver = new Activity(ActivityNames.IncomingMessageActivityName); + receiver.SetIdFormat(ActivityIdFormat.W3C); + receiver.Start(); + read(receiver, headers); + return receiver.GetBaggageItem("key"); + } + + static string Transmit(string value, Writer write, Reader read) => Receive(Send(value, write), read); + + [Test] + public void Legacy_sender_to_new_receiver_preserves_the_value() + { + var received = Transmit(AllSpecialCharacters, LegacyWrite, NewRead); + Assert.That(received, Is.EqualTo(AllSpecialCharacters)); + } + + [Test] + public void New_sender_to_legacy_receiver_prepends_a_leading_space_but_keeps_the_special_characters() + { + var received = Transmit(AllSpecialCharacters, NewWrite, LegacyRead); + + Assert.That(received, Is.EqualTo(" " + AllSpecialCharacters), + "ignoring the leading space, every special character round-trips correctly"); + } + + [Test] + public void New_propagation_loses_leading_whitespace_in_a_value() + { + const string valueWithLeadingSpace = " hasLeadingSpace"; + + var legacyRoundTrip = Transmit(valueWithLeadingSpace, LegacyWrite, LegacyRead); + var newRoundTrip = Transmit(valueWithLeadingSpace, NewWrite, NewRead); + + using (Assert.EnterMultipleScope()) + { + Assert.That(legacyRoundTrip, Is.EqualTo(valueWithLeadingSpace), + "legacy propagation preserves leading whitespace via percent-encoding"); + Assert.That(newRoundTrip, Is.EqualTo("hasLeadingSpace"), + "new propagation strips the leading whitespace from the value"); + } + } +} diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationTests.cs index 6de0d1ade7a..48f95c1fe86 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationTests.cs @@ -1,6 +1,5 @@ namespace NServiceBus.Core.Tests.OpenTelemetry; -using System; using System.Collections; using System.Collections.Generic; using System.Diagnostics; @@ -94,7 +93,9 @@ public void Can_propagate_baggage_from_header_to_activity(ContextPropagationTest headers[Headers.DiagnosticsBaggage] = testCase.BaggageHeaderValue; } - var activity = new Activity(ActivityNames.IncomingMessageActivityName); + using var activity = new Activity(ActivityNames.IncomingMessageActivityName); + activity.SetIdFormat(ActivityIdFormat.W3C); + activity.Start(); ContextPropagation.PropagateContextFromHeaders(activity, headers); @@ -114,7 +115,9 @@ public void Can_propagate_baggage_from_activity_to_header(ContextPropagationTest var headers = new Dictionary(); - var activity = new Activity(ActivityNames.OutgoingMessageActivityName); + using var activity = new Activity(ActivityNames.OutgoingMessageActivityName); + activity.SetIdFormat(ActivityIdFormat.W3C); + activity.Start(); foreach (var baggageItem in testCase.ExpectedBaggageItems.Reverse()) { @@ -131,7 +134,7 @@ public void Can_propagate_baggage_from_activity_to_header(ContextPropagationTest { Assert.That(baggageHeaderSet, Is.True, "Should have a baggage header if there is baggage"); - Assert.That(baggageValue, Is.EqualTo(testCase.BaggageHeaderValueWithoutOptionalWhitespace), "baggage header is set but is not correct"); + Assert.That(baggageValue, Is.EqualTo(testCase.BaggageHeaderValue), "baggage header is set but is not correct"); } } else @@ -146,7 +149,9 @@ public void Can_roundtrip_baggage(ContextPropagationTestCase testCase) TestContext.Out.WriteLine($"Baggage header: {testCase.BaggageHeaderValue}"); var outgoingHeaders = new Dictionary(); - var outgoingActivity = new Activity(ActivityNames.OutgoingMessageActivityName); + using var outgoingActivity = new Activity(ActivityNames.OutgoingMessageActivityName); + outgoingActivity.SetIdFormat(ActivityIdFormat.W3C); + outgoingActivity.Start(); foreach (var baggageItem in testCase.ExpectedBaggageItems.Reverse()) { @@ -157,7 +162,9 @@ public void Can_roundtrip_baggage(ContextPropagationTestCase testCase) // Simulate wire transfer var incomingHeaders = outgoingHeaders; - var incomingActivity = new Activity(ActivityNames.IncomingMessageActivityName); + using var incomingActivity = new Activity(ActivityNames.IncomingMessageActivityName); + incomingActivity.SetIdFormat(ActivityIdFormat.W3C); + incomingActivity.Start(); ContextPropagation.PropagateContextFromHeaders(incomingActivity, incomingHeaders); @@ -170,55 +177,87 @@ public void Can_roundtrip_baggage(ContextPropagationTestCase testCase) } } + [Test] + public void Can_not_roundtrip_baggage_value_with_optional_whitespaces() + { + var outgoingHeaders = new Dictionary(); + using var outgoingActivity = new Activity(ActivityNames.OutgoingMessageActivityName); + outgoingActivity.SetIdFormat(ActivityIdFormat.W3C); + outgoingActivity.Start(); + + outgoingActivity.AddBaggage("key1", " value1"); + outgoingActivity.AddBaggage("key2", "value2 "); + + ContextPropagation.PropagateContextToHeaders(outgoingActivity, outgoingHeaders, new ContextBag()); + + // Simulate wire transfer + var incomingHeaders = outgoingHeaders; + using var incomingActivity = new Activity(ActivityNames.IncomingMessageActivityName); + incomingActivity.SetIdFormat(ActivityIdFormat.W3C); + incomingActivity.Start(); + + ContextPropagation.PropagateContextFromHeaders(incomingActivity, incomingHeaders); + + using (Assert.EnterMultipleScope()) + { + foreach (var baggageItem in outgoingActivity.Baggage) + { + var key = baggageItem.Key; + var actualValue = incomingActivity.GetBaggageItem(key); + Assert.That(actualValue, Is.Not.Null, $"Baggage is missing item with key |{key}|"); + Assert.That(actualValue, Is.EqualTo(baggageItem.Value.Trim()), $"Baggage item |{key}| has the wrong value"); + } + } + } + // HINT: Many of these test cases are given as examples in the spec https://www.w3.org/TR/baggage/#example static IEnumerable TestCases => new object[] { new ContextPropagationTestCase("without any baggage"), new ContextPropagationTestCase("with a single key") - .WithBaggage("key1", "value1"), + .WithBaggage("key1", "value1") + .WithHeaderValue("key1 = value1"), new ContextPropagationTestCase("with multiple keys") .WithBaggage("key1", "value1") - .WithBaggage("key2", "value2"), - - new ContextPropagationTestCase("with whitespace") - .WithBaggage("key1 ", " value1") - .WithBaggage(" key2", "value2 ") - .WithBaggage(" key3 ", " value3 "), + .WithBaggage("key2", "value2") + .WithHeaderValue("key1 = value1, key2 = value2"), new ContextPropagationTestCase("with properties that do not have keys") - .WithBaggage("key1", "value1;property1;property2"), + .WithBaggage("key1", "value1;property1;property2") + .WithHeaderValue("key1 = value1%3Bproperty1%3Bproperty2"), new ContextPropagationTestCase("with properties that have keys") - .WithBaggage("key3", "value3; propertyKey=propertyValue"), + .WithBaggage("key3", "value3; propertyKey=propertyValue") + .WithHeaderValue("key3 = value3%3B%20propertyKey=propertyValue"), new ContextPropagationTestCase("with values containing whitespace") - .WithBaggage("serverNode", "DF 28"), + .WithBaggage("serverNode", "DF 28") + .WithHeaderValue("serverNode = DF%2028"), new ContextPropagationTestCase("with values containing unicode") .WithBaggage("userId", "Amélie") + .WithHeaderValue("userId = Am%C3%A9lie") }; - public class ContextPropagationTestCase + public class ContextPropagationTestCase(string caseName) { - string caseName; - Dictionary baggageItems = []; + readonly Dictionary baggageItems = []; - public ContextPropagationTestCase(string caseName) + public ContextPropagationTestCase WithBaggage(string key, string value) { - this.caseName = caseName; + baggageItems.Add(key, value); + return this; } - public ContextPropagationTestCase WithBaggage(string key, string value) + public ContextPropagationTestCase WithHeaderValue(string headerValue) { - baggageItems.Add(key, value); + BaggageHeaderValue = headerValue; return this; } - public string BaggageHeaderValue => string.Join(",", from kvp in baggageItems select $"{kvp.Key}={Uri.EscapeDataString(kvp.Value)}"); - public string BaggageHeaderValueWithoutOptionalWhitespace - => string.Join(",", from kvp in baggageItems select $"{kvp.Key.Trim()}={Uri.EscapeDataString(kvp.Value)}"); + public string BaggageHeaderValue { get; private set; } public IEnumerable> ExpectedBaggageItems => from kvp in baggageItems select new KeyValuePair( kvp.Key.Trim(), diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagator.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagator.cs new file mode 100644 index 00000000000..dc35002215b --- /dev/null +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagator.cs @@ -0,0 +1,87 @@ +#nullable enable + +namespace NServiceBus.Core.Tests.OpenTelemetry; + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Extensibility; + +static class LegacyContextPropagator +{ + public static void PropagateContextToHeaders(Activity activity, Dictionary headers, ContextBag contextBag) + { + if (activity is null) + { + return; + } + + if (activity.Id is not null) + { + headers[Headers.DiagnosticsTraceParent] = activity.Id; + } + + if (activity.TraceStateString is not null) + { + headers[Headers.DiagnosticsTraceState] = activity.TraceStateString; + } + + // Check whether the startnewtrace setting was set in the context, if so, add it to the headers now the trace parent was added + if (contextBag.TryGet(Headers.StartNewTrace, out var headerContent)) + { + headers[Headers.StartNewTrace] = headerContent; + } + + var baggage = string.Join(",", activity.Baggage.Select(item => $"{item.Key}={Uri.EscapeDataString(item.Value ?? string.Empty)}")); + if (!string.IsNullOrEmpty(baggage)) + { + headers[Headers.DiagnosticsBaggage] = baggage; + } + } + + public static void PropagateContextFromHeaders(Activity? activity, IDictionary headers) + { + if (activity is null) + { + return; + } + + if (headers.TryGetValue(Headers.DiagnosticsTraceState, out var traceState)) + { + activity.TraceStateString = traceState; + } + + if (headers.TryGetValue(Headers.DiagnosticsBaggage, out var baggageValue)) + { + var baggageSpan = baggageValue.AsSpan(); + // HINT: Iterate in reverse order because Activity baggage is LIFO + while (!baggageSpan.IsEmpty) + { + var lastComma = baggageSpan.LastIndexOf(','); + ReadOnlySpan baggageItem; + + if (lastComma >= 0) + { + baggageItem = baggageSpan[(lastComma + 1)..]; + baggageSpan = baggageSpan[..lastComma]; + } + else + { + baggageItem = baggageSpan; + baggageSpan = []; + } + + var firstEquals = baggageItem.IndexOf('='); + if (firstEquals < 0 || firstEquals >= baggageItem.Length) + { + continue; + } + + var key = baggageItem[..firstEquals].Trim(); + var value = baggageItem[(firstEquals + 1)..]; + activity.AddBaggage(key.ToString(), Uri.UnescapeDataString(value)); + } + } + } +} \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs index 04e287679c0..30b0bf1066f 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs @@ -41,6 +41,8 @@ public static void SetErrorStatus(this Activity activity, Exception ex) activity.SetStatus(ActivityStatusCode.Error, ex.Message); activity.SetTag("otel.status_code", "ERROR"); activity.SetTag("otel.status_description", ex.Message); + + activity.AddEvent(new ActivityEvent("exception", DateTimeOffset.UtcNow, [ new KeyValuePair("exception.escaped", true), diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs index 0f1d8868270..b6ac0314b4c 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs @@ -2,10 +2,8 @@ namespace NServiceBus; -using System; using System.Collections.Generic; using System.Diagnostics; -using System.Linq; using Extensibility; static class ContextPropagation @@ -17,26 +15,14 @@ public static void PropagateContextToHeaders(Activity? activity, Dictionary(Headers.StartNewTrace, out var startNewTrace); - // Check whether the startnewtrace setting was set in the context, if so, add it to the headers now the trace parent was added - if (contextBag.TryGet(Headers.StartNewTrace, out var headerContent)) + if (traceParentExists && startNewTraceOnReceive) { - headers[Headers.StartNewTrace] = headerContent; - } - - var baggage = string.Join(",", activity.Baggage.Select(item => $"{item.Key}={Uri.EscapeDataString(item.Value ?? string.Empty)}")); - if (!string.IsNullOrEmpty(baggage)) - { - headers[Headers.DiagnosticsBaggage] = baggage; + headers[Headers.StartNewTrace] = startNewTrace!; } } @@ -47,41 +33,33 @@ public static void PropagateContextFromHeaders(Activity? activity, IDictionary baggageItem; + var baggage = DistributedContextPropagator.Current.ExtractBaggage(headers, Getter); - if (lastComma >= 0) - { - baggageItem = baggageSpan[(lastComma + 1)..]; - baggageSpan = baggageSpan[..lastComma]; - } - else - { - baggageItem = baggageSpan; - baggageSpan = []; - } - - var firstEquals = baggageItem.IndexOf('='); - if (firstEquals < 0 || firstEquals >= baggageItem.Length) - { - continue; - } + if (baggage is null) + { + return; + } - var key = baggageItem[..firstEquals].Trim(); - var value = baggageItem[(firstEquals + 1)..]; - activity.AddBaggage(key.ToString(), Uri.UnescapeDataString(value)); - } + foreach (var baggageItem in baggage) + { + activity.AddBaggage(baggageItem.Key, baggageItem.Value); } } + + static readonly DistributedContextPropagator.PropagatorSetterCallback Setter = static (carrier, key, value) => + ((IDictionary)carrier!)[key] = value; + + static readonly DistributedContextPropagator.PropagatorGetterCallback Getter = + static (carrier, key, out value, out values) => + { + values = null; + value = ((IReadOnlyDictionary)carrier!).GetValueOrDefault(key); + }; } \ No newline at end of file From ca842e67166c25f183fc0da3744d7b1e9e001df5 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 17 Jun 2026 12:47:23 +0200 Subject: [PATCH 07/37] =?UTF-8?q?=F0=9F=90=9B=20Add=20regression=20test=20?= =?UTF-8?q?for=20null=20baggage=20value=20(#6983)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Activity baggage with a null value used to throw ArgumentNullException during context propagation (Uri.EscapeDataString(null)). The switch to DistributedContextPropagator on this branch fixes the root cause; this test pins the behavior so it cannot regress. --- .../OpenTelemetry/ContextPropagationTests.cs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationTests.cs index 48f95c1fe86..0141d6c9a15 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationTests.cs @@ -81,6 +81,22 @@ public void Overwrites_existing_propagation_header() Assert.That(activity.Id, Is.EqualTo(headers[Headers.DiagnosticsTraceParent])); } + [Test] + public void Should_not_throw_when_baggage_value_is_null() + { + // Reproduces https://github.com/Particular/NServiceBus/issues/6983 + // A baggage item with a null value used to make the hand-written propagator call + // Uri.EscapeDataString(null), throwing ArgumentNullException while sending a message. + using var activity = new Activity(ActivityNames.OutgoingMessageActivityName); + activity.SetIdFormat(ActivityIdFormat.W3C); + activity.Start(); + activity.AddBaggage("test", null); + + var headers = new Dictionary(); + + Assert.DoesNotThrow(() => ContextPropagation.PropagateContextToHeaders(activity, headers, new ContextBag())); + } + [TestCaseSource(nameof(TestCases))] public void Can_propagate_baggage_from_header_to_activity(ContextPropagationTestCase testCase) { From a33484eaa8812a4d148c801bfc1f2f18b4f4091d Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Fri, 19 Jun 2026 14:10:30 +0200 Subject: [PATCH 08/37] Make DistributedContextPropagator opt-in (keep OTel propagation backwards compatible until v11) (#7825) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ Make DistributedContextPropagator opt-in via AppContext switch The switch to System.Diagnostics.DistributedContextPropagator changes the OpenTelemetry baggage wire format (W3C OWS encoding + whitespace trimming), which is breaking on rolling upgrades. Keep the legacy percent-encoded propagator as the default and gate the new propagator behind the NServiceBus.Core.OpenTelemetry.UseDistributedContextPropagator AppContext switch (default in v11). All temporary code (switch plumbing + legacy propagator) lives in obsolete_v11.cs so v11 cleanup is a single file deletion plus removing the two delegation blocks in ContextPropagation.cs. Follows the existing AppContextSwitches.UseV2DeterministicGuid / PreObsolete pattern. - ContextPropagation: delegate to the legacy propagator unless the switch is on - obsolete_v11.cs: switch + byte-for-byte revert of the pre-10.3 propagator - Tests asserting the new W3C format enable the switch per-fixture - New ContextPropagationDefaultBehaviorTests locks in legacy default behavior - Acceptance baggage assertion reverted to the legacy default wire format Span naming (UseMessageDestinationInSpanNames) is already opt-in and unchanged. * Refactor: Replace `ObsoleteV11` with `LegacyContextPropagation` * Add test to verify null baggage value does not throw in legacy propagator * Added comments about preserve legacy baggage handling behavior when escaping and trimming values * Add test to verify that we are preserving whitespace in legacy propagator baggage values. * add comments to clarify intent behind legacy propagator handling * update ContextPropagationCompatibilityTests to use correct LegacyContextPropagation delegates * rename ContextPropagationTests to LegacyContextPropagationTests and remove outdated baggage handling tests * allow changing the propagator implementation at runtime --------- Co-authored-by: Tomasz Masternak --- .../When_outgoing_activity_has_baggage.cs | 5 +- ...> ContextPropagationCompatibilityTests.cs} | 42 +++++- .../ContextPropagationDefaultBehaviorTests.cs | 72 +++++++++ ...ts.cs => LegacyContextPropagationTests.cs} | 48 +----- .../OpenTelemetry/LegacyContextPropagator.cs | 87 ----------- .../Tracing/ContextPropagation.cs | 19 +++ .../OpenTelemetry/Tracing/obsolete_v11.cs | 139 ++++++++++++++++++ 7 files changed, 279 insertions(+), 133 deletions(-) rename src/NServiceBus.Core.Tests/OpenTelemetry/{ContextPropagationIncompatibilityTests.cs => ContextPropagationCompatibilityTests.cs} (68%) create mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs rename src/NServiceBus.Core.Tests/OpenTelemetry/{ContextPropagationTests.cs => LegacyContextPropagationTests.cs} (83%) delete mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagator.cs create mode 100644 src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_outgoing_activity_has_baggage.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_outgoing_activity_has_baggage.cs index e8bed02db26..b6a2310ad25 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_outgoing_activity_has_baggage.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_outgoing_activity_has_baggage.cs @@ -33,7 +33,10 @@ public async Task Should_propagate_baggage_to_headers() ) .Run(); - Assert.That(context.BaggageHeader, Is.EqualTo("key3 = , key2 = value2, key1 = value1")); + // Default (backwards-compatible) propagation produces the legacy comma-separated, percent-encoded format. + // The W3C OWS format ("key3 = , key2 = value2, key1 = value1") is produced only when the + // NServiceBus.Core.OpenTelemetry.UseDistributedContextPropagator AppContext switch is enabled (default in v11). + Assert.That(context.BaggageHeader, Is.EqualTo("key3=,key2=value2,key1=value1")); } public class TestEndpoint : EndpointConfigurationBuilder diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationIncompatibilityTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationCompatibilityTests.cs similarity index 68% rename from src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationIncompatibilityTests.cs rename to src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationCompatibilityTests.cs index 4aa93147137..7f4332e0cde 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationIncompatibilityTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationCompatibilityTests.cs @@ -1,18 +1,33 @@ namespace NServiceBus.Core.Tests.OpenTelemetry; +using System; using System.Collections.Generic; using System.Diagnostics; using Extensibility; using NUnit.Framework; [TestFixture] -public class ContextPropagationIncompatibilityTests +public class ContextPropagationCompatibilityTests { + [SetUp] + public void EnableDistributedContextPropagator() + { + AppContext.SetSwitch(LegacyContextPropagation.UseDistributedContextPropagatorSwitchName, true); + LegacyContextPropagation.ResetUseDistributedContextPropagator(); + } + + [TearDown] + public void ResetDistributedContextPropagator() + { + AppContext.SetSwitch(LegacyContextPropagation.UseDistributedContextPropagatorSwitchName, false); + LegacyContextPropagation.ResetUseDistributedContextPropagator(); + } + delegate void Writer(Activity activity, Dictionary headers, ContextBag context); delegate void Reader(Activity activity, IDictionary headers); - static readonly Writer LegacyWrite = LegacyContextPropagator.PropagateContextToHeaders; - static readonly Reader LegacyRead = LegacyContextPropagator.PropagateContextFromHeaders; + static readonly Writer LegacyWrite = LegacyContextPropagation.PropagateContextToHeaders; + static readonly Reader LegacyRead = LegacyContextPropagation.PropagateContextFromHeaders; static readonly Writer NewWrite = ContextPropagation.PropagateContextToHeaders; static readonly Reader NewRead = ContextPropagation.PropagateContextFromHeaders; @@ -22,7 +37,7 @@ public class ContextPropagationIncompatibilityTests // value isolates "what happens to special characters" from the separate edge-whitespace // issue covered by New_propagation_loses_leading_whitespace_in_a_value. // This already includes property-like syntax (the ';' and '=' delimiters), so a value such as - // "zone=eu;sensitive" is just a subset and needs no separate case here. + // "zone=eu;sensitive" is just a subset and needs no separate case here. const string AllSpecialCharacters = "a b,c;d=e&f'g\"h\\i(j)k{l}m[n]o%p/q?r:s@t~u|vx é ü 😀 z"; static Dictionary Send(string value, Writer write) @@ -81,4 +96,21 @@ public void New_propagation_loses_leading_whitespace_in_a_value() "new propagation strips the leading whitespace from the value"); } } -} + + [TestCase(null, "", "")] + [TestCase("", "", "")] + [TestCase(" ", "", " ")] + [TestCase(" x ", "x", " x ")] + [TestCase(" x x ", "x x", " x x ")] + public void ValidateThatLegacyPropagatorPreservesLeadingAndTrailingWhitespaceInBaggageValues(string input, string expectedNew, string expectedLegacy) + { + var outputNew = Transmit(input, NewWrite, NewRead); + var outputLegacy = Transmit(input, LegacyWrite, LegacyRead); + + using (Assert.EnterMultipleScope()) + { + Assert.That(expectedNew, Is.EqualTo(outputNew), "Native propagator isn't trimming all leading and trailing whitespaces"); + Assert.That(expectedLegacy, Is.EqualTo(outputLegacy), "Legacy propagator isn't preserving leading and trailing whitespace for backwards compatibility"); + } + } +} \ No newline at end of file diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs new file mode 100644 index 00000000000..fbe65df4b51 --- /dev/null +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs @@ -0,0 +1,72 @@ +namespace NServiceBus.Core.Tests.OpenTelemetry; + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using Extensibility; +using NUnit.Framework; + +[TestFixture] +public class ContextPropagationDefaultBehaviorTests +{ + // Without the opt-in switch, the endpoint default must remain the backwards-compatible + // legacy propagator (percent-encoded, comma-separated, whitespace preserved). + [SetUp] + public void EnsureDefault() + { + AppContext.SetSwitch(LegacyContextPropagation.UseDistributedContextPropagatorSwitchName, false); + LegacyContextPropagation.ResetUseDistributedContextPropagator(); + } + + [Test] + public void Default_uses_legacy_percent_encoded_baggage_format() + { + using var activity = new Activity(ActivityNames.OutgoingMessageActivityName); + activity.SetIdFormat(ActivityIdFormat.W3C); + activity.Start(); + activity.AddBaggage("serverNode", "DF 28"); + + var headers = new Dictionary(); + ContextPropagation.PropagateContextToHeaders(activity, headers, new ContextBag()); + + Assert.That(headers[Headers.DiagnosticsBaggage], Is.EqualTo("serverNode=DF%2028")); + } + + [Test] + public void Default_does_not_throw_when_baggage_value_is_null() + { + // Reproduces https://github.com/Particular/NServiceBus/issues/6983 on the legacy propagator. + // A null baggage value must not make the legacy propagator call Uri.EscapeDataString(null). + // Calls LegacyContextPropagation directly so the assertion is independent of the AppContext switch. + using var activity = new Activity(ActivityNames.OutgoingMessageActivityName); + activity.SetIdFormat(ActivityIdFormat.W3C); + activity.Start(); + activity.AddBaggage("test", null); + + var headers = new Dictionary(); + + Assert.DoesNotThrow(() => LegacyContextPropagation.PropagateContextToHeaders(activity, headers, new ContextBag())); + Assert.That(headers[Headers.DiagnosticsBaggage], Is.EqualTo("test=")); + } + + [Test] + public void Default_round_trip_preserves_value_whitespace() + { + using var outgoing = new Activity(ActivityNames.OutgoingMessageActivityName); + outgoing.SetIdFormat(ActivityIdFormat.W3C); + outgoing.Start(); + outgoing.AddBaggage("key1", " leading-and-trailing "); + + var headers = new Dictionary(); + ContextPropagation.PropagateContextToHeaders(outgoing, headers, new ContextBag()); + + using var incoming = new Activity(ActivityNames.IncomingMessageActivityName); + incoming.SetIdFormat(ActivityIdFormat.W3C); + incoming.Start(); + ContextPropagation.PropagateContextFromHeaders(incoming, headers); + + // Legacy propagation preserves leading/trailing whitespace via percent-encoding; + // the DistributedContextPropagator (opt-in) would trim it. + Assert.That(incoming.GetBaggageItem("key1"), Is.EqualTo(" leading-and-trailing ")); + } +} diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagationTests.cs similarity index 83% rename from src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationTests.cs rename to src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagationTests.cs index 0141d6c9a15..4c62f70cd71 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagationTests.cs @@ -1,5 +1,6 @@ namespace NServiceBus.Core.Tests.OpenTelemetry; +using System; using System.Collections; using System.Collections.Generic; using System.Diagnostics; @@ -8,7 +9,7 @@ using NUnit.Framework; [TestFixture] -public class ContextPropagationTests +public class LegacyContextPropagationTests { [Test] public void Propagate_activity_id_to_header() @@ -193,39 +194,6 @@ public void Can_roundtrip_baggage(ContextPropagationTestCase testCase) } } - [Test] - public void Can_not_roundtrip_baggage_value_with_optional_whitespaces() - { - var outgoingHeaders = new Dictionary(); - using var outgoingActivity = new Activity(ActivityNames.OutgoingMessageActivityName); - outgoingActivity.SetIdFormat(ActivityIdFormat.W3C); - outgoingActivity.Start(); - - outgoingActivity.AddBaggage("key1", " value1"); - outgoingActivity.AddBaggage("key2", "value2 "); - - ContextPropagation.PropagateContextToHeaders(outgoingActivity, outgoingHeaders, new ContextBag()); - - // Simulate wire transfer - var incomingHeaders = outgoingHeaders; - using var incomingActivity = new Activity(ActivityNames.IncomingMessageActivityName); - incomingActivity.SetIdFormat(ActivityIdFormat.W3C); - incomingActivity.Start(); - - ContextPropagation.PropagateContextFromHeaders(incomingActivity, incomingHeaders); - - using (Assert.EnterMultipleScope()) - { - foreach (var baggageItem in outgoingActivity.Baggage) - { - var key = baggageItem.Key; - var actualValue = incomingActivity.GetBaggageItem(key); - Assert.That(actualValue, Is.Not.Null, $"Baggage is missing item with key |{key}|"); - Assert.That(actualValue, Is.EqualTo(baggageItem.Value.Trim()), $"Baggage item |{key}| has the wrong value"); - } - } - } - // HINT: Many of these test cases are given as examples in the spec https://www.w3.org/TR/baggage/#example static IEnumerable TestCases => new object[] { @@ -233,28 +201,28 @@ public void Can_not_roundtrip_baggage_value_with_optional_whitespaces() new ContextPropagationTestCase("with a single key") .WithBaggage("key1", "value1") - .WithHeaderValue("key1 = value1"), + .WithHeaderValue("key1=value1"), new ContextPropagationTestCase("with multiple keys") .WithBaggage("key1", "value1") .WithBaggage("key2", "value2") - .WithHeaderValue("key1 = value1, key2 = value2"), + .WithHeaderValue("key1=value1,key2=value2"), new ContextPropagationTestCase("with properties that do not have keys") .WithBaggage("key1", "value1;property1;property2") - .WithHeaderValue("key1 = value1%3Bproperty1%3Bproperty2"), + .WithHeaderValue("key1=value1%3Bproperty1%3Bproperty2"), new ContextPropagationTestCase("with properties that have keys") .WithBaggage("key3", "value3; propertyKey=propertyValue") - .WithHeaderValue("key3 = value3%3B%20propertyKey=propertyValue"), + .WithHeaderValue("key3=value3%3B%20propertyKey%3DpropertyValue"), new ContextPropagationTestCase("with values containing whitespace") .WithBaggage("serverNode", "DF 28") - .WithHeaderValue("serverNode = DF%2028"), + .WithHeaderValue("serverNode=DF%2028"), new ContextPropagationTestCase("with values containing unicode") .WithBaggage("userId", "Amélie") - .WithHeaderValue("userId = Am%C3%A9lie") + .WithHeaderValue("userId=Am%C3%A9lie") }; public class ContextPropagationTestCase(string caseName) diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagator.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagator.cs deleted file mode 100644 index dc35002215b..00000000000 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagator.cs +++ /dev/null @@ -1,87 +0,0 @@ -#nullable enable - -namespace NServiceBus.Core.Tests.OpenTelemetry; - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using Extensibility; - -static class LegacyContextPropagator -{ - public static void PropagateContextToHeaders(Activity activity, Dictionary headers, ContextBag contextBag) - { - if (activity is null) - { - return; - } - - if (activity.Id is not null) - { - headers[Headers.DiagnosticsTraceParent] = activity.Id; - } - - if (activity.TraceStateString is not null) - { - headers[Headers.DiagnosticsTraceState] = activity.TraceStateString; - } - - // Check whether the startnewtrace setting was set in the context, if so, add it to the headers now the trace parent was added - if (contextBag.TryGet(Headers.StartNewTrace, out var headerContent)) - { - headers[Headers.StartNewTrace] = headerContent; - } - - var baggage = string.Join(",", activity.Baggage.Select(item => $"{item.Key}={Uri.EscapeDataString(item.Value ?? string.Empty)}")); - if (!string.IsNullOrEmpty(baggage)) - { - headers[Headers.DiagnosticsBaggage] = baggage; - } - } - - public static void PropagateContextFromHeaders(Activity? activity, IDictionary headers) - { - if (activity is null) - { - return; - } - - if (headers.TryGetValue(Headers.DiagnosticsTraceState, out var traceState)) - { - activity.TraceStateString = traceState; - } - - if (headers.TryGetValue(Headers.DiagnosticsBaggage, out var baggageValue)) - { - var baggageSpan = baggageValue.AsSpan(); - // HINT: Iterate in reverse order because Activity baggage is LIFO - while (!baggageSpan.IsEmpty) - { - var lastComma = baggageSpan.LastIndexOf(','); - ReadOnlySpan baggageItem; - - if (lastComma >= 0) - { - baggageItem = baggageSpan[(lastComma + 1)..]; - baggageSpan = baggageSpan[..lastComma]; - } - else - { - baggageItem = baggageSpan; - baggageSpan = []; - } - - var firstEquals = baggageItem.IndexOf('='); - if (firstEquals < 0 || firstEquals >= baggageItem.Length) - { - continue; - } - - var key = baggageItem[..firstEquals].Trim(); - var value = baggageItem[(firstEquals + 1)..]; - activity.AddBaggage(key.ToString(), Uri.UnescapeDataString(value)); - } - } - } -} \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs index b6ac0314b4c..8df14752012 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs @@ -10,6 +10,16 @@ static class ContextPropagation { public static void PropagateContextToHeaders(Activity? activity, Dictionary headers, ContextBag contextBag) { + // TODO: investigate if we need to improve the switch check for better performance + // Removed in v11, see obsolete_v11.cs + if (!LegacyContextPropagation.UseDistributedContextPropagator) + { + LegacyContextPropagation.PropagateContextToHeaders(activity, headers, contextBag); + return; + } + + // The following part was intentionally not extracted to a separate class to prevent + // accidental leftovers when because that the legacy propagator will be removed in v11 if (activity is null) { return; @@ -28,6 +38,15 @@ public static void PropagateContextToHeaders(Activity? activity, Dictionary headers) { + // Removed in v11, see obsolete_v11.cs + if (!LegacyContextPropagation.UseDistributedContextPropagator) + { + LegacyContextPropagation.PropagateContextFromHeaders(activity, headers); + return; + } + + // The following part was intentionally not extracted to a separate class to prevent + // accidental leftovers when because that the legacy propagator will be removed in v11 if (activity is null) { return; diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs new file mode 100644 index 00000000000..b1dec663683 --- /dev/null +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs @@ -0,0 +1,139 @@ +#nullable enable + +namespace NServiceBus; + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Extensibility; +using Particular.Obsoletes; + +// ============================================================================= +// EVERYTHING IN THIS FILE IS TEMPORARY AND WILL BE REMOVED IN v11. +// +// In v10.3, switching to System.Diagnostics.DistributedContextPropagator for +// OpenTelemetry trace-context/baggage propagation changes the baggage wire +// format (W3C OWS encoding + whitespace trimming) and is therefore breaking on +// rolling upgrades. To stay backwards compatible it is opt-in via an AppContext +// switch and the legacy propagator below remains the default. +// +// In v11 the new propagator becomes the default: delete this entire file and +// remove the two `if (!ObsoleteV11.UseDistributedContextPropagator)` delegation +// blocks in ContextPropagation.cs. +// ============================================================================= +static class LegacyContextPropagation +{ + enum SwitchState : byte + { + Unchecked = 0, + Enabled = 1, + Disabled = 2 + } + + static SwitchState cachedUseDistributedContextPropagator; + + [PreObsolete("https://github.com/Particular/NServiceBus/issues/7825", + Note = "In v11, DistributedContextPropagator-based context propagation becomes the default and this switch will be removed together with the legacy propagator in obsolete_v11.cs.", + ReplacementTypeOrMember = "ContextPropagation")] + public const string UseDistributedContextPropagatorSwitchName = "NServiceBus.Core.OpenTelemetry.UseDistributedContextPropagator"; + + [PreObsolete("https://github.com/Particular/NServiceBus/issues/7825", + Note = "In v11, DistributedContextPropagator-based context propagation becomes the default and this switch will be removed together with the legacy propagator in obsolete_v11.cs.", + ReplacementTypeOrMember = "ContextPropagation")] + public static bool UseDistributedContextPropagator + { + get + { + var state = cachedUseDistributedContextPropagator; + if (state != SwitchState.Unchecked) + { + return state == SwitchState.Enabled; + } + + state = AppContext.TryGetSwitch(UseDistributedContextPropagatorSwitchName, out var isEnabled) && isEnabled + ? SwitchState.Enabled + : SwitchState.Disabled; + cachedUseDistributedContextPropagator = state; + + return state == SwitchState.Enabled; + } + } + + internal static void ResetUseDistributedContextPropagator() => cachedUseDistributedContextPropagator = SwitchState.Unchecked; + + public static void PropagateContextToHeaders(Activity? activity, Dictionary headers, ContextBag contextBag) + { + if (activity is null) + { + return; + } + + if (activity.Id is not null) + { + headers[Headers.DiagnosticsTraceParent] = activity.Id; + } + + if (activity.TraceStateString is not null) + { + headers[Headers.DiagnosticsTraceState] = activity.TraceStateString; + } + + // Check whether the startnewtrace setting was set in the context, if so, add it to the headers now the trace parent was added + if (contextBag.TryGet(Headers.StartNewTrace, out var headerContent)) + { + headers[Headers.StartNewTrace] = headerContent; + } + + var baggage = string.Join(",", activity.Baggage.Select(item => $"{item.Key}={Uri.EscapeDataString(item.Value ?? string.Empty)}")); + if (!string.IsNullOrEmpty(baggage)) + { + headers[Headers.DiagnosticsBaggage] = baggage; + } + } + + public static void PropagateContextFromHeaders(Activity? activity, IDictionary headers) + { + if (activity is null) + { + return; + } + + if (headers.TryGetValue(Headers.DiagnosticsTraceState, out var traceState)) + { + activity.TraceStateString = traceState; + } + + if (headers.TryGetValue(Headers.DiagnosticsBaggage, out var baggageValue)) + { + var baggageSpan = baggageValue.AsSpan(); + // HINT: Iterate in reverse order because Activity baggage is LIFO + while (!baggageSpan.IsEmpty) + { + var lastComma = baggageSpan.LastIndexOf(','); + ReadOnlySpan baggageItem; + + if (lastComma >= 0) + { + baggageItem = baggageSpan[(lastComma + 1)..]; + baggageSpan = baggageSpan[..lastComma]; + } + else + { + baggageItem = baggageSpan; + baggageSpan = []; + } + + var firstEquals = baggageItem.IndexOf('='); + if (firstEquals < 0 || firstEquals >= baggageItem.Length) + { + continue; + } + + var key = baggageItem[..firstEquals].Trim(); + var value = baggageItem[(firstEquals + 1)..]; + activity.AddBaggage(key.ToString(), Uri.UnescapeDataString(value)); + } + } + } +} \ No newline at end of file From 68274cf0706a08a2e33b50c4b58bc15c2bd6c919 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 8 Jul 2026 14:14:58 +0200 Subject: [PATCH 09/37] Emit handler spans from a dedicated NServiceBus.Core.Handler ActivitySource (#7844) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Handler (InvokeHandler) spans can now be emitted from a dedicated "NServiceBus.Core.Handler" ActivitySource so they can be filtered or sampled independently of the pipeline spans (#7284). Opting in without subscribing to the new source suppresses handler spans entirely, making Activity.Current inside handlers/behaviors the "process message" span — the flattened trace requested in the issue. Existing OpenTelemetry configurations only subscribe to "NServiceBus.Core" and would silently lose handler spans, so the new source is opt-in via the NServiceBus.Core.OpenTelemetry.UseHandlerActivitySource AppContext switch until v11, following the DistributedContextPropagator pattern: - ActivitySources: add permanent Handler source - ActivityFactory.StartHandlerActivity: pick source based on the switch - obsolete_v11.cs: cached switch with PreObsolete markers; in v11 delete the class and make ActivitySources.Handler unconditional - Unit tests cover default source, opt-in source, preserved span data, and no-listener suppression; acceptance test locks in the default --- ...g_message_with_default_activity_sources.cs | 48 ++++++++++ .../HandlerActivitySourceTests.cs | 95 +++++++++++++++++++ .../OpenTelemetry/Tracing/ActivityFactory.cs | 8 +- .../OpenTelemetry/Tracing/ActivitySources.cs | 4 + .../OpenTelemetry/Tracing/obsolete_v11.cs | 50 ++++++++++ 5 files changed, 204 insertions(+), 1 deletion(-) create mode 100644 src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_message_with_default_activity_sources.cs create mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/HandlerActivitySourceTests.cs diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_message_with_default_activity_sources.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_message_with_default_activity_sources.cs new file mode 100644 index 00000000000..a233aa178b1 --- /dev/null +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_message_with_default_activity_sources.cs @@ -0,0 +1,48 @@ +namespace NServiceBus.AcceptanceTests.Core.OpenTelemetry.Traces; + +using System.Linq; +using System.Threading.Tasks; +using EndpointTemplates; +using NServiceBus.AcceptanceTesting; +using NUnit.Framework; + +public class When_processing_message_with_default_activity_sources : OpenTelemetryAcceptanceTest +{ + // Until v11, handler spans are emitted from the "NServiceBus.Core" ActivitySource by default + // for backwards compatibility. The dedicated "NServiceBus.Core.Handler" source is opt-in via + // the NServiceBus.Core.OpenTelemetry.UseHandlerActivitySource AppContext switch (default in v11). + [Test] + public async Task Should_emit_handler_span_from_main_source() + { + await Scenario.Define() + .WithEndpoint(b => + b.When(session => session.SendLocal(new SomeMessage())) + ) + .Run(); + + var invokedHandlerActivities = NServiceBusActivityListener.CompletedActivities.GetInvokedHandlerActivities(); + + Assert.That(invokedHandlerActivities, Has.Count.EqualTo(1)); + Assert.That(invokedHandlerActivities.Single().Source.Name, Is.EqualTo("NServiceBus.Core"), + "without the opt-in switch, handler spans must keep coming from the main source so existing OpenTelemetry configurations keep seeing them"); + } + + public class Context : ScenarioContext; + + public class ReceivingEndpoint : EndpointConfigurationBuilder + { + public ReceivingEndpoint() => EndpointSetup(); + + [Handler] + public class MessageHandler(Context testContext) : IHandleMessages + { + public Task Handle(SomeMessage message, IMessageHandlerContext context) + { + testContext.MarkAsCompleted(); + return Task.CompletedTask; + } + } + } + + public class SomeMessage : IMessage; +} diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/HandlerActivitySourceTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/HandlerActivitySourceTests.cs new file mode 100644 index 00000000000..8af728a9c8a --- /dev/null +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/HandlerActivitySourceTests.cs @@ -0,0 +1,95 @@ +#nullable enable + +namespace NServiceBus.Core.Tests.OpenTelemetry; + +using System; +using System.Collections.Immutable; +using System.Diagnostics; +using System.Linq; +using Helpers; +using NServiceBus.Pipeline; +using NUnit.Framework; + +[TestFixture] +public class HandlerActivitySourceTests +{ + readonly ActivityFactory activityFactory = new(new InstrumentationOptions()); + + TestingActivityListener mainListener; + + [SetUp] + public void SetUp() => mainListener = TestingActivityListener.SetupNServiceBusDiagnosticListener(); + + [TearDown] + public void TearDown() + { + mainListener.Dispose(); + AppContext.SetSwitch(HandlerActivitySourceSwitch.UseHandlerActivitySourceSwitchName, false); + HandlerActivitySourceSwitch.ResetUseHandlerActivitySource(); + } + + static void OptIn() + { + AppContext.SetSwitch(HandlerActivitySourceSwitch.UseHandlerActivitySourceSwitchName, true); + HandlerActivitySourceSwitch.ResetUseHandlerActivitySource(); + } + + [Test] + public void Default_emits_handler_activity_from_main_source() + { + using var ambientActivity = new Activity("ambient activity"); + ambientActivity.Start(); + + var activity = activityFactory.StartHandlerActivity(new MessageHandler { HandlerType = typeof(HandlerActivitySourceTests) }); + + Assert.That(activity, Is.Not.Null); + Assert.That(activity!.Source.Name, Is.EqualTo("NServiceBus.Core")); + } + + [Test] + public void Opt_in_emits_handler_activity_from_handler_source() + { + OptIn(); + using var handlerListener = TestingActivityListener.SetupDiagnosticListener("NServiceBus.Core.Handler"); + + using var ambientActivity = new Activity("ambient activity"); + ambientActivity.Start(); + + var activity = activityFactory.StartHandlerActivity(new MessageHandler { HandlerType = typeof(HandlerActivitySourceTests) }); + + Assert.That(activity, Is.Not.Null); + Assert.That(activity!.Source.Name, Is.EqualTo("NServiceBus.Core.Handler")); + } + + [Test] + public void Opt_in_preserves_display_name_and_handler_type_tag() + { + OptIn(); + using var handlerListener = TestingActivityListener.SetupDiagnosticListener("NServiceBus.Core.Handler"); + + using var ambientActivity = new Activity("ambient activity"); + ambientActivity.Start(); + + Type handlerType = typeof(HandlerActivitySourceTests); + var activity = activityFactory.StartHandlerActivity(new MessageHandler { HandlerType = handlerType }); + + Assert.That(activity, Is.Not.Null); + Assert.That(activity!.DisplayName, Is.EqualTo(handlerType.Name)); + var tags = activity.Tags.ToImmutableDictionary(); + Assert.That(tags[ActivityTags.HandlerType], Is.EqualTo(handlerType.FullName)); + } + + [Test] + public void Opt_in_without_handler_source_listener_does_not_create_handler_activity() + { + OptIn(); + + using var ambientActivity = new Activity("ambient activity"); + ambientActivity.Start(); + + var activity = activityFactory.StartHandlerActivity(new MessageHandler { HandlerType = typeof(HandlerActivitySourceTests) }); + + Assert.That(activity, Is.Null, "handler activity must not be created when the dedicated source has no listeners"); + Assert.That(Activity.Current, Is.SameAs(ambientActivity), "user tags must land on the parent (process message) activity"); + } +} diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs index b6548eef2f0..d5dcf4ec07a 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs @@ -111,7 +111,13 @@ public ActivityFactory(InstrumentationOptions options) return null; } - var activity = ActivitySources.Main.StartActivity(ActivityNames.InvokeHandlerActivityName); + // Until v11 the dedicated handler source is opt-in; existing configurations only + // subscribe to the main source and must keep receiving handler spans from it. + var source = HandlerActivitySourceSwitch.UseHandlerActivitySource + ? ActivitySources.Handler + : ActivitySources.Main; + + var activity = source.StartActivity(ActivityNames.InvokeHandlerActivityName); if (activity is null) { diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivitySources.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivitySources.cs index 8f6e26858e1..939305dd0eb 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivitySources.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivitySources.cs @@ -9,4 +9,8 @@ static class ActivitySources public static readonly ActivitySource Main = new("NServiceBus.Core", "0.1.0"); + + public static readonly ActivitySource Handler = + new("NServiceBus.Core.Handler", + "0.1.0"); } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs index b1dec663683..499be45157c 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs @@ -136,4 +136,54 @@ public static void PropagateContextFromHeaders(Activity? activity, IDictionary cachedUseHandlerActivitySource = SwitchState.Unchecked; } \ No newline at end of file From beeadd0241d8ed0807557009baddc0267f76d407 Mon Sep 17 00:00:00 2001 From: "Irina Dominte(Scurtu)" Date: Thu, 16 Jul 2026 14:11:47 +0300 Subject: [PATCH 10/37] Optout on dispatching events (#7846) * Optout on dispatching events * Aproval tests * Update src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt Co-authored-by: Tomasz Masternak * Renamed Suppress to Emmit * Wrap `AddEvent` call in null-check for `TryGetRecordingIncomingPipelineActivity`. --------- Co-authored-by: Tomasz Masternak Co-authored-by: Tomasz Masternak --- .../APIApprovals.ApproveNServiceBus.approved.txt | 1 + ...ransportReceiveToPhysicalMessageConnectorTests.cs | 2 +- .../OpenTelemetry/InstrumentationOptions.cs | 8 ++++++++ .../TransportReceiveToPhysicalMessageConnector.cs | 12 ++++++++---- src/NServiceBus.Core/Receiving/ReceiveComponent.cs | 2 +- 5 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt index a5bdc1d3893..67152cc72d5 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt @@ -635,6 +635,7 @@ namespace NServiceBus public class InstrumentationOptions { public InstrumentationOptions() { } + public bool EmitMessageDispatchingEvents { get; set; } public bool UseMessageDestinationInSpanNames { get; set; } } public sealed class KeyedServiceKey diff --git a/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs b/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs index 96e3f57b8e8..2431d74b3ab 100644 --- a/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs +++ b/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs @@ -192,7 +192,7 @@ public void SetUp() fakeOutbox = new FakeOutboxStorage(); fakeBatchPipeline = new FakeBatchPipeline(); - behavior = new TransportReceiveToPhysicalMessageConnector(fakeOutbox, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc")); + behavior = new TransportReceiveToPhysicalMessageConnector(fakeOutbox, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc"), new InstrumentationOptions()); } Task Invoke(ITransportReceiveContext context, Func next = null) => behavior.Invoke(context, next ?? (_ => Task.CompletedTask)); diff --git a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs index f920a28b93a..a725e1f0f1c 100644 --- a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs @@ -14,4 +14,12 @@ public class InstrumentationOptions /// Disabled by default for backward compatibility. /// public bool UseMessageDestinationInSpanNames { get; set; } + + /// + /// Controls whether the "Start dispatching" and "Finished dispatching" activity events + /// are added to the incoming message span when outgoing messages are dispatched. + /// Enabled by default for backward compatibility. Disable to avoid the ingestion cost + /// of these events when they add no diagnostic value. + /// + public bool EmitMessageDispatchingEvents { get; set; } = true; } diff --git a/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs b/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs index 0504c24f16e..96f31beecc9 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs @@ -15,7 +15,8 @@ namespace NServiceBus; class TransportReceiveToPhysicalMessageConnector( IOutboxStorage outboxStorage, - IncomingPipelineMetrics incomingPipelineMetrics) + IncomingPipelineMetrics incomingPipelineMetrics, + InstrumentationOptions instrumentationOptions) : IStageForkConnector { public async Task Invoke(ITransportReceiveContext context, Func next) @@ -65,10 +66,13 @@ public async Task Invoke(ITransportReceiveContext context, Func { var storage = b.GetService() ?? new NoOpOutboxStorage(); - return new TransportReceiveToPhysicalMessageConnector(storage, b.GetRequiredService()); + return new TransportReceiveToPhysicalMessageConnector(storage, b.GetRequiredService(), hostingConfiguration.ActivityFactory.Options); }, "Allows to abort processing the message"); pipelineSettings.Register("LoadHandlersConnector", b => new LoadHandlersConnector(b.GetRequiredService(), hostingConfiguration.ActivityFactory), "Gets all the handlers to invoke from the MessageHandler registry based on the message type."); From 338ea5f79bd0bd9d431e0bf9ae1edf314dc08d72 Mon Sep 17 00:00:00 2001 From: Tomasz Masternak Date: Thu, 16 Jul 2026 13:28:00 +0200 Subject: [PATCH 11/37] Gauge meter for active message processings (#7841) * gauge meter counting active handler invocations * update MeterAPI approval file to reflect version 0.3.0 changes and new active handlers metric * Improvements * switching to active messages gauge * remove obsolete MessageMetadataRegistry dependency from IncomingPipelineMetrics and capture enclosed message types on the gauge meter * remove unused Unicast.Messages imports from test files * Apply suggestion from @tmasternak * move `ActiveMessageScope` definition to correct location in `IncomingPipelineMetrics` file --------- Co-authored-by: Irina Dominte --- ...hen_messages_are_processed_concurrently.cs | 72 +++++++++++++++++++ .../MeterTests.Verify_MeterAPI.approved.txt | 4 +- .../Envelopes/EnvelopeUnwrapperTests.cs | 2 - .../Incoming/InvokeHandlerTerminatorTest.cs | 2 +- ...tReceiveToPhysicalMessageConnectorTests.cs | 2 +- .../OpenTelemetry/Metrics/MeterTags.cs | 1 + .../Incoming/IncomingPipelineMetrics.cs | 36 ++++++++-- .../Pipeline/MainPipelineExecutor.cs | 3 + 8 files changed, 113 insertions(+), 9 deletions(-) create mode 100644 src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_messages_are_processed_concurrently.cs diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_messages_are_processed_concurrently.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_messages_are_processed_concurrently.cs new file mode 100644 index 00000000000..da6b3382f0e --- /dev/null +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_messages_are_processed_concurrently.cs @@ -0,0 +1,72 @@ +namespace NServiceBus.AcceptanceTests.Core.OpenTelemetry.Metrics; + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using EndpointTemplates; +using NServiceBus; +using AcceptanceTesting; +using NUnit.Framework; +using Conventions = AcceptanceTesting.Customization.Conventions; + +public class When_messages_are_processed_concurrently : OpenTelemetryAcceptanceTest +{ + const string ActiveMessagesMetric = "nservicebus.messaging.active_messages"; + const int numberOfMessages = 5; + + [Test] + public async Task Should_report_active_messages_gauge_that_balances_once_idle() + { + using var metricsListener = TestingMetricListener.SetupNServiceBusMetricsListener(); + + _ = await Scenario.Define() + .WithEndpoint(b => b.CustomConfig(c => + { + c.MakeInstanceUniquelyAddressable("instanceId"); + c.LimitMessageProcessingConcurrencyTo(10); + }).When(async (session, ctx) => + { + for (var x = 0; x < numberOfMessages; x++) + { + await session.SendLocal(new OutgoingMessage()); + } + })) + .Run(); + + Assert.That(metricsListener.ReportedMeters.TryGetValue(ActiveMessagesMetric, out var net), Is.True, + $"'{ActiveMessagesMetric}' gauge should be reported"); + Assert.That(net, Is.EqualTo(0), + "increments and decrements should balance once all messages have been processed"); + + metricsListener.AssertTags(ActiveMessagesMetric, + new Dictionary + { + ["nservicebus.queue"] = Conventions.EndpointNamingConvention(typeof(EndpointWithMetrics)), + ["nservicebus.discriminator"] = "instanceId", + ["nservicebus.enclosed_message_types"] = typeof(OutgoingMessage).AssemblyQualifiedName + }); + } + + public class Context : ScenarioContext + { + public int OutgoingMessagesReceived; + } + + public class EndpointWithMetrics : EndpointConfigurationBuilder + { + public EndpointWithMetrics() => EndpointSetup(); + + [Handler] + public class MessageHandler(Context testContext) : IHandleMessages + { + public Task Handle(OutgoingMessage message, IMessageHandlerContext context) + { + var messagesHandled = Interlocked.Increment(ref testContext.OutgoingMessagesReceived); + testContext.MarkAsCompleted(messagesHandled == numberOfMessages); + return Task.CompletedTask; + } + } + } + + public class OutgoingMessage : IMessage; +} diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/MeterTests.Verify_MeterAPI.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/MeterTests.Verify_MeterAPI.approved.txt index a572d43098f..d97f8c61a26 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/MeterTests.Verify_MeterAPI.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/MeterTests.Verify_MeterAPI.approved.txt @@ -1,11 +1,12 @@ { "Note": "Changes to metrics API should result in an update to NServiceBusMeter version.", "MetricsSourceName": "NServiceBus.Core.Pipeline.Incoming", - "MetricsSourceVersion": "0.2.0", + "MetricsSourceVersion": "0.3.0", "Tags": [ "error.type", "execution.result", "nservicebus.discriminator", + "nservicebus.enclosed_message_types", "nservicebus.envelope.unwrapper_type", "nservicebus.message_handler_type", "nservicebus.message_handler_types", @@ -14,6 +15,7 @@ ], "Metrics": [ "nservicebus.envelope.unwrapped => Counter", + "nservicebus.messaging.active_messages => UpDownCounter", "nservicebus.messaging.critical_time => Histogram, Unit: s", "nservicebus.messaging.failures => Counter", "nservicebus.messaging.fetches => Counter", diff --git a/src/NServiceBus.Core.Tests/Envelopes/EnvelopeUnwrapperTests.cs b/src/NServiceBus.Core.Tests/Envelopes/EnvelopeUnwrapperTests.cs index 1f2460a8b9c..1824dc35f2c 100644 --- a/src/NServiceBus.Core.Tests/Envelopes/EnvelopeUnwrapperTests.cs +++ b/src/NServiceBus.Core.Tests/Envelopes/EnvelopeUnwrapperTests.cs @@ -4,12 +4,10 @@ namespace NServiceBus.Core.Tests.Envelopes; using System; using System.Buffers; using System.Collections.Generic; -using System.Text; using Extensibility; using NUnit.Framework; using Transport; - public class EnvelopeUnwrapperTests { string nativeId; diff --git a/src/NServiceBus.Core.Tests/Pipeline/Incoming/InvokeHandlerTerminatorTest.cs b/src/NServiceBus.Core.Tests/Pipeline/Incoming/InvokeHandlerTerminatorTest.cs index f36af8a855a..47276e798e4 100644 --- a/src/NServiceBus.Core.Tests/Pipeline/Incoming/InvokeHandlerTerminatorTest.cs +++ b/src/NServiceBus.Core.Tests/Pipeline/Incoming/InvokeHandlerTerminatorTest.cs @@ -11,7 +11,7 @@ [TestFixture] public class InvokeHandlerTerminatorTest { - InvokeHandlerTerminator terminator = new(new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc")); + readonly InvokeHandlerTerminator terminator = new(new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc")); [Test] public async Task When_saga_found_and_handler_is_saga_should_invoke_handler() diff --git a/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs b/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs index 2431d74b3ab..52d0fb8ad11 100644 --- a/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs +++ b/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs @@ -1,4 +1,4 @@ -namespace NServiceBus.Core.Tests.Reliability.Outbox; +namespace NServiceBus.Core.Tests.Reliability.Outbox; using System; using System.Collections.Generic; diff --git a/src/NServiceBus.Core/OpenTelemetry/Metrics/MeterTags.cs b/src/NServiceBus.Core/OpenTelemetry/Metrics/MeterTags.cs index 9eb8795c87f..d99b135197e 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Metrics/MeterTags.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Metrics/MeterTags.cs @@ -7,6 +7,7 @@ static class MeterTags public const string EndpointDiscriminator = "nservicebus.discriminator"; public const string QueueName = "nservicebus.queue"; public const string MessageType = "nservicebus.message_type"; + public const string EnclosedMessageTypes = "nservicebus.enclosed_message_types"; public const string MessageHandlerTypes = "nservicebus.message_handler_types"; public const string MessageHandlerType = "nservicebus.message_handler_type"; public const string ExecutionResult = "execution.result"; diff --git a/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs b/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs index b842e912a3c..6a063b2abee 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs @@ -21,10 +21,11 @@ class IncomingPipelineMetrics const string RecoverabilityDelayed = "nservicebus.recoverability.delayed"; const string RecoverabilityError = "nservicebus.recoverability.error"; const string EnvelopeUnwrapping = "nservicebus.envelope.unwrapped"; + const string ActiveMessages = "nservicebus.messaging.active_messages"; public IncomingPipelineMetrics(IMeterFactory meterFactory, string queueName, string discriminator) { - var meter = meterFactory.Create("NServiceBus.Core.Pipeline.Incoming", "0.2.0"); + var meter = meterFactory.Create("NServiceBus.Core.Pipeline.Incoming", "0.3.0"); totalProcessedSuccessfully = meter.CreateCounter(TotalProcessedSuccessfully, description: "Total number of messages processed successfully by the endpoint."); totalFetched = meter.CreateCounter(TotalFetched, @@ -45,6 +46,8 @@ public IncomingPipelineMetrics(IMeterFactory meterFactory, string queueName, str description: "Total number of messages sent to the error queue."); totalEnvelopeUnwrapping = meter.CreateCounter(EnvelopeUnwrapping, description: "Total number of unwrapping attempts by the endpoint."); + activeMessages = meter.CreateUpDownCounter(ActiveMessages, + description: "Number of messages currently being processed by the endpoint."); queueNameBase = queueName; endpointDiscriminator = discriminator; @@ -53,7 +56,7 @@ public IncomingPipelineMetrics(IMeterFactory meterFactory, string queueName, str public void AddDefaultIncomingPipelineMetricTags(IncomingPipelineMetricTags incomingPipelineMetricsTags) { incomingPipelineMetricsTags.Add(MeterTags.QueueName, queueNameBase); - incomingPipelineMetricsTags.Add(MeterTags.EndpointDiscriminator, endpointDiscriminator ?? ""); + incomingPipelineMetricsTags.Add(MeterTags.EndpointDiscriminator, endpointDiscriminator); } public void RecordProcessingTime(ITransportReceiveContext context, TimeSpan elapsed) @@ -243,6 +246,24 @@ public void RecordSendToErrorQueue(IRecoverabilityContext recoverabilityContext) totalSentToErrorQueue.Add(1, meterTags); } + public ActiveMessageScope TrackMessageProcessing(IncomingPipelineMetricTags incomingPipelineMetricTags, IncomingMessage message) + { + if (!activeMessages.Enabled) + { + return default; + } + + TagList tags; + incomingPipelineMetricTags.ApplyTags(ref tags, [MeterTags.QueueName, MeterTags.EndpointDiscriminator]); + if (message.Headers.TryGetValue(Headers.EnclosedMessageTypes, out var enclosedMessageTypes)) + { + tags.Add(new KeyValuePair(MeterTags.EnclosedMessageTypes, enclosedMessageTypes)); + } + + activeMessages.Add(1, tags); + return new ActiveMessageScope(activeMessages, tags); + } + public void EnvelopeUnwrappingSucceeded(MessageContext messageContext, IEnvelopeHandler type) => RecordEnvelopeUnwrapping(messageContext, type, true, null); public void EnvelopeUnwrappingFailed(MessageContext messageContext, IEnvelopeHandler type, Exception? exception) => RecordEnvelopeUnwrapping(messageContext, type, false, exception); void RecordEnvelopeUnwrapping(MessageContext messageContext, IEnvelopeHandler type, bool succeeded, Exception? exception) @@ -266,6 +287,11 @@ void RecordEnvelopeUnwrapping(MessageContext messageContext, IEnvelopeHandler ty totalEnvelopeUnwrapping.Add(succeeded ? 0 : 1, meterTags); } + public readonly struct ActiveMessageScope(UpDownCounter? counter, TagList tags) : IDisposable + { + public void Dispose() => counter?.Add(-1, tags); + } + readonly Counter totalProcessedSuccessfully; readonly Counter totalFetched; readonly Counter totalFailures; @@ -276,6 +302,8 @@ void RecordEnvelopeUnwrapping(MessageContext messageContext, IEnvelopeHandler ty readonly Counter totalDelayedRetries; readonly Counter totalSentToErrorQueue; readonly Counter totalEnvelopeUnwrapping; - string queueNameBase; - string endpointDiscriminator; + readonly UpDownCounter activeMessages; + + readonly string queueNameBase; + readonly string endpointDiscriminator; } \ No newline at end of file diff --git a/src/NServiceBus.Core/Pipeline/MainPipelineExecutor.cs b/src/NServiceBus.Core/Pipeline/MainPipelineExecutor.cs index 07560a36c13..7bab6ffb264 100644 --- a/src/NServiceBus.Core/Pipeline/MainPipelineExecutor.cs +++ b/src/NServiceBus.Core/Pipeline/MainPipelineExecutor.cs @@ -35,6 +35,9 @@ public async Task Invoke(MessageContext messageContext, CancellationToken cancel using var incomingMessageHandle = envelopeUnwrapper.UnwrapEnvelope(messageContext); IncomingMessage message = incomingMessageHandle; + //This needs to happen after envelope unwrapping to ensure the proper value of the EnclosedMessageTypes header + using var activeMessageScope = incomingPipelineMetrics.TrackMessageProcessing(incomingPipelineMetricsTags, message); + var transportReceiveContext = new TransportReceiveContext( childScope.ServiceProvider, messageOperations, From 7d8cc9342a8f329e65b735543b1539935f54fde3 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 16 Jul 2026 14:06:23 +0200 Subject: [PATCH 12/37] Endpoint-level trace connector defaults for sends and publishes (#7867) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ Add TraceConnector enum and endpoint-level trace connector defaults to InstrumentationOptions * ✨ Honor endpoint-level trace connector defaults in send/publish behaviors with per-message overrides * ✨ Add acceptance tests for endpoint-level trace connector defaults and per-message overrides * ✨ Approve public API additions for trace connector configuration * Replace `TraceConnector` with `TraceMode` for clearer terminology and updated tracing behavior. * Update approval file after replacing `TraceConnector` with `TraceMode` in public API --------- Co-authored-by: Tomasz Masternak --- .../Traces/When_publishing_messages.cs | 120 ++++++++++++++++++ .../Traces/When_sending_messages.cs | 80 ++++++++++++ ...IApprovals.ApproveNServiceBus.approved.txt | 9 ++ .../InstrumentationOptionsTests.cs | 19 +++ .../OpenTelemetryExtensionsTests.cs | 63 +++++++++ .../OpenTelemetryPublishBehaviorTests.cs | 55 ++++++++ .../OpenTelemetrySendBehaviorTests.cs | 55 ++++++++ .../OpenTelemetry/InstrumentationOptions.cs | 16 +++ .../OpenTelemetry/OpenTelemetryExtensions.cs | 36 +++++- .../OpenTelemetry/OpenTelemetryFeature.cs | 6 +- .../OpenTelemetryPublishBehavior.cs | 17 +-- .../OpenTelemetrySendBehavior.cs | 17 +-- .../OpenTelemetry/TraceMode.cs | 19 +++ 13 files changed, 484 insertions(+), 28 deletions(-) create mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/InstrumentationOptionsTests.cs create mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetryExtensionsTests.cs create mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetryPublishBehaviorTests.cs create mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetrySendBehaviorTests.cs create mode 100644 src/NServiceBus.Core/OpenTelemetry/TraceMode.cs diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs index 4281ddc170a..7aed06a30a8 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs @@ -242,5 +242,125 @@ public Task Handle(ThisIsAnEvent @event, IMessageHandlerContext context) } } + [Test] + public async Task Should_create_child_on_receive_when_endpoint_defaults_to_child_span() + { + var context = await Scenario.Define() + .WithEndpoint(b => b + .When(ctx => ctx.SomeEventSubscribed, s => s.Publish(new ThisIsAnEvent()))) + .WithEndpoint(b => b.When((session, ctx) => + { + if (ctx.HasNativePubSubSupport) + { + ctx.SomeEventSubscribed = true; + } + + return Task.CompletedTask; + })) + .Run(); + + var publishMessageActivities = NServiceBusActivityListener.CompletedActivities.GetPublishEventActivities(); + var receiveMessageActivities = NServiceBusActivityListener.CompletedActivities.GetReceiveMessageActivities(); + using (Assert.EnterMultipleScope()) + { + Assert.That(publishMessageActivities, Has.Count.EqualTo(1), "1 message is published as part of this test"); + Assert.That(receiveMessageActivities, Has.Count.EqualTo(1), "1 message is received as part of this test"); + } + + var publishRequest = publishMessageActivities[0]; + var receiveRequest = receiveMessageActivities[0]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(receiveRequest.RootId, Is.EqualTo(publishRequest.RootId), "publish and receive operations are part the same root activity"); + Assert.That(receiveRequest.ParentId, Is.Not.Null, "incoming message does have a parent"); + } + + Assert.That(receiveRequest.Links, Is.Empty, "receive does not have links"); + } + + [Test] + public async Task Should_create_new_linked_trace_on_receive_when_option_overrides_endpoint_connector() + { + var context = await Scenario.Define() + .WithEndpoint(b => b + .When(ctx => ctx.SomeEventSubscribed, s => + { + var publishOptions = new PublishOptions(); + publishOptions.StartNewTraceOnReceive(); + return s.Publish(new ThisIsAnEvent(), publishOptions); + })) + .WithEndpoint(b => b.When((session, ctx) => + { + if (ctx.HasNativePubSubSupport) + { + ctx.SomeEventSubscribed = true; + } + + return Task.CompletedTask; + })) + .Run(); + + var publishMessageActivities = NServiceBusActivityListener.CompletedActivities.GetPublishEventActivities(); + var receiveMessageActivities = NServiceBusActivityListener.CompletedActivities.GetReceiveMessageActivities(); + using (Assert.EnterMultipleScope()) + { + Assert.That(publishMessageActivities, Has.Count.EqualTo(1), "1 message is published as part of this test"); + Assert.That(receiveMessageActivities, Has.Count.EqualTo(1), "1 message is received as part of this test"); + } + + var publishRequest = publishMessageActivities[0]; + var receiveRequest = receiveMessageActivities[0]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(receiveRequest.RootId, Is.Not.EqualTo(publishRequest.RootId), "publish and receive operations are part of different root activities"); + Assert.That(receiveRequest.ParentId, Is.Null, "incoming message does not have a parent, it's a root"); + } + + ActivityLink link = receiveRequest.Links.FirstOrDefault(); + Assert.That(link, Is.Not.EqualTo(default(ActivityLink)), "Receive has a link"); + Assert.That(link.Context.TraceId, Is.EqualTo(publishRequest.TraceId), "receive is linked to publish operation"); + } + + public class PublisherWithChildSpanConnector : EndpointConfigurationBuilder + { + public PublisherWithChildSpanConnector() => + EndpointSetup(b => + { + b.Tracing().PublishTraceMode = TraceMode.ContinueExisting; + b.OnEndpointSubscribed((s, context) => + { + if (s.SubscriberEndpoint.Contains(Conventions.EndpointNamingConvention(typeof(SubscriberForPublisherWithChildSpanConnector)))) + { + if (s.MessageType == typeof(ThisIsAnEvent).AssemblyQualifiedName) + { + context.SomeEventSubscribed = true; + } + } + }); + }); + } + + public class SubscriberForPublisherWithChildSpanConnector : EndpointConfigurationBuilder + { + public SubscriberForPublisherWithChildSpanConnector() => + EndpointSetup(c => { }, + metadata => + { + metadata.RegisterPublisherFor(); + }); + + [Handler] + public class ThisHandlesSomethingHandler(Context testContext) : IHandleMessages + { + public Task Handle(ThisIsAnEvent @event, IMessageHandlerContext context) + { + testContext.MarkAsCompleted(); + return Task.CompletedTask; + } + } + } + public class ThisIsAnEvent : IEvent; } \ No newline at end of file diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_messages.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_messages.cs index e14df1d05e6..c08e660e883 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_messages.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_messages.cs @@ -163,5 +163,85 @@ public Task Handle(OutgoingMessage message, IMessageHandlerContext context) } } + [Test] + public async Task Should_create_new_linked_trace_on_receive_when_endpoint_defaults_to_span_link() + { + await Scenario.Define() + .WithEndpoint(b => b + .When(s => s.SendLocal(new OutgoingMessage()))) + .Run(); + + var sendMessageActivities = NServiceBusActivityListener.CompletedActivities.GetSendMessageActivities(); + var receiveMessageActivities = NServiceBusActivityListener.CompletedActivities.GetReceiveMessageActivities(); + using (Assert.EnterMultipleScope()) + { + Assert.That(sendMessageActivities, Has.Count.EqualTo(1), "1 message is sent as part of this test"); + Assert.That(receiveMessageActivities, Has.Count.EqualTo(1), "1 message is received as part of this test"); + } + + var sendRequest = sendMessageActivities[0]; + var receiveRequest = receiveMessageActivities[0]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(receiveRequest.RootId, Is.Not.EqualTo(sendRequest.RootId), "send and receive operations are part of different root activities"); + Assert.That(receiveRequest.ParentId, Is.Null, "incoming message does not have a parent, it's a root"); + } + + ActivityLink link = receiveRequest.Links.FirstOrDefault(); + Assert.That(link, Is.Not.EqualTo(default(ActivityLink)), "Receive has a link"); + Assert.That(link.Context.TraceId, Is.EqualTo(sendRequest.TraceId), "receive is linked to send operation"); + } + + [Test] + public async Task Should_create_child_on_receive_when_option_overrides_endpoint_connector() + { + await Scenario.Define() + .WithEndpoint(b => b + .When(s => + { + var sendOptions = new SendOptions(); + sendOptions.RouteToThisEndpoint(); + sendOptions.ContinueExistingTraceOnReceive(); + return s.Send(new OutgoingMessage(), sendOptions); + })) + .Run(); + + var sendMessageActivities = NServiceBusActivityListener.CompletedActivities.GetSendMessageActivities(); + var receiveMessageActivities = NServiceBusActivityListener.CompletedActivities.GetReceiveMessageActivities(); + using (Assert.EnterMultipleScope()) + { + Assert.That(sendMessageActivities, Has.Count.EqualTo(1), "1 message is sent as part of this test"); + Assert.That(receiveMessageActivities, Has.Count.EqualTo(1), "1 message is received as part of this test"); + } + + var sendRequest = sendMessageActivities[0]; + var receiveRequest = receiveMessageActivities[0]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(receiveRequest.RootId, Is.EqualTo(sendRequest.RootId), "send and receive operations are part of the same root activity"); + Assert.That(receiveRequest.ParentId, Is.Not.Null, "incoming message does have a parent"); + } + + Assert.That(receiveRequest.Links, Is.Empty, "receive does not have links"); + } + + public class TestEndpointWithSpanLinkConnector : EndpointConfigurationBuilder + { + public TestEndpointWithSpanLinkConnector() => + EndpointSetup(b => b.Tracing().SendTraceMode = TraceMode.StartNew); + + [Handler] + public class MessageHandler(Context testContext) : IHandleMessages + { + public Task Handle(OutgoingMessage message, IMessageHandlerContext context) + { + testContext.MarkAsCompleted(); + return Task.CompletedTask; + } + } + } + public class OutgoingMessage : IMessage; } \ No newline at end of file diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt index 67152cc72d5..30981403a3d 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt @@ -636,6 +636,8 @@ namespace NServiceBus { public InstrumentationOptions() { } public bool EmitMessageDispatchingEvents { get; set; } + public NServiceBus.TraceMode PublishTraceMode { get; set; } + public NServiceBus.TraceMode SendTraceMode { get; set; } public bool UseMessageDestinationInSpanNames { get; set; } } public sealed class KeyedServiceKey @@ -777,6 +779,8 @@ namespace NServiceBus public static class OpenTelemetryExtensions { public static void ContinueExistingTraceOnReceive(this NServiceBus.PublishOptions publishOptions) { } + public static void ContinueExistingTraceOnReceive(this NServiceBus.SendOptions sendOptions) { } + public static void StartNewTraceOnReceive(this NServiceBus.PublishOptions publishOptions) { } public static void StartNewTraceOnReceive(this NServiceBus.SendOptions sendOptions) { } public static NServiceBus.InstrumentationOptions Tracing(this NServiceBus.EndpointConfiguration config) { } } @@ -1174,6 +1178,11 @@ namespace NServiceBus public ToSagaExpression(NServiceBus.IConfigureHowToFindSagaWithMessage sagaMessageFindingConfiguration, System.Linq.Expressions.Expression> messageProperty) { } public void ToSaga(System.Linq.Expressions.Expression> sagaEntityProperty) { } } + public enum TraceMode + { + ContinueExisting = 0, + StartNew = 1, + } public static class TransportConfig { extension(NServiceBus.EndpointConfiguration endpointConfiguration) diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/InstrumentationOptionsTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/InstrumentationOptionsTests.cs new file mode 100644 index 00000000000..ded2e6a41c2 --- /dev/null +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/InstrumentationOptionsTests.cs @@ -0,0 +1,19 @@ +namespace NServiceBus.Core.Tests.OpenTelemetry; + +using NUnit.Framework; + +[TestFixture] +public class InstrumentationOptionsTests +{ + [Test] + public void Should_default_trace_connectors_to_current_behavior() + { + var options = new InstrumentationOptions(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(options.SendTraceMode, Is.EqualTo(TraceMode.ContinueExisting), "sends continue the trace by default"); + Assert.That(options.PublishTraceMode, Is.EqualTo(TraceMode.StartNew), "publishes start a new linked trace by default"); + } + } +} diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetryExtensionsTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetryExtensionsTests.cs new file mode 100644 index 00000000000..14752802842 --- /dev/null +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetryExtensionsTests.cs @@ -0,0 +1,63 @@ +namespace NServiceBus.Core.Tests.OpenTelemetry; + +using NUnit.Framework; + +[TestFixture] +public class OpenTelemetryExtensionsTests +{ + [Test] + public void StartNewTraceOnReceive_should_set_span_link_override_on_send_options() + { + var options = new SendOptions(); + + options.StartNewTraceOnReceive(); + + Assert.That(options.Context.TryGet(OpenTelemetryExtensions.TraceConnectorOverrideKey, out TraceMode connector), Is.True); + Assert.That(connector, Is.EqualTo(TraceMode.StartNew)); + } + + [Test] + public void ContinueExistingTraceOnReceive_should_set_child_span_override_on_send_options() + { + var options = new SendOptions(); + + options.ContinueExistingTraceOnReceive(); + + Assert.That(options.Context.TryGet(OpenTelemetryExtensions.TraceConnectorOverrideKey, out TraceMode connector), Is.True); + Assert.That(connector, Is.EqualTo(TraceMode.ContinueExisting)); + } + + [Test] + public void StartNewTraceOnReceive_should_set_span_link_override_on_publish_options() + { + var options = new PublishOptions(); + + options.StartNewTraceOnReceive(); + + Assert.That(options.Context.TryGet(OpenTelemetryExtensions.TraceConnectorOverrideKey, out TraceMode connector), Is.True); + Assert.That(connector, Is.EqualTo(TraceMode.StartNew)); + } + + [Test] + public void ContinueExistingTraceOnReceive_should_set_child_span_override_on_publish_options() + { + var options = new PublishOptions(); + + options.ContinueExistingTraceOnReceive(); + + Assert.That(options.Context.TryGet(OpenTelemetryExtensions.TraceConnectorOverrideKey, out TraceMode connector), Is.True); + Assert.That(connector, Is.EqualTo(TraceMode.ContinueExisting)); + } + + [Test] + public void Last_override_call_wins() + { + var options = new PublishOptions(); + + options.ContinueExistingTraceOnReceive(); + options.StartNewTraceOnReceive(); + + Assert.That(options.Context.TryGet(OpenTelemetryExtensions.TraceConnectorOverrideKey, out TraceMode connector), Is.True); + Assert.That(connector, Is.EqualTo(TraceMode.StartNew)); + } +} diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetryPublishBehaviorTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetryPublishBehaviorTests.cs new file mode 100644 index 00000000000..3cc9e44ee06 --- /dev/null +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetryPublishBehaviorTests.cs @@ -0,0 +1,55 @@ +namespace NServiceBus.Core.Tests.OpenTelemetry; + +using System.Threading.Tasks; +using NUnit.Framework; +using Testing; + +[TestFixture] +public class OpenTelemetryPublishBehaviorTests +{ + [Test] + public async Task Should_start_new_trace_on_receive_by_default() + { + var behavior = new OpenTelemetryPublishBehavior(new InstrumentationOptions()); + var context = new TestableOutgoingPublishContext(); + + await behavior.Invoke(context, _ => Task.CompletedTask); + + Assert.That(context.Headers[Headers.StartNewTrace], Is.EqualTo(bool.TrueString)); + } + + [Test] + public async Task Should_continue_trace_on_receive_when_endpoint_connector_is_child_span() + { + var behavior = new OpenTelemetryPublishBehavior(new InstrumentationOptions { PublishTraceMode = TraceMode.ContinueExisting }); + var context = new TestableOutgoingPublishContext(); + + await behavior.Invoke(context, _ => Task.CompletedTask); + + Assert.That(context.Headers[Headers.StartNewTrace], Is.EqualTo(bool.FalseString)); + } + + [Test] + public async Task Should_prefer_child_span_option_over_endpoint_connector() + { + var behavior = new OpenTelemetryPublishBehavior(new InstrumentationOptions { PublishTraceMode = TraceMode.StartNew }); + var context = new TestableOutgoingPublishContext(); + context.Extensions.Set(OpenTelemetryExtensions.TraceConnectorOverrideKey, TraceMode.ContinueExisting); + + await behavior.Invoke(context, _ => Task.CompletedTask); + + Assert.That(context.Headers[Headers.StartNewTrace], Is.EqualTo(bool.FalseString)); + } + + [Test] + public async Task Should_prefer_span_link_option_over_endpoint_connector() + { + var behavior = new OpenTelemetryPublishBehavior(new InstrumentationOptions { PublishTraceMode = TraceMode.ContinueExisting }); + var context = new TestableOutgoingPublishContext(); + context.Extensions.Set(OpenTelemetryExtensions.TraceConnectorOverrideKey, TraceMode.StartNew); + + await behavior.Invoke(context, _ => Task.CompletedTask); + + Assert.That(context.Headers[Headers.StartNewTrace], Is.EqualTo(bool.TrueString)); + } +} diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetrySendBehaviorTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetrySendBehaviorTests.cs new file mode 100644 index 00000000000..8e26d213544 --- /dev/null +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetrySendBehaviorTests.cs @@ -0,0 +1,55 @@ +namespace NServiceBus.Core.Tests.OpenTelemetry; + +using System.Threading.Tasks; +using NUnit.Framework; +using Testing; + +[TestFixture] +public class OpenTelemetrySendBehaviorTests +{ + [Test] + public async Task Should_continue_trace_on_receive_by_default() + { + var behavior = new OpenTelemetrySendBehavior(new InstrumentationOptions()); + var context = new TestableOutgoingSendContext(); + + await behavior.Invoke(context, _ => Task.CompletedTask); + + Assert.That(context.Headers[Headers.StartNewTrace], Is.EqualTo(bool.FalseString)); + } + + [Test] + public async Task Should_start_new_trace_on_receive_when_endpoint_connector_is_span_link() + { + var behavior = new OpenTelemetrySendBehavior(new InstrumentationOptions { SendTraceMode = TraceMode.StartNew }); + var context = new TestableOutgoingSendContext(); + + await behavior.Invoke(context, _ => Task.CompletedTask); + + Assert.That(context.Headers[Headers.StartNewTrace], Is.EqualTo(bool.TrueString)); + } + + [Test] + public async Task Should_prefer_span_link_option_over_endpoint_connector() + { + var behavior = new OpenTelemetrySendBehavior(new InstrumentationOptions { SendTraceMode = TraceMode.ContinueExisting }); + var context = new TestableOutgoingSendContext(); + context.Extensions.Set(OpenTelemetryExtensions.TraceConnectorOverrideKey, TraceMode.StartNew); + + await behavior.Invoke(context, _ => Task.CompletedTask); + + Assert.That(context.Headers[Headers.StartNewTrace], Is.EqualTo(bool.TrueString)); + } + + [Test] + public async Task Should_prefer_child_span_option_over_endpoint_connector() + { + var behavior = new OpenTelemetrySendBehavior(new InstrumentationOptions { SendTraceMode = TraceMode.StartNew }); + var context = new TestableOutgoingSendContext(); + context.Extensions.Set(OpenTelemetryExtensions.TraceConnectorOverrideKey, TraceMode.ContinueExisting); + + await behavior.Invoke(context, _ => Task.CompletedTask); + + Assert.That(context.Headers[Headers.StartNewTrace], Is.EqualTo(bool.FalseString)); + } +} diff --git a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs index a725e1f0f1c..dcdd85a1c93 100644 --- a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs @@ -22,4 +22,20 @@ public class InstrumentationOptions /// of these events when they add no diagnostic value. /// public bool EmitMessageDispatchingEvents { get; set; } = true; + + /// + /// Controls how the receive-side processing span relates to the send span for messages sent by this endpoint. + /// Defaults to : receivers continue the trace. + /// Can be overridden per message via + /// or . + /// + public TraceMode SendTraceMode { get; set; } = TraceMode.ContinueExisting; + + /// + /// Controls how the receive-side processing span relates to the publish span for events published by this endpoint. + /// Defaults to : receivers start a new trace linked back to the publish span. + /// Can be overridden per message via + /// or . + /// + public TraceMode PublishTraceMode { get; set; } = TraceMode.StartNew; } diff --git a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryExtensions.cs b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryExtensions.cs index 9c3ad6e63e8..6db97f31694 100644 --- a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryExtensions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryExtensions.cs @@ -21,20 +21,48 @@ public static InstrumentationOptions Tracing(this EndpointConfiguration config) } /// - /// Start a new OpenTelemetry trace conversation. + /// Start a new OpenTelemetry trace on receive of this message, linked back to the send span. + /// Overrides for this message. /// /// The option being extended. public static void StartNewTraceOnReceive(this SendOptions sendOptions) { - sendOptions.Context.Set(OpenTelemetrySendBehavior.StartNewTraceOnReceive, true); + ArgumentNullException.ThrowIfNull(sendOptions); + sendOptions.Context.Set(TraceConnectorOverrideKey, TraceMode.StartNew); } /// - /// Start a new OpenTelemetry trace conversation. + /// Continue the existing OpenTelemetry trace on receive of this message. + /// Overrides for this message. + /// + /// The option being extended. + public static void ContinueExistingTraceOnReceive(this SendOptions sendOptions) + { + ArgumentNullException.ThrowIfNull(sendOptions); + sendOptions.Context.Set(TraceConnectorOverrideKey, TraceMode.ContinueExisting); + } + + /// + /// Start a new OpenTelemetry trace on receive of this event, linked back to the publish span. + /// Overrides for this message. + /// + /// The option being extended. + public static void StartNewTraceOnReceive(this PublishOptions publishOptions) + { + ArgumentNullException.ThrowIfNull(publishOptions); + publishOptions.Context.Set(TraceConnectorOverrideKey, TraceMode.StartNew); + } + + /// + /// Continue the existing OpenTelemetry trace on receive of this event. + /// Overrides for this message. /// /// The option being extended. public static void ContinueExistingTraceOnReceive(this PublishOptions publishOptions) { - publishOptions.Context.Set(OpenTelemetryPublishBehavior.ContinueTraceOnReceive, true); + ArgumentNullException.ThrowIfNull(publishOptions); + publishOptions.Context.Set(TraceConnectorOverrideKey, TraceMode.ContinueExisting); } + + internal const string TraceConnectorOverrideKey = "NServiceBus.OpenTelemetry.TraceConnectorOverride"; } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs index c485129eee0..b108b409c92 100644 --- a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs +++ b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs @@ -8,13 +8,15 @@ sealed class OpenTelemetryFeature : Feature { protected override void Setup(FeatureConfigurationContext context) { + var instrumentationOptions = context.Settings.GetOrDefault() ?? new InstrumentationOptions(); + context.Pipeline.Register( - new OpenTelemetryPublishBehavior(), + new OpenTelemetryPublishBehavior(instrumentationOptions), "Manages the depth of the trace for publishes" ); context.Pipeline.Register( - new OpenTelemetrySendBehavior(), + new OpenTelemetrySendBehavior(instrumentationOptions), "Manages the depth of the trace for sends" ); diff --git a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryPublishBehavior.cs b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryPublishBehavior.cs index 55617620d7f..367647a0148 100644 --- a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryPublishBehavior.cs +++ b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryPublishBehavior.cs @@ -6,22 +6,17 @@ namespace NServiceBus; using System.Threading.Tasks; using Pipeline; -class OpenTelemetryPublishBehavior : IBehavior +class OpenTelemetryPublishBehavior(InstrumentationOptions instrumentationOptions) : IBehavior { public Task Invoke(IOutgoingPublishContext context, Func next) { - // publishes always start a new trace on receive - context.Headers[Headers.StartNewTrace] = bool.TrueString; + // the per-message override wins over the endpoint-level default + var connector = context.Extensions.TryGet(OpenTelemetryExtensions.TraceConnectorOverrideKey, out TraceMode requestedConnector) + ? requestedConnector + : instrumentationOptions.PublishTraceMode; - // unless the user explicitly requests to continue the trace - bool continueTraceWasSet = context.Extensions.TryGet(ContinueTraceOnReceive, out var continueTraceRequested); - if (continueTraceWasSet && continueTraceRequested) - { - context.Headers[Headers.StartNewTrace] = bool.FalseString; - } + context.Headers[Headers.StartNewTrace] = connector == TraceMode.StartNew ? bool.TrueString : bool.FalseString; return next(context); } - - public const string ContinueTraceOnReceive = "ContinueTraceRequested"; } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetrySendBehavior.cs b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetrySendBehavior.cs index b8ca6de993c..75338de54ea 100644 --- a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetrySendBehavior.cs +++ b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetrySendBehavior.cs @@ -6,22 +6,17 @@ namespace NServiceBus; using System.Threading.Tasks; using Pipeline; -class OpenTelemetrySendBehavior : IBehavior +class OpenTelemetrySendBehavior(InstrumentationOptions instrumentationOptions) : IBehavior { public Task Invoke(IOutgoingSendContext context, Func next) { - // sends never start a new trace on receive - context.Headers[Headers.StartNewTrace] = bool.FalseString; + // the per-message override wins over the endpoint-level default + var connector = context.Extensions.TryGet(OpenTelemetryExtensions.TraceConnectorOverrideKey, out TraceMode requestedConnector) + ? requestedConnector + : instrumentationOptions.SendTraceMode; - // unless the user explicitly requests to start a new trace - bool breakTraceWasSet = context.Extensions.TryGet(StartNewTraceOnReceive, out var breakTraceWasRequested); - if (breakTraceWasSet && breakTraceWasRequested) - { - context.Headers[Headers.StartNewTrace] = bool.TrueString; - } + context.Headers[Headers.StartNewTrace] = connector == TraceMode.StartNew ? bool.TrueString : bool.FalseString; return next(context); } - - public const string StartNewTraceOnReceive = "BreakTraceRequested"; } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/TraceMode.cs b/src/NServiceBus.Core/OpenTelemetry/TraceMode.cs new file mode 100644 index 00000000000..590258b3b71 --- /dev/null +++ b/src/NServiceBus.Core/OpenTelemetry/TraceMode.cs @@ -0,0 +1,19 @@ +#nullable enable + +namespace NServiceBus; + +/// +/// Controls how the receive-side processing span relates to the outgoing send or publish span. +/// +public enum TraceMode +{ + /// + /// The receiving endpoint continues the trace: the processing span becomes a child of the outgoing span. + /// + ContinueExisting, + + /// + /// The receiving endpoint starts a new trace: the processing span becomes the root of a new trace with a link back to the outgoing span. + /// + StartNew +} From b7eb4692fbcae16c52c5f3b26f0950561d65dd5b Mon Sep 17 00:00:00 2001 From: "Irina Dominte(Scurtu)" Date: Thu, 23 Jul 2026 15:13:49 +0300 Subject: [PATCH 13/37] Add meter for total number of messages deduplicated via Outbox (#7864) * first stab at metric counter * Text key fix * Apply suggestion from @tmasternak * update metric name for deduplication tracking * rename meterFactory to fakeMeterFactory in unit tests * subscribe test meter listern to all available instruments to handle instruments with duplicated names caused by multiple creations of IncomingPipelineMetrics objects --------- Co-authored-by: Tomasz Masternak Co-authored-by: Tomasz Masternak --- .../MeterTests.Verify_MeterAPI.approved.txt | 1 + .../Helpers/TestingMetricListener.cs | 7 ------ ...tReceiveToPhysicalMessageConnectorTests.cs | 23 ++++++++++++++++++- .../Incoming/IncomingPipelineMetrics.cs | 23 ++++++++++++++++++- ...nsportReceiveToPhysicalMessageConnector.cs | 1 + 5 files changed, 46 insertions(+), 9 deletions(-) diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/MeterTests.Verify_MeterAPI.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/MeterTests.Verify_MeterAPI.approved.txt index d97f8c61a26..8cfd1afa89c 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/MeterTests.Verify_MeterAPI.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/MeterTests.Verify_MeterAPI.approved.txt @@ -22,6 +22,7 @@ "nservicebus.messaging.handler_time => Histogram, Unit: s", "nservicebus.messaging.processing_time => Histogram, Unit: s", "nservicebus.messaging.successes => Counter", + "nservicebus.outbox.duplicates => Counter", "nservicebus.recoverability.delayed => Counter", "nservicebus.recoverability.error => Counter", "nservicebus.recoverability.immediate => Counter" diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/Helpers/TestingMetricListener.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/Helpers/TestingMetricListener.cs index cbae1435e61..f288d4f4476 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/Helpers/TestingMetricListener.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/Helpers/TestingMetricListener.cs @@ -10,7 +10,6 @@ namespace NServiceBus.AcceptanceTests.Core.OpenTelemetry.Metrics; class TestingMetricListener : IDisposable { readonly MeterListener meterListener; - readonly ConcurrentDictionary subscribedInstruments = new(StringComparer.Ordinal); public readonly List metrics = []; public string version = ""; public string metricsSourceName = ""; @@ -26,12 +25,6 @@ class TestingMetricListener : IDisposable return; } - var instrumentKey = $"{instrument.Meter.Name}|{instrument.Name}|{instrument.GetType().FullName}"; - if (!subscribedInstruments.TryAdd(instrumentKey, 0)) - { - return; - } - TestContext.Out.WriteLine($"Subscribing to {instrument.Meter.Name}\\{instrument.Name}"); listener.EnableMeasurementEvents(instrument); metrics.Add(instrument); diff --git a/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs b/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs index 52d0fb8ad11..8f66cf3bf60 100644 --- a/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs +++ b/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs @@ -1,4 +1,4 @@ -namespace NServiceBus.Core.Tests.Reliability.Outbox; +namespace NServiceBus.Core.Tests.Reliability.Outbox; using System; using System.Collections.Generic; @@ -6,6 +6,7 @@ namespace NServiceBus.Core.Tests.Reliability.Outbox; using System.Diagnostics; using System.Linq; using System.Threading.Tasks; +using AcceptanceTests.Core.OpenTelemetry.Metrics; using NServiceBus.Outbox; using NServiceBus.Pipeline; using NServiceBus.Routing; @@ -130,6 +131,20 @@ public async Task Should_add_outbox_span_tag_when_deduplicating() Assert.That(pipelineActivity.TagObjects.ToImmutableDictionary()["nservicebus.outbox.deduplicate-message"], Is.EqualTo(true)); } + [Test] + public async Task Should_report_deduplicated_message_metric_when_deduplicating() + { + using var metricsListener = TestingMetricListener.SetupNServiceBusMetricsListener(); + + string messageId = Guid.NewGuid().ToString(); + fakeOutbox.ExistingMessage = new OutboxMessage(messageId, Array.Empty()); + var context = CreateContext(fakeBatchPipeline, messageId); + + await Invoke(context); + + metricsListener.AssertMetric("nservicebus.outbox.duplicates", 1); + } + [Test] public async Task Should_add_batch_dispatch_events_when_sending_batched_messages() { @@ -182,6 +197,7 @@ static TestableTransportReceiveContext CreateContext(FakeBatchPipeline pipeline, }; context.Extensions.Set(new FakePipelineCache(pipeline)); + context.Extensions.Set(new IncomingPipelineMetricTags()); return context; } @@ -191,16 +207,21 @@ public void SetUp() { fakeOutbox = new FakeOutboxStorage(); fakeBatchPipeline = new FakeBatchPipeline(); + fakeMeterFactory = new TestMeterFactory(); behavior = new TransportReceiveToPhysicalMessageConnector(fakeOutbox, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc"), new InstrumentationOptions()); } + [TearDown] + public void TearDown() => fakeMeterFactory.Dispose(); + Task Invoke(ITransportReceiveContext context, Func next = null) => behavior.Invoke(context, next ?? (_ => Task.CompletedTask)); TransportReceiveToPhysicalMessageConnector behavior; FakeBatchPipeline fakeBatchPipeline; FakeOutboxStorage fakeOutbox; + TestMeterFactory fakeMeterFactory; class MyEvent; diff --git a/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs b/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs index 6a063b2abee..8fd252bf161 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs @@ -1,4 +1,4 @@ -#nullable enable +#nullable enable namespace NServiceBus; @@ -22,6 +22,7 @@ class IncomingPipelineMetrics const string RecoverabilityError = "nservicebus.recoverability.error"; const string EnvelopeUnwrapping = "nservicebus.envelope.unwrapped"; const string ActiveMessages = "nservicebus.messaging.active_messages"; + const string TotalDeduplicated = "nservicebus.outbox.duplicates"; public IncomingPipelineMetrics(IMeterFactory meterFactory, string queueName, string discriminator) { @@ -32,6 +33,8 @@ public IncomingPipelineMetrics(IMeterFactory meterFactory, string queueName, str description: "Total number of messages fetched from the queue by the endpoint."); totalFailures = meter.CreateCounter(TotalFailures, description: "Total number of messages processed unsuccessfully by the endpoint."); + totalDeduplicated = meter.CreateCounter(TotalDeduplicated, + description: "Total number of duplicate messages detected by the Outbox."); messageHandlerTime = meter.CreateHistogram(MessageHandlerTime, "s", "The time in seconds for the execution of the business code."); criticalTime = meter.CreateHistogram(CriticalTime, "s", @@ -148,6 +151,23 @@ public void RecordFetchedMessage(IncomingPipelineMetricTags incomingPipelineMetr totalFetched.Add(1, tags); } + public void RecordDeduplicatedMessage(ITransportReceiveContext context) + { + if (!totalDeduplicated.Enabled) + { + return; + } + + var incomingPipelineMetricTags = context.Extensions.Get(); + TagList tags; + incomingPipelineMetricTags.ApplyTags(ref tags, [ + MeterTags.EndpointDiscriminator, + MeterTags.QueueName, + MeterTags.MessageType]); + + totalDeduplicated.Add(1, tags); + } + public void RecordSuccessfulMessageHandlerTime(IInvokeHandlerContext invokeHandlerContext, TimeSpan elapsed) { if (!messageHandlerTime.Enabled) @@ -295,6 +315,7 @@ public readonly struct ActiveMessageScope(UpDownCounter? counter, TagList readonly Counter totalProcessedSuccessfully; readonly Counter totalFetched; readonly Counter totalFailures; + readonly Counter totalDeduplicated; readonly Histogram messageHandlerTime; readonly Histogram criticalTime; readonly Histogram processingTime; diff --git a/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs b/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs index 96f31beecc9..6e6a3d05cf1 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs @@ -59,6 +59,7 @@ public async Task Invoke(ITransportReceiveContext context, Func Date: Wed, 29 Jul 2026 10:52:46 +0200 Subject: [PATCH 14/37] Allow changing trace continuation behavior for delayed messages (#7845) * Add tracing for recoverability pipeline using ActivityFactory * Refactor recoverability pipeline to use `Activity` fallback for tagging recoverability actions * Replace `new NoOpActivityFactory()` with `NoOpActivityFactory.Instance` in tests and implementation to enforce singleton usage. * Normalize recoverability action tags and update Activity display names for better trace clarity in recoverability pipeline. * Normalize recoverability action tags and update Activity display names to improve trace clarity in the recoverability pipeline. * Fixed approval file * Update ActivityTagsTests.Verify_ActivityTags.approved.txt * Ensure that tracemode operations for delayed messages (user operations, sagas, and recoverability) are happening before RoutingContext stage, and that StartNewTrace header is properly set before reaching that state. * Replace RecoverabilityTraceMode with the existing TraceMode enum; only set recoverability trace metadata for DelayedRetry * Remove custom recoverability action spans; keep this PR focused on delayed-message trace mode The dedicated "Recoverability" span/activity work (ActivityFactory.StartRecoverabilityActivity and its wiring/tagging) has been split out to the recoverability-action-spans branch. This PR now only concerns how trace mode is decided for delayed sends, saga timeouts, and delayed retries via InstrumentationOptions. * Simplify `InstrumentationOptions` by removing `MoveToErrorTraceMode`; refactor `ActivityFactory` constructor for improved initialization consistency * Add acceptance tests for delayed-message trace mode; fix MoveToError regression Covers the three delayed-message origins this PR configures: explicit delayed sends (SendOperationTraceMode), saga timeouts (SagaTimeoutTraceMode), and recoverability delayed retries (Recoverability.DelayedRetryTraceMode). Each area verifies the backward-compatible default (start a new linked trace) and that the new option flips it to continuing the existing trace. The delayed-send tests also cover per-message override precedence: an explicit StartNewTraceOnReceive always wins, and a ContinueExistingTraceOnReceive request cannot defeat the delayed-send backward-compatible default. Also fixes When_retrying_messages.Should_correlate_delayed_retry_with_send, which asserted the opposite (same-trace) behavior and never actually ran due to Requires.DelayedDelivery() gating - switched to the in-memory transport's native delayed delivery support so it exercises the real code path. Restores PopulateRecoverabilityTraceMetadataBehavior's MoveToError handling (dropped when DelayedRetry-only gating was introduced), which had silently broken When_incoming_message_moved_to_error_queue.Should_add_start_new_trace_header. MoveToError itself stays hardcoded to always start a new trace, matching pre-existing behavior; it was not part of this PR's configurable scope after MoveToErrorTraceMode was removed from InstrumentationOptions. * Fix trailing whitespace flagged by IDE0055 * test tweaks * Apply suggestion from @tmasternak * Update delayed message trace mode configuration in acceptance tests Replaces `TestEndpointContinuingTrace` with a unified `TestEndpoint` that dynamically configures `TraceMode` for delayed sends and saga timeouts. Simplifies `RetryingEndpoint` initialization. Removes redundant handler setup in trace tests. * Apply suggestions from code review Co-authored-by: Tomasz Masternak * fixing att: fix endpoint setups for delayed retries and fix access modifiers in otel tests saga * Simplify delayed message trace tests by consolidating scenarios and removing redundant per-message overrides * better test names --------- Co-authored-by: Irina Dominte Co-authored-by: Tomasz Masternak Co-authored-by: Tomasz Masternak --- .../Traces/When_retrying_messages.cs | 64 +++++++-- .../Traces/When_saga_requests_a_timeout.cs | 115 ++++++++++++++++ .../Traces/When_sending_a_delayed_message.cs | 124 ++++++++++++++++++ ...IApprovals.ApproveNServiceBus.approved.txt | 13 ++ .../ContextPropagationCompatibilityTests.cs | 5 +- .../ContextPropagationDefaultBehaviorTests.cs | 7 +- .../LegacyContextPropagationTests.cs | 43 +----- ...ecoverabilityTraceMetadataBehaviorTests.cs | 64 ++++++++- .../OpenTelemetry/InstrumentationOptions.cs | 43 ++++++ .../OpenTelemetry/OpenTelemetryFeature.cs | 2 +- .../OpenTelemetrySendBehavior.cs | 31 ++++- .../OpenTelemetry/Tracing/ActivityFactory.cs | 9 +- .../Tracing/ContextPropagation.cs | 13 +- ...lateRecoverabilityTraceMetadataBehavior.cs | 18 ++- .../OpenTelemetry/Tracing/obsolete_v11.cs | 9 +- ...ttachSenderRelatedInfoOnMessageBehavior.cs | 2 - .../Outgoing/RoutingToDispatchConnector.cs | 2 +- .../MessageDrivenSubscribeTerminator.cs | 2 +- .../MessageDrivenUnsubscribeTerminator.cs | 2 +- .../MigrationSubscribeTerminator.cs | 2 +- .../MigrationUnsubscribeTerminator.cs | 2 +- 21 files changed, 471 insertions(+), 101 deletions(-) create mode 100644 src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_saga_requests_a_timeout.cs create mode 100644 src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_a_delayed_message.cs diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_retrying_messages.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_retrying_messages.cs index b0f02c3c4b5..76f76a5f3e9 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_retrying_messages.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_retrying_messages.cs @@ -1,10 +1,11 @@ namespace NServiceBus.AcceptanceTests.Core.OpenTelemetry.Traces; using System; +using System.Diagnostics; using System.Linq; using System.Threading.Tasks; using EndpointTemplates; -using NServiceBus.AcceptanceTesting; +using AcceptanceTesting; using NUnit.Framework; public class When_retrying_messages : OpenTelemetryAcceptanceTest @@ -37,10 +38,8 @@ await Scenario.Define() } [Test] - public async Task Should_correlate_delayed_retry_with_send() + public async Task Should_start_new_trace_on_receive_by_default() { - Requires.DelayedDelivery(); - await Scenario.Define() .WithEndpoint(e => e .CustomConfig(c => c.Recoverability().Delayed(i => i.NumberOfRetries(1).TimeIncrease(TimeSpan.FromMilliseconds(1)))) @@ -48,21 +47,58 @@ await Scenario.Define() .When(s => s.SendLocal(new FailingMessage()))) .Run(); - var receiveActivities = NServiceBusActivityListener.CompletedActivities.GetReceiveMessageActivities(); - var sendActivities = NServiceBusActivityListener.CompletedActivities.GetSendMessageActivities(); + var (sendRequest, firstAttempt, retryAttempt) = GetDelayedRetryActivities(); using (Assert.EnterMultipleScope()) { - Assert.That(sendActivities, Has.Count.EqualTo(1)); - Assert.That(receiveActivities, Has.Count.EqualTo(2), "the message should be processed twice due to one immediate retry"); + Assert.That(firstAttempt.TraceId, Is.EqualTo(sendRequest.TraceId), "the first attempt is part of the original send's trace"); + Assert.That(firstAttempt.ParentId, Is.EqualTo(sendRequest.Id)); + + Assert.That(retryAttempt.TraceId, Is.Not.EqualTo(sendRequest.TraceId), "a delayed retry should start a new trace on receive by default (backward compatible)"); + Assert.That(retryAttempt.ParentId, Is.Null, "the retry attempt should be a new root"); } + + var link = retryAttempt.Links.FirstOrDefault(); + Assert.That(link, Is.Not.Default, "the retry attempt should be linked back to the original send operation"); + Assert.That(link.Context.TraceId, Is.EqualTo(sendRequest.TraceId)); + } + + [Test] + public async Task Should_continue_existing_trace_on_receive_when_configured() + { + await Scenario.Define() + .WithEndpoint(e => e + .CustomConfig(c => + { + c.Recoverability().Delayed(i => i.NumberOfRetries(1).TimeIncrease(TimeSpan.FromMilliseconds(1))); + c.Tracing().Recoverability.DelayedRetryTraceMode = TraceMode.ContinueExisting; + }) + .DoNotFailOnErrorMessages() + .When(s => s.SendLocal(new FailingMessage()))) + .Run(); + + var (sendRequest, _, retryAttempt) = GetDelayedRetryActivities(); + using (Assert.EnterMultipleScope()) { - Assert.That(receiveActivities[0].ParentId, Is.EqualTo(sendActivities[0].Id), "should not change parent span"); - Assert.That(receiveActivities[1].ParentId, Is.EqualTo(sendActivities[0].Id), "should not change parent span"); + Assert.That(retryAttempt.TraceId, Is.EqualTo(sendRequest.TraceId), "a delayed retry should continue the existing trace when DelayedRetryTraceMode is set to ContinueExisting"); + Assert.That(retryAttempt.ParentId, Is.EqualTo(sendRequest.Id)); + Assert.That(retryAttempt.Links, Is.Empty); + } + } - Assert.That(sendActivities.Concat(receiveActivities).All(a => a.TraceId == sendActivities[0].TraceId), Is.True, "all activities should be part of the same trace"); + (Activity SendRequest, Activity FirstAttempt, Activity RetryAttempt) GetDelayedRetryActivities() + { + var receiveActivities = NServiceBusActivityListener.CompletedActivities.GetReceiveMessageActivities(); + var sendActivities = NServiceBusActivityListener.CompletedActivities.GetSendMessageActivities(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(sendActivities, Has.Count.EqualTo(1)); + Assert.That(receiveActivities, Has.Count.EqualTo(2), "the message should be processed twice due to one delayed retry"); } + + return (sendActivities[0], receiveActivities[0], receiveActivities[1]); } public class Context : ScenarioContext @@ -72,7 +108,11 @@ public class Context : ScenarioContext public class RetryingEndpoint : EndpointConfigurationBuilder { - public RetryingEndpoint() => EndpointSetup(); + public RetryingEndpoint() => + EndpointSetup(new DefaultServer + { + TransportConfiguration = new ConfigureEndpointAcceptanceTestingTransport(false, true) + }, (c, _) => { }, _ => { }); [Handler] public class Handler(Context testContext) : IHandleMessages diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_saga_requests_a_timeout.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_saga_requests_a_timeout.cs new file mode 100644 index 00000000000..d0dfc71b143 --- /dev/null +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_saga_requests_a_timeout.cs @@ -0,0 +1,115 @@ +namespace NServiceBus.AcceptanceTests.Core.OpenTelemetry.Traces; + +using System; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using AcceptanceTesting; +using EndpointTemplates; +using NUnit.Framework; + +public class When_saga_requests_a_timeout : OpenTelemetryAcceptanceTest +{ + [Test] + public async Task Should_start_new_trace_on_receive_by_default() + { + await Scenario.Define() + .WithEndpoint(b => b + .When(s => s.SendLocal(new StartSagaMessage { SomeId = Guid.NewGuid().ToString() }))) + .Run(); + + var (timeoutSend, timeoutReceive) = GetTimeoutActivities(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(timeoutReceive.TraceId, Is.Not.EqualTo(timeoutSend.TraceId), "a saga timeout should start a new trace on receive by default (backward compatible)"); + Assert.That(timeoutReceive.ParentId, Is.Null, "timeout receive should be a new root"); + } + + var link = timeoutReceive.Links.FirstOrDefault(); + Assert.That(link, Is.Not.Default, "timeout receive should be linked back to the timeout send operation"); + Assert.That(link.Context.TraceId, Is.EqualTo(timeoutSend.TraceId)); + } + + [Test] + public async Task Should_continue_existing_trace_on_receive_when_configured() + { + await Scenario.Define() + .WithEndpoint(b => b.CustomConfig(c => + { + c.Tracing().DelayedDelivery.SagaTimeoutTraceMode = TraceMode.ContinueExisting; + }) + .When(s => s.SendLocal(new StartSagaMessage { SomeId = Guid.NewGuid().ToString() }))) + .Run(); + + var (timeoutSend, timeoutReceive) = GetTimeoutActivities(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(timeoutReceive.TraceId, Is.EqualTo(timeoutSend.TraceId), "a saga timeout should continue the existing trace when SagaTimeoutTraceMode is set to ContinueExisting"); + Assert.That(timeoutReceive.ParentId, Is.EqualTo(timeoutSend.Id)); + Assert.That(timeoutReceive.Links, Is.Empty); + } + } + + (Activity TimeoutSend, Activity TimeoutReceive) GetTimeoutActivities() + { + var sendActivities = NServiceBusActivityListener.CompletedActivities.GetSendMessageActivities(); + var receiveActivities = NServiceBusActivityListener.CompletedActivities.GetReceiveMessageActivities(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(sendActivities, Has.Count.EqualTo(2), "start-saga send and timeout send"); + Assert.That(receiveActivities, Has.Count.EqualTo(2), "start-saga receive and timeout receive"); + } + + return (sendActivities[1], receiveActivities[1]); + } + + public class Context : ScenarioContext + { + public bool SagaMarkedComplete { get; set; } + } + + public class SagaEndpoint : EndpointConfigurationBuilder + { + public SagaEndpoint() => + EndpointSetup(new DefaultServer + { + TransportConfiguration = new ConfigureEndpointAcceptanceTestingTransport(false, true) + }, (c, _) => { }, _ => { }); + + [Saga] + public class TimeoutSaga(Context testContext) : Saga, IAmStartedByMessages, IHandleTimeouts + { + protected override void ConfigureHowToFindSaga(SagaPropertyMapper mapper) => + mapper.MapSaga(s => s.SomeId).ToMessage(m => m.SomeId); + + public Task Handle(StartSagaMessage message, IMessageHandlerContext context) + { + Data.SomeId = message.SomeId; + return RequestTimeout(context, DateTimeOffset.UtcNow.AddMilliseconds(2)); + } + + public Task Timeout(SagaTimeout state, IMessageHandlerContext context) + { + MarkAsComplete(); + testContext.SagaMarkedComplete = true; + testContext.MarkAsCompleted(); + return Task.CompletedTask; + } + } + } + + public class TimeoutSagaData : ContainSagaData + { + public virtual string SomeId { get; set; } + } + + public class StartSagaMessage : IMessage + { + public string SomeId { get; set; } + } + + public class SagaTimeout; +} diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_a_delayed_message.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_a_delayed_message.cs new file mode 100644 index 00000000000..1e0e72e17cc --- /dev/null +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_a_delayed_message.cs @@ -0,0 +1,124 @@ +namespace NServiceBus.AcceptanceTests.Core.OpenTelemetry.Traces; + +using System; +using System.Linq; +using System.Threading.Tasks; +using AcceptanceTesting; +using EndpointTemplates; +using NUnit.Framework; + +public class When_sending_a_delayed_message : OpenTelemetryAcceptanceTest +{ + [Test] + public async Task Should_start_new_trace_on_receive_by_default() + { + await Scenario.Define() + .WithEndpoint(b => b + .When(s => s.Send(new DelayedMessage(), DelayedSend()))) + .Run(); + + var (send, receive) = GetActivities(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(receive.TraceId, Is.Not.EqualTo(send.TraceId), "a delayed send should start a new trace on receive by default (backward compatible)"); + Assert.That(receive.ParentId, Is.Null, "receive should be a new root"); + } + + var link = receive.Links.FirstOrDefault(); + Assert.That(link, Is.Not.Default, "receive should be linked back to the send operation"); + Assert.That(link.Context.TraceId, Is.EqualTo(send.TraceId)); + } + + [Test] + public async Task Should_continue_existing_trace_on_receive_when_configured() + { + await Scenario.Define() + .WithEndpoint(b => b.CustomConfig(c => + { + c.Tracing().DelayedDelivery.SendOperationTraceMode = TraceMode.ContinueExisting; + }) + .When(s => s.Send(new DelayedMessage(), DelayedSend()))) + .Run(); + + var (send, receive) = GetActivities(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(receive.TraceId, Is.EqualTo(send.TraceId), "a delayed send should continue the existing trace when SendOperationTraceMode is set to ContinueExisting"); + Assert.That(receive.ParentId, Is.EqualTo(send.Id)); + Assert.That(receive.Links, Is.Empty); + } + } + + [Test] + public async Task Should_start_new_trace_by_default_no_matter_per_message_option() + { + await Scenario.Define() + .WithEndpoint(b => b + .When(s => + { + var sendOptions = DelayedSend(); + sendOptions.ContinueExistingTraceOnReceive(); + return s.Send(new DelayedMessage(), sendOptions); + })) + .Run(); + + var (send, receive) = GetActivities(); + + Assert.That(receive.TraceId, Is.Not.EqualTo(send.TraceId), + "a per-message request to continue the existing trace must not defeat the backward-compatible default for delayed sends"); + } + + static SendOptions DelayedSend() + { + var sendOptions = new SendOptions(); + sendOptions.RouteToThisEndpoint(); + sendOptions.DelayDeliveryWith(TimeSpan.FromMilliseconds(1)); + return sendOptions; + } + + (System.Diagnostics.Activity Send, System.Diagnostics.Activity Receive) GetActivities() + { + var sendActivities = NServiceBusActivityListener.CompletedActivities.GetSendMessageActivities(); + var receiveActivities = NServiceBusActivityListener.CompletedActivities.GetReceiveMessageActivities(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(sendActivities, Has.Count.EqualTo(1), "1 message is sent as part of this test"); + Assert.That(receiveActivities, Has.Count.EqualTo(1), "1 message is received as part of this test"); + } + + return (sendActivities[0], receiveActivities[0]); + } + + public class Context : ScenarioContext + { + public bool DelayedMessageReceived { get; set; } + } + + public class TestEndpoint : EndpointConfigurationBuilder + { + public TestEndpoint() + { + var template = new DefaultServer + { + TransportConfiguration = new ConfigureEndpointAcceptanceTestingTransport(false, true) + }; + EndpointSetup(template, (c, _) => { }, metadata => { }); + } + + [Handler] + public class DelayedMessageHandler(Context testContext) : IHandleMessages + { + public Task Handle(DelayedMessage message, IMessageHandlerContext context) + { + testContext.DelayedMessageReceived = true; + testContext.MarkAsCompleted(); + return Task.CompletedTask; + } + } + } + + public class DelayedMessage : IMessage; +} diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt index 30981403a3d..59a401d4526 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt @@ -214,6 +214,12 @@ namespace NServiceBus public int MaxNumberOfRetries { get; } public System.TimeSpan TimeIncrease { get; } } + public class DelayedDeliveryInstrumentationOptions + { + public DelayedDeliveryInstrumentationOptions() { } + public NServiceBus.TraceMode SagaTimeoutTraceMode { get; set; } + public NServiceBus.TraceMode SendOperationTraceMode { get; set; } + } public static class DelayedDeliveryOptionExtensions { public static void DelayDeliveryWith(this NServiceBus.SendOptions options, System.TimeSpan delay) { } @@ -635,8 +641,10 @@ namespace NServiceBus public class InstrumentationOptions { public InstrumentationOptions() { } + public NServiceBus.DelayedDeliveryInstrumentationOptions DelayedDelivery { get; } public bool EmitMessageDispatchingEvents { get; set; } public NServiceBus.TraceMode PublishTraceMode { get; set; } + public NServiceBus.RecoverabilityInstrumentationOptions Recoverability { get; } public NServiceBus.TraceMode SendTraceMode { get; set; } public bool UseMessageDestinationInSpanNames { get; set; } } @@ -900,6 +908,11 @@ namespace NServiceBus { public static NServiceBus.RecoverabilitySettings Recoverability(this NServiceBus.EndpointConfiguration configuration) { } } + public class RecoverabilityInstrumentationOptions + { + public RecoverabilityInstrumentationOptions() { } + public NServiceBus.TraceMode DelayedRetryTraceMode { get; set; } + } public class RecoverabilitySettings : NServiceBus.Configuration.AdvancedExtensibility.ExposeSettings { public NServiceBus.RecoverabilitySettings AddUnrecoverableException(System.Type exceptionType) { } diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationCompatibilityTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationCompatibilityTests.cs index 7f4332e0cde..3dee5e145f4 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationCompatibilityTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationCompatibilityTests.cs @@ -3,7 +3,6 @@ namespace NServiceBus.Core.Tests.OpenTelemetry; using System; using System.Collections.Generic; using System.Diagnostics; -using Extensibility; using NUnit.Framework; [TestFixture] @@ -23,7 +22,7 @@ public void ResetDistributedContextPropagator() LegacyContextPropagation.ResetUseDistributedContextPropagator(); } - delegate void Writer(Activity activity, Dictionary headers, ContextBag context); + delegate void Writer(Activity activity, Dictionary headers); delegate void Reader(Activity activity, IDictionary headers); static readonly Writer LegacyWrite = LegacyContextPropagation.PropagateContextToHeaders; @@ -48,7 +47,7 @@ static Dictionary Send(string value, Writer write) sender.AddBaggage("key", value); var headers = new Dictionary(); - write(sender, headers, new ContextBag()); + write(sender, headers); sender.Stop(); return headers; } diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs index fbe65df4b51..0984daf14fd 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs @@ -3,7 +3,6 @@ namespace NServiceBus.Core.Tests.OpenTelemetry; using System; using System.Collections.Generic; using System.Diagnostics; -using Extensibility; using NUnit.Framework; [TestFixture] @@ -27,7 +26,7 @@ public void Default_uses_legacy_percent_encoded_baggage_format() activity.AddBaggage("serverNode", "DF 28"); var headers = new Dictionary(); - ContextPropagation.PropagateContextToHeaders(activity, headers, new ContextBag()); + ContextPropagation.PropagateContextToHeaders(activity, headers); Assert.That(headers[Headers.DiagnosticsBaggage], Is.EqualTo("serverNode=DF%2028")); } @@ -45,7 +44,7 @@ public void Default_does_not_throw_when_baggage_value_is_null() var headers = new Dictionary(); - Assert.DoesNotThrow(() => LegacyContextPropagation.PropagateContextToHeaders(activity, headers, new ContextBag())); + Assert.DoesNotThrow(() => LegacyContextPropagation.PropagateContextToHeaders(activity, headers)); Assert.That(headers[Headers.DiagnosticsBaggage], Is.EqualTo("test=")); } @@ -58,7 +57,7 @@ public void Default_round_trip_preserves_value_whitespace() outgoing.AddBaggage("key1", " leading-and-trailing "); var headers = new Dictionary(); - ContextPropagation.PropagateContextToHeaders(outgoing, headers, new ContextBag()); + ContextPropagation.PropagateContextToHeaders(outgoing, headers); using var incoming = new Activity(ActivityNames.IncomingMessageActivityName); incoming.SetIdFormat(ActivityIdFormat.W3C); diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagationTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagationTests.cs index 4c62f70cd71..4e90b6b3e1b 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagationTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/LegacyContextPropagationTests.cs @@ -5,7 +5,6 @@ using System.Collections.Generic; using System.Diagnostics; using System.Linq; -using Extensibility; using NUnit.Framework; [TestFixture] @@ -20,7 +19,7 @@ public void Propagate_activity_id_to_header() var headers = new Dictionary(); - ContextPropagation.PropagateContextToHeaders(activity, headers, new ContextBag()); + ContextPropagation.PropagateContextToHeaders(activity, headers); Assert.That(activity.Id, Is.EqualTo(headers[Headers.DiagnosticsTraceParent])); } @@ -30,41 +29,11 @@ public void Should_not_set_header_without_activity() { var headers = new Dictionary(); - ContextPropagation.PropagateContextToHeaders(null, headers, new ContextBag()); + ContextPropagation.PropagateContextToHeaders(null, headers); Assert.That(headers, Is.Empty); } - [Test] - public void Should_set_start_new_trace_header_when_adding_trace_parent_header() - { - using var activity = new Activity("test"); - activity.SetIdFormat(ActivityIdFormat.W3C); - activity.Start(); - - var headers = new Dictionary(); - var contextBag = new ContextBag(); - contextBag.Set(Headers.StartNewTrace, bool.TrueString); - ContextPropagation.PropagateContextToHeaders(activity, headers, contextBag); - - using (Assert.EnterMultipleScope()) - { - Assert.That(headers.ContainsKey(Headers.StartNewTrace), Is.True, bool.TrueString); - Assert.That(bool.TrueString, Is.EqualTo(headers[Headers.StartNewTrace])); - } - } - - [Test] - public void Should_not_set_start_new_trace_header_when_no_trace_parent_header_is_added() - { - var headers = new Dictionary(); - var contextBag = new ContextBag(); - contextBag.Set(Headers.StartNewTrace, bool.TrueString); - ContextPropagation.PropagateContextToHeaders(null, headers, contextBag); - - Assert.That(headers.ContainsKey(Headers.StartNewTrace), Is.False); - } - [Test] public void Overwrites_existing_propagation_header() { @@ -77,7 +46,7 @@ public void Overwrites_existing_propagation_header() { Headers.DiagnosticsTraceParent, "some existing id" } }; - ContextPropagation.PropagateContextToHeaders(activity, headers, new ContextBag()); + ContextPropagation.PropagateContextToHeaders(activity, headers); Assert.That(activity.Id, Is.EqualTo(headers[Headers.DiagnosticsTraceParent])); } @@ -95,7 +64,7 @@ public void Should_not_throw_when_baggage_value_is_null() var headers = new Dictionary(); - Assert.DoesNotThrow(() => ContextPropagation.PropagateContextToHeaders(activity, headers, new ContextBag())); + Assert.DoesNotThrow(() => ContextPropagation.PropagateContextToHeaders(activity, headers)); } [TestCaseSource(nameof(TestCases))] @@ -141,7 +110,7 @@ public void Can_propagate_baggage_from_activity_to_header(ContextPropagationTest activity.AddBaggage(baggageItem.Key, baggageItem.Value); } - ContextPropagation.PropagateContextToHeaders(activity, headers, new ContextBag()); + ContextPropagation.PropagateContextToHeaders(activity, headers); var baggageHeaderSet = headers.TryGetValue(Headers.DiagnosticsBaggage, out var baggageValue); @@ -175,7 +144,7 @@ public void Can_roundtrip_baggage(ContextPropagationTestCase testCase) outgoingActivity.AddBaggage(baggageItem.Key, baggageItem.Value); } - ContextPropagation.PropagateContextToHeaders(outgoingActivity, outgoingHeaders, new ContextBag()); + ContextPropagation.PropagateContextToHeaders(outgoingActivity, outgoingHeaders); // Simulate wire transfer var incomingHeaders = outgoingHeaders; diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/PopulateRecoverabilityTraceMetadataBehaviorTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/PopulateRecoverabilityTraceMetadataBehaviorTests.cs index cee304d848d..cb8e2d5823b 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/PopulateRecoverabilityTraceMetadataBehaviorTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/PopulateRecoverabilityTraceMetadataBehaviorTests.cs @@ -12,7 +12,7 @@ public class PopulateRecoverabilityTraceMetadataBehaviorTests [Test] public async Task Should_not_write_metadata_when_trace_not_present() { - var behavior = new PopulateRecoverabilityTraceMetadataBehavior(); + var behavior = new PopulateRecoverabilityTraceMetadataBehavior(new InstrumentationOptions()); var context = new TestableRecoverabilityContext(); await behavior.Invoke(context, _ => Task.CompletedTask); @@ -25,10 +25,10 @@ public async Task Should_not_write_metadata_when_trace_not_present() } [Test] - [TestCaseSource(nameof(Actions))] + [TestCaseSource(nameof(ActionsThatWriteMetadata))] public async Task Should_write_metadata_when_trace_present(RecoverabilityAction recoverabilityAction) { - var behavior = new PopulateRecoverabilityTraceMetadataBehavior(); + var behavior = new PopulateRecoverabilityTraceMetadataBehavior(new InstrumentationOptions()); var context = new TestableRecoverabilityContext { @@ -45,9 +45,63 @@ public async Task Should_write_metadata_when_trace_present(RecoverabilityAction } } - static IEnumerable Actions() + [Test] + public async Task Should_not_write_metadata_for_immediate_retry() + { + var behavior = new PopulateRecoverabilityTraceMetadataBehavior(new InstrumentationOptions()); + + var context = new TestableRecoverabilityContext + { + Headers = { { Headers.DiagnosticsTraceParent, "traceparent" } }, + RecoverabilityAction = new ImmediateRetry() + }; + + await behavior.Invoke(context, _ => Task.CompletedTask); + + using (Assert.EnterMultipleScope()) + { + Assert.That(context.Headers, Does.Not.ContainKey(Headers.StartNewTrace)); + Assert.That(context.Metadata, Does.Not.ContainKey(Headers.StartNewTrace)); + } + } + + [Test] + public async Task Should_always_start_new_trace_for_move_to_error() + { + var behavior = new PopulateRecoverabilityTraceMetadataBehavior(new InstrumentationOptions()); + + var context = new TestableRecoverabilityContext + { + Headers = { { Headers.DiagnosticsTraceParent, "traceparent" } }, + RecoverabilityAction = new MoveToError("errorqueue") + }; + + await behavior.Invoke(context, _ => Task.CompletedTask); + + Assert.That(context.Metadata[Headers.StartNewTrace], Is.EqualTo(bool.TrueString)); + } + + [Test] + public async Task Should_honor_delayed_retry_trace_mode() + { + var behavior = new PopulateRecoverabilityTraceMetadataBehavior(new InstrumentationOptions + { + Recoverability = { DelayedRetryTraceMode = TraceMode.ContinueExisting } + }); + + var context = new TestableRecoverabilityContext + { + Headers = { { Headers.DiagnosticsTraceParent, "traceparent" } }, + RecoverabilityAction = new DelayedRetry(TimeSpan.FromSeconds(10)) + }; + + await behavior.Invoke(context, _ => Task.CompletedTask); + + Assert.That(context.Metadata[Headers.StartNewTrace], Is.EqualTo(bool.FalseString)); + } + + static IEnumerable ActionsThatWriteMetadata() { - yield return new ImmediateRetry(); yield return new DelayedRetry(TimeSpan.FromSeconds(10)); yield return new MoveToError("errorqueue"); } diff --git a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs index dcdd85a1c93..78f80d8ead1 100644 --- a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs @@ -15,6 +15,19 @@ public class InstrumentationOptions /// public bool UseMessageDestinationInSpanNames { get; set; } + /// + /// Controls instrumentation of the recoverability pipeline (retries and error handling). + /// + public RecoverabilityInstrumentationOptions Recoverability { get; } = new(); + + /// + /// Controls instrumentation of explicitly delayed messages (SendOptions.DelayDeliveryWith + /// / DoNotDeliverBefore and saga timeouts. + /// Recoverability-driven delayed retries are controlled separately via + /// . + /// + public DelayedDeliveryInstrumentationOptions DelayedDelivery { get; } = new(); + /// /// Controls whether the "Start dispatching" and "Finished dispatching" activity events /// are added to the incoming message span when outgoing messages are dispatched. @@ -39,3 +52,33 @@ public class InstrumentationOptions /// public TraceMode PublishTraceMode { get; set; } = TraceMode.StartNew; } + +/// +/// Controls instrumentation of the recoverability pipeline (retries and error handling). +/// +public class RecoverabilityInstrumentationOptions +{ + /// + /// Controls how the span for a delayed retry relates to the failed + /// attempt's trace. + /// + public TraceMode DelayedRetryTraceMode { get; set; } = TraceMode.StartNew; +} + +/// +/// Controls instrumentation of delayed messages. +/// +public class DelayedDeliveryInstrumentationOptions +{ + /// + /// Controls how a delayed Send relates to the sender's trace, when requested directly + /// by application code via SendOptions.DelayDeliveryWith/DoNotDeliverBefore. + /// + public TraceMode SendOperationTraceMode { get; set; } = TraceMode.StartNew; + + /// + /// Controls how a saga timeout (Saga.RequestTimeout) relates to the trace of the + /// message that requested it. + /// + public TraceMode SagaTimeoutTraceMode { get; set; } = TraceMode.StartNew; +} diff --git a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs index b108b409c92..e207ee6ce57 100644 --- a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs +++ b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs @@ -21,7 +21,7 @@ protected override void Setup(FeatureConfigurationContext context) ); context.Pipeline.Register( - new PopulateRecoverabilityTraceMetadataBehavior(), + new PopulateRecoverabilityTraceMetadataBehavior(instrumentationOptions), "Populates the recoverability metadata" ); } diff --git a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetrySendBehavior.cs b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetrySendBehavior.cs index 75338de54ea..ef86db89c94 100644 --- a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetrySendBehavior.cs +++ b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetrySendBehavior.cs @@ -5,17 +5,44 @@ namespace NServiceBus; using System; using System.Threading.Tasks; using Pipeline; +using Transport; class OpenTelemetrySendBehavior(InstrumentationOptions instrumentationOptions) : IBehavior { public Task Invoke(IOutgoingSendContext context, Func next) { // the per-message override wins over the endpoint-level default - var connector = context.Extensions.TryGet(OpenTelemetryExtensions.TraceConnectorOverrideKey, out TraceMode requestedConnector) + var operationTraceMode = context.Extensions.TryGet(OpenTelemetryExtensions.TraceConnectorOverrideKey, out TraceMode requestedConnector) ? requestedConnector : instrumentationOptions.SendTraceMode; - context.Headers[Headers.StartNewTrace] = connector == TraceMode.StartNew ? bool.TrueString : bool.FalseString; + if (operationTraceMode == TraceMode.StartNew) + { + context.Headers[Headers.StartNewTrace] = bool.TrueString; + } + else + { + // This is needed to ensure the trace continuation behavior is backwards compatible. + // If the message is delayed, we always start a new trace unless different behavior is explicitly configured. + var isDelayed = context.Extensions.TryGet(out var dispatchProperties) && + (dispatchProperties.DelayDeliveryWith != null || dispatchProperties.DoNotDeliverBefore != null); + + bool startNewTrace; + if (isDelayed) + { + var isSagaTimeout = context.Headers.ContainsKey(Headers.IsSagaTimeoutMessage); + var mode = isSagaTimeout + ? instrumentationOptions.DelayedDelivery.SagaTimeoutTraceMode + : instrumentationOptions.DelayedDelivery.SendOperationTraceMode; + startNewTrace = mode == TraceMode.StartNew; + } + else + { + startNewTrace = false; + } + + context.Headers[Headers.StartNewTrace] = startNewTrace ? bool.TrueString : bool.FalseString; + } return next(context); } diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs index d5dcf4ec07a..bca611216d9 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs @@ -6,14 +6,9 @@ namespace NServiceBus; using Pipeline; using Transport; -sealed class ActivityFactory : IActivityFactory +sealed class ActivityFactory(InstrumentationOptions options) : IActivityFactory { - public ActivityFactory(InstrumentationOptions options) - { - Options = options; - } - - public InstrumentationOptions Options { get; } + public InstrumentationOptions Options { get; } = options; public Activity? StartIncomingPipelineActivity(MessageContext context) { diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs index 8df14752012..dc0e568dd87 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs @@ -4,17 +4,16 @@ namespace NServiceBus; using System.Collections.Generic; using System.Diagnostics; -using Extensibility; static class ContextPropagation { - public static void PropagateContextToHeaders(Activity? activity, Dictionary headers, ContextBag contextBag) + public static void PropagateContextToHeaders(Activity? activity, Dictionary headers) { // TODO: investigate if we need to improve the switch check for better performance // Removed in v11, see obsolete_v11.cs if (!LegacyContextPropagation.UseDistributedContextPropagator) { - LegacyContextPropagation.PropagateContextToHeaders(activity, headers, contextBag); + LegacyContextPropagation.PropagateContextToHeaders(activity, headers); return; } @@ -26,14 +25,6 @@ public static void PropagateContextToHeaders(Activity? activity, Dictionary(Headers.StartNewTrace, out var startNewTrace); - - if (traceParentExists && startNewTraceOnReceive) - { - headers[Headers.StartNewTrace] = startNewTrace!; - } } public static void PropagateContextFromHeaders(Activity? activity, IDictionary headers) diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/PopulateRecoverabilityTraceMetadataBehavior.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/PopulateRecoverabilityTraceMetadataBehavior.cs index 0bb256ecef2..0a1a5c50881 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/PopulateRecoverabilityTraceMetadataBehavior.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/PopulateRecoverabilityTraceMetadataBehavior.cs @@ -4,7 +4,7 @@ namespace NServiceBus; using System.Threading.Tasks; using Pipeline; -class PopulateRecoverabilityTraceMetadataBehavior : IBehavior +class PopulateRecoverabilityTraceMetadataBehavior(InstrumentationOptions instrumentationOptions) : IBehavior { public Task Invoke(IRecoverabilityContext context, Func next) { @@ -13,9 +13,19 @@ public Task Invoke(IRecoverabilityContext context, Func cachedUseDistributedContextPropagator = SwitchState.Unchecked; - public static void PropagateContextToHeaders(Activity? activity, Dictionary headers, ContextBag contextBag) + public static void PropagateContextToHeaders(Activity? activity, Dictionary headers) { if (activity is null) { @@ -79,12 +78,6 @@ public static void PropagateContextToHeaders(Activity? activity, Dictionary(Headers.StartNewTrace, out var headerContent)) - { - headers[Headers.StartNewTrace] = headerContent; - } - var baggage = string.Join(",", activity.Baggage.Select(item => $"{item.Key}={Uri.EscapeDataString(item.Value ?? string.Empty)}")); if (!string.IsNullOrEmpty(baggage)) { diff --git a/src/NServiceBus.Core/Pipeline/Outgoing/AttachSenderRelatedInfoOnMessageBehavior.cs b/src/NServiceBus.Core/Pipeline/Outgoing/AttachSenderRelatedInfoOnMessageBehavior.cs index 3472a9355f5..7ba822fbc6e 100644 --- a/src/NServiceBus.Core/Pipeline/Outgoing/AttachSenderRelatedInfoOnMessageBehavior.cs +++ b/src/NServiceBus.Core/Pipeline/Outgoing/AttachSenderRelatedInfoOnMessageBehavior.cs @@ -35,12 +35,10 @@ public Task Invoke(IRoutingContext context, Func next) { var timeDelay = dispatchProperties.DelayDeliveryWith.Delay; message.Headers[Headers.DeliverAt] = DateTimeOffsetHelper.ToWireFormattedString(utcNow.Add(timeDelay)); - context.Extensions.Set(Headers.StartNewTrace, bool.TrueString); } else if (dispatchProperties.DoNotDeliverBefore != null) { message.Headers[Headers.DeliverAt] = DateTimeOffsetHelper.ToWireFormattedString(dispatchProperties.DoNotDeliverBefore.At); - context.Extensions.Set(Headers.StartNewTrace, bool.TrueString); } } } diff --git a/src/NServiceBus.Core/Pipeline/Outgoing/RoutingToDispatchConnector.cs b/src/NServiceBus.Core/Pipeline/Outgoing/RoutingToDispatchConnector.cs index 55f62023b4b..3243dd7b150 100644 --- a/src/NServiceBus.Core/Pipeline/Outgoing/RoutingToDispatchConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Outgoing/RoutingToDispatchConnector.cs @@ -33,7 +33,7 @@ public override Task Invoke(IRoutingContext context, Func Date: Wed, 29 Jul 2026 13:57:16 +0300 Subject: [PATCH 15/37] Added error.type (#7885) --- ...ctivityTagsTests.Verify_ActivityTags.approved.txt | 3 ++- .../OpenTelemetry/Tracing/ActivityExtensions.cs | 12 ++---------- .../OpenTelemetry/Tracing/ActivityTags.cs | 1 + 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/ActivityTagsTests.Verify_ActivityTags.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/ActivityTagsTests.Verify_ActivityTags.approved.txt index 27d7bc2fc19..69a83e1f04c 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/ActivityTagsTests.Verify_ActivityTags.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/ActivityTagsTests.Verify_ActivityTags.approved.txt @@ -38,6 +38,7 @@ "HandlerType => nservicebus.handler.handler_type", "HandlerSagaId => nservicebus.handler.saga_id", "EventTypes => nservicebus.event_types", - "CancelledTask => nservicebus.cancelled" + "CancelledTask => nservicebus.cancelled", + "ErrorType => error.type" ] } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs index 30b0bf1066f..4d7586fae2a 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs @@ -3,7 +3,6 @@ namespace NServiceBus; using System; -using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Threading.Tasks; @@ -41,15 +40,8 @@ public static void SetErrorStatus(this Activity activity, Exception ex) activity.SetStatus(ActivityStatusCode.Error, ex.Message); activity.SetTag("otel.status_code", "ERROR"); activity.SetTag("otel.status_description", ex.Message); - - - activity.AddEvent(new ActivityEvent("exception", DateTimeOffset.UtcNow, - [ - new KeyValuePair("exception.escaped", true), - new KeyValuePair("exception.type", ex.GetType()), - new KeyValuePair("exception.message", ex.Message), - new KeyValuePair("exception.stacktrace", ex.ToString()) - ])); + activity.SetTag(ActivityTags.ErrorType, ex.GetType().FullName); + activity.AddException(ex, new TagList { { "exception.escaped", true } }); if (ex is TaskCanceledException) { diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityTags.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityTags.cs index c7597a63f2e..fbbb2e0caad 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityTags.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityTags.cs @@ -41,4 +41,5 @@ static class ActivityTags public const string HandlerSagaId = "nservicebus.handler.saga_id"; public const string EventTypes = "nservicebus.event_types"; public const string CancelledTask = "nservicebus.cancelled"; + public const string ErrorType = "error.type"; } \ No newline at end of file From 7e6873bd7c990d51357207a0d47a8e865974b5cd Mon Sep 17 00:00:00 2001 From: Tomasz Masternak Date: Fri, 31 Jul 2026 08:44:17 +0200 Subject: [PATCH 16/37] Spans for recoverability actions (#7890) * Add tracing for recoverability pipeline using ActivityFactory * Refactor recoverability pipeline to use `Activity` fallback for tagging recoverability actions * Replace `new NoOpActivityFactory()` with `NoOpActivityFactory.Instance` in tests and implementation to enforce singleton usage. * Normalize recoverability action tags and update Activity display names for better trace clarity in recoverability pipeline. * Normalize recoverability action tags and update Activity display names to improve trace clarity in the recoverability pipeline. * Fixed approval file * Update ActivityTagsTests.Verify_ActivityTags.approved.txt * recoverabilty span should not be parenting the follow-up processing attempts. Span tags and display names update moved directly after recoverability aciton resolution * refactor: centralize recoverability span updates and add display name adjustments * fix tests * fix: correct typo in DelayedRetryOperation constant * Enable the Recoverability ActivitySource in acceptance tests; add dedicated recoverability span tests TestingActivityListener now accepts multiple source names so OpenTelemetryAcceptanceTest can subscribe to both "NServiceBus.Core" and "NServiceBus.Core.Recoverability" - previously only Main was subscribed, so recoverability spans were silently never created in any acceptance test. Adds When_recoverability_action_occurs.cs covering all four recoverability outcomes (immediate retry, delayed retry, move to error, discard): span creation, display name, and the nservicebus.recoverability_action tag, plus the UseMessageDestinationInSpanNames toggle for immediate retry. * Simplify When_recoverability_action_occurs: one endpoint, no ErrorSpy Collapsed all five scenarios onto a single RecoverabilityEndpoint, moving the per-test differences (Recoverability policy, UseMessageDestinationInSpanNames) into each test's CustomConfig callback instead of separate endpoint classes. Replaced the ErrorSpy endpoint (used only to signal test completion for the move-to-error case) with a Done() predicate that waits directly on the recoverability span being captured - the actual signal the test cares about, rather than an indirect one requiring a second endpoint and message hop. * Combine immediate/delayed/move-to-error/discard into one recoverability test One endpoint config now drives a single message through the default policy's full immediate-retry -> delayed-retry -> move-to-error cascade, plus a second message routed straight to Discard via a CustomPolicy branch that otherwise delegates to DefaultRecoverabilityPolicy.Invoke for the first message. Asserts by matching on the nservicebus.recoverability_action tag rather than activity position/count, since the exact number of immediate-retry attempts before falling through isn't a contract worth pinning down. Kept the destination-naming test separate since it needs a different Tracing() configuration that can't share the same CustomConfig block. * bug fixes and att for recoverability spans * Fix ActivitySource versioning and make sure the versiosn are checked in tests * small ActivityFactory refactor * fix: add missing newline at end of ActivityTagsTests.cs --------- Co-authored-by: Ramon Smits Co-authored-by: Irina Dominte --- .../OpenTelemetryAcceptanceTest.cs | 2 +- .../Traces/TestingActivityListener.cs | 9 +- .../When_recoverability_action_occurs.cs | 112 ++++++++++++++++++ ...TagsTests.Verify_ActivityTags.approved.txt | 18 ++- .../OpenTelemetry/ActivityTagsTests.cs | 12 +- .../RecoverabilityExecutorTests.cs | 7 +- .../RoutingToDispatchConnectorTests.cs | 20 ++-- .../Unicast/LoadHandlersConnectorTests.cs | 8 +- .../Tracing/ActivityDisplayNames.cs | 5 + .../OpenTelemetry/Tracing/ActivityFactory.cs | 110 ++++++++++++++--- .../OpenTelemetry/Tracing/ActivityNames.cs | 1 + .../OpenTelemetry/Tracing/ActivitySources.cs | 4 + .../OpenTelemetry/Tracing/ActivityTags.cs | 1 + .../OpenTelemetry/Tracing/IActivityFactory.cs | 2 + .../Tracing/NoOpActivityFactory.cs | 8 +- .../Receiving/ReceiveComponent.cs | 4 +- .../Recoverability/RecoverabilityComponent.cs | 7 +- .../RecoverabilityPipelineExecutor.cs | 70 +++++------ .../RecoverabilityRoutingConnector.cs | 2 +- 19 files changed, 314 insertions(+), 88 deletions(-) create mode 100644 src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_recoverability_action_occurs.cs diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/OpenTelemetryAcceptanceTest.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/OpenTelemetryAcceptanceTest.cs index 99b8b43ab54..a2748753077 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/OpenTelemetryAcceptanceTest.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/OpenTelemetryAcceptanceTest.cs @@ -9,7 +9,7 @@ public abstract class OpenTelemetryAcceptanceTest : NServiceBusAcceptanceTest protected TestingActivityListener NServiceBusActivityListener { get; private set; } [SetUp] - public void Setup() => NServiceBusActivityListener = TestingActivityListener.SetupDiagnosticListener("NServiceBus.Core"); + public void Setup() => NServiceBusActivityListener = TestingActivityListener.SetupDiagnosticListener("NServiceBus.Core", "NServiceBus.Core.Recoverability"); [TearDown] public void Cleanup() diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/TestingActivityListener.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/TestingActivityListener.cs index a5fdb5ce06a..38dc6113d83 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/TestingActivityListener.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/TestingActivityListener.cs @@ -11,19 +11,19 @@ public class TestingActivityListener : IDisposable { readonly ActivityListener activityListener; - public static TestingActivityListener SetupDiagnosticListener(string sourceName) + public static TestingActivityListener SetupDiagnosticListener(params string[] sourceNames) { - var testingListener = new TestingActivityListener(sourceName); + var testingListener = new TestingActivityListener(sourceNames); ActivitySource.AddActivityListener(testingListener.activityListener); return testingListener; } - TestingActivityListener(string sourceName = null) + TestingActivityListener(params string[] sourceNames) { activityListener = new ActivityListener { - ShouldListenTo = source => string.IsNullOrEmpty(sourceName) || source.Name == sourceName, + ShouldListenTo = source => sourceNames.Length == 0 || sourceNames.Contains(source.Name), Sample = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllData, SampleUsingParentId = (ref ActivityCreationOptions options) => ActivitySamplingResult.AllData }; @@ -62,4 +62,5 @@ public static List GetReceiveMessageActivities(this ConcurrentQueue GetSendMessageActivities(this ConcurrentQueue activities) => activities.Where(a => a.OperationName == "NServiceBus.Diagnostics.SendMessage").ToList(); public static List GetPublishEventActivities(this ConcurrentQueue activities) => activities.Where(a => a.OperationName == "NServiceBus.Diagnostics.PublishMessage").ToList(); public static List GetInvokedHandlerActivities(this ConcurrentQueue activities) => activities.Where(a => a.OperationName == "NServiceBus.Diagnostics.InvokeHandler").ToList(); + public static List GetRecoverabilityActivities(this ConcurrentQueue activities) => activities.Where(a => a.OperationName == "NServiceBus.Diagnostics.Recoverability").ToList(); } diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_recoverability_action_occurs.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_recoverability_action_occurs.cs new file mode 100644 index 00000000000..ec458ed6ba7 --- /dev/null +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_recoverability_action_occurs.cs @@ -0,0 +1,112 @@ +namespace NServiceBus.AcceptanceTests.Core.OpenTelemetry.Traces; + +using System; +using System.Linq; +using System.Threading.Tasks; +using AcceptanceTesting; +using AcceptanceTesting.Customization; +using EndpointTemplates; +using NUnit.Framework; + +public class When_recoverability_action_occurs : OpenTelemetryAcceptanceTest +{ + [Test] + public async Task Should_create_spans_for_all_recoverability_actions() + { + await Scenario.Define() + .WithEndpoint(b => b + .CustomConfig(c => c.Recoverability() + .Immediate(i => i.NumberOfRetries(1)) + .Delayed(i => i.NumberOfRetries(1).TimeIncrease(TimeSpan.FromMilliseconds(1))) + .CustomPolicy((cfg, errorContext) => + errorContext.Headers[Headers.EnclosedMessageTypes].Contains(nameof(DiscardMessage)) + ? RecoverabilityAction.Discard("test discard reason") + : DefaultRecoverabilityPolicy.Invoke(cfg, errorContext))) + .DoNotFailOnErrorMessages() + .When(async s => + { + await s.SendLocal(new FailingMessage()); + await s.SendLocal(new DiscardMessage()); + })) + .Done(_ => ActionTags().Contains("move_to_error") && ActionTags().Contains("discard")) + .Run(); + + var activities = NServiceBusActivityListener.CompletedActivities.GetRecoverabilityActivities(); + + var immediateRetry = activities.FirstOrDefault(a => (string)a.GetTagItem(ActivityTagName) == "immediate_retry"); + var delayedRetry = activities.FirstOrDefault(a => (string)a.GetTagItem(ActivityTagName) == "delayed_retry"); + var moveToError = activities.Single(a => (string)a.GetTagItem(ActivityTagName) == "move_to_error"); + var discard = activities.Single(a => (string)a.GetTagItem(ActivityTagName) == "discard"); + + using (Assert.EnterMultipleScope()) + { + Assert.That(immediateRetry, Is.Not.Null, "expected at least one immediate retry span"); + Assert.That(immediateRetry.DisplayName, Is.EqualTo("immediate retry")); + + Assert.That(delayedRetry, Is.Not.Null, "expected at least one delayed retry span"); + Assert.That(delayedRetry.DisplayName, Is.EqualTo("delayed retry")); + + Assert.That(moveToError.DisplayName, Does.StartWith("move to ")); + Assert.That(discard.DisplayName, Is.EqualTo("discard")); + } + } + + [Test] + public async Task Should_include_destination_in_display_name_when_opted_in() + { + await Scenario.Define() + .WithEndpoint(b => b + .CustomConfig(c => + { + c.Recoverability().Immediate(i => i.NumberOfRetries(1)).Delayed(i => i.NumberOfRetries(0)); + c.Tracing().UseMessageDestinationInSpanNames = true; + }) + .DoNotFailOnErrorMessages() + .When(s => s.SendLocal(new FailingMessage()))) + .Done(_ => ActionTags().Contains("move_to_error")) + .Run(); + + var immediateRetry = NServiceBusActivityListener.CompletedActivities.GetRecoverabilityActivities() + .First(a => (string)a.GetTagItem(ActivityTagName) == "immediate_retry"); + + var endpointName = Conventions.EndpointNamingConvention(typeof(RecoverabilityEndpoint)); + Assert.That(immediateRetry.DisplayName, Is.EqualTo($"immediate retry {endpointName}")); + } + + string[] ActionTags() => + NServiceBusActivityListener.CompletedActivities.GetRecoverabilityActivities() + .Select(a => (string)a.GetTagItem(ActivityTagName)) + .ToArray(); + + const string ActivityTagName = "nservicebus.recoverability_action"; + + public class Context : ScenarioContext; + + public class RecoverabilityEndpoint : EndpointConfigurationBuilder + { + public RecoverabilityEndpoint() + { + var template = new DefaultServer + { + TransportConfiguration = new ConfigureEndpointAcceptanceTestingTransport(false, true) + }; + EndpointSetup(template, (c, _) => { }, metadata => { }); + } + + [Handler] + public class FailingMessageHandler : IHandleMessages + { + public Task Handle(FailingMessage message, IMessageHandlerContext context) => throw new SimulatedException("always fails"); + } + + [Handler] + public class DiscardMessageHandler : IHandleMessages + { + public Task Handle(DiscardMessage message, IMessageHandlerContext context) => throw new SimulatedException("always fails"); + } + } + + public class FailingMessage : IMessage; + + public class DiscardMessage : IMessage; +} diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/ActivityTagsTests.Verify_ActivityTags.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/ActivityTagsTests.Verify_ActivityTags.approved.txt index 69a83e1f04c..c012daa39d0 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/ActivityTagsTests.Verify_ActivityTags.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/ActivityTagsTests.Verify_ActivityTags.approved.txt @@ -1,6 +1,5 @@ { "Note": "Changes to activity tags should result in ActivitySource version updates", - "ActivitySourceVersion": "0.1.0", "Tags": [ "SagaId => nservicebus.saga.saga_id", "MessageId => nservicebus.message_id", @@ -39,6 +38,21 @@ "HandlerSagaId => nservicebus.handler.saga_id", "EventTypes => nservicebus.event_types", "CancelledTask => nservicebus.cancelled", - "ErrorType => error.type" + "ErrorType => error.type", + "RecoverabilityAction => nservicebus.recoverability_action" + ], + "ActivitySourceVersions": [ + { + "Name": "Main", + "Version": "0.1.0" + }, + { + "Name": "Handler", + "Version": "0.1.0" + }, + { + "Name": "Recoverability", + "Version": "0.1.0" + } ] } \ No newline at end of file diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityTagsTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityTagsTests.cs index 5503c58bbef..a8adbd49f86 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityTagsTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityTagsTests.cs @@ -14,14 +14,18 @@ public void Verify_ActivityTags() var activityTags = typeof(ActivityTags) .GetFields(BindingFlags.Public | BindingFlags.Static) .Where(fi => fi.IsLiteral && !fi.IsInitOnly) - .Select(x => $"{x.Name} => {x.GetRawConstantValue()}") - .ToList(); + .Select(x => $"{x.Name} => {x.GetRawConstantValue()}"); Approver.Verify(new { Note = "Changes to activity tags should result in ActivitySource version updates", - ActivitySourceVersion = ActivitySources.Main.Version, - Tags = activityTags + Tags = activityTags, + ActivitySourceVersions = new[] + { + new { Name = nameof(ActivitySources.Main), ActivitySources.Main.Version }, + new { Name = nameof(ActivitySources.Handler), ActivitySources.Handler.Version }, + new { Name = nameof(ActivitySources.Recoverability), ActivitySources.Recoverability.Version } + } }); } } \ No newline at end of file diff --git a/src/NServiceBus.Core.Tests/Recoverability/RecoverabilityExecutorTests.cs b/src/NServiceBus.Core.Tests/Recoverability/RecoverabilityExecutorTests.cs index 1b3a92cf0a9..af8c3d265cb 100644 --- a/src/NServiceBus.Core.Tests/Recoverability/RecoverabilityExecutorTests.cs +++ b/src/NServiceBus.Core.Tests/Recoverability/RecoverabilityExecutorTests.cs @@ -67,10 +67,13 @@ static RecoverabilityPipelineExecutor CreateRecoverabilityExecutor(Testa new ServiceCollection().BuildServiceProvider(), // TODO: Does not get disposed new ThrowingPipelineCache(), new TestableMessageOperations(), - null, (_, _) => RecoverabilityAction.Discard("test"), + null, + (_, _) => RecoverabilityAction.Discard("test"), recoverabilityPipeline, new FaultMetadataExtractor([], _ => { }), - null); + null, + NoOpActivityFactory.Instance + ); return executor; } diff --git a/src/NServiceBus.Core.Tests/Routing/RoutingToDispatchConnectorTests.cs b/src/NServiceBus.Core.Tests/Routing/RoutingToDispatchConnectorTests.cs index da97e86e33f..0d8293f2340 100644 --- a/src/NServiceBus.Core.Tests/Routing/RoutingToDispatchConnectorTests.cs +++ b/src/NServiceBus.Core.Tests/Routing/RoutingToDispatchConnectorTests.cs @@ -17,7 +17,7 @@ public class RoutingToDispatchConnectorTests [Test] public async Task Should_preserve_message_state_for_one_routing_strategy_for_allocation_reasons() { - var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); + var behavior = new RoutingToDispatchConnector(NoOpActivityFactory.Instance); IEnumerable operations = null; var testableRoutingContext = new TestableRoutingContext { @@ -59,7 +59,7 @@ await behavior.Invoke(testableRoutingContext, context => [Test] public async Task Should_copy_message_state_for_multiple_routing_strategies() { - var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); + var behavior = new RoutingToDispatchConnector(NoOpActivityFactory.Instance); List operations = null; var testableRoutingContext = new TestableRoutingContext { @@ -135,7 +135,7 @@ await behavior.Invoke(testableRoutingContext, context => [Test] public async Task Should_preserve_headers_generated_by_custom_routing_strategy() { - var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); + var behavior = new RoutingToDispatchConnector(NoOpActivityFactory.Instance); Dictionary headers = null; await behavior.Invoke(new TestableRoutingContext { RoutingStrategies = [new HeaderModifyingRoutingStrategy()] }, context => { @@ -153,7 +153,7 @@ public async Task Should_dispatch_immediately_if_user_requested() options.RequireImmediateDispatch(); var dispatched = false; - var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); + var behavior = new RoutingToDispatchConnector(NoOpActivityFactory.Instance); var message = new OutgoingMessage("ID", [], Array.Empty()); await behavior.Invoke(new RoutingContext(message, @@ -170,7 +170,7 @@ await behavior.Invoke(new RoutingContext(message, public async Task Should_dispatch_immediately_if_not_sending_from_a_handler() { var dispatched = false; - var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); + var behavior = new RoutingToDispatchConnector(NoOpActivityFactory.Instance); var message = new OutgoingMessage("ID", [], Array.Empty()); await behavior.Invoke(new RoutingContext(message, @@ -187,7 +187,7 @@ await behavior.Invoke(new RoutingContext(message, public async Task Should_not_dispatch_by_default() { var dispatched = false; - var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); + var behavior = new RoutingToDispatchConnector(NoOpActivityFactory.Instance); var message = new OutgoingMessage("ID", [], Array.Empty()); await behavior.Invoke(new RoutingContext(message, @@ -203,7 +203,7 @@ await behavior.Invoke(new RoutingContext(message, [Test] public async Task Should_promote_message_headers_to_pipeline_activity() { - var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); + var behavior = new RoutingToDispatchConnector(NoOpActivityFactory.Instance); var routingContext = new TestableRoutingContext(); routingContext.Message.Headers[Headers.ContentType] = "test content type"; // one of the headers that will be mapped to tags @@ -257,7 +257,7 @@ class MyMessage : IMessage; [Test] public async Task Should_merge_receive_properties_when_declared_by_transport() { - var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); + var behavior = new RoutingToDispatchConnector(NoOpActivityFactory.Instance); var receiveProperties = new ReceiveProperties(new Dictionary { @@ -290,7 +290,7 @@ await behavior.Invoke(routingContext, context => [Test] public async Task Should_not_override_user_set_dispatch_property() { - var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); + var behavior = new RoutingToDispatchConnector(NoOpActivityFactory.Instance); var receiveProperties = new ReceiveProperties(new Dictionary { @@ -324,7 +324,7 @@ await behavior.Invoke(routingContext, context => [Test] public async Task Should_preserve_user_dispatch_properties_even_with_receive_properties() { - var behavior = new RoutingToDispatchConnector(new NoOpActivityFactory()); + var behavior = new RoutingToDispatchConnector(NoOpActivityFactory.Instance); var receiveProperties = new ReceiveProperties(new Dictionary { diff --git a/src/NServiceBus.Core.Tests/Unicast/LoadHandlersConnectorTests.cs b/src/NServiceBus.Core.Tests/Unicast/LoadHandlersConnectorTests.cs index 42c5a2cd0d2..60b8a81cca0 100644 --- a/src/NServiceBus.Core.Tests/Unicast/LoadHandlersConnectorTests.cs +++ b/src/NServiceBus.Core.Tests/Unicast/LoadHandlersConnectorTests.cs @@ -16,7 +16,7 @@ public class LoadHandlersConnectorTests [Test] public void Should_throw_when_there_are_no_registered_message_handlers() { - var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), new NoOpActivityFactory()); + var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance); var context = new TestableIncomingLogicalMessageContext(); @@ -29,7 +29,7 @@ public void Should_throw_when_there_are_no_registered_message_handlers() [Test] public void Should_throw_if_ambient_transaction_is_different_from_scope_used_by_transport() { - var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), new NoOpActivityFactory()); + var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance); var context = new TestableIncomingLogicalMessageContext(); @@ -49,7 +49,7 @@ public void Should_throw_if_ambient_transaction_is_different_from_scope_used_by_ [Test] public void Should_throw_if_ambient_transaction_suppressed_when_transport_uses_a_scope() { - var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), new NoOpActivityFactory()); + var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance); var context = new TestableIncomingLogicalMessageContext(); @@ -77,7 +77,7 @@ public void Should_not_throw_if_ambient_scope_is_same_as_transport_scope() context.Services.AddSingleton(); context.Extensions.Set(new NoOpOutboxTransaction()); - var behavior = new LoadHandlersConnector(messageHandlerRegistry, new NoOpActivityFactory()); + var behavior = new LoadHandlersConnector(messageHandlerRegistry, NoOpActivityFactory.Instance); using (new TransactionScope(TransactionScopeAsyncFlowOption.Enabled)) { diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs index cf56402f4dd..50b874bfac3 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs @@ -10,9 +10,14 @@ static class ActivityDisplayNames public const string UnsubscribeEvent = "unsubscribe event"; public const string SendMessage = "send message"; public const string ReplyMessage = "reply"; + public const string Recoverability = "recover"; // Operation-only prefixes used when UseMessageDestinationInSpanNames is enabled internal const string ProcessOperation = "process"; internal const string PublishOperation = "publish"; internal const string SendOperation = "send"; + internal const string ImmediateRetryOperation = "immediate retry"; + internal const string DelayedRetryOperation = "delayed retry"; + internal const string MoveToErrorOperation = "move to"; + internal const string DiscardOperation = "discard"; } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs index bca611216d9..ec6bce50648 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs @@ -2,7 +2,9 @@ namespace NServiceBus; +using System.Collections.Generic; using System.Diagnostics; +using Extensibility; using Pipeline; using Transport; @@ -10,20 +12,20 @@ sealed class ActivityFactory(InstrumentationOptions options) : IActivityFactory { public InstrumentationOptions Options { get; } = options; - public Activity? StartIncomingPipelineActivity(MessageContext context) + static Activity? CreateActivityFromIncomingMessage(ActivitySource activitySource, string activityName, Dictionary headers, string nativeMessageId, ContextBag extensions) { // CreateActivity is a no-op if there are no listeners but we are doing a fast path check // here nonetheless to avoid having to parse headers, access the extension bag, etc. - if (!ActivitySources.Main.HasListeners()) + if (!activitySource.HasListeners()) { return null; } Activity? activity; - var incomingTraceParentExists = context.Headers.TryGetValue(Headers.DiagnosticsTraceParent, out var sendSpanId); + var incomingTraceParentExists = headers.TryGetValue(Headers.DiagnosticsTraceParent, out var sendSpanId); var activityContextCreatedFromIncomingTraceParent = ActivityContext.TryParse(sendSpanId, null, out var sendSpanContext); - if (context.Extensions.TryGet(out var transportActivity)) // attach to transport span but link receive pipeline span to send pipeline span + if (extensions.TryGet(out var transportActivity)) // attach to transport span but link receive pipeline span to send pipeline span { ActivityLink[]? links = null; if (incomingTraceParentExists && sendSpanId != transportActivity.Id) @@ -34,31 +36,31 @@ sealed class ActivityFactory(InstrumentationOptions options) : IActivityFactory } } - activity = ActivitySources.Main.CreateActivity(name: ActivityNames.IncomingMessageActivityName, + activity = activitySource.CreateActivity(name: activityName, ActivityKind.Consumer, transportActivity.Context, links: links, idFormat: ActivityIdFormat.W3C); } else if (incomingTraceParentExists && activityContextCreatedFromIncomingTraceParent) // otherwise directly create child from logical send { - var isStartNewTraceHeaderAvailable = context.Headers.TryGetValue(Headers.StartNewTrace, out var shouldStartNewTrace); + var isStartNewTraceHeaderAvailable = headers.TryGetValue(Headers.StartNewTrace, out var shouldStartNewTrace); if (isStartNewTraceHeaderAvailable && shouldStartNewTrace?.Equals(bool.TrueString) is true) { // create a new trace or root activity - ActivityLink[] links = [new ActivityLink(sendSpanContext)]; + ActivityLink[] links = [new(sendSpanContext)]; //null the current activity so that the new one is created as root https://github.com/dotnet/runtime/issues/65528#issuecomment-2613486896 Activity.Current = null; - activity = ActivitySources.Main.StartActivity(name: ActivityNames.IncomingMessageActivityName, ActivityKind.Consumer, parentContext: default, tags: null, links: links); + activity = activitySource.StartActivity(name: activityName, ActivityKind.Consumer, parentContext: default, tags: null, links: links); } else { // no new trace was requested, so start a child trace ActivityContext.TryParse(sendSpanId, null, true, out var remoteParentActivityContext); - activity = ActivitySources.Main.CreateActivity(name: ActivityNames.IncomingMessageActivityName, ActivityKind.Consumer, remoteParentActivityContext); + activity = activitySource.CreateActivity(name: activityName, ActivityKind.Consumer, remoteParentActivityContext); } } - else // otherwise start new trace + else // otherwise start a new trace { // This will set Activity.Current as parent if available - activity = ActivitySources.Main.CreateActivity(name: ActivityNames.IncomingMessageActivityName, ActivityKind.Consumer); + activity = activitySource.CreateActivity(name: activityName, ActivityKind.Consumer); } if (activity is null) @@ -66,15 +68,33 @@ sealed class ActivityFactory(InstrumentationOptions options) : IActivityFactory return activity; } - ContextPropagation.PropagateContextFromHeaders(activity, context.Headers); + ContextPropagation.PropagateContextFromHeaders(activity, headers); + + activity.SetIdFormat(ActivityIdFormat.W3C); + activity.AddTag(ActivityTags.NativeMessageId, nativeMessageId); + + ActivityDecorator.PromoteHeadersToTags(activity, headers); + + return activity; + } + + public Activity? StartIncomingPipelineActivity(MessageContext context) + { + var activity = CreateActivityFromIncomingMessage( + ActivitySources.Main, + ActivityNames.IncomingMessageActivityName, + context.Headers, + context.NativeMessageId, + context.Extensions); + + if (activity is null) + { + return activity; + } activity.DisplayName = Options.UseMessageDestinationInSpanNames ? $"{ActivityDisplayNames.ProcessOperation} {context.ReceiveAddress}" : ActivityDisplayNames.ProcessMessage; - activity.SetIdFormat(ActivityIdFormat.W3C); - activity.AddTag(ActivityTags.NativeMessageId, context.NativeMessageId); - - ActivityDecorator.PromoteHeadersToTags(activity, context.Headers); activity.Start(); @@ -123,4 +143,62 @@ sealed class ActivityFactory(InstrumentationOptions options) : IActivityFactory activity.AddTag(ActivityTags.HandlerType, messageHandler.HandlerType.FullName); return activity; } + + public Activity? StartRecoverabilityActivity(ErrorContext context) + { + var activity = CreateActivityFromIncomingMessage( + ActivitySources.Recoverability, + ActivityNames.RecoverabilityActivityName, + context.Headers, + context.NativeMessageId, + context.Extensions); + + if (activity is null) + { + return activity; + } + + activity.DisplayName = ActivityDisplayNames.Recoverability; + + activity.Start(); + + return activity; + } + + public void UpdateActivityFromRecoverabilityAction(Activity activity, RecoverabilityAction recoverabilityAction, string receiveAddress) + { + if (recoverabilityAction is ImmediateRetry) + { + activity.AddTag(ActivityTags.RecoverabilityAction, "immediate_retry"); + activity.DisplayName = ActivityDisplayNames.ImmediateRetryOperation; + + if (Options.UseMessageDestinationInSpanNames) + { + activity.DisplayName += $" {receiveAddress}"; + } + } + else if (recoverabilityAction is DelayedRetry) + { + activity.AddTag(ActivityTags.RecoverabilityAction, "delayed_retry"); + activity.DisplayName = ActivityDisplayNames.DelayedRetryOperation; + + if (Options.UseMessageDestinationInSpanNames) + { + activity.DisplayName += $" {receiveAddress}"; + } + } + else if (recoverabilityAction is MoveToError moveToError) + { + activity.AddTag(ActivityTags.RecoverabilityAction, "move_to_error"); + + activity.DisplayName = Options.UseMessageDestinationInSpanNames + ? $"{ActivityDisplayNames.MoveToErrorOperation} {moveToError.ErrorQueue}" + : $"{ActivityDisplayNames.MoveToErrorOperation} error"; + } + else if (recoverabilityAction is Discard) + { + activity.AddTag(ActivityTags.RecoverabilityAction, "discard"); + activity.DisplayName = ActivityDisplayNames.DiscardOperation; + } + } } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityNames.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityNames.cs index 0399bb931bd..5472c817385 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityNames.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityNames.cs @@ -10,4 +10,5 @@ static class ActivityNames public const string SubscribeActivityName = "NServiceBus.Diagnostics.Subscribe"; public const string UnsubscribeActivityName = "NServiceBus.Diagnostics.Unsubscribe"; public const string InvokeHandlerActivityName = "NServiceBus.Diagnostics.InvokeHandler"; + public const string RecoverabilityActivityName = "NServiceBus.Diagnostics.Recoverability"; } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivitySources.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivitySources.cs index 939305dd0eb..b3d86aeaf63 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivitySources.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivitySources.cs @@ -13,4 +13,8 @@ static class ActivitySources public static readonly ActivitySource Handler = new("NServiceBus.Core.Handler", "0.1.0"); + + public static readonly ActivitySource Recoverability = + new("NServiceBus.Core.Recoverability", + "0.1.0"); } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityTags.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityTags.cs index fbbb2e0caad..ff03de38003 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityTags.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityTags.cs @@ -42,4 +42,5 @@ static class ActivityTags public const string EventTypes = "nservicebus.event_types"; public const string CancelledTask = "nservicebus.cancelled"; public const string ErrorType = "error.type"; + public const string RecoverabilityAction = "nservicebus.recoverability_action"; } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs index daa6553a2eb..b7f536a7133 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs @@ -12,4 +12,6 @@ interface IActivityFactory Activity? StartIncomingPipelineActivity(MessageContext context); Activity? StartOutgoingPipelineActivity(string activityName, string displayName, IBehaviorContext outgoingContext); Activity? StartHandlerActivity(MessageHandler messageHandler); + Activity? StartRecoverabilityActivity(ErrorContext context); + void UpdateActivityFromRecoverabilityAction(Activity activity, RecoverabilityAction recoverabilityAction, string receiveAddress); } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs index fe1f2f82c12..6d9c92706a9 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs @@ -8,11 +8,17 @@ namespace NServiceBus; sealed class NoOpActivityFactory : IActivityFactory { - public InstrumentationOptions Options { get; } = new InstrumentationOptions(); + NoOpActivityFactory() { } + public static readonly NoOpActivityFactory Instance = new(); + public InstrumentationOptions Options { get; } = new(); public Activity? StartIncomingPipelineActivity(MessageContext context) => null; public Activity? StartOutgoingPipelineActivity(string activityName, string displayName, IBehaviorContext outgoingContext) => null; public Activity? StartHandlerActivity(MessageHandler messageHandler) => null; + public Activity? StartRecoverabilityActivity(ErrorContext context) => null; + public void UpdateActivityFromRecoverabilityAction(Activity activity, RecoverabilityAction recoverabilityAction, string receiveAddress) + { + } } \ No newline at end of file diff --git a/src/NServiceBus.Core/Receiving/ReceiveComponent.cs b/src/NServiceBus.Core/Receiving/ReceiveComponent.cs index 1ad0c78d989..4a0a5a8c1c6 100644 --- a/src/NServiceBus.Core/Receiving/ReceiveComponent.cs +++ b/src/NServiceBus.Core/Receiving/ReceiveComponent.cs @@ -180,7 +180,9 @@ public async Task Initialize( builder, pipelineCache, pipelineComponent, - messageOperations); + messageOperations, + activityFactory + ); await mainPump.Initialize( configuration.PushRuntimeSettings, diff --git a/src/NServiceBus.Core/Recoverability/RecoverabilityComponent.cs b/src/NServiceBus.Core/Recoverability/RecoverabilityComponent.cs index a3be93fa090..972bcba47d2 100644 --- a/src/NServiceBus.Core/Recoverability/RecoverabilityComponent.cs +++ b/src/NServiceBus.Core/Recoverability/RecoverabilityComponent.cs @@ -74,7 +74,8 @@ public IRecoverabilityPipelineExecutor CreateRecoverabilityPipelineExecutor( IServiceProvider serviceProvider, IPipelineCache pipelineCache, PipelineComponent pipeline, - MessageOperations messageOperations) + MessageOperations messageOperations, + IActivityFactory activityFactory) { ArgumentNullException.ThrowIfNull(recoverabilityConfig); ArgumentNullException.ThrowIfNull(faultMetadataExtractor); @@ -103,7 +104,9 @@ public IRecoverabilityPipelineExecutor CreateRecoverabilityPipelineExecutor( }, recoverabilityPipeline, faultMetadataExtractor, - (this, policy)); + (this, policy), + activityFactory + ); } public IRecoverabilityPipelineExecutor CreateSatelliteRecoverabilityExecutor( diff --git a/src/NServiceBus.Core/Recoverability/RecoverabilityPipelineExecutor.cs b/src/NServiceBus.Core/Recoverability/RecoverabilityPipelineExecutor.cs index 70f9c4ffb63..88373e89027 100644 --- a/src/NServiceBus.Core/Recoverability/RecoverabilityPipelineExecutor.cs +++ b/src/NServiceBus.Core/Recoverability/RecoverabilityPipelineExecutor.cs @@ -9,60 +9,50 @@ namespace NServiceBus; using NServiceBus.Pipeline; using Transport; -class RecoverabilityPipelineExecutor : IRecoverabilityPipelineExecutor +class RecoverabilityPipelineExecutor( + IServiceProvider serviceProvider, + IPipelineCache pipelineCache, + MessageOperations messageOperations, + RecoverabilityConfig recoverabilityConfig, + Func recoverabilityPolicy, + IPipeline recoverabilityPipeline, + FaultMetadataExtractor faultMetadataExtractor, + TState state, + IActivityFactory activityFactory) : IRecoverabilityPipelineExecutor { - public RecoverabilityPipelineExecutor( - IServiceProvider serviceProvider, - IPipelineCache pipelineCache, - MessageOperations messageOperations, - RecoverabilityConfig recoverabilityConfig, - Func recoverabilityPolicy, - IPipeline recoverabilityPipeline, - FaultMetadataExtractor faultMetadataExtractor, - TState state) - { - this.state = state; - this.serviceProvider = serviceProvider; - this.pipelineCache = pipelineCache; - this.messageOperations = messageOperations; - this.recoverabilityConfig = recoverabilityConfig; - this.recoverabilityPolicy = recoverabilityPolicy; - this.recoverabilityPipeline = recoverabilityPipeline; - this.faultMetadataExtractor = faultMetadataExtractor; - } - public async Task Invoke(ErrorContext errorContext, CancellationToken cancellationToken = default) { var childScope = serviceProvider.CreateAsyncScope(); await using (childScope.ConfigureAwait(false)) { - var recoverabilityAction = recoverabilityPolicy(errorContext, state); + RecoverabilityAction? recoverabilityAction; + + using (var activity = activityFactory.StartRecoverabilityActivity(errorContext)) + { + recoverabilityAction = recoverabilityPolicy(errorContext, state); + + if (activity is not null) + { + activityFactory.UpdateActivityFromRecoverabilityAction(activity, recoverabilityAction, errorContext.ReceiveAddress); + } + } var metadata = faultMetadataExtractor.Extract(errorContext); var recoverabilityContext = new RecoverabilityContext( - childScope.ServiceProvider, - messageOperations, - pipelineCache, - errorContext, - recoverabilityConfig, - metadata, - recoverabilityAction, - errorContext.Extensions, - cancellationToken); + childScope.ServiceProvider, + messageOperations, + pipelineCache, + errorContext, + recoverabilityConfig, + metadata, + recoverabilityAction, + errorContext.Extensions, + cancellationToken); await recoverabilityPipeline.Invoke(recoverabilityContext).ConfigureAwait(false); return recoverabilityContext.RecoverabilityAction.ErrorHandleResult; } } - - readonly IServiceProvider serviceProvider; - readonly IPipelineCache pipelineCache; - readonly MessageOperations messageOperations; - readonly RecoverabilityConfig recoverabilityConfig; - readonly Func recoverabilityPolicy; - readonly IPipeline recoverabilityPipeline; - readonly FaultMetadataExtractor faultMetadataExtractor; - readonly TState state; } \ No newline at end of file diff --git a/src/NServiceBus.Core/Recoverability/RecoverabilityRoutingConnector.cs b/src/NServiceBus.Core/Recoverability/RecoverabilityRoutingConnector.cs index b81ad021265..62f10fb27c2 100644 --- a/src/NServiceBus.Core/Recoverability/RecoverabilityRoutingConnector.cs +++ b/src/NServiceBus.Core/Recoverability/RecoverabilityRoutingConnector.cs @@ -56,4 +56,4 @@ public override async Task Invoke(IRecoverabilityContext context, Func Date: Wed, 5 Aug 2026 13:29:20 +0200 Subject: [PATCH 17/37] Add option for exception details capturing via logging (#7899) * Introduce log based exception recording mode * in the dup exception capturing mode the exception deatils are captured only on the inner most span. * Rename ExceptionRecordingMode.Dup to SpanAndLogs for clarity "Dup" didn't convey what the mode actually does; SpanAndLogs states it directly. * Relocate legacy exception tags into obsolete_v11 with removal notes otel.status_code/otel.status_description predate Activity.SetStatus and exception.escaped is Deprecated by the OTel semantic conventions. Neither belongs long-term; group them in LegacyExceptionTags alongside the file's other v11-scheduled removals and document exactly what to delete then. * Default ExceptionRecordingMode from OTEL_SEMCONV_EXCEPTION_SIGNAL_OPT_IN unless explicitly configured Lets users opt into the exceptions-as-logs model ahead of NServiceBus adopting it as the default, per the (unratified) OTel migration convention. Tracks whether the user explicitly touched ExceptionRecordingMode via a field-backed setter so an explicit setting always wins over the environment variable. Scaffolding lives in obsolete_v12.cs since the whole mechanism is meant to be removed once the migration settles. * Remove RecoverabilityAction.LogMessage, build log messages via switch in RecoverabilityPipelineExecutor Keeps the message-formatting logic out of the public RecoverabilityAction API surface; RecoverabilityPipelineExecutor is the only consumer, so it can own the mapping from action type to log message directly. * Migrate recoverability logging from static ILog to DI-resolved MEL ILogger Starts the move away from NServiceBus.Logging's static LogManager pattern in favor of Microsoft.Extensions.Logging, resolved from the DI container that's already flowing through the pipeline. Extracts all of it into a dedicated RecoverabilityActionLogger: each recoverability action type gets its own ILogger (ImmediateRetry, DelayedRetry, MoveToError, Discard, falling back to ILogger for anything else), so hosts can filter/level recoverability logs per action type. Message templates are literal per LoggerMessage-generated method to satisfy CA2254 rather than building a variable string, which also gives cheap structured logging for free. RecoverabilityPipelineExecutor now just delegates to RecoverabilityActionLogger.LogRecoverabilityAction in one line instead of owning the switch/DI-resolution logic itself. * Give OTEL_SEMCONV_EXCEPTION_SIGNAL_OPT_IN precedence over explicit ExceptionRecordingMode configuration Lets operators force the exception-signal behavior via environment variable without a code change/redeploy, overriding whatever the endpoint explicitly configured. Drops ExceptionRecordingModeSetByUser, which is no longer needed now that precedence doesn't depend on whether the user touched the setting. * fixing att test after minor changes to the recoverability action log messages --- .../Traces/When_processing_fails.cs | 7 ++ .../When_immediate_retries_are_enabled.cs | 2 +- .../When_message_fails_retries.cs | 2 +- ...IApprovals.ApproveNServiceBus.approved.txt | 6 ++ .../OpenTelemetryExtensionsTests.cs | 76 +++++++++++++++++++ .../OpenTelemetry/TracingExtensionsTests.cs | 23 +++++- .../RecoverabilityExecutorTests.cs | 3 +- .../OpenTelemetry/ExceptionRecordingMode.cs | 20 +++++ .../OpenTelemetry/InstrumentationOptions.cs | 8 +- .../OpenTelemetry/OpenTelemetryFeature.cs | 2 + .../Tracing/ActivityExtensions.cs | 16 ---- .../OpenTelemetry/Tracing/ActivityFactory.cs | 33 ++++++++ .../OpenTelemetry/Tracing/IActivityFactory.cs | 3 + .../Tracing/NoOpActivityFactory.cs | 6 ++ .../Tracing/RecordedExceptions.cs | 19 +++++ .../Tracing/TracingExtensions.cs | 8 +- .../OpenTelemetry/Tracing/obsolete_v11.cs | 29 +++++++ .../OpenTelemetry/Tracing/obsolete_v12.cs | 39 ++++++++++ .../Incoming/LoadHandlersConnector.cs | 5 +- .../Pipeline/MainPipelineExecutor.cs | 2 +- .../Recoverability/DelayedRetry.cs | 5 -- .../Recoverability/Discard.cs | 9 +-- .../Recoverability/ImmediateRetry.cs | 6 +- .../Recoverability/MoveToError.cs | 5 -- .../Recoverability/RecoverabilityAction.cs | 2 +- .../RecoverabilityActionLogger.cs | 65 ++++++++++++++++ .../RecoverabilityPipelineExecutor.cs | 10 ++- .../Transports/MessageContext.cs | 1 + .../Unicast/MessageOperations.cs | 10 +-- 29 files changed, 360 insertions(+), 62 deletions(-) create mode 100644 src/NServiceBus.Core/OpenTelemetry/ExceptionRecordingMode.cs create mode 100644 src/NServiceBus.Core/OpenTelemetry/Tracing/RecordedExceptions.cs create mode 100644 src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v12.cs create mode 100644 src/NServiceBus.Core/Recoverability/RecoverabilityActionLogger.cs diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_fails.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_fails.cs index a752b82ef29..52ab5ba660e 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_fails.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_fails.cs @@ -44,6 +44,13 @@ public async Task Should_mark_span_as_failed() handlerActivityTags.VerifyTag("otel.status_code", "ERROR"); handlerActivityTags.VerifyTag("otel.status_description", ErrorMessage); + using (Assert.EnterMultipleScope()) + { + Assert.That(failedHandlerActivity.Events, Has.Exactly(1).Items, + "the innermost span (the handler invocation) should record the exception details"); + Assert.That(failedPipelineActivity.Events, Is.Empty, + "the outer span should not duplicate the exception details already recorded on the inner span"); + } } public class Context : ScenarioContext; diff --git a/src/NServiceBus.AcceptanceTests/Recoverability/When_immediate_retries_are_enabled.cs b/src/NServiceBus.AcceptanceTests/Recoverability/When_immediate_retries_are_enabled.cs index 51002130852..7f01e523cd3 100644 --- a/src/NServiceBus.AcceptanceTests/Recoverability/When_immediate_retries_are_enabled.cs +++ b/src/NServiceBus.AcceptanceTests/Recoverability/When_immediate_retries_are_enabled.cs @@ -23,7 +23,7 @@ public async Task Should_do_the_configured_number_of_retries() Assert.That(context.ForwardedToErrorQueue, Is.True); Assert.That(context.NumberOfTimesInvoked, Is.EqualTo(numberOfRetries + 1), "Message should be retried 5 times immediately"); Assert.That(context.Logs.Count(l => l.Message - .StartsWith($"Immediate Retry is going to retry message '{context.MessageId}' because of an exception:")), Is.EqualTo(numberOfRetries)); + .StartsWith($"Immediate Retry is going to retry message '{context.MessageId}' because of an exception.")), Is.EqualTo(numberOfRetries)); } } diff --git a/src/NServiceBus.AcceptanceTests/Recoverability/When_message_fails_retries.cs b/src/NServiceBus.AcceptanceTests/Recoverability/When_message_fails_retries.cs index 8177397224a..ba014c87f9b 100644 --- a/src/NServiceBus.AcceptanceTests/Recoverability/When_message_fails_retries.cs +++ b/src/NServiceBus.AcceptanceTests/Recoverability/When_message_fails_retries.cs @@ -28,7 +28,7 @@ public void Should_forward_message_to_error_queue() } Assert.That(testContext.Logs.Count(l => l.Message - .StartsWith($"Moving message '{testContext.PhysicalMessageId}' to the error queue 'error' because processing failed due to an exception:")), Is.EqualTo(1)); + .StartsWith($"Moving message '{testContext.PhysicalMessageId}' to the error queue 'error' because processing failed due to an exception.")), Is.EqualTo(1)); } public class RetryEndpoint : EndpointConfigurationBuilder diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt index 59a401d4526..0c7e19fe1f1 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt @@ -331,6 +331,11 @@ namespace NServiceBus public bool TryGetExplicitlyConfiguredErrorQueueAddress([System.Diagnostics.CodeAnalysis.NotNullWhen(true)] out string? errorQueue) { } } } + public enum ExceptionRecordingMode + { + Logs = 0, + SpanAndLogs = 1, + } public class FailedConfig { public FailedConfig(string errorQueue, System.Collections.Generic.HashSet unrecoverableExceptionTypes) { } @@ -643,6 +648,7 @@ namespace NServiceBus public InstrumentationOptions() { } public NServiceBus.DelayedDeliveryInstrumentationOptions DelayedDelivery { get; } public bool EmitMessageDispatchingEvents { get; set; } + public NServiceBus.ExceptionRecordingMode ExceptionRecordingMode { get; set; } public NServiceBus.TraceMode PublishTraceMode { get; set; } public NServiceBus.RecoverabilityInstrumentationOptions Recoverability { get; } public NServiceBus.TraceMode SendTraceMode { get; set; } diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetryExtensionsTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetryExtensionsTests.cs index 14752802842..7428d6e660f 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetryExtensionsTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/OpenTelemetryExtensionsTests.cs @@ -1,6 +1,8 @@ namespace NServiceBus.Core.Tests.OpenTelemetry; +using System.Collections.Generic; using NUnit.Framework; +using Settings; [TestFixture] public class OpenTelemetryExtensionsTests @@ -60,4 +62,78 @@ public void Last_override_call_wins() Assert.That(options.Context.TryGet(OpenTelemetryExtensions.TraceConnectorOverrideKey, out TraceMode connector), Is.True); Assert.That(connector, Is.EqualTo(TraceMode.StartNew)); } + + [Test] + public void Defaults_to_span_and_logs_when_opt_in_environment_variable_is_not_set() + { + var settingsHolder = new SettingsHolder(); + settingsHolder.Set(new FakeEnvironment { ValueToReturn = [] }); + + InstrumentationOptions.SetExceptionRecordingModeDefault(settingsHolder); + + Assert.That(settingsHolder.Get().ExceptionRecordingMode, Is.EqualTo(ExceptionRecordingMode.SpanAndLogs)); + } + + [Test] + public void Uses_logs_only_when_opt_in_environment_variable_is_logs() + { + var settingsHolder = new SettingsHolder(); + settingsHolder.Set(new FakeEnvironment + { + ValueToReturn = new Dictionary { { InstrumentationOptions.ExceptionSignalOptInEnvironmentVariableKey, "logs" } } + }); + + InstrumentationOptions.SetExceptionRecordingModeDefault(settingsHolder); + + Assert.That(settingsHolder.Get().ExceptionRecordingMode, Is.EqualTo(ExceptionRecordingMode.Logs)); + } + + [Test] + public void Uses_span_and_logs_when_opt_in_environment_variable_is_logs_dup() + { + var settingsHolder = new SettingsHolder(); + settingsHolder.Set(new FakeEnvironment + { + ValueToReturn = new Dictionary { { InstrumentationOptions.ExceptionSignalOptInEnvironmentVariableKey, "logs/dup" } } + }); + + InstrumentationOptions.SetExceptionRecordingModeDefault(settingsHolder); + + Assert.That(settingsHolder.Get().ExceptionRecordingMode, Is.EqualTo(ExceptionRecordingMode.SpanAndLogs)); + } + + [Test] + public void Environment_variable_takes_precedence_over_explicit_configuration() + { + var settingsHolder = new SettingsHolder(); + settingsHolder.Set(new FakeEnvironment + { + ValueToReturn = new Dictionary { { InstrumentationOptions.ExceptionSignalOptInEnvironmentVariableKey, "logs" } } + }); + + // explicitly configured to something other than what the environment variable resolves to + settingsHolder.Set(new InstrumentationOptions { ExceptionRecordingMode = ExceptionRecordingMode.SpanAndLogs }); + InstrumentationOptions.SetExceptionRecordingModeDefault(settingsHolder); + + Assert.That(settingsHolder.Get().ExceptionRecordingMode, Is.EqualTo(ExceptionRecordingMode.Logs)); + } + + [Test] + public void Explicit_configuration_is_preserved_when_environment_variable_is_not_set() + { + var settingsHolder = new SettingsHolder(); + settingsHolder.Set(new FakeEnvironment { ValueToReturn = [] }); + + settingsHolder.Set(new InstrumentationOptions { ExceptionRecordingMode = ExceptionRecordingMode.Logs }); + InstrumentationOptions.SetExceptionRecordingModeDefault(settingsHolder); + + Assert.That(settingsHolder.Get().ExceptionRecordingMode, Is.EqualTo(ExceptionRecordingMode.Logs)); + } + + class FakeEnvironment : SystemEnvironment + { + public Dictionary ValueToReturn { get; set; } + + public override string GetEnvironmentVariable(string variable) => ValueToReturn.GetValueOrDefault(variable); + } } diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/TracingExtensionsTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/TracingExtensionsTests.cs index 64c1947dbf5..4b6473de6f5 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/TracingExtensionsTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/TracingExtensionsTests.cs @@ -21,7 +21,7 @@ public async Task Invoke_should_invoke_pipeline_when_activity_null() return Task.CompletedTask; }); - await pipeline.Invoke(new FakeRootContext(), null); + await pipeline.Invoke(new FakeRootContext(), null, new ActivityFactory(new InstrumentationOptions())); Assert.That(invokedPipeline, Is.True); } @@ -33,7 +33,7 @@ public async Task Invoke_should_set_success_status_when_no_exception() using var activity = new Activity("test activity"); activity.Start(); - await pipeline.Invoke(new FakeRootContext(), activity); + await pipeline.Invoke(new FakeRootContext(), activity, new ActivityFactory(new InstrumentationOptions())); Assert.That(activity.Status, Is.EqualTo(ActivityStatusCode.Ok)); } @@ -46,7 +46,7 @@ public void Invoke_should_set_error_status_and_tags_when_exception() using var activity = new Activity("test activity"); activity.Start(); - Assert.ThrowsAsync(() => pipeline.Invoke(new FakeRootContext(), activity)); + Assert.ThrowsAsync(() => pipeline.Invoke(new FakeRootContext(), activity, new ActivityFactory(new InstrumentationOptions()))); Assert.That(activity.Status, Is.EqualTo(ActivityStatusCode.Error)); @@ -61,6 +61,23 @@ public void Invoke_should_set_error_status_and_tags_when_exception() Assert.That(errorEvent.Name, Is.EqualTo("exception")); } + [Test] + public void Invoke_should_set_error_status_without_exception_event_when_log_mode() + { + var exception = new Exception("test exception"); + var pipeline = new FakePipeline(() => throw exception); + using var activity = new Activity("test activity"); + activity.Start(); + + Assert.ThrowsAsync(() => pipeline.Invoke(new FakeRootContext(), activity, new ActivityFactory(new InstrumentationOptions { ExceptionRecordingMode = ExceptionRecordingMode.Logs }))); + + using (Assert.EnterMultipleScope()) + { + Assert.That(activity.Status, Is.EqualTo(ActivityStatusCode.Error)); + Assert.That(activity.Events, Is.Empty, "no exception event should be added when recording via the log instead"); + } + } + class FakePipeline : IPipeline { readonly Func pipelineAction; diff --git a/src/NServiceBus.Core.Tests/Recoverability/RecoverabilityExecutorTests.cs b/src/NServiceBus.Core.Tests/Recoverability/RecoverabilityExecutorTests.cs index af8c3d265cb..139a3ff7185 100644 --- a/src/NServiceBus.Core.Tests/Recoverability/RecoverabilityExecutorTests.cs +++ b/src/NServiceBus.Core.Tests/Recoverability/RecoverabilityExecutorTests.cs @@ -1,7 +1,6 @@ namespace NServiceBus.Core.Tests.Recoverability; using System; -using System.Collections.Generic; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; using Pipeline; @@ -64,7 +63,7 @@ public async Task Should_use_error_context_extensions_as_extensions_root() static RecoverabilityPipelineExecutor CreateRecoverabilityExecutor(TestableMessageOperations.Pipeline recoverabilityPipeline) { var executor = new RecoverabilityPipelineExecutor( - new ServiceCollection().BuildServiceProvider(), // TODO: Does not get disposed + new ServiceCollection().AddLogging().BuildServiceProvider(), // TODO: Does not get disposed new ThrowingPipelineCache(), new TestableMessageOperations(), null, diff --git a/src/NServiceBus.Core/OpenTelemetry/ExceptionRecordingMode.cs b/src/NServiceBus.Core/OpenTelemetry/ExceptionRecordingMode.cs new file mode 100644 index 00000000000..32c9da69655 --- /dev/null +++ b/src/NServiceBus.Core/OpenTelemetry/ExceptionRecordingMode.cs @@ -0,0 +1,20 @@ +#nullable enable + +namespace NServiceBus; + +/// +/// Controls how exception details are recorded when an operation represented by an activity fails. +/// +public enum ExceptionRecordingMode +{ + /// + /// Records the exception details via NServiceBus's logging infrastructure instead of adding an event to the + /// activity. + /// + Logs, + + /// + /// Records the exception details via NServiceBus's logging infrastructure and as an event the activity. + /// + SpanAndLogs +} diff --git a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs index 78f80d8ead1..7028795c9aa 100644 --- a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs @@ -6,7 +6,7 @@ namespace NServiceBus; /// Controls opt-in OpenTelemetry instrumentation behaviors. /// Accessed via endpointConfiguration.Tracing(). /// -public class InstrumentationOptions +public partial class InstrumentationOptions { /// /// Appends the destination to span names following the OTel messaging convention @@ -51,6 +51,12 @@ public class InstrumentationOptions /// or . /// public TraceMode PublishTraceMode { get; set; } = TraceMode.StartNew; + + /// + /// Controls how exception details are recorded when an operation fails. + /// Defaults to : exceptions are recorded as an event on the activity. + /// + public ExceptionRecordingMode ExceptionRecordingMode { get; set; } = ExceptionRecordingMode.SpanAndLogs; } /// diff --git a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs index e207ee6ce57..7ae9288c9a7 100644 --- a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs +++ b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs @@ -6,6 +6,8 @@ namespace NServiceBus; sealed class OpenTelemetryFeature : Feature { + public OpenTelemetryFeature() => Defaults(InstrumentationOptions.SetExceptionRecordingModeDefault); + protected override void Setup(FeatureConfigurationContext context) { var instrumentationOptions = context.Settings.GetOrDefault() ?? new InstrumentationOptions(); diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs index 4d7586fae2a..3f784962a9b 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs @@ -2,10 +2,8 @@ namespace NServiceBus; -using System; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; -using System.Threading.Tasks; using Extensibility; static class ActivityExtensions @@ -34,18 +32,4 @@ static bool TryGetRecordingPipelineActivity(this ContextBag pipelineContext, str public static void SetOutgoingPipelineActivity(this ContextBag pipelineContext, Activity activity) => pipelineContext.Set(OutgoingActivityKey, activity); public static void SetIncomingPipelineActivity(this ContextBag pipelineContext, Activity activity) => pipelineContext.Set(IncomingActivityKey, activity); - - public static void SetErrorStatus(this Activity activity, Exception ex) - { - activity.SetStatus(ActivityStatusCode.Error, ex.Message); - activity.SetTag("otel.status_code", "ERROR"); - activity.SetTag("otel.status_description", ex.Message); - activity.SetTag(ActivityTags.ErrorType, ex.GetType().FullName); - activity.AddException(ex, new TagList { { "exception.escaped", true } }); - - if (ex is TaskCanceledException) - { - activity.SetTag(ActivityTags.CancelledTask, true); - } - } } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs index ec6bce50648..9c407ce77b5 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs @@ -2,9 +2,12 @@ namespace NServiceBus; +using System; using System.Collections.Generic; using System.Diagnostics; +using System.Threading.Tasks; using Extensibility; +using Logging; using Pipeline; using Transport; @@ -201,4 +204,34 @@ public void UpdateActivityFromRecoverabilityAction(Activity activity, Recoverabi activity.DisplayName = ActivityDisplayNames.DiscardOperation; } } + + public void RecordError(Activity activity, Exception exception, ContextBag context) + { + activity.SetStatus(ActivityStatusCode.Error, exception.Message); + activity.SetTag(ActivityTags.ErrorType, exception.GetType().FullName); + + LegacyExceptionTags.SetLegacyStatusTags(activity, exception); + + var recordedExceptions = context.GetOrCreate(); + if (!recordedExceptions.HasBeenRecorded(exception)) + { + if (Options.ExceptionRecordingMode == ExceptionRecordingMode.Logs) + { + Logger.Error($"An exception occurred while executing '{activity.DisplayName}'.", exception); + } + else + { + activity.AddException(exception, LegacyExceptionTags.EscapedTagList); + } + + recordedExceptions.MarkAsRecorded(exception); + } + + if (exception is TaskCanceledException) + { + activity.SetTag(ActivityTags.CancelledTask, true); + } + } + + static readonly ILog Logger = LogManager.GetLogger(); } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs index b7f536a7133..bccdf87faf6 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs @@ -2,7 +2,9 @@ namespace NServiceBus; +using System; using System.Diagnostics; +using Extensibility; using Pipeline; using Transport; @@ -14,4 +16,5 @@ interface IActivityFactory Activity? StartHandlerActivity(MessageHandler messageHandler); Activity? StartRecoverabilityActivity(ErrorContext context); void UpdateActivityFromRecoverabilityAction(Activity activity, RecoverabilityAction recoverabilityAction, string receiveAddress); + void RecordError(Activity activity, Exception exception, ContextBag context); } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs index 6d9c92706a9..6e2e183c637 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs @@ -2,7 +2,9 @@ namespace NServiceBus; +using System; using System.Diagnostics; +using Extensibility; using Pipeline; using Transport; @@ -21,4 +23,8 @@ sealed class NoOpActivityFactory : IActivityFactory public void UpdateActivityFromRecoverabilityAction(Activity activity, RecoverabilityAction recoverabilityAction, string receiveAddress) { } + + public void RecordError(Activity activity, Exception exception, ContextBag context) + { + } } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/RecordedExceptions.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/RecordedExceptions.cs new file mode 100644 index 00000000000..4ecfafb8152 --- /dev/null +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/RecordedExceptions.cs @@ -0,0 +1,19 @@ +#nullable enable + +namespace NServiceBus; + +using System; +using System.Collections.Generic; + +sealed class RecordedExceptions +{ + // Reference equality is intentional: this tracks specific exception instances as they + // propagate, not exceptions that merely look alike. +#pragma warning disable PS0025 + readonly HashSet recorded = new(ReferenceEqualityComparer.Instance); +#pragma warning restore PS0025 + + public bool HasBeenRecorded(Exception exception) => recorded.Contains(exception); + + public void MarkAsRecorded(Exception exception) => recorded.Add(exception); +} diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/TracingExtensions.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/TracingExtensions.cs index ad8557b98a4..afdad2c908a 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/TracingExtensions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/TracingExtensions.cs @@ -9,11 +9,11 @@ namespace NServiceBus; static class TracingExtensions { - public static Task Invoke(this IPipeline pipeline, TContext context, Activity? activity) where TContext : IBehaviorContext + public static Task Invoke(this IPipeline pipeline, TContext context, Activity? activity, IActivityFactory activityFactory) where TContext : IBehaviorContext { - return activity is null ? pipeline.Invoke(context) : TracePipelineStatus(pipeline, context, activity); + return activity is null ? pipeline.Invoke(context) : TracePipelineStatus(pipeline, context, activity, activityFactory); - static async Task TracePipelineStatus(IPipeline pipeline, TContext context, Activity activity) + static async Task TracePipelineStatus(IPipeline pipeline, TContext context, Activity activity, IActivityFactory activityFactory) { #pragma warning disable PS0019 // When catching System.Exception, cancellation needs to be properly accounted for try @@ -23,7 +23,7 @@ static async Task TracePipelineStatus(IPipeline pipeline, TContext con } catch (Exception ex) { - activity.SetErrorStatus(ex); + activityFactory.RecordError(activity, ex, context.Extensions); throw; } #pragma warning restore PS0019 // When catching System.Exception, cancellation needs to be properly accounted for diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs index 74dac3aadbe..0e8dd5f4d4f 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs @@ -179,4 +179,33 @@ public static bool UseHandlerActivitySource } internal static void ResetUseHandlerActivitySource() => cachedUseHandlerActivitySource = SwitchState.Unchecked; +} + + +// This class bridges two independent legacy exception-tagging behaviors, both +// scheduled for removal in v11: +// +// - SetLegacyStatusTags sets the "otel.status_code"/"otel.status_description" +// tags, which predate native support for Activity.SetStatus/Activity.Status +// and are now redundant with it. Kept only for consumers still reading the +// tags directly instead of Activity.Status. +// - EscapedTagList carries "exception.escaped", an attribute the OTel semantic +// conventions have marked Deprecated: +// https://opentelemetry.io/docs/specs/semconv/exceptions/exceptions-logs/ +// It's added to the exception event for backward compatibility with +// existing consumers of that attribute. +// +// In v11, delete this entire class, remove the +// `LegacyExceptionTags.SetLegacyStatusTags(activity, exception);` call in +// ActivityFactory.RecordError, and stop passing EscapedTagList to +// activity.AddException in the same method. +static class LegacyExceptionTags +{ + public static void SetLegacyStatusTags(Activity activity, Exception exception) + { + activity.SetTag("otel.status_code", "ERROR"); + activity.SetTag("otel.status_description", exception.Message); + } + + public static TagList EscapedTagList { get; } = new() { { "exception.escaped", true } }; } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v12.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v12.cs new file mode 100644 index 00000000000..13e193f8957 --- /dev/null +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v12.cs @@ -0,0 +1,39 @@ +#nullable enable + +namespace NServiceBus; + +using Settings; + +// This scaffolds a temporary, environment-variable-driven override for +// ExceptionRecordingMode so users can opt in ahead of time to the "logs" +// model described by the OTEL_SEMCONV_EXCEPTION_SIGNAL_OPT_IN convention: +// https://opentelemetry.io/docs/specs/semconv/exceptions/exceptions-logs/ +// The environment variable is the highest-priority signal for this setting: +// when it's set to a recognized value it always wins, even over an +// explicitly configured ExceptionRecordingMode, so operators can force the +// exception-signal behavior without a code change/redeploy. +// +// In v12, once the exceptions-as-logs migration has settled, delete this +// entire file (SetExceptionRecordingModeDefault and +// ExceptionSignalOptInEnvironmentVariableKey), and remove the +// `Defaults(InstrumentationOptions.SetExceptionRecordingModeDefault);` call in +// OpenTelemetryFeature's constructor. +public partial class InstrumentationOptions +{ + internal static void SetExceptionRecordingModeDefault(SettingsHolder settings) + { + var options = settings.GetOrCreate(); + + var environment = settings.Get(); + var variableValue = environment.GetEnvironmentVariable(ExceptionSignalOptInEnvironmentVariableKey); + + options.ExceptionRecordingMode = variableValue switch + { + "logs" => ExceptionRecordingMode.Logs, + "logs/dup" => ExceptionRecordingMode.SpanAndLogs, + _ => options.ExceptionRecordingMode + }; + } + + internal static readonly string ExceptionSignalOptInEnvironmentVariableKey = "OTEL_SEMCONV_EXCEPTION_SIGNAL_OPT_IN"; +} diff --git a/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs b/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs index 1fce445695a..e11af72114e 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs @@ -64,7 +64,10 @@ public override async Task Invoke(IIncomingLogicalMessageContext context, Func GetRoutingContexts(IRecover { var exception = context.Exception; - Logger.Warn($"Delayed Retry will reschedule message '{context.MessageId}' after a delay of {Delay} because of an exception:", exception); - var outgoingMessage = new OutgoingMessage(context.MessageId, new Dictionary(context.Headers), context.Body); var currentDelayedRetriesAttempt = context.DelayedDeliveriesPerformed + 1; @@ -73,6 +70,4 @@ public override IReadOnlyCollection GetRoutingContexts(IRecover }); return [routingContext]; } - - static readonly ILog Logger = LogManager.GetLogger(); } \ No newline at end of file diff --git a/src/NServiceBus.Core/Recoverability/Discard.cs b/src/NServiceBus.Core/Recoverability/Discard.cs index 7ecc8e0b6ea..86862add7dc 100644 --- a/src/NServiceBus.Core/Recoverability/Discard.cs +++ b/src/NServiceBus.Core/Recoverability/Discard.cs @@ -3,7 +3,6 @@ namespace NServiceBus; using System.Collections.Generic; -using Logging; using Pipeline; using Transport; @@ -28,11 +27,5 @@ public class Discard : RecoverabilityAction public override ErrorHandleResult ErrorHandleResult => ErrorHandleResult.Handled; /// - public override IReadOnlyCollection GetRoutingContexts(IRecoverabilityActionContext context) - { - Logger.Info($"Discarding message with id '{context.MessageId}'. Reason: {Reason}", context.Exception); - return []; - } - - static readonly ILog Logger = LogManager.GetLogger(); + public override IReadOnlyCollection GetRoutingContexts(IRecoverabilityActionContext context) => []; } \ No newline at end of file diff --git a/src/NServiceBus.Core/Recoverability/ImmediateRetry.cs b/src/NServiceBus.Core/Recoverability/ImmediateRetry.cs index 2b43ca652eb..71b7fb9cdc9 100644 --- a/src/NServiceBus.Core/Recoverability/ImmediateRetry.cs +++ b/src/NServiceBus.Core/Recoverability/ImmediateRetry.cs @@ -4,8 +4,7 @@ namespace NServiceBus; using System; using System.Collections.Generic; -using NServiceBus.Logging; -using NServiceBus.Transport; +using Transport; using Pipeline; /// @@ -30,7 +29,6 @@ public override IReadOnlyCollection GetRoutingContexts(IRecover { var exception = context.Exception; - Logger.Info($"Immediate Retry is going to retry message '{context.MessageId}' because of an exception:", exception); if (context is IRecoverabilityActionContextNotifications notifications) { notifications.Add(new MessageToBeRetried( @@ -46,6 +44,4 @@ public override IReadOnlyCollection GetRoutingContexts(IRecover } return []; } - - static readonly ILog Logger = LogManager.GetLogger(); } \ No newline at end of file diff --git a/src/NServiceBus.Core/Recoverability/MoveToError.cs b/src/NServiceBus.Core/Recoverability/MoveToError.cs index 8c805833676..cbb148a7a46 100644 --- a/src/NServiceBus.Core/Recoverability/MoveToError.cs +++ b/src/NServiceBus.Core/Recoverability/MoveToError.cs @@ -3,7 +3,6 @@ namespace NServiceBus; using System.Collections.Generic; -using Logging; using Pipeline; using Recoverability; using Routing; @@ -35,8 +34,6 @@ public override IReadOnlyCollection GetRoutingContexts(IRecover var metadata = context.Metadata; var exception = context.Exception; - Logger.Error($"Moving message '{context.MessageId}' to the error queue '{ErrorQueue}' because processing failed due to an exception:", exception); - if (context is IRecoverabilityActionContextNotifications notifications) { notifications.Add(new MessageFaulted(ErrorQueue, context.NativeMessageId, context.MessageId, context.Headers, context.Body, context.ReceiveProperties, exception)); @@ -56,6 +53,4 @@ public override IReadOnlyCollection GetRoutingContexts(IRecover context.CreateRoutingContext(outgoingMessage, new UnicastRoutingStrategy(ErrorQueue)) ]; } - - static readonly ILog Logger = LogManager.GetLogger(); } \ No newline at end of file diff --git a/src/NServiceBus.Core/Recoverability/RecoverabilityAction.cs b/src/NServiceBus.Core/Recoverability/RecoverabilityAction.cs index 9d6992336f3..cbdb330cde1 100644 --- a/src/NServiceBus.Core/Recoverability/RecoverabilityAction.cs +++ b/src/NServiceBus.Core/Recoverability/RecoverabilityAction.cs @@ -74,5 +74,5 @@ public static Discard Discard(string reason) /// public abstract ErrorHandleResult ErrorHandleResult { get; } - static readonly ImmediateRetry CachedImmediateRetry = new ImmediateRetry(); + static readonly ImmediateRetry CachedImmediateRetry = new(); } \ No newline at end of file diff --git a/src/NServiceBus.Core/Recoverability/RecoverabilityActionLogger.cs b/src/NServiceBus.Core/Recoverability/RecoverabilityActionLogger.cs new file mode 100644 index 00000000000..40878f9730c --- /dev/null +++ b/src/NServiceBus.Core/Recoverability/RecoverabilityActionLogger.cs @@ -0,0 +1,65 @@ +#nullable enable + +namespace NServiceBus; + +using System; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Transport; + +sealed class RecoverabilityActionLogger(IServiceProvider serviceProvider) +{ + public void LogRecoverabilityAction(RecoverabilityAction recoverabilityAction, ErrorContext errorContext, ExceptionRecordingMode exceptionRecordingMode) + { + // ExceptionRecordingMode.SpanAndLogs option preserves exception details behavior from the older version of NSB + // In this mode the exception details are captured both on the span and here in the logs. + // The other option (Logs) emits details in logs. Not here but rather in the proper OTel scope i.e., in the span + // in which the exception was thrown. + var exceptionToLog = exceptionRecordingMode == ExceptionRecordingMode.SpanAndLogs + ? errorContext.Exception + : null; + + switch (recoverabilityAction) + { + case ImmediateRetry: + immediateRetryLogger.ImmediateRetryLogged(exceptionToLog, errorContext.MessageId); + break; + case DelayedRetry delayedRetry: + delayedRetryLogger.DelayedRetryLogged(exceptionToLog, errorContext.MessageId, delayedRetry.Delay); + break; + case MoveToError moveToError: + moveToErrorLogger.MoveToErrorLogged(exceptionToLog, errorContext.MessageId, moveToError.ErrorQueue); + break; + case Discard discard: + discardLogger.DiscardLogged(exceptionToLog, errorContext.MessageId, discard.Reason); + break; + default: + unknownActionLogger.UnknownRecoverabilityActionLogged(exceptionToLog, recoverabilityAction.GetType().Name, errorContext.MessageId); + break; + } + } + + readonly ILogger immediateRetryLogger = serviceProvider.GetRequiredService>(); + readonly ILogger delayedRetryLogger = serviceProvider.GetRequiredService>(); + readonly ILogger moveToErrorLogger = serviceProvider.GetRequiredService>(); + readonly ILogger discardLogger = serviceProvider.GetRequiredService>(); + readonly ILogger unknownActionLogger = serviceProvider.GetRequiredService>(); +} + +static partial class RecoverabilityActionLoggerMessages +{ + [LoggerMessage(Level = LogLevel.Information, Message = "Immediate Retry is going to retry message '{MessageId}' because of an exception.")] + public static partial void ImmediateRetryLogged(this ILogger logger, Exception? exception, string messageId); + + [LoggerMessage(Level = LogLevel.Information, Message = "Delayed Retry will reschedule message '{MessageId}' after a delay of {Delay} because of an exception.")] + public static partial void DelayedRetryLogged(this ILogger logger, Exception? exception, string messageId, TimeSpan delay); + + [LoggerMessage(Level = LogLevel.Information, Message = "Moving message '{MessageId}' to the error queue '{ErrorQueue}' because processing failed due to an exception.")] + public static partial void MoveToErrorLogged(this ILogger logger, Exception? exception, string messageId, string errorQueue); + + [LoggerMessage(Level = LogLevel.Information, Message = "Discarding message with id '{MessageId}'. Reason: {Reason}.")] + public static partial void DiscardLogged(this ILogger logger, Exception? exception, string messageId, string reason); + + [LoggerMessage(Level = LogLevel.Information, Message = "Recoverability action '{ActionType}' invoked for message '{MessageId}'.")] + public static partial void UnknownRecoverabilityActionLogged(this ILogger logger, Exception? exception, string actionType, string messageId); +} diff --git a/src/NServiceBus.Core/Recoverability/RecoverabilityPipelineExecutor.cs b/src/NServiceBus.Core/Recoverability/RecoverabilityPipelineExecutor.cs index 88373e89027..e6c0e8ba9e1 100644 --- a/src/NServiceBus.Core/Recoverability/RecoverabilityPipelineExecutor.cs +++ b/src/NServiceBus.Core/Recoverability/RecoverabilityPipelineExecutor.cs @@ -1,4 +1,4 @@ -#nullable enable +#nullable enable namespace NServiceBus; @@ -6,7 +6,7 @@ namespace NServiceBus; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; -using NServiceBus.Pipeline; +using Pipeline; using Transport; class RecoverabilityPipelineExecutor( @@ -35,6 +35,8 @@ public async Task Invoke(ErrorContext errorContext, Cancellat { activityFactory.UpdateActivityFromRecoverabilityAction(activity, recoverabilityAction, errorContext.ReceiveAddress); } + + recoverabilityActionLogger.LogRecoverabilityAction(recoverabilityAction, errorContext, activityFactory.Options.ExceptionRecordingMode); } var metadata = faultMetadataExtractor.Extract(errorContext); @@ -55,4 +57,6 @@ public async Task Invoke(ErrorContext errorContext, Cancellat return recoverabilityContext.RecoverabilityAction.ErrorHandleResult; } } -} \ No newline at end of file + + readonly RecoverabilityActionLogger recoverabilityActionLogger = new(serviceProvider); +} diff --git a/src/NServiceBus.Core/Transports/MessageContext.cs b/src/NServiceBus.Core/Transports/MessageContext.cs index 7d191154923..a48e6ec3b15 100644 --- a/src/NServiceBus.Core/Transports/MessageContext.cs +++ b/src/NServiceBus.Core/Transports/MessageContext.cs @@ -51,6 +51,7 @@ public MessageContext(string nativeMessageId, Dictionary headers TransportTransaction = transportTransaction; context.GetOrCreate(); + context.GetOrCreate(); } /// diff --git a/src/NServiceBus.Core/Unicast/MessageOperations.cs b/src/NServiceBus.Core/Unicast/MessageOperations.cs index 7f3af927a2a..cfceeb4c684 100644 --- a/src/NServiceBus.Core/Unicast/MessageOperations.cs +++ b/src/NServiceBus.Core/Unicast/MessageOperations.cs @@ -71,7 +71,7 @@ async Task Publish(IBehaviorContext context, Type messageType, object message, P using var activity = activityFactory.StartOutgoingPipelineActivity(ActivityNames.OutgoingEventActivityName, publishDisplayName, publishContext); - await publishPipeline.Invoke(publishContext, activity).ConfigureAwait(false); + await publishPipeline.Invoke(publishContext, activity, activityFactory).ConfigureAwait(false); } public Task Subscribe(IBehaviorContext context, Type eventType, SubscribeOptions options) @@ -90,7 +90,7 @@ public async Task Subscribe(IBehaviorContext context, Type[] eventTypes, Subscri using var activity = activityFactory.StartOutgoingPipelineActivity(ActivityNames.SubscribeActivityName, ActivityDisplayNames.SubscribeEvent, context); - await subscribePipeline.Invoke(subscribeContext, activity).ConfigureAwait(false); + await subscribePipeline.Invoke(subscribeContext, activity, activityFactory).ConfigureAwait(false); } public async Task Unsubscribe(IBehaviorContext context, Type eventType, UnsubscribeOptions options) @@ -104,7 +104,7 @@ public async Task Unsubscribe(IBehaviorContext context, Type eventType, Unsubscr using var activity = activityFactory.StartOutgoingPipelineActivity(ActivityNames.UnsubscribeActivityName, ActivityDisplayNames.UnsubscribeEvent, context); - await unsubscribePipeline.Invoke(unsubscribeContext, activity).ConfigureAwait(false); + await unsubscribePipeline.Invoke(unsubscribeContext, activity, activityFactory).ConfigureAwait(false); } public Task Send(IBehaviorContext context, Action messageConstructor, SendOptions options) @@ -138,7 +138,7 @@ async Task SendMessage(IBehaviorContext context, Type messageType, object messag using var activity = activityFactory.StartOutgoingPipelineActivity(ActivityNames.OutgoingMessageActivityName, ActivityDisplayNames.SendMessage, outgoingContext); - await sendPipeline.Invoke(outgoingContext, activity).ConfigureAwait(false); + await sendPipeline.Invoke(outgoingContext, activity, activityFactory).ConfigureAwait(false); } public Task Reply(IBehaviorContext context, object message, ReplyOptions options) @@ -172,7 +172,7 @@ async Task ReplyMessage(IBehaviorContext context, Type messageType, object messa using var activity = activityFactory.StartOutgoingPipelineActivity(ActivityNames.OutgoingMessageActivityName, ActivityDisplayNames.ReplyMessage, outgoingContext); - await replyPipeline.Invoke(outgoingContext, activity).ConfigureAwait(false); + await replyPipeline.Invoke(outgoingContext, activity, activityFactory).ConfigureAwait(false); } static void MergeDispatchProperties(ContextBag context, DispatchProperties dispatchProperties) From 3345a883eb33b4e53ec78700efd86e062dd57eb8 Mon Sep 17 00:00:00 2001 From: "Irina Dominte(Scurtu)" Date: Mon, 10 Aug 2026 15:06:18 +0300 Subject: [PATCH 18/37] Additional performance-related instruments (#7898) * Started performance metrics * Removed the Performance metrics from public API * Renamed a metric * nullable enable * remove nullable * added a line * test fix * trailing whitespace * Removed execution result * add enclosed message type to the deserialization insturmentation * Renamed classes and made the tags backwards compatible * Apply suggestions from code review Co-authored-by: Tomasz Masternak * fix: correct tag formatting and indentation in OpenTelemetry tests * undo the type renamings for now * fix: rename IncomingPipelineMetricsTags to IncomingPipelineMetricTags * add MetersOptions to API approvals * fix: correct variable name from messagingMetricsMetricses to messagingMetricsMeters --------- Co-authored-by: Tomasz Masternak Co-authored-by: Tomasz Masternak --- .../Metrics/When_message_processing_fails.cs | 2 +- ...IApprovals.ApproveNServiceBus.approved.txt | 6 + .../MeterTests.Verify_MeterAPI.approved.txt | 13 +- .../Envelopes/EnvelopeUnwrapperTests.cs | 2 +- .../OpenTelemetry/MeterTests.cs | 4 +- .../Incoming/InvokeHandlerTerminatorTest.cs | 2 +- .../SerializeMessageConnectorTests.cs | 3 +- .../IncomingPipelineMetricTagsTests.cs | 3 +- .../Pipeline/MainPipelineExecutorTests.cs | 2 +- ...tReceiveToPhysicalMessageConnectorTests.cs | 2 +- .../Unicast/LoadHandlersConnectorTests.cs | 9 +- src/NServiceBus.Core/EndpointCreator.cs | 2 +- .../OpenTelemetry/InstrumentationOptions.cs | 5 + .../OpenTelemetry/MetersOptions.cs | 17 ++ .../OpenTelemetry/Metrics/MeterTags.cs | 1 + .../Incoming/DeserializeMessageConnector.cs | 24 ++- .../Incoming/IncomingPipelineMetrics.cs | 188 +++++++++++++++++- .../Incoming/LoadHandlersConnector.cs | 4 +- ...nsportReceiveToPhysicalMessageConnector.cs | 4 + .../Outgoing/SerializeMessageConnector.cs | 21 +- .../Pipeline/PipelineComponent.cs | 5 +- .../Receiving/ReceiveComponent.cs | 2 +- .../Sagas/SagaPersistenceBehavior.cs | 17 +- src/NServiceBus.Core/Sagas/Sagas.cs | 2 +- .../Serialization/SerializationFeature.cs | 4 +- 25 files changed, 300 insertions(+), 44 deletions(-) create mode 100644 src/NServiceBus.Core/OpenTelemetry/MetersOptions.cs diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_message_processing_fails.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_message_processing_fails.cs index 15a7172bde4..9c1aadacc89 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_message_processing_fails.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_message_processing_fails.cs @@ -51,7 +51,7 @@ public async Task Should_report_failing_message_metrics() ["nservicebus.discriminator"] = "disc", ["nservicebus.message_type"] = typeof(FailingMessage).FullName, ["execution.result"] = "failure", - ["error.type"] = typeof(SimulatedException).FullName, + ["error.type"] = typeof(SimulatedException).FullName }); } diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt index 0c7e19fe1f1..78f0fc99ed0 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt @@ -649,6 +649,7 @@ namespace NServiceBus public NServiceBus.DelayedDeliveryInstrumentationOptions DelayedDelivery { get; } public bool EmitMessageDispatchingEvents { get; set; } public NServiceBus.ExceptionRecordingMode ExceptionRecordingMode { get; set; } + public NServiceBus.MetersOptions Meters { get; } public NServiceBus.TraceMode PublishTraceMode { get; set; } public NServiceBus.RecoverabilityInstrumentationOptions Recoverability { get; } public NServiceBus.TraceMode SendTraceMode { get; set; } @@ -768,6 +769,11 @@ namespace NServiceBus public static System.Threading.Tasks.Task Unsubscribe(this NServiceBus.IMessageSession session, System.Type messageType, System.Threading.CancellationToken cancellationToken = default) { } public static System.Threading.Tasks.Task Unsubscribe(this NServiceBus.IMessageSession session, System.Threading.CancellationToken cancellationToken = default) { } } + public class MetersOptions + { + public MetersOptions() { } + public bool EmitExecutionResultTags { get; set; } + } public class MoveToError : NServiceBus.RecoverabilityAction { protected MoveToError(string errorQueue) { } diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/MeterTests.Verify_MeterAPI.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/MeterTests.Verify_MeterAPI.approved.txt index 8cfd1afa89c..b632b411cce 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/MeterTests.Verify_MeterAPI.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/MeterTests.Verify_MeterAPI.approved.txt @@ -1,7 +1,7 @@ { "Note": "Changes to metrics API should result in an update to NServiceBusMeter version.", "MetricsSourceName": "NServiceBus.Core.Pipeline.Incoming", - "MetricsSourceVersion": "0.3.0", + "MetricsSourceVersion": "0.4.0", "Tags": [ "error.type", "execution.result", @@ -11,20 +11,27 @@ "nservicebus.message_handler_type", "nservicebus.message_handler_types", "nservicebus.message_type", - "nservicebus.queue" + "nservicebus.queue", + "nservicebus.saga_type" ], "Metrics": [ "nservicebus.envelope.unwrapped => Counter", "nservicebus.messaging.active_messages => UpDownCounter", "nservicebus.messaging.critical_time => Histogram, Unit: s", + "nservicebus.messaging.deserialize_time => Histogram, Unit: s", "nservicebus.messaging.failures => Counter", "nservicebus.messaging.fetches => Counter", "nservicebus.messaging.handler_time => Histogram, Unit: s", "nservicebus.messaging.processing_time => Histogram, Unit: s", + "nservicebus.messaging.serialize_time => Histogram, Unit: s", "nservicebus.messaging.successes => Counter", "nservicebus.outbox.duplicates => Counter", + "nservicebus.outbox.fetch_time => Histogram, Unit: s", + "nservicebus.outbox.store_time => Histogram, Unit: s", + "nservicebus.persistence.commit_time => Histogram, Unit: s", "nservicebus.recoverability.delayed => Counter", "nservicebus.recoverability.error => Counter", - "nservicebus.recoverability.immediate => Counter" + "nservicebus.recoverability.immediate => Counter", + "nservicebus.sagas.fetch_time => Histogram, Unit: s" ] } \ No newline at end of file diff --git a/src/NServiceBus.Core.Tests/Envelopes/EnvelopeUnwrapperTests.cs b/src/NServiceBus.Core.Tests/Envelopes/EnvelopeUnwrapperTests.cs index 1824dc35f2c..2b52437cca8 100644 --- a/src/NServiceBus.Core.Tests/Envelopes/EnvelopeUnwrapperTests.cs +++ b/src/NServiceBus.Core.Tests/Envelopes/EnvelopeUnwrapperTests.cs @@ -29,7 +29,7 @@ public void Setup() originalBody = "payload"u8.ToArray().AsMemory(); messageContext = new MessageContext(nativeId, originalHeaders, originalBody, new TransportTransaction(), "receiveAddress", new ContextBag()); meterFactory = new TestMeterFactory(); - incomingPipelineMetrics = new IncomingPipelineMetrics(meterFactory, "queue", "disc"); + incomingPipelineMetrics = new IncomingPipelineMetrics(meterFactory, "queue", "disc", new MetersOptions()); } [TearDown] diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/MeterTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/MeterTests.cs index b59ee8b556e..83b1e16e1d3 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/MeterTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/MeterTests.cs @@ -20,9 +20,9 @@ public void Verify_MeterAPI() .ToList(); using var meterFactory = new TestMeterFactory(); - //The IncomingPipelineMetrics constructor creates the meters, therefore a new instance before collecting the metrics. + //The IncomingPipelineMeter constructor creates the meters, therefore a new instance before collecting the metrics. #pragma warning disable CA1806 - new IncomingPipelineMetrics(meterFactory, "queue", "disc"); + new IncomingPipelineMetrics(meterFactory, "queue", "disc", new MetersOptions()); #pragma warning restore CA1806 using var metricsListener = TestingMetricListener.SetupNServiceBusMetricsListener(); diff --git a/src/NServiceBus.Core.Tests/Pipeline/Incoming/InvokeHandlerTerminatorTest.cs b/src/NServiceBus.Core.Tests/Pipeline/Incoming/InvokeHandlerTerminatorTest.cs index 47276e798e4..b794eb25413 100644 --- a/src/NServiceBus.Core.Tests/Pipeline/Incoming/InvokeHandlerTerminatorTest.cs +++ b/src/NServiceBus.Core.Tests/Pipeline/Incoming/InvokeHandlerTerminatorTest.cs @@ -11,7 +11,7 @@ [TestFixture] public class InvokeHandlerTerminatorTest { - readonly InvokeHandlerTerminator terminator = new(new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc")); + readonly InvokeHandlerTerminator terminator = new(new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); [Test] public async Task When_saga_found_and_handler_is_saga_should_invoke_handler() diff --git a/src/NServiceBus.Core.Tests/Pipeline/Incoming/SerializeMessageConnectorTests.cs b/src/NServiceBus.Core.Tests/Pipeline/Incoming/SerializeMessageConnectorTests.cs index 12b7ed19bc6..76e0efb2e00 100644 --- a/src/NServiceBus.Core.Tests/Pipeline/Incoming/SerializeMessageConnectorTests.cs +++ b/src/NServiceBus.Core.Tests/Pipeline/Incoming/SerializeMessageConnectorTests.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.IO; using System.Threading.Tasks; +using NServiceBus.Core.Tests.OpenTelemetry; using NServiceBus.Pipeline; using NUnit.Framework; using Serialization; @@ -29,7 +30,7 @@ public async Task Should_set_content_type_header() Message = new OutgoingLogicalMessage(typeof(MyMessage), new MyMessage()) }; - var behavior = new SerializeMessageConnector(new FakeSerializer("myContentType"), registry); + var behavior = new SerializeMessageConnector(new FakeSerializer("myContentType"), registry, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); await behavior.Invoke(context, c => Task.CompletedTask); diff --git a/src/NServiceBus.Core.Tests/Pipeline/IncomingPipelineMetricTagsTests.cs b/src/NServiceBus.Core.Tests/Pipeline/IncomingPipelineMetricTagsTests.cs index 52576e26016..e04e3999d2c 100644 --- a/src/NServiceBus.Core.Tests/Pipeline/IncomingPipelineMetricTagsTests.cs +++ b/src/NServiceBus.Core.Tests/Pipeline/IncomingPipelineMetricTagsTests.cs @@ -5,6 +5,7 @@ namespace NServiceBus.Core.Tests.Pipeline.Incoming; using System.IO; using System.Threading.Tasks; using MessageInterfaces.MessageMapper.Reflection; +using NServiceBus.Core.Tests.OpenTelemetry; using NServiceBus.Pipeline; using NUnit.Framework; using Serialization; @@ -35,7 +36,7 @@ public void Should_not_fail_when_handling_more_than_one_logical_message() }; var messageMapper = new MessageMapper(); - var behavior = new DeserializeMessageConnector(new MessageDeserializerResolver(new FakeSerializer(), []), new LogicalMessageFactory(registry, messageMapper), registry, messageMapper, false); + var behavior = new DeserializeMessageConnector(new MessageDeserializerResolver(new FakeSerializer(), []), new LogicalMessageFactory(registry, messageMapper), registry, messageMapper, false, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); Assert.DoesNotThrowAsync(async () => await behavior.Invoke(context, c => { diff --git a/src/NServiceBus.Core.Tests/Pipeline/MainPipelineExecutorTests.cs b/src/NServiceBus.Core.Tests/Pipeline/MainPipelineExecutorTests.cs index ee6b76d9e3b..8befd4e48c9 100644 --- a/src/NServiceBus.Core.Tests/Pipeline/MainPipelineExecutorTests.cs +++ b/src/NServiceBus.Core.Tests/Pipeline/MainPipelineExecutorTests.cs @@ -123,7 +123,7 @@ static MessageContext CreateMessageContext() => static MainPipelineExecutor CreateMainPipelineExecutor(ServiceProvider serviceProvider, IPipeline receivePipeline) { - var incomingPipelineMetrics = new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc"); + var incomingPipelineMetrics = new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions()); var executor = new MainPipelineExecutor( serviceProvider, new PipelineCache(serviceProvider, new PipelineModifications()), diff --git a/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs b/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs index 8f66cf3bf60..e33a9ef34a2 100644 --- a/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs +++ b/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs @@ -209,7 +209,7 @@ public void SetUp() fakeBatchPipeline = new FakeBatchPipeline(); fakeMeterFactory = new TestMeterFactory(); - behavior = new TransportReceiveToPhysicalMessageConnector(fakeOutbox, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc"), new InstrumentationOptions()); + behavior = new TransportReceiveToPhysicalMessageConnector(fakeOutbox, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions()), new InstrumentationOptions()); } [TearDown] diff --git a/src/NServiceBus.Core.Tests/Unicast/LoadHandlersConnectorTests.cs b/src/NServiceBus.Core.Tests/Unicast/LoadHandlersConnectorTests.cs index 60b8a81cca0..67cb7048465 100644 --- a/src/NServiceBus.Core.Tests/Unicast/LoadHandlersConnectorTests.cs +++ b/src/NServiceBus.Core.Tests/Unicast/LoadHandlersConnectorTests.cs @@ -4,6 +4,7 @@ using System.Threading.Tasks; using System.Transactions; using Core.Tests.Fakes; +using Core.Tests.OpenTelemetry; using Microsoft.Extensions.DependencyInjection; using NServiceBus.Transport; using NUnit.Framework; @@ -16,7 +17,7 @@ public class LoadHandlersConnectorTests [Test] public void Should_throw_when_there_are_no_registered_message_handlers() { - var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance); + var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); var context = new TestableIncomingLogicalMessageContext(); @@ -29,7 +30,7 @@ public void Should_throw_when_there_are_no_registered_message_handlers() [Test] public void Should_throw_if_ambient_transaction_is_different_from_scope_used_by_transport() { - var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance); + var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); var context = new TestableIncomingLogicalMessageContext(); @@ -49,7 +50,7 @@ public void Should_throw_if_ambient_transaction_is_different_from_scope_used_by_ [Test] public void Should_throw_if_ambient_transaction_suppressed_when_transport_uses_a_scope() { - var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance); + var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); var context = new TestableIncomingLogicalMessageContext(); @@ -77,7 +78,7 @@ public void Should_not_throw_if_ambient_scope_is_same_as_transport_scope() context.Services.AddSingleton(); context.Extensions.Set(new NoOpOutboxTransaction()); - var behavior = new LoadHandlersConnector(messageHandlerRegistry, NoOpActivityFactory.Instance); + var behavior = new LoadHandlersConnector(messageHandlerRegistry, NoOpActivityFactory.Instance, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); using (new TransactionScope(TransactionScopeAsyncFlowOption.Enabled)) { diff --git a/src/NServiceBus.Core/EndpointCreator.cs b/src/NServiceBus.Core/EndpointCreator.cs index 71c893904df..627887e0ff7 100644 --- a/src/NServiceBus.Core/EndpointCreator.cs +++ b/src/NServiceBus.Core/EndpointCreator.cs @@ -157,7 +157,7 @@ void Configure() pipelineSettings); receiveComponent.AddManifest(hostingConfiguration, settings); - pipelineComponent = PipelineComponent.Initialize(pipelineSettings, hostingConfiguration, receiveConfiguration); + pipelineComponent = PipelineComponent.Initialize(pipelineSettings, hostingConfiguration, receiveConfiguration, hostingConfiguration.ActivityFactory.Options.Meters); // The settings can only be locked after initializing the feature component since it uses the settings to store & share feature state. // As well as all the other components have been initialized diff --git a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs index 7028795c9aa..4930b5d2ffe 100644 --- a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs @@ -20,6 +20,11 @@ public partial class InstrumentationOptions /// public RecoverabilityInstrumentationOptions Recoverability { get; } = new(); + /// + /// Controls meter instruments behaviors. + /// + public MetersOptions Meters { get; } = new(); + /// /// Controls instrumentation of explicitly delayed messages (SendOptions.DelayDeliveryWith /// / DoNotDeliverBefore and saga timeouts. diff --git a/src/NServiceBus.Core/OpenTelemetry/MetersOptions.cs b/src/NServiceBus.Core/OpenTelemetry/MetersOptions.cs new file mode 100644 index 00000000000..0838da8aea8 --- /dev/null +++ b/src/NServiceBus.Core/OpenTelemetry/MetersOptions.cs @@ -0,0 +1,17 @@ +#nullable enable + +namespace NServiceBus; + +/// +/// Controls opt-in meter instruments behaviors. +/// Accessed via endpointConfiguration.Tracing().Meters. +/// +public class MetersOptions +{ + /// + /// Emits the legacy execution.result tag with values "success" or "failure" + /// on handler time, processing time, saga fetch time, deserialize time, and serialize time metrics. + /// Enabled by default for backwards compatibility. Disable to reduce tag cardinality. + /// + public bool EmitExecutionResultTags { get; set; } = true; +} diff --git a/src/NServiceBus.Core/OpenTelemetry/Metrics/MeterTags.cs b/src/NServiceBus.Core/OpenTelemetry/Metrics/MeterTags.cs index d99b135197e..f3fc9666fdb 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Metrics/MeterTags.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Metrics/MeterTags.cs @@ -13,4 +13,5 @@ static class MeterTags public const string ExecutionResult = "execution.result"; public const string ErrorType = "error.type"; public const string EnvelopeUnwrapperType = "nservicebus.envelope.unwrapper_type"; + public const string SagaType = "nservicebus.saga_type"; } \ No newline at end of file diff --git a/src/NServiceBus.Core/Pipeline/Incoming/DeserializeMessageConnector.cs b/src/NServiceBus.Core/Pipeline/Incoming/DeserializeMessageConnector.cs index dc01b394b78..573645d8750 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/DeserializeMessageConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/DeserializeMessageConnector.cs @@ -1,10 +1,11 @@ -#nullable enable +#nullable enable namespace NServiceBus; using System; using System.Collections.Concurrent; using System.Collections.Generic; +using System.Diagnostics; using System.Threading.Tasks; using Logging; using MessageInterfaces; @@ -17,14 +18,29 @@ class DeserializeMessageConnector( LogicalMessageFactory logicalMessageFactory, MessageMetadataRegistry messageMetadataRegistry, IMessageMapper mapper, - bool allowContentTypeInference) + bool allowContentTypeInference, + IncomingPipelineMetrics incomingPipelineMetrics) : StageConnector { public override async Task Invoke(IIncomingPhysicalMessageContext context, Func stage) { var incomingMessage = context.Message; - var messages = ExtractWithExceptionHandling(incomingMessage); + LogicalMessage[] messages; + var deserializeStart = Stopwatch.GetTimestamp(); + try + { + messages = ExtractWithExceptionHandling(incomingMessage); + } +#pragma warning disable PS0019 + catch (Exception ex) +#pragma warning restore PS0019 + { + incomingPipelineMetrics.RecordDeserializeTime(context, Stopwatch.GetElapsedTime(deserializeStart), error: ex); + throw; + } + + incomingPipelineMetrics.RecordDeserializeTime(context, Stopwatch.GetElapsedTime(deserializeStart)); bool first = true; foreach (var message in messages) @@ -137,4 +153,4 @@ LogicalMessage[] Extract(IncomingMessage physicalMessage) static readonly ILog log = LogManager.GetLogger(); static ReadOnlySpan ImplSuffix => "__impl".AsSpan(); static ReadOnlySpan EnclosedMessageTypeSeparator => ";".AsSpan(); -} \ No newline at end of file +} diff --git a/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs b/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs index 8fd252bf161..5f9a7cff744 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs @@ -23,10 +23,17 @@ class IncomingPipelineMetrics const string EnvelopeUnwrapping = "nservicebus.envelope.unwrapped"; const string ActiveMessages = "nservicebus.messaging.active_messages"; const string TotalDeduplicated = "nservicebus.outbox.duplicates"; - - public IncomingPipelineMetrics(IMeterFactory meterFactory, string queueName, string discriminator) + const string SagaFetchTime = "nservicebus.sagas.fetch_time"; + const string MessageDeserializeTime = "nservicebus.messaging.deserialize_time"; + const string MessageSerializeTime = "nservicebus.messaging.serialize_time"; + const string OutboxFetchTime = "nservicebus.outbox.fetch_time"; + const string OutboxStoreTime = "nservicebus.outbox.store_time"; + const string CommitTime = "nservicebus.persistence.commit_time"; + + public IncomingPipelineMetrics(IMeterFactory meterFactory, string queueName, string discriminator, MetersOptions metersOptions) { - var meter = meterFactory.Create("NServiceBus.Core.Pipeline.Incoming", "0.3.0"); + emitExecutionResultTags = metersOptions.EmitExecutionResultTags; + var meter = meterFactory.Create("NServiceBus.Core.Pipeline.Incoming", "0.4.0"); totalProcessedSuccessfully = meter.CreateCounter(TotalProcessedSuccessfully, description: "Total number of messages processed successfully by the endpoint."); totalFetched = meter.CreateCounter(TotalFetched, @@ -51,15 +58,27 @@ public IncomingPipelineMetrics(IMeterFactory meterFactory, string queueName, str description: "Total number of unwrapping attempts by the endpoint."); activeMessages = meter.CreateUpDownCounter(ActiveMessages, description: "Number of messages currently being processed by the endpoint."); + sagaFetchTime = meter.CreateHistogram(SagaFetchTime, "s", + "The time in seconds for loading saga data from the persister."); + messageDeserializeTime = meter.CreateHistogram(MessageDeserializeTime, "s", + "The time in seconds for deserializing an incoming message."); + messageSerializeTime = meter.CreateHistogram(MessageSerializeTime, "s", + "The time in seconds for serializing an outgoing message."); + outboxFetchTime = meter.CreateHistogram(OutboxFetchTime, "s", + "The time in seconds for querying the outbox storage for deduplication."); + outboxStoreTime = meter.CreateHistogram(OutboxStoreTime, "s", + "The time in seconds for storing a message in the outbox storage."); + persistenceTime = meter.CreateHistogram(CommitTime, "s", + "The time in seconds for completing the synchronized storage session."); queueNameBase = queueName; endpointDiscriminator = discriminator; } - public void AddDefaultIncomingPipelineMetricTags(IncomingPipelineMetricTags incomingPipelineMetricsTags) + public void AddDefaultIncomingPipelineMetricTags(IncomingPipelineMetricTags incomingPipelineMetricTags) { - incomingPipelineMetricsTags.Add(MeterTags.QueueName, queueNameBase); - incomingPipelineMetricsTags.Add(MeterTags.EndpointDiscriminator, endpointDiscriminator); + incomingPipelineMetricTags.Add(MeterTags.QueueName, queueNameBase); + incomingPipelineMetricTags.Add(MeterTags.EndpointDiscriminator, endpointDiscriminator); } public void RecordProcessingTime(ITransportReceiveContext context, TimeSpan elapsed) @@ -72,13 +91,16 @@ public void RecordProcessingTime(ITransportReceiveContext context, TimeSpan elap var incomingPipelineMetricTags = context.Extensions.Get(); TagList tags; - tags.Add(new(MeterTags.ExecutionResult, "success")); incomingPipelineMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.MessageType, MeterTags.MessageHandlerTypes]); + if (emitExecutionResultTags) + { + tags.Add(new KeyValuePair(MeterTags.ExecutionResult, "success")); + } processingTime.Record(elapsed.TotalSeconds, tags); } @@ -92,13 +114,17 @@ public void RecordCriticalTimeAndTotalProcessed(ITransportReceiveContext context var incomingPipelineMetricTags = context.Extensions.Get(); TagList tags; - tags.Add(new(MeterTags.ExecutionResult, "success")); incomingPipelineMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.MessageType, MeterTags.MessageHandlerTypes]); + if (emitExecutionResultTags) + { + tags.Add(new KeyValuePair(MeterTags.ExecutionResult, "success")); + } + if (totalProcessedSuccessfully.Enabled) { totalProcessedSuccessfully.Add(1, tags); @@ -124,12 +150,15 @@ public void RecordMessageProcessingFailure(IncomingPipelineMetricTags incomingPi TagList tags; tags.Add(new(MeterTags.ErrorType, error.GetType().FullName)); - tags.Add(new(MeterTags.ExecutionResult, "failure")); incomingPipelineMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.MessageType, MeterTags.MessageHandlerTypes]); + if (emitExecutionResultTags) + { + tags.Add(new KeyValuePair(MeterTags.ExecutionResult, "failure")); + } totalFailures.Add(1, tags); // the processing and critical time are intentionally not recorded in case of failure @@ -184,7 +213,10 @@ public void RecordSuccessfulMessageHandlerTime(IInvokeHandlerContext invokeHandl MeterTags.MessageHandlerType]); // This is what Add(string, object) does so skipping an unnecessary stack frame meterTags.Add(new KeyValuePair(MeterTags.MessageHandlerType, invokeHandlerContext.MessageHandler.HandlerType.FullName)); - meterTags.Add(new KeyValuePair(MeterTags.ExecutionResult, "success")); + if (emitExecutionResultTags) + { + meterTags.Add(new KeyValuePair(MeterTags.ExecutionResult, "success")); + } messageHandlerTime.Record(elapsed.TotalSeconds, meterTags); } @@ -204,8 +236,11 @@ public void RecordFailedMessageHandlerTime(IInvokeHandlerContext invokeHandlerCo MeterTags.MessageHandlerType]); // This is what Add(string, object) does so skipping an unnecessary stack frame meterTags.Add(new KeyValuePair(MeterTags.MessageHandlerType, invokeHandlerContext.MessageHandler.HandlerType.FullName)); - meterTags.Add(new KeyValuePair(MeterTags.ExecutionResult, "failure")); meterTags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); + if (emitExecutionResultTags) + { + meterTags.Add(new KeyValuePair(MeterTags.ExecutionResult, "failure")); + } messageHandlerTime.Record(elapsed.TotalSeconds, meterTags); } @@ -284,6 +319,130 @@ public ActiveMessageScope TrackMessageProcessing(IncomingPipelineMetricTags inco return new ActiveMessageScope(activeMessages, tags); } + public void RecordSagaFetchTime(IInvokeHandlerContext context, TimeSpan elapsed, string sagaType, Exception? error = null) + { + if (!sagaFetchTime.Enabled) + { + return; + } + + var incomingPipelineMetricTags = context.Extensions.Get(); + TagList tags; + incomingPipelineMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator, + MeterTags.MessageType]); + tags.Add(new KeyValuePair(MeterTags.SagaType, sagaType)); + if (error != null) + { + tags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); + } + if (emitExecutionResultTags) + { + tags.Add(new KeyValuePair(MeterTags.ExecutionResult, error != null ? "failure" : "success")); + } + sagaFetchTime.Record(elapsed.TotalSeconds, tags); + } + + public void RecordDeserializeTime(IIncomingPhysicalMessageContext context, TimeSpan elapsed, Exception? error = null) + { + if (!messageDeserializeTime.Enabled) + { + return; + } + + var incomingPipelineMetricTags = context.Extensions.Get(); + TagList tags; + incomingPipelineMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator + ]); + if (error != null) + { + tags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); + } + if (context.Message.Headers.TryGetValue(Headers.EnclosedMessageTypes, out var messageTypes)) + { + tags.Add(new KeyValuePair(MeterTags.EnclosedMessageTypes, messageTypes)); + } + if (emitExecutionResultTags) + { + tags.Add(new KeyValuePair(MeterTags.ExecutionResult, error != null ? "failure" : "success")); + } + + messageDeserializeTime.Record(elapsed.TotalSeconds, tags); + } + + public void RecordSerializeTime(TimeSpan elapsed, string? messageType, Exception? error = null) + { + if (!messageSerializeTime.Enabled) + { + return; + } + + TagList tags; + if (messageType != null) + { + tags.Add(new KeyValuePair(MeterTags.MessageType, messageType)); + } + if (error != null) + { + tags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); + } + if (emitExecutionResultTags) + { + tags.Add(new KeyValuePair(MeterTags.ExecutionResult, error != null ? "failure" : "success")); + } + messageSerializeTime.Record(elapsed.TotalSeconds, tags); + } + + public void RecordOutboxFetchTime(ITransportReceiveContext context, TimeSpan elapsed) + { + if (!outboxFetchTime.Enabled) + { + return; + } + + var incomingPipelineMetricTags = context.Extensions.Get(); + TagList tags; + incomingPipelineMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator]); + outboxFetchTime.Record(elapsed.TotalSeconds, tags); + } + + public void RecordOutboxStoreTime(ITransportReceiveContext context, TimeSpan elapsed) + { + if (!outboxStoreTime.Enabled) + { + return; + } + + var incomingPipelineMetricTags = context.Extensions.Get(); + TagList tags; + incomingPipelineMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator]); + outboxStoreTime.Record(elapsed.TotalSeconds, tags); + } + + public void RecordPersistenceTime(IIncomingLogicalMessageContext context, TimeSpan elapsed) + { + if (!persistenceTime.Enabled) + { + return; + } + + var incomingPipelineMetricTags = context.Extensions.Get(); + TagList tags; + incomingPipelineMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator, + MeterTags.MessageType, + MeterTags.MessageHandlerTypes]); + persistenceTime.Record(elapsed.TotalSeconds, tags); + } + public void EnvelopeUnwrappingSucceeded(MessageContext messageContext, IEnvelopeHandler type) => RecordEnvelopeUnwrapping(messageContext, type, true, null); public void EnvelopeUnwrappingFailed(MessageContext messageContext, IEnvelopeHandler type, Exception? exception) => RecordEnvelopeUnwrapping(messageContext, type, false, exception); void RecordEnvelopeUnwrapping(MessageContext messageContext, IEnvelopeHandler type, bool succeeded, Exception? exception) @@ -324,7 +483,14 @@ public readonly struct ActiveMessageScope(UpDownCounter? counter, TagList readonly Counter totalSentToErrorQueue; readonly Counter totalEnvelopeUnwrapping; readonly UpDownCounter activeMessages; + readonly Histogram sagaFetchTime; + readonly Histogram messageDeserializeTime; + readonly Histogram messageSerializeTime; + readonly Histogram outboxFetchTime; + readonly Histogram outboxStoreTime; + readonly Histogram persistenceTime; readonly string queueNameBase; readonly string endpointDiscriminator; + readonly bool emitExecutionResultTags; } \ No newline at end of file diff --git a/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs b/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs index e11af72114e..26a3b0178bf 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs @@ -16,7 +16,7 @@ namespace NServiceBus; using Pipeline; using Unicast; -class LoadHandlersConnector(MessageHandlerRegistry messageHandlerRegistry, IActivityFactory activityFactory) : StageConnector +class LoadHandlersConnector(MessageHandlerRegistry messageHandlerRegistry, IActivityFactory activityFactory, IncomingPipelineMetrics incomingPipelineMetrics) : StageConnector { public override async Task Invoke(IIncomingLogicalMessageContext context, Func stage) { @@ -80,7 +80,9 @@ public override async Task Invoke(IIncomingLogicalMessageContext context, Func(); await outboxTransaction.Commit(context.CancellationToken).ConfigureAwait(false); diff --git a/src/NServiceBus.Core/Pipeline/Outgoing/SerializeMessageConnector.cs b/src/NServiceBus.Core/Pipeline/Outgoing/SerializeMessageConnector.cs index 38e3cc223cc..14bbad12000 100644 --- a/src/NServiceBus.Core/Pipeline/Outgoing/SerializeMessageConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Outgoing/SerializeMessageConnector.cs @@ -3,6 +3,7 @@ namespace NServiceBus; using System; +using System.Diagnostics; using System.IO; using System.Threading.Tasks; using Logging; @@ -12,10 +13,11 @@ namespace NServiceBus; class SerializeMessageConnector : StageConnector { - public SerializeMessageConnector(IMessageSerializer messageSerializer, MessageMetadataRegistry messageMetadataRegistry) + public SerializeMessageConnector(IMessageSerializer messageSerializer, MessageMetadataRegistry messageMetadataRegistry, IncomingPipelineMetrics incomingPipelineMetrics) { this.messageSerializer = messageSerializer; this.messageMetadataRegistry = messageMetadataRegistry; + this.incomingPipelineMetrics = incomingPipelineMetrics; } public override async Task Invoke(IOutgoingLogicalMessageContext context, Func stage) @@ -39,7 +41,19 @@ public override async Task Invoke(IOutgoingLogicalMessageContext context, Func(); -} \ No newline at end of file +} diff --git a/src/NServiceBus.Core/Pipeline/PipelineComponent.cs b/src/NServiceBus.Core/Pipeline/PipelineComponent.cs index fba1beea0e1..03326dc412b 100644 --- a/src/NServiceBus.Core/Pipeline/PipelineComponent.cs +++ b/src/NServiceBus.Core/Pipeline/PipelineComponent.cs @@ -12,14 +12,15 @@ sealed class PipelineComponent PipelineComponent(PipelineModifications modifications) => this.modifications = modifications; public static PipelineComponent Initialize(PipelineSettings settings, - HostingComponent.Configuration hostingConfiguration, ReceiveComponent.Configuration receiveConfiguration) + HostingComponent.Configuration hostingConfiguration, ReceiveComponent.Configuration receiveConfiguration, + MetersOptions metersOptions) { // make the PipelineMetrics available to the Pipeline hostingConfiguration.Services.AddSingleton(sp => { var meterFactory = sp.GetRequiredService(); string discriminator = receiveConfiguration.InstanceSpecificQueueAddress?.Discriminator ?? ""; - return new IncomingPipelineMetrics(meterFactory, receiveConfiguration.LocalQueueAddress.BaseAddress, discriminator); + return new IncomingPipelineMetrics(meterFactory, receiveConfiguration.LocalQueueAddress.BaseAddress, discriminator, metersOptions); }); return new PipelineComponent(settings.modifications); diff --git a/src/NServiceBus.Core/Receiving/ReceiveComponent.cs b/src/NServiceBus.Core/Receiving/ReceiveComponent.cs index 4a0a5a8c1c6..5d859914cef 100644 --- a/src/NServiceBus.Core/Receiving/ReceiveComponent.cs +++ b/src/NServiceBus.Core/Receiving/ReceiveComponent.cs @@ -70,7 +70,7 @@ public static ReceiveComponent Configure( return new TransportReceiveToPhysicalMessageConnector(storage, b.GetRequiredService(), hostingConfiguration.ActivityFactory.Options); }, "Allows to abort processing the message"); - pipelineSettings.Register("LoadHandlersConnector", b => new LoadHandlersConnector(b.GetRequiredService(), hostingConfiguration.ActivityFactory), "Gets all the handlers to invoke from the MessageHandler registry based on the message type."); + pipelineSettings.Register("LoadHandlersConnector", b => new LoadHandlersConnector(b.GetRequiredService(), hostingConfiguration.ActivityFactory, b.GetRequiredService()), "Gets all the handlers to invoke from the MessageHandler registry based on the message type."); pipelineSettings.Register("InvokeHandlers", sp => new InvokeHandlerTerminator(sp.GetRequiredService()), "Calls the IHandleMessages.Handle(T)"); diff --git a/src/NServiceBus.Core/Sagas/SagaPersistenceBehavior.cs b/src/NServiceBus.Core/Sagas/SagaPersistenceBehavior.cs index 577a748e28a..66376382029 100644 --- a/src/NServiceBus.Core/Sagas/SagaPersistenceBehavior.cs +++ b/src/NServiceBus.Core/Sagas/SagaPersistenceBehavior.cs @@ -9,7 +9,7 @@ using Pipeline; using Sagas; -class SagaPersistenceBehavior(ISagaPersister persister, ISagaIdGenerator sagaIdGenerator, SagaMetadataCollection sagaMetadataCollection, IServiceProvider serviceProvider) +class SagaPersistenceBehavior(ISagaPersister persister, ISagaIdGenerator sagaIdGenerator, SagaMetadataCollection sagaMetadataCollection, IServiceProvider serviceProvider, IncomingPipelineMetrics incomingPipelineMetrics) : IBehavior { public async Task Invoke(IInvokeHandlerContext context, Func next) @@ -67,7 +67,20 @@ public async Task Invoke(IInvokeHandlerContext context, Func(); // Register the Saga related behaviors for incoming messages - context.Pipeline.Register("InvokeSaga", b => new SagaPersistenceBehavior(b.GetRequiredService(), sagaIdGenerator, sagaMetaModel, b), "Invokes the saga logic"); + context.Pipeline.Register("InvokeSaga", b => new SagaPersistenceBehavior(b.GetRequiredService(), sagaIdGenerator, sagaMetaModel, b, b.GetRequiredService()), "Invokes the saga logic"); context.Pipeline.Register("AttachSagaDetailsToOutGoingMessage", new AttachSagaDetailsToOutGoingMessageBehavior(), "Makes sure that outgoing messages have saga info attached to them"); } } \ No newline at end of file diff --git a/src/NServiceBus.Core/Serialization/SerializationFeature.cs b/src/NServiceBus.Core/Serialization/SerializationFeature.cs index a332f50bd37..41bec7b6d5a 100644 --- a/src/NServiceBus.Core/Serialization/SerializationFeature.cs +++ b/src/NServiceBus.Core/Serialization/SerializationFeature.cs @@ -48,8 +48,8 @@ protected override void Setup(FeatureConfigurationContext context) var allowMessageTypeInference = settings.IsMessageTypeInferenceEnabled(); var resolver = new MessageDeserializerResolver(mainSerializer, additionalDeserializers); var logicalMessageFactory = new LogicalMessageFactory(messageMetadataRegistry, mapper); - context.Pipeline.Register("DeserializeLogicalMessagesConnector", new DeserializeMessageConnector(resolver, logicalMessageFactory, messageMetadataRegistry, mapper, allowMessageTypeInference), "Deserializes the physical message body into logical messages"); - context.Pipeline.Register("SerializeMessageConnector", new SerializeMessageConnector(mainSerializer, messageMetadataRegistry), "Converts a logical message into a physical message"); + context.Pipeline.Register("DeserializeLogicalMessagesConnector", b => new DeserializeMessageConnector(resolver, logicalMessageFactory, messageMetadataRegistry, mapper, allowMessageTypeInference, b.GetRequiredService()), "Deserializes the physical message body into logical messages"); + context.Pipeline.Register("SerializeMessageConnector", b => new SerializeMessageConnector(mainSerializer, messageMetadataRegistry, b.GetRequiredService()), "Converts a logical message into a physical message"); context.Services.AddSingleton(mapper); context.Services.AddSingleton(mapper); From a1b03374dbc96b96d334dfe46c4543a6b34d6ec8 Mon Sep 17 00:00:00 2001 From: Tomasz Masternak Date: Tue, 11 Aug 2026 14:35:33 +0200 Subject: [PATCH 19/37] Minor tweaks to the OpenTelemetry featue (#7908) * adjust log levels for recoverability actions * split exception handling in recoverability action log messages to ensure backwards compatibility * updated exception log message formats in recoverability tests for consistency --- .../When_immediate_retries_are_enabled.cs | 2 +- .../When_message_fails_retries.cs | 2 +- .../RecoverabilityActionLogger.cs | 76 +++++++++++++++++-- 3 files changed, 72 insertions(+), 8 deletions(-) diff --git a/src/NServiceBus.AcceptanceTests/Recoverability/When_immediate_retries_are_enabled.cs b/src/NServiceBus.AcceptanceTests/Recoverability/When_immediate_retries_are_enabled.cs index 7f01e523cd3..51002130852 100644 --- a/src/NServiceBus.AcceptanceTests/Recoverability/When_immediate_retries_are_enabled.cs +++ b/src/NServiceBus.AcceptanceTests/Recoverability/When_immediate_retries_are_enabled.cs @@ -23,7 +23,7 @@ public async Task Should_do_the_configured_number_of_retries() Assert.That(context.ForwardedToErrorQueue, Is.True); Assert.That(context.NumberOfTimesInvoked, Is.EqualTo(numberOfRetries + 1), "Message should be retried 5 times immediately"); Assert.That(context.Logs.Count(l => l.Message - .StartsWith($"Immediate Retry is going to retry message '{context.MessageId}' because of an exception.")), Is.EqualTo(numberOfRetries)); + .StartsWith($"Immediate Retry is going to retry message '{context.MessageId}' because of an exception:")), Is.EqualTo(numberOfRetries)); } } diff --git a/src/NServiceBus.AcceptanceTests/Recoverability/When_message_fails_retries.cs b/src/NServiceBus.AcceptanceTests/Recoverability/When_message_fails_retries.cs index ba014c87f9b..8177397224a 100644 --- a/src/NServiceBus.AcceptanceTests/Recoverability/When_message_fails_retries.cs +++ b/src/NServiceBus.AcceptanceTests/Recoverability/When_message_fails_retries.cs @@ -28,7 +28,7 @@ public void Should_forward_message_to_error_queue() } Assert.That(testContext.Logs.Count(l => l.Message - .StartsWith($"Moving message '{testContext.PhysicalMessageId}' to the error queue 'error' because processing failed due to an exception.")), Is.EqualTo(1)); + .StartsWith($"Moving message '{testContext.PhysicalMessageId}' to the error queue 'error' because processing failed due to an exception:")), Is.EqualTo(1)); } public class RetryEndpoint : EndpointConfigurationBuilder diff --git a/src/NServiceBus.Core/Recoverability/RecoverabilityActionLogger.cs b/src/NServiceBus.Core/Recoverability/RecoverabilityActionLogger.cs index 40878f9730c..ca722b5550f 100644 --- a/src/NServiceBus.Core/Recoverability/RecoverabilityActionLogger.cs +++ b/src/NServiceBus.Core/Recoverability/RecoverabilityActionLogger.cs @@ -48,17 +48,81 @@ public void LogRecoverabilityAction(RecoverabilityAction recoverabilityAction, E static partial class RecoverabilityActionLoggerMessages { + // Exception is only attached when it hasn't already been logged separately by + // ActivityFactory.RecordError (see RecoverabilityActionLogger.LogRecoverabilityAction). + // The trailing punctuation differs in both scenarios, and we need to keep the colon when + // an exception is logged to ensure backwards compatibility + public static void ImmediateRetryLogged(this ILogger logger, Exception? exception, string messageId) + { + if (exception is not null) + { + ImmediateRetryLoggedWithException(logger, exception, messageId); + } + else + { + ImmediateRetryLoggedWithoutException(logger, messageId); + } + } + + [LoggerMessage(Level = LogLevel.Information, Message = "Immediate Retry is going to retry message '{MessageId}' because of an exception:")] + static partial void ImmediateRetryLoggedWithException(ILogger logger, Exception exception, string messageId); + [LoggerMessage(Level = LogLevel.Information, Message = "Immediate Retry is going to retry message '{MessageId}' because of an exception.")] - public static partial void ImmediateRetryLogged(this ILogger logger, Exception? exception, string messageId); + static partial void ImmediateRetryLoggedWithoutException(ILogger logger, string messageId); + + public static void DelayedRetryLogged(this ILogger logger, Exception? exception, string messageId, TimeSpan delay) + { + if (exception is not null) + { + DelayedRetryLoggedWithException(logger, exception, messageId, delay); + } + else + { + DelayedRetryLoggedWithoutException(logger, messageId, delay); + } + } + + [LoggerMessage(Level = LogLevel.Warning, Message = "Delayed Retry will reschedule message '{MessageId}' after a delay of {Delay} because of an exception:")] + static partial void DelayedRetryLoggedWithException(ILogger logger, Exception exception, string messageId, TimeSpan delay); - [LoggerMessage(Level = LogLevel.Information, Message = "Delayed Retry will reschedule message '{MessageId}' after a delay of {Delay} because of an exception.")] - public static partial void DelayedRetryLogged(this ILogger logger, Exception? exception, string messageId, TimeSpan delay); + [LoggerMessage(Level = LogLevel.Warning, Message = "Delayed Retry will reschedule message '{MessageId}' after a delay of {Delay} because of an exception.")] + static partial void DelayedRetryLoggedWithoutException(ILogger logger, string messageId, TimeSpan delay); + + public static void MoveToErrorLogged(this ILogger logger, Exception? exception, string messageId, string errorQueue) + { + if (exception is not null) + { + MoveToErrorLoggedWithException(logger, exception, messageId, errorQueue); + } + else + { + MoveToErrorLoggedWithoutException(logger, messageId, errorQueue); + } + } + + [LoggerMessage(Level = LogLevel.Error, Message = "Moving message '{MessageId}' to the error queue '{ErrorQueue}' because processing failed due to an exception:")] + static partial void MoveToErrorLoggedWithException(ILogger logger, Exception exception, string messageId, string errorQueue); + + [LoggerMessage(Level = LogLevel.Error, Message = "Moving message '{MessageId}' to the error queue '{ErrorQueue}' because processing failed due to an exception.")] + static partial void MoveToErrorLoggedWithoutException(ILogger logger, string messageId, string errorQueue); + + public static void DiscardLogged(this ILogger logger, Exception? exception, string messageId, string reason) + { + if (exception is not null) + { + DiscardLoggedWithException(logger, exception, messageId, reason); + } + else + { + DiscardLoggedWithoutException(logger, messageId, reason); + } + } - [LoggerMessage(Level = LogLevel.Information, Message = "Moving message '{MessageId}' to the error queue '{ErrorQueue}' because processing failed due to an exception.")] - public static partial void MoveToErrorLogged(this ILogger logger, Exception? exception, string messageId, string errorQueue); + [LoggerMessage(Level = LogLevel.Information, Message = "Discarding message with id '{MessageId}'. Reason: {Reason}")] + static partial void DiscardLoggedWithException(ILogger logger, Exception exception, string messageId, string reason); [LoggerMessage(Level = LogLevel.Information, Message = "Discarding message with id '{MessageId}'. Reason: {Reason}.")] - public static partial void DiscardLogged(this ILogger logger, Exception? exception, string messageId, string reason); + static partial void DiscardLoggedWithoutException(ILogger logger, string messageId, string reason); [LoggerMessage(Level = LogLevel.Information, Message = "Recoverability action '{ActionType}' invoked for message '{MessageId}'.")] public static partial void UnknownRecoverabilityActionLogged(this ILogger logger, Exception? exception, string actionType, string messageId); From 0c135cdde93772c45ed054972a0c5e77d7e51ecd Mon Sep 17 00:00:00 2001 From: Tomasz Masternak Date: Thu, 13 Aug 2026 11:33:41 +0200 Subject: [PATCH 20/37] Removed RecordedExceptions tracking in favor of directly using exception Data (#7911) * removed RecordedExceptions tracking in favor of directly using exception.Data * moved ExceptionRecordedFlag where it's used * Update src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs Co-authored-by: Daniel Marbach --------- Co-authored-by: Daniel Marbach --- .../OpenTelemetry/Tracing/ActivityFactory.cs | 7 ++++--- .../Tracing/RecordedExceptions.cs | 19 ------------------- .../Transports/MessageContext.cs | 1 - 3 files changed, 4 insertions(+), 23 deletions(-) delete mode 100644 src/NServiceBus.Core/OpenTelemetry/Tracing/RecordedExceptions.cs diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs index 9c407ce77b5..c8a958733fd 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs @@ -212,8 +212,7 @@ public void RecordError(Activity activity, Exception exception, ContextBag conte LegacyExceptionTags.SetLegacyStatusTags(activity, exception); - var recordedExceptions = context.GetOrCreate(); - if (!recordedExceptions.HasBeenRecorded(exception)) + if (!exception.Data.Contains(ExceptionRecordedFlag)) { if (Options.ExceptionRecordingMode == ExceptionRecordingMode.Logs) { @@ -224,7 +223,7 @@ public void RecordError(Activity activity, Exception exception, ContextBag conte activity.AddException(exception, LegacyExceptionTags.EscapedTagList); } - recordedExceptions.MarkAsRecorded(exception); + exception.Data[ExceptionRecordedFlag] = true; } if (exception is TaskCanceledException) @@ -233,5 +232,7 @@ public void RecordError(Activity activity, Exception exception, ContextBag conte } } + const string ExceptionRecordedFlag = "otel.exception.recorded"; + static readonly ILog Logger = LogManager.GetLogger(); } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/RecordedExceptions.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/RecordedExceptions.cs deleted file mode 100644 index 4ecfafb8152..00000000000 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/RecordedExceptions.cs +++ /dev/null @@ -1,19 +0,0 @@ -#nullable enable - -namespace NServiceBus; - -using System; -using System.Collections.Generic; - -sealed class RecordedExceptions -{ - // Reference equality is intentional: this tracks specific exception instances as they - // propagate, not exceptions that merely look alike. -#pragma warning disable PS0025 - readonly HashSet recorded = new(ReferenceEqualityComparer.Instance); -#pragma warning restore PS0025 - - public bool HasBeenRecorded(Exception exception) => recorded.Contains(exception); - - public void MarkAsRecorded(Exception exception) => recorded.Add(exception); -} diff --git a/src/NServiceBus.Core/Transports/MessageContext.cs b/src/NServiceBus.Core/Transports/MessageContext.cs index a48e6ec3b15..7d191154923 100644 --- a/src/NServiceBus.Core/Transports/MessageContext.cs +++ b/src/NServiceBus.Core/Transports/MessageContext.cs @@ -51,7 +51,6 @@ public MessageContext(string nativeMessageId, Dictionary headers TransportTransaction = transportTransaction; context.GetOrCreate(); - context.GetOrCreate(); } /// From 96f8a168ea6effb47af4fbeac16cdcd615e79632 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Wed, 26 Aug 2026 14:48:30 +0200 Subject: [PATCH 21/37] Fix edge case where InstrumentionOptions could not be a shared instance (#7914) --- ...essing_fails_with_exception_logs_opt_in.cs | 59 +++++++++++++++++++ .../Hosting/HostingComponent.Settings.cs | 2 +- 2 files changed, 60 insertions(+), 1 deletion(-) create mode 100644 src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_fails_with_exception_logs_opt_in.cs diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_fails_with_exception_logs_opt_in.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_fails_with_exception_logs_opt_in.cs new file mode 100644 index 00000000000..c5d6b664333 --- /dev/null +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_fails_with_exception_logs_opt_in.cs @@ -0,0 +1,59 @@ +namespace NServiceBus.AcceptanceTests.Core.OpenTelemetry.Traces; + +using System.Linq; +using System.Threading.Tasks; +using AcceptanceTesting; +using Configuration.AdvancedExtensibility; +using EndpointTemplates; +using NServiceBus; +using NUnit.Framework; + +// The OTEL_SEMCONV_EXCEPTION_SIGNAL_OPT_IN override is applied while the OpenTelemetryFeature defaults +// run, which is AFTER the endpoint's activity factory has already been built from the instrumentation +// options. The opt-in only takes effect when the activity factory and the settings share a single +// InstrumentationOptions instance. +public class When_processing_fails_with_exception_logs_opt_in : OpenTelemetryAcceptanceTest +{ + [Test] + public async Task Should_record_the_exception_as_a_log_instead_of_a_span_event() + { + var context = await Scenario.Define() + .WithEndpoint(e => e + .DoNotFailOnErrorMessages() + .When(s => s.SendLocal(new FailingMessage()))) + .Run(); + + Assert.That(context.FailedMessages, Has.Count.EqualTo(1), "the message should have failed"); + + var handlerActivity = NServiceBusActivityListener.CompletedActivities.GetInvokedHandlerActivities().Single(); + + using (Assert.EnterMultipleScope()) + { + Assert.That(handlerActivity.Events, Is.Empty, "the exception should not be recorded as a span event when the endpoint opted in to exceptions as logs"); + Assert.That(context.Logs.Any(l => l.LoggerName == "NServiceBus.ActivityFactory" && l.Level == Logging.LogLevel.Error && l.Message.Contains(ErrorMessage)), Is.True, "the exception should be recorded as an error log instead"); + } + } + + public class Context : ScenarioContext; + + public class FailingEndpoint : EndpointConfigurationBuilder + { + // Does not call endpointConfiguration.Tracing(): the instrumentation options only come into existence while the endpoint is being created. + public FailingEndpoint() => EndpointSetup(c => c.GetSettings().Set($"ACCEPTANCETEST_ENV:{OptInEnvironmentVariable}", "logs")); + + [Handler] + public class FailingMessageHandler(Context testContext) : IHandleMessages + { + public Task Handle(FailingMessage message, IMessageHandlerContext context) + { + testContext.MarkAsCompleted(); + throw new SimulatedException(ErrorMessage); + } + } + } + + public class FailingMessage : IMessage; + + const string OptInEnvironmentVariable = "OTEL_SEMCONV_EXCEPTION_SIGNAL_OPT_IN"; + const string ErrorMessage = "boom!"; +} \ No newline at end of file diff --git a/src/NServiceBus.Core/Hosting/HostingComponent.Settings.cs b/src/NServiceBus.Core/Hosting/HostingComponent.Settings.cs index 8e85864c2be..d2b8bdb4027 100644 --- a/src/NServiceBus.Core/Hosting/HostingComponent.Settings.cs +++ b/src/NServiceBus.Core/Hosting/HostingComponent.Settings.cs @@ -95,7 +95,7 @@ public bool WriteDiagnosticsToLog get; set; } - public InstrumentationOptions InstrumentationOptions => settings.GetOrDefault() ?? new InstrumentationOptions(); + public InstrumentationOptions InstrumentationOptions => settings.GetOrCreate(); internal void ConfigureHostLogging(object? endpointIdentifier) { From e3eaa5c763dc84292660ac78776597ae520bfe38 Mon Sep 17 00:00:00 2001 From: Tomasz Masternak Date: Tue, 1 Sep 2026 10:42:00 +0200 Subject: [PATCH 22/37] Add support for instrument-specific metric tags in the incoming pipeline (#7912) * Add support for instrument-specific metric tags in the incoming pipeline * Introduce support for optional instrument names in metric tag methods and improve metric handling logic * Simplify metric tagging logic and remove `IncomingPipelineMetricsTagBagConventionTests`. * Simplify `RecordSerializeTime` by removing unnecessary context dependency and refactoring metric tagging logic. * fix to outgoing pipeline metric tagging * Improve outgoing pipeline metric tagging and refactor `RecordSerializeTime` to handle context-based tags. * Remove unused metric tags in `IncomingPipelineMetrics`. * Refactor metric tagging by replacing `Get` with `MetricTags` extension property. Remove IncomingPipelineMetrics dependency from fakes * Introduce `IMetricsTags` interface as the public api for changing metric tags. * Refactor metric tagging logic in `IncomingPipelineMetrics` to simplify tag application and enforce consistent usage of `TagList`. * Refactor `criticalTime` calculation to ensure `completedAt` is only set when needed * Fix incorrect usage of `messageDeserializeTime` to `messageSerializeTime` in metric tagging logic * Rename `Add` to `AddOrOverride` in `IMetricsTags * Remove unnecessary call to `GetOrCreate` in `MessageContext` constructor. All calls are now GetOrCreate via extension property. * change the base type for MetricTagsExtensions to concrete type from ExtensionBag. Makes accessing tags requrie less nesting * Refactor metric tagging tests to use constants for tag keys and simplify assertions * fixing formatting errors --- .../Metrics/When_customizing_metric_tags.cs | 110 +++++++ .../NServiceBus.AcceptanceTests.csproj | 1 + ...IApprovals.ApproveNServiceBus.approved.txt | 21 +- .../NServiceBus.Core.Tests.csproj | 1 - .../IncomingPipelineMetricTagsTests.cs | 2 +- .../Incoming/DeserializeMessageConnector.cs | 2 +- .../Pipeline/Incoming/IMetricsTags.cs | 23 ++ .../Incoming/IncomingPipelineMetricTags.cs | 84 ++++-- .../Incoming/IncomingPipelineMetrics.cs | 279 ++++++++++-------- .../Incoming/LoadHandlersConnector.cs | 2 +- .../Pipeline/Incoming/MetricTagsExtensions.cs | 29 ++ .../Pipeline/MainPipelineExecutor.cs | 2 +- .../Outgoing/SerializeMessageConnector.cs | 4 +- .../Transports/MessageContext.cs | 2 - .../TestablePipelineContext.cs | 6 +- 15 files changed, 398 insertions(+), 170 deletions(-) create mode 100644 src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_customizing_metric_tags.cs create mode 100644 src/NServiceBus.Core/Pipeline/Incoming/IMetricsTags.cs create mode 100644 src/NServiceBus.Core/Pipeline/Incoming/MetricTagsExtensions.cs diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_customizing_metric_tags.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_customizing_metric_tags.cs new file mode 100644 index 00000000000..e964f5faf57 --- /dev/null +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_customizing_metric_tags.cs @@ -0,0 +1,110 @@ +namespace NServiceBus.AcceptanceTests.Core.OpenTelemetry.Metrics; + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using EndpointTemplates; +using NServiceBus; +using AcceptanceTesting; +using NServiceBus.Pipeline; +using NUnit.Framework; +using global::OpenTelemetry; +using global::OpenTelemetry.Metrics; + +public class When_customizing_metric_tags : OpenTelemetryAcceptanceTest +{ + const string TotalFetched = "nservicebus.messaging.fetches"; + const string MessageDeserializeTime = "nservicebus.messaging.deserialize_time"; + const string EndpointDiscriminatorTag = "nservicebus.discriminator"; + const string EnclosedMessageTypesTag = "nservicebus.enclosed_message_types"; + const string TenantTag = "acceptance.tenant_id"; + const string FriendlyMessageTypeName = "Order placed (friendly name)"; + + [Test] + public async Task Should_allow_adding_removing_and_overriding_tags_per_instrument() + { + using var metricsListener = TestingMetricListener.SetupNServiceBusMetricsListener(); + + List exportedMetrics = []; + using var meterProvider = Sdk.CreateMeterProviderBuilder() + .AddMeter("NServiceBus.Core.Pipeline.Incoming") + .AddView(TotalFetched, new MetricStreamConfiguration + { + TagKeys = ["nservicebus.queue", "nservicebus.message_type", TenantTag] + }) + .AddReader(new BaseExportingMetricReader(new CapturingExporter(exportedMetrics))) + .Build(); + + await Scenario.Define() + .WithEndpoint(b => b.CustomConfig(c => c.MakeInstanceUniquelyAddressable("disc")) + .When(async session => + { + var sendOptions = new SendOptions(); + sendOptions.RouteToThisEndpoint(); + sendOptions.SetHeader(TenantTag, "acme-corp"); + await session.Send(new MyMessage(), sendOptions); + })) + .Run(); + + meterProvider.ForceFlush(); + + metricsListener.AssertTags(TotalFetched, new Dictionary { [TenantTag] = "acme-corp" }); + + metricsListener.AssertTagKeyExists(TotalFetched, EndpointDiscriminatorTag); + + var overriddenValue = metricsListener.AssertTagKeyExists(MessageDeserializeTime, EnclosedMessageTypesTag); + Assert.That(overriddenValue, Is.EqualTo(FriendlyMessageTypeName)); + } + + public class Context : ScenarioContext; + + public class EndpointWithCustomTags : EndpointConfigurationBuilder + { + public EndpointWithCustomTags() => + EndpointSetup(c => c.Pipeline.Register( + new CustomizeMetricTagsBehavior(), "Adds a tenant tag from a header and overrides the enclosed message type tag")); + + [Handler] + public class MyHandler(Context testContext) : IHandleMessages + { + public Task Handle(MyMessage message, IMessageHandlerContext context) + { + testContext.MarkAsCompleted(); + return Task.CompletedTask; + } + } + } + + class CustomizeMetricTagsBehavior : Behavior + { + public override Task Invoke(IIncomingPhysicalMessageContext context, Func next) + { + var tags = context.MetricTags; + + if (context.Message.Headers.TryGetValue(TenantTag, out var tenantId)) + { + tags.AddOrOverride(TenantTag, tenantId, TotalFetched); + } + + tags.AddOrOverride(EnclosedMessageTypesTag, FriendlyMessageTypeName, MessageDeserializeTime); + + return next(); + } + } + + class CapturingExporter(List exportedMetrics) : BaseExporter + { + public override ExportResult Export(in Batch batch) + { + foreach (var metric in batch) + { + exportedMetrics.Add(metric); + } + + return ExportResult.Success; + } + } + + public class MyMessage : IMessage; +} diff --git a/src/NServiceBus.AcceptanceTests/NServiceBus.AcceptanceTests.csproj b/src/NServiceBus.AcceptanceTests/NServiceBus.AcceptanceTests.csproj index ea546b1c257..1af349bf801 100644 --- a/src/NServiceBus.AcceptanceTests/NServiceBus.AcceptanceTests.csproj +++ b/src/NServiceBus.AcceptanceTests/NServiceBus.AcceptanceTests.csproj @@ -14,6 +14,7 @@ + diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt index 78f0fc99ed0..c7302ebe466 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt @@ -554,6 +554,10 @@ namespace NServiceBus System.Threading.Tasks.Task Subscribe(System.Type eventType, NServiceBus.SubscribeOptions subscribeOptions, System.Threading.CancellationToken cancellationToken = default); System.Threading.Tasks.Task Unsubscribe(System.Type eventType, NServiceBus.UnsubscribeOptions unsubscribeOptions, System.Threading.CancellationToken cancellationToken = default); } + public interface IMetricsTags + { + void AddOrOverride(string tagKey, object value, string instrumentName); + } public interface INeedInitialization { void Customize(NServiceBus.EndpointConfiguration configuration); @@ -618,13 +622,6 @@ namespace NServiceBus public override NServiceBus.Transport.ErrorHandleResult ErrorHandleResult { get; } public override System.Collections.Generic.IReadOnlyCollection GetRoutingContexts(NServiceBus.Pipeline.IRecoverabilityActionContext context) { } } - public sealed class IncomingPipelineMetricTags - { - public IncomingPipelineMetricTags() { } - public void Add(string tagKey, object value) { } - public void ApplyTag(ref System.Diagnostics.TagList tagList, string tagKey) { } - public void ApplyTags(ref System.Diagnostics.TagList tagList, System.ReadOnlySpan tagKeys) { } - } public static class InstallConfigExtensions { public static void AddInstaller<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembers(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.None | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicParameterlessConstructor | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicMethods | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.Interfaces)] TInstaller>(this NServiceBus.EndpointConfiguration config) @@ -774,6 +771,16 @@ namespace NServiceBus public MetersOptions() { } public bool EmitExecutionResultTags { get; set; } } + public static class MetricTagsExtensions + { + extension(NServiceBus.Pipeline.IBehaviorContext context) + { + public NServiceBus.IMetricsTags MetricTags { get; } + } + extension(NServiceBus.Transport.MessageContext context) + { + } + } public class MoveToError : NServiceBus.RecoverabilityAction { protected MoveToError(string errorQueue) { } diff --git a/src/NServiceBus.Core.Tests/NServiceBus.Core.Tests.csproj b/src/NServiceBus.Core.Tests/NServiceBus.Core.Tests.csproj index 52ccab6f296..9d3ef87ddac 100644 --- a/src/NServiceBus.Core.Tests/NServiceBus.Core.Tests.csproj +++ b/src/NServiceBus.Core.Tests/NServiceBus.Core.Tests.csproj @@ -4,7 +4,6 @@ net10.0 true ..\NServiceBusTests.snk - 13.0 diff --git a/src/NServiceBus.Core.Tests/Pipeline/IncomingPipelineMetricTagsTests.cs b/src/NServiceBus.Core.Tests/Pipeline/IncomingPipelineMetricTagsTests.cs index e04e3999d2c..1ce103a841c 100644 --- a/src/NServiceBus.Core.Tests/Pipeline/IncomingPipelineMetricTagsTests.cs +++ b/src/NServiceBus.Core.Tests/Pipeline/IncomingPipelineMetricTagsTests.cs @@ -40,7 +40,7 @@ public void Should_not_fail_when_handling_more_than_one_logical_message() Assert.DoesNotThrowAsync(async () => await behavior.Invoke(context, c => { - c.Extensions.Get().Add("Same", "Same"); + c.IncomingMetricTags.Add("Same", "Same"); return Task.CompletedTask; })); } diff --git a/src/NServiceBus.Core/Pipeline/Incoming/DeserializeMessageConnector.cs b/src/NServiceBus.Core/Pipeline/Incoming/DeserializeMessageConnector.cs index 573645d8750..1afa8f9986c 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/DeserializeMessageConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/DeserializeMessageConnector.cs @@ -47,7 +47,7 @@ public override async Task Invoke(IIncomingPhysicalMessageContext context, Func< { if (first) // ignore the legacy case in which a single message payload contained multiple messages { - var availableMetricTags = context.Extensions.Get(); + var availableMetricTags = context.IncomingMetricTags; availableMetricTags.Add(MeterTags.MessageType, message.MessageType.FullName!); first = false; } diff --git a/src/NServiceBus.Core/Pipeline/Incoming/IMetricsTags.cs b/src/NServiceBus.Core/Pipeline/Incoming/IMetricsTags.cs new file mode 100644 index 00000000000..d5b90d6446a --- /dev/null +++ b/src/NServiceBus.Core/Pipeline/Incoming/IMetricsTags.cs @@ -0,0 +1,23 @@ +#nullable enable + +namespace NServiceBus; + +/// +/// The tags applied to the metrics reported for the message currently being processed. +/// +public interface IMetricsTags +{ + /// + /// Adds the specified tag and value to , overwriting any value previously added + /// for that instrument and tag key, and taking precedence over the value NServiceBus reports for that tag. + /// + /// + /// Tags are scoped to a single instrument because a tag value is frequently only valid for one measurement (for + /// example, which handler just ran) rather than a fact that holds for every metric reported for the message. For + /// the same reason a later call for the same instrument and tag key replaces an earlier one. + /// + /// The tag to add. + /// The value assigned to the tag. + /// The name of the instrument the tag applies to. + void AddOrOverride(string tagKey, object value, string instrumentName); +} diff --git a/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetricTags.cs b/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetricTags.cs index 534d735e08b..f1f0f4d7a47 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetricTags.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetricTags.cs @@ -9,9 +9,26 @@ namespace NServiceBus; /// /// Captures possible metric tags that can be applied to a metric throughout the incoming processing pipeline. /// -public sealed class IncomingPipelineMetricTags +sealed class IncomingPipelineMetricTags : IMetricsTags { - Dictionary>? tags; + readonly Dictionary> tags = []; + readonly Dictionary>> instrumentTags = []; + + /// + /// + /// Unlike , this overwrites rather than keeping the first value, and the tag + /// isn't visible to other instruments applying tags from this collection. + /// + public void AddOrOverride(string tagKey, object value, string instrumentName) + { + if (!instrumentTags.TryGetValue(instrumentName, out var perInstrumentTags)) + { + perInstrumentTags = []; + instrumentTags.Add(instrumentName, perInstrumentTags); + } + + perInstrumentTags[tagKey] = new(tagKey, value); + } /// /// Adds the specified tag and value to the collection if not already present. @@ -19,48 +36,55 @@ public sealed class IncomingPipelineMetricTags /// The tag to add. /// The value assigned to the tag. public void Add(string tagKey, object value) - { - tags ??= []; - // We are using tryAdd to mitigate multiple logical messages transmitted in a single physical message - tags.TryAdd(tagKey, new(tagKey, value)); - } + => tags.TryAdd(tagKey, new KeyValuePair(tagKey, value)); /// - /// Applies the specified tag to the . + /// Applies the specified tags to the , replacing any tag already in + /// with a matching key - so a caller can populate with its + /// own computed defaults before calling this, and have any matching tag from this collection take precedence. + /// General tags (from ) are only applied when their key is in + /// . When is provided, every tag added for that + /// instrument via is applied unconditionally - regardless of whether its + /// key is in - and takes precedence over a general tag with the same key: naming an + /// instrument when adding a tag is already an explicit statement of intent for that one instrument, so callers + /// don't also need to know about it to pull it in. /// - /// The tagList to apply the specified tag to. - /// The tag to add to the . - public void ApplyTag(ref TagList tagList, string tagKey) + /// The tagList to add the tags to. + /// The collection of tag keys to apply to the . + /// The instrument to apply instrument-specific tags for, if any. + public void ApplyTags(ref TagList tagList, ReadOnlySpan tagKeys, string? instrumentName = null) { - if (tags == null) + foreach (var tagKey in tagKeys) { - return; + if (tags.TryGetValue(tagKey, out var keyValuePair)) + { + SetOrAdd(ref tagList, keyValuePair); + } } - if (tags.TryGetValue(tagKey, out var keyValuePair)) + if (instrumentName != null && instrumentTags.TryGetValue(instrumentName, out var perInstrumentTags)) { - tagList.Add(keyValuePair); + foreach (var (_, keyValuePair) in perInstrumentTags) + { + SetOrAdd(ref tagList, keyValuePair); + } } } - /// - /// Applies the specified tags to the . - /// - /// The tagList to add the tags to. - /// The collection of tag keys to apply to the . - public void ApplyTags(ref TagList tagList, ReadOnlySpan tagKeys) + // A caller may have already added a computed default for this key directly to tagList before calling + // ApplyTag/ApplyTags. Replacing it in place - rather than appending a duplicate - is what lets a tag from this + // collection act as an override regardless of how a consumer of the recorded measurement handles duplicate keys. + static void SetOrAdd(ref TagList tagList, KeyValuePair tag) { - if (tags == null || tagKeys.IsEmpty) + for (var i = 0; i < tagList.Count; i++) { - return; - } - - foreach (var tagKey in tagKeys) - { - if (tags.TryGetValue(tagKey, out var keyValuePair)) + if (tagList[i].Key == tag.Key) { - tagList.Add(keyValuePair); + tagList[i] = tag; + return; } } + + tagList.Add(tag); } -} \ No newline at end of file +} diff --git a/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs b/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs index 5f9a7cff744..45ee4da5c5e 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs @@ -88,19 +88,20 @@ public void RecordProcessingTime(ITransportReceiveContext context, TimeSpan elap return; } - var incomingPipelineMetricTags = context.Extensions.Get(); - TagList tags; - incomingPipelineMetricTags.ApplyTags(ref tags, [ - MeterTags.QueueName, - MeterTags.EndpointDiscriminator, - MeterTags.MessageType, - MeterTags.MessageHandlerTypes]); - if (emitExecutionResultTags) { + // Execution result is to be removed so we don't support overriding it by the user tags.Add(new KeyValuePair(MeterTags.ExecutionResult, "success")); } + + context.IncomingMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator, + MeterTags.MessageType, + MeterTags.MessageHandlerTypes], + processingTime.Name); + processingTime.Record(elapsed.TotalSeconds, tags); } @@ -110,32 +111,34 @@ public void RecordCriticalTimeAndTotalProcessed(ITransportReceiveContext context { return; } - - var incomingPipelineMetricTags = context.Extensions.Get(); - TagList tags; - incomingPipelineMetricTags.ApplyTags(ref tags, [ - MeterTags.QueueName, - MeterTags.EndpointDiscriminator, - MeterTags.MessageType, - MeterTags.MessageHandlerTypes]); - if (emitExecutionResultTags) { + // Execution result is to be removed so we don't support overriding it by the user tags.Add(new KeyValuePair(MeterTags.ExecutionResult, "success")); } + // totalProcessedSuccessfully and criticalTime always share the same tags in this method, so overrides are + // looked up under criticalTime's instrument name. + context.IncomingMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator, + MeterTags.MessageType, + MeterTags.MessageHandlerTypes], + criticalTime.Name); + if (totalProcessedSuccessfully.Enabled) { totalProcessedSuccessfully.Add(1, tags); } - var completedAt = DateTimeOffset.UtcNow; if (criticalTime.Enabled) { if (context.Message.Headers.TryGetDeliverAt(out var startTime) || context.Message.Headers.TryGetTimeSent(out startTime)) { + var completedAt = DateTimeOffset.UtcNow; var criticalTimeElapsed = completedAt - startTime; + criticalTime.Record(criticalTimeElapsed.TotalSeconds, tags); } } @@ -149,16 +152,21 @@ public void RecordMessageProcessingFailure(IncomingPipelineMetricTags incomingPi } TagList tags; - tags.Add(new(MeterTags.ErrorType, error.GetType().FullName)); - incomingPipelineMetricTags.ApplyTags(ref tags, [ - MeterTags.QueueName, - MeterTags.EndpointDiscriminator, - MeterTags.MessageType, - MeterTags.MessageHandlerTypes]); if (emitExecutionResultTags) { + // Execution result is to be removed so we don't support overriding it by the user tags.Add(new KeyValuePair(MeterTags.ExecutionResult, "failure")); } + + tags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); + + incomingPipelineMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator, + MeterTags.MessageType, + MeterTags.MessageHandlerTypes, + MeterTags.ErrorType], + totalFailures.Name); totalFailures.Add(1, tags); // the processing and critical time are intentionally not recorded in case of failure @@ -175,7 +183,8 @@ public void RecordFetchedMessage(IncomingPipelineMetricTags incomingPipelineMetr incomingPipelineMetricTags.ApplyTags(ref tags, [ MeterTags.EndpointDiscriminator, MeterTags.QueueName, - MeterTags.MessageType]); + MeterTags.MessageType], + totalFetched.Name); totalFetched.Add(1, tags); } @@ -187,118 +196,126 @@ public void RecordDeduplicatedMessage(ITransportReceiveContext context) return; } - var incomingPipelineMetricTags = context.Extensions.Get(); TagList tags; - incomingPipelineMetricTags.ApplyTags(ref tags, [ + context.IncomingMetricTags.ApplyTags(ref tags, [ MeterTags.EndpointDiscriminator, MeterTags.QueueName, - MeterTags.MessageType]); + MeterTags.MessageType], + totalDeduplicated.Name); totalDeduplicated.Add(1, tags); } - public void RecordSuccessfulMessageHandlerTime(IInvokeHandlerContext invokeHandlerContext, TimeSpan elapsed) + public void RecordSuccessfulMessageHandlerTime(IInvokeHandlerContext context, TimeSpan elapsed) { if (!messageHandlerTime.Enabled) { return; } - var incomingPipelineMetricTags = invokeHandlerContext.Extensions.Get(); - TagList meterTags; - incomingPipelineMetricTags.ApplyTags(ref meterTags, [ - MeterTags.QueueName, - MeterTags.EndpointDiscriminator, - MeterTags.MessageType, - MeterTags.MessageHandlerType]); - // This is what Add(string, object) does so skipping an unnecessary stack frame - meterTags.Add(new KeyValuePair(MeterTags.MessageHandlerType, invokeHandlerContext.MessageHandler.HandlerType.FullName)); + TagList tags; if (emitExecutionResultTags) { - meterTags.Add(new KeyValuePair(MeterTags.ExecutionResult, "success")); + // Execution result is to be removed so we don't support overriding it by the user + tags.Add(new KeyValuePair(MeterTags.ExecutionResult, "success")); } - messageHandlerTime.Record(elapsed.TotalSeconds, meterTags); + + tags.Add(new KeyValuePair(MeterTags.MessageHandlerType, context.MessageHandler.HandlerType.FullName)); + + context.IncomingMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator, + MeterTags.MessageType, + MeterTags.MessageHandlerType], + messageHandlerTime.Name); + messageHandlerTime.Record(elapsed.TotalSeconds, tags); } - public void RecordFailedMessageHandlerTime(IInvokeHandlerContext invokeHandlerContext, TimeSpan elapsed, Exception error) + public void RecordFailedMessageHandlerTime(IInvokeHandlerContext context, TimeSpan elapsed, Exception error) { if (!messageHandlerTime.Enabled) { return; } - var incomingPipelineMetricTags = invokeHandlerContext.Extensions.Get(); - TagList meterTags; - incomingPipelineMetricTags.ApplyTags(ref meterTags, [ - MeterTags.QueueName, - MeterTags.EndpointDiscriminator, - MeterTags.MessageType, - MeterTags.MessageHandlerType]); - // This is what Add(string, object) does so skipping an unnecessary stack frame - meterTags.Add(new KeyValuePair(MeterTags.MessageHandlerType, invokeHandlerContext.MessageHandler.HandlerType.FullName)); - meterTags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); + TagList tags; if (emitExecutionResultTags) { - meterTags.Add(new KeyValuePair(MeterTags.ExecutionResult, "failure")); + // Execution result is to be removed so we don't support overriding it by the user + tags.Add(new KeyValuePair(MeterTags.ExecutionResult, "failure")); } - messageHandlerTime.Record(elapsed.TotalSeconds, meterTags); + + tags.Add(new KeyValuePair(MeterTags.MessageHandlerType, context.MessageHandler.HandlerType.FullName)); + tags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); + + context.IncomingMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator, + MeterTags.MessageType, + MeterTags.MessageHandlerType, + MeterTags.ErrorType], + messageHandlerTime.Name); + messageHandlerTime.Record(elapsed.TotalSeconds, tags); } - public void RecordImmediateRetry(IRecoverabilityContext recoverabilityContext) + public void RecordImmediateRetry(IRecoverabilityContext context) { if (!totalImmediateRetries.Enabled) { return; } - var incomingPipelineMetricTags = recoverabilityContext.Extensions.Get(); - TagList meterTags; - incomingPipelineMetricTags.ApplyTags(ref meterTags, [ + TagList tags; + tags.Add(new KeyValuePair(MeterTags.ErrorType, context.Exception.GetType().FullName)); + + context.IncomingMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.MessageType, - MeterTags.MessageHandlerType]); - // This is what Add(string, object) does so skipping an unnecessary stack frame - meterTags.Add(new KeyValuePair(MeterTags.ErrorType, recoverabilityContext.Exception.GetType().FullName)); - totalImmediateRetries.Add(1, meterTags); + MeterTags.MessageHandlerType, + MeterTags.ErrorType], + totalImmediateRetries.Name); + totalImmediateRetries.Add(1, tags); } - public void RecordDelayedRetry(IRecoverabilityContext recoverabilityContext) + public void RecordDelayedRetry(IRecoverabilityContext context) { if (!totalDelayedRetries.Enabled) { return; } - var incomingPipelineMetricTags = recoverabilityContext.Extensions.Get(); - TagList meterTags; - incomingPipelineMetricTags.ApplyTags(ref meterTags, [ + TagList tags; + tags.Add(new KeyValuePair(MeterTags.ErrorType, context.Exception.GetType().FullName)); + + context.IncomingMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.MessageType, - MeterTags.MessageHandlerType]); - // This is what Add(string, object) does so skipping an unnecessary stack frame - meterTags.Add(new KeyValuePair(MeterTags.ErrorType, recoverabilityContext.Exception.GetType().FullName)); - totalDelayedRetries.Add(1, meterTags); + MeterTags.MessageHandlerType, + MeterTags.ErrorType], + totalDelayedRetries.Name); + totalDelayedRetries.Add(1, tags); } - public void RecordSendToErrorQueue(IRecoverabilityContext recoverabilityContext) + public void RecordSendToErrorQueue(IRecoverabilityContext context) { if (!totalSentToErrorQueue.Enabled) { return; } - var incomingPipelineMetricTags = recoverabilityContext.Extensions.Get(); - TagList meterTags; - incomingPipelineMetricTags.ApplyTags(ref meterTags, [ + TagList tags; + tags.Add(new KeyValuePair(MeterTags.ErrorType, context.Exception.GetType().FullName)); + + context.IncomingMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.MessageType, - MeterTags.MessageHandlerType]); - // This is what Add(string, object) does so skipping an unnecessary stack frame - meterTags.Add(new KeyValuePair(MeterTags.ErrorType, recoverabilityContext.Exception.GetType().FullName)); - totalSentToErrorQueue.Add(1, meterTags); + MeterTags.MessageHandlerType, + MeterTags.ErrorType], + totalSentToErrorQueue.Name); + totalSentToErrorQueue.Add(1, tags); } public ActiveMessageScope TrackMessageProcessing(IncomingPipelineMetricTags incomingPipelineMetricTags, IncomingMessage message) @@ -309,11 +326,15 @@ public ActiveMessageScope TrackMessageProcessing(IncomingPipelineMetricTags inco } TagList tags; - incomingPipelineMetricTags.ApplyTags(ref tags, [MeterTags.QueueName, MeterTags.EndpointDiscriminator]); if (message.Headers.TryGetValue(Headers.EnclosedMessageTypes, out var enclosedMessageTypes)) { tags.Add(new KeyValuePair(MeterTags.EnclosedMessageTypes, enclosedMessageTypes)); } + incomingPipelineMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator, + MeterTags.EnclosedMessageTypes], + activeMessages.Name); activeMessages.Add(1, tags); return new ActiveMessageScope(activeMessages, tags); @@ -326,21 +347,25 @@ public void RecordSagaFetchTime(IInvokeHandlerContext context, TimeSpan elapsed, return; } - var incomingPipelineMetricTags = context.Extensions.Get(); TagList tags; - incomingPipelineMetricTags.ApplyTags(ref tags, [ - MeterTags.QueueName, - MeterTags.EndpointDiscriminator, - MeterTags.MessageType]); + if (emitExecutionResultTags) + { + // Execution result is to be removed so we don't support overriding it by the user + tags.Add(new KeyValuePair(MeterTags.ExecutionResult, error != null ? "failure" : "success")); + } tags.Add(new KeyValuePair(MeterTags.SagaType, sagaType)); if (error != null) { tags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); } - if (emitExecutionResultTags) - { - tags.Add(new KeyValuePair(MeterTags.ExecutionResult, error != null ? "failure" : "success")); - } + + context.IncomingMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator, + MeterTags.MessageType, + MeterTags.SagaType, + MeterTags.ErrorType], + sagaFetchTime.Name); sagaFetchTime.Record(elapsed.TotalSeconds, tags); } @@ -351,30 +376,35 @@ public void RecordDeserializeTime(IIncomingPhysicalMessageContext context, TimeS return; } - var incomingPipelineMetricTags = context.Extensions.Get(); TagList tags; - incomingPipelineMetricTags.ApplyTags(ref tags, [ - MeterTags.QueueName, - MeterTags.EndpointDiscriminator - ]); - if (error != null) - { - tags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); - } if (context.Message.Headers.TryGetValue(Headers.EnclosedMessageTypes, out var messageTypes)) { tags.Add(new KeyValuePair(MeterTags.EnclosedMessageTypes, messageTypes)); } + if (error != null) + { + tags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); + } if (emitExecutionResultTags) { + // Execution result is to be removed so we don't support overriding it by the user tags.Add(new KeyValuePair(MeterTags.ExecutionResult, error != null ? "failure" : "success")); } + context.IncomingMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator, + MeterTags.EnclosedMessageTypes, + MeterTags.ErrorType], + messageDeserializeTime.Name); + messageDeserializeTime.Record(elapsed.TotalSeconds, tags); } - public void RecordSerializeTime(TimeSpan elapsed, string? messageType, Exception? error = null) + public void RecordSerializeTime(IOutgoingLogicalMessageContext context, TimeSpan elapsed, string? messageType, Exception? error = null) { + // No incoming pipeline context is available here (this fires from the outgoing send pipeline, which may + // run with no incoming message at all), so there's no IncomingPipelineMetricTags to route these through. if (!messageSerializeTime.Enabled) { return; @@ -383,16 +413,22 @@ public void RecordSerializeTime(TimeSpan elapsed, string? messageType, Exception TagList tags; if (messageType != null) { - tags.Add(new KeyValuePair(MeterTags.MessageType, messageType)); + tags.Add(new KeyValuePair(MeterTags.MessageType, messageType)); // tag-bag-bypass: see comment above } if (error != null) { - tags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); + tags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); // tag-bag-bypass: see comment above } if (emitExecutionResultTags) { - tags.Add(new KeyValuePair(MeterTags.ExecutionResult, error != null ? "failure" : "success")); + tags.Add(new KeyValuePair(MeterTags.ExecutionResult, error != null ? "failure" : "success")); // tag-bag-bypass: see comment above } + + context.IncomingMetricTags.ApplyTags(ref tags, [ + MeterTags.MessageType, + MeterTags.ErrorType], + messageSerializeTime.Name); + messageSerializeTime.Record(elapsed.TotalSeconds, tags); } @@ -403,11 +439,12 @@ public void RecordOutboxFetchTime(ITransportReceiveContext context, TimeSpan ela return; } - var incomingPipelineMetricTags = context.Extensions.Get(); TagList tags; - incomingPipelineMetricTags.ApplyTags(ref tags, [ + context.IncomingMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, - MeterTags.EndpointDiscriminator]); + MeterTags.EndpointDiscriminator], + outboxFetchTime.Name); + outboxFetchTime.Record(elapsed.TotalSeconds, tags); } @@ -418,11 +455,12 @@ public void RecordOutboxStoreTime(ITransportReceiveContext context, TimeSpan ela return; } - var incomingPipelineMetricTags = context.Extensions.Get(); TagList tags; - incomingPipelineMetricTags.ApplyTags(ref tags, [ + context.IncomingMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, - MeterTags.EndpointDiscriminator]); + MeterTags.EndpointDiscriminator], + outboxStoreTime.Name); + outboxStoreTime.Record(elapsed.TotalSeconds, tags); } @@ -433,13 +471,14 @@ public void RecordPersistenceTime(IIncomingLogicalMessageContext context, TimeSp return; } - var incomingPipelineMetricTags = context.Extensions.Get(); TagList tags; - incomingPipelineMetricTags.ApplyTags(ref tags, [ + context.IncomingMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.MessageType, - MeterTags.MessageHandlerTypes]); + MeterTags.MessageHandlerTypes], + persistenceTime.Name); + persistenceTime.Record(elapsed.TotalSeconds, tags); } @@ -451,19 +490,21 @@ void RecordEnvelopeUnwrapping(MessageContext messageContext, IEnvelopeHandler ty { return; } - - var incomingPipelineMetricTags = messageContext.Extensions.Get(); - TagList meterTags; - incomingPipelineMetricTags.ApplyTags(ref meterTags, [ - MeterTags.QueueName, - MeterTags.EndpointDiscriminator]); - meterTags.Add(new KeyValuePair(MeterTags.EnvelopeUnwrapperType, type.GetType().FullName)); + TagList tags; + tags.Add(new KeyValuePair(MeterTags.EnvelopeUnwrapperType, type.GetType().FullName)); if (exception != null) { - meterTags.Add(new KeyValuePair(MeterTags.ErrorType, exception.GetType().FullName)); + tags.Add(new KeyValuePair(MeterTags.ErrorType, exception.GetType().FullName)); } - totalEnvelopeUnwrapping.Add(succeeded ? 0 : 1, meterTags); + messageContext.IncomingMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator, + MeterTags.EnvelopeUnwrapperType, + MeterTags.ErrorType], + totalEnvelopeUnwrapping.Name); + + totalEnvelopeUnwrapping.Add(succeeded ? 0 : 1, tags); } public readonly struct ActiveMessageScope(UpDownCounter? counter, TagList tags) : IDisposable @@ -493,4 +534,4 @@ public readonly struct ActiveMessageScope(UpDownCounter? counter, TagList readonly string queueNameBase; readonly string endpointDiscriminator; readonly bool emitExecutionResultTags; -} \ No newline at end of file +} diff --git a/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs b/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs index 26a3b0178bf..bb140dcb96c 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs @@ -43,7 +43,7 @@ public override async Task Invoke(IIncomingLogicalMessageContext context, Func(); + var availableMetricTags = context.IncomingMetricTags; availableMetricTags.Add(MeterTags.MessageHandlerTypes, string.Join(';', handlersToInvoke.Select(x => x.HandlerType.FullName))); foreach (var messageHandler in handlersToInvoke) diff --git a/src/NServiceBus.Core/Pipeline/Incoming/MetricTagsExtensions.cs b/src/NServiceBus.Core/Pipeline/Incoming/MetricTagsExtensions.cs new file mode 100644 index 00000000000..46b05278214 --- /dev/null +++ b/src/NServiceBus.Core/Pipeline/Incoming/MetricTagsExtensions.cs @@ -0,0 +1,29 @@ +#nullable enable + +namespace NServiceBus; + +using Pipeline; +using Transport; + +/// +/// Provides access to the metric tags captured for the message currently being processed. +/// +public static class MetricTagsExtensions +{ + /// The context to extend. + extension(IBehaviorContext context) + { + /// + /// The collected for the message currently being processed. Add to this + /// collection to have the tags applied to the metrics emitted for that message. + /// + public IMetricsTags MetricTags => context.Extensions.GetOrCreate(); + + internal IncomingPipelineMetricTags IncomingMetricTags => context.Extensions.GetOrCreate(); + } + + extension(MessageContext context) + { + internal IncomingPipelineMetricTags IncomingMetricTags => context.Extensions.GetOrCreate(); + } +} diff --git a/src/NServiceBus.Core/Pipeline/MainPipelineExecutor.cs b/src/NServiceBus.Core/Pipeline/MainPipelineExecutor.cs index f6098dd0664..da6bd6ccd10 100644 --- a/src/NServiceBus.Core/Pipeline/MainPipelineExecutor.cs +++ b/src/NServiceBus.Core/Pipeline/MainPipelineExecutor.cs @@ -25,7 +25,7 @@ public async Task Invoke(MessageContext messageContext, CancellationToken cancel var pipelineStartedAt = DateTimeOffset.UtcNow; using var activity = activityFactory.StartIncomingPipelineActivity(messageContext); - var incomingPipelineMetricsTags = messageContext.Extensions.Get(); + var incomingPipelineMetricsTags = messageContext.IncomingMetricTags; incomingPipelineMetrics.AddDefaultIncomingPipelineMetricTags(incomingPipelineMetricsTags); diff --git a/src/NServiceBus.Core/Pipeline/Outgoing/SerializeMessageConnector.cs b/src/NServiceBus.Core/Pipeline/Outgoing/SerializeMessageConnector.cs index 14bbad12000..f8fe0d459e8 100644 --- a/src/NServiceBus.Core/Pipeline/Outgoing/SerializeMessageConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Outgoing/SerializeMessageConnector.cs @@ -45,13 +45,13 @@ public override async Task Invoke(IOutgoingLogicalMessageContext context, Func headers Extensions = context; ReceiveAddress = receiveAddress; TransportTransaction = transportTransaction; - - context.GetOrCreate(); } /// diff --git a/src/NServiceBus.Testing.Fakes/TestablePipelineContext.cs b/src/NServiceBus.Testing.Fakes/TestablePipelineContext.cs index ca256ad6b42..4090f548108 100644 --- a/src/NServiceBus.Testing.Fakes/TestablePipelineContext.cs +++ b/src/NServiceBus.Testing.Fakes/TestablePipelineContext.cs @@ -16,11 +16,7 @@ public partial class TestablePipelineContext : IPipelineContext /// /// Creates a new instance. /// - public TestablePipelineContext(IMessageCreator messageCreator = null) - { - this.messageCreator = messageCreator ?? new MessageMapper(); - Extensions.GetOrCreate(); - } + public TestablePipelineContext(IMessageCreator messageCreator = null) => this.messageCreator = messageCreator ?? new MessageMapper(); /// /// A list of all messages sent with a saga timeout header. From 14152cbe4e1f720f95921f67440a9216d381a800 Mon Sep 17 00:00:00 2001 From: "Irina Dominte(Scurtu)" Date: Thu, 24 Sep 2026 13:47:24 +0300 Subject: [PATCH 23/37] Provide discriminator and queueName tags to the outgoing pipeline instruments (#7936) * Rename `IncomingPipelineMetrics` to `PipelineMetrics` The class now serves both incoming and outgoing pipelines (e.g. serialization), so the `Incoming` prefix no longer applies. * Add `nservicebus.queue` and `nservicebus.discriminator` default metric tags with send-only support Include queue name and discriminator as default metric tags on PipelineMetrics whenever they are available. For send-only endpoints, which never create an input queue, both tags are omitted to avoid misleading values. Also adds serialize time metric tagging with queue, discriminator, and message type, with new acceptance tests covering both regular and send-only scenarios. * Rename `UseMessageDestinationInSpanNames` to `UseMessageTypeNamesInSpanNames` The new name better describes the intent: span names include the message type name, not the transport destination address. * Make `RecordError` accept a nullable `Activity?` Allow callers to pass a null activity when tracing is disabled, avoiding null-checks at every call site. * Remove `TracingExtensions` and inline pipeline tracing into `MessageOperations` Replace the TracingExtensions.Invoke extension method with explicit try/catch blocks in each MessageOperations method and MainPipelineExecutor. This makes tracing flow visible at each call site. Also converts MessageOperations to a primary constructor for consistency. * Extend span display names to include message type for all outgoing operations When UseMessageTypeNamesInSpanNames is enabled, subscribe, unsubscribe, send, and reply spans now include the event/message type name in their display name, matching the existing behavior for publish spans. * Rename `IncomingPipelineMetricTags` to `PipelineMetricTags` * Prepare new opt-in configuration options for proper obsoletion * introduce pre-v11 defaults for backward compatibility of trace modes --------- Co-authored-by: Ramon Smits Co-authored-by: Tomasz Masternak --- .../Metrics/TestingMetricListener.cs | 10 ++ .../When_serializing_outgoing_messages.cs | 85 +++++++++ .../When_processing_incoming_message.cs | 2 +- .../Traces/When_publishing_messages.cs | 2 +- .../When_recoverability_action_occurs.cs | 2 +- .../Traces/When_sending_messages.cs | 2 +- .../Traces/When_sending_replies.cs | 2 +- ...IApprovals.ApproveNServiceBus.approved.txt | 2 +- .../Envelopes/EnvelopeUnwrapperTests.cs | 6 +- .../Helpers/TestingMetricListener.cs | 27 ++- .../InstrumentationOptionsTests.cs | 4 +- .../OpenTelemetry/MeterTests.cs | 2 +- .../OpenTelemetry/TracingExtensionsTests.cs | 94 ---------- .../Incoming/InvokeHandlerTerminatorTest.cs | 2 +- .../SerializeMessageConnectorTests.cs | 53 ++++-- .../Pipeline/MainPipelineExecutorTests.cs | 2 +- ...agsTests.cs => PipelineMetricTagsTests.cs} | 6 +- .../Pipeline/TestableMessageOperations.cs | 14 +- ...tReceiveToPhysicalMessageConnectorTests.cs | 4 +- .../Unicast/LoadHandlersConnectorTests.cs | 8 +- .../Envelopes/EnvelopeComponent.cs | 2 +- .../Envelopes/EnvelopeUnwrapper.cs | 2 +- .../OpenTelemetry/InstrumentationOptions.cs | 34 +--- .../OpenTelemetry/MetersOptions.cs | 17 -- .../Tracing/ActivityDisplayNames.cs | 2 +- .../OpenTelemetry/Tracing/ActivityFactory.cs | 15 +- .../OpenTelemetry/Tracing/IActivityFactory.cs | 2 +- .../Tracing/NoOpActivityFactory.cs | 2 +- .../Tracing/TracingExtensions.cs | 32 ---- .../OpenTelemetry/Tracing/obsolete_v11.cs | 54 +++++- .../OpenTelemetry/Tracing/obsolete_v12.cs | 6 + .../Incoming/DeserializeMessageConnector.cs | 9 +- .../Incoming/InvokeHandlerTerminator.cs | 2 +- .../Incoming/LoadHandlersConnector.cs | 7 +- .../Pipeline/Incoming/MetricTagsExtensions.cs | 6 +- ...ineMetricTags.cs => PipelineMetricTags.cs} | 2 +- ...gPipelineMetrics.cs => PipelineMetrics.cs} | 164 +++++++++--------- ...nsportReceiveToPhysicalMessageConnector.cs | 12 +- .../Pipeline/MainPipelineExecutor.cs | 19 +- .../Outgoing/RoutingToDispatchConnector.cs | 2 +- .../Outgoing/SerializeMessageConnector.cs | 10 +- .../Pipeline/PipelineComponent.cs | 8 +- .../Receiving/ReceiveComponent.cs | 8 +- .../Recoverability/RecoverabilityComponent.cs | 2 +- .../RecoverabilityRoutingConnector.cs | 12 +- .../Sagas/SagaPersistenceBehavior.cs | 6 +- src/NServiceBus.Core/Sagas/Sagas.cs | 2 +- .../Serialization/SerializationFeature.cs | 4 +- .../Unicast/MessageOperations.cs | 136 +++++++++++---- 49 files changed, 513 insertions(+), 395 deletions(-) create mode 100644 src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_serializing_outgoing_messages.cs delete mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/TracingExtensionsTests.cs rename src/NServiceBus.Core.Tests/Pipeline/{IncomingPipelineMetricTagsTests.cs => PipelineMetricTagsTests.cs} (90%) delete mode 100644 src/NServiceBus.Core/OpenTelemetry/MetersOptions.cs delete mode 100644 src/NServiceBus.Core/OpenTelemetry/Tracing/TracingExtensions.cs rename src/NServiceBus.Core/Pipeline/Incoming/{IncomingPipelineMetricTags.cs => PipelineMetricTags.cs} (98%) rename src/NServiceBus.Core/Pipeline/Incoming/{IncomingPipelineMetrics.cs => PipelineMetrics.cs} (74%) diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/TestingMetricListener.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/TestingMetricListener.cs index 3b7e9a10932..218652d1c9e 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/TestingMetricListener.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/TestingMetricListener.cs @@ -96,6 +96,16 @@ public object AssertTagKeyExists(string metricName, string tagKey) return meterTag.Value; } + public void AssertTagKeyDoesNotExist(string metricName, string tagKey) + { + if (!Tags.ContainsKey(metricName)) + { + Assert.Fail($"'{metricName}' metric was not reported"); + } + + Assert.That(Tags[metricName].Select(t => t.Key), Does.Not.Contain(tagKey)); + } + public void AssertTags(string metricName, Dictionary expectedTags) { foreach (var kvp in expectedTags) diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_serializing_outgoing_messages.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_serializing_outgoing_messages.cs new file mode 100644 index 00000000000..04349df4f0c --- /dev/null +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_serializing_outgoing_messages.cs @@ -0,0 +1,85 @@ +namespace NServiceBus.AcceptanceTests.Core.OpenTelemetry.Metrics; + +using System.Collections.Generic; +using System.Threading.Tasks; +using EndpointTemplates; +using NServiceBus; +using NServiceBus.AcceptanceTesting; +using NServiceBus.AcceptanceTesting.Customization; +using NUnit.Framework; +using Conventions = AcceptanceTesting.Customization.Conventions; + +public class When_serializing_outgoing_messages : OpenTelemetryAcceptanceTest +{ + const string MessageSerializeTime = "nservicebus.messaging.serialize_time"; + + [Test] + public async Task Should_add_the_queue_name_and_discriminator_tags_to_serialize_time_instrument_when_full_endpoint() + { + using var metricsListener = TestingMetricListener.SetupNServiceBusMetricsListener(); + + _ = await Scenario.Define() + .WithEndpoint(b => b + .CustomConfig(x => x.MakeInstanceUniquelyAddressable("disc")) + .When(session => session.SendLocal(new OutgoingMessage()))) + .Run(); + + metricsListener.AssertTags(MessageSerializeTime, + new Dictionary + { + ["nservicebus.queue"] = Conventions.EndpointNamingConvention(typeof(EndpointWithMetrics)), + ["nservicebus.discriminator"] = "disc", + ["nservicebus.message_type"] = typeof(OutgoingMessage).FullName + }); + } + + [Test] + public async Task Should_not_add_the_queue_name_and_discriminator_tags_to_serialize_time_instrument_when_send_only() + { + // A send-only endpoint never creates its input queue, so reporting one would be misleading. + using var metricsListener = TestingMetricListener.SetupNServiceBusMetricsListener(); + + _ = await Scenario.Define() + .WithEndpoint(b => b.When(session => session.Send(new OutgoingMessage()))) + .WithEndpoint() + .Run(); + + //using (Assert.EnterMultipleScope()) + { + metricsListener.AssertTagKeyDoesNotExist(MessageSerializeTime, "nservicebus.queue"); + metricsListener.AssertTagKeyDoesNotExist(MessageSerializeTime, "nservicebus.discriminator"); + // the metric itself is still recorded, only the two tags are unavailable + Assert.That(metricsListener.AssertTagKeyExists(MessageSerializeTime, "nservicebus.message_type"), + Is.EqualTo(typeof(OutgoingMessage).FullName)); + } + } + + public class Context : ScenarioContext; + + public class SendOnlyEndpoint : EndpointConfigurationBuilder + { + public SendOnlyEndpoint() => + EndpointSetup(c => + { + c.SendOnly(); + c.ConfigureRouting().RouteToEndpoint(typeof(OutgoingMessage), typeof(EndpointWithMetrics)); + }); + } + + public class EndpointWithMetrics : EndpointConfigurationBuilder + { + public EndpointWithMetrics() => EndpointSetup(); + + [Handler] + public class MessageHandler(Context testContext) : IHandleMessages + { + public Task Handle(OutgoingMessage message, IMessageHandlerContext context) + { + testContext.MarkAsCompleted(); + return Task.CompletedTask; + } + } + } + + public class OutgoingMessage : IMessage; +} diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_incoming_message.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_incoming_message.cs index d386a9906d2..86fc87529a4 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_incoming_message.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_incoming_message.cs @@ -99,7 +99,7 @@ await Scenario.Define() public class ReceivingEndpointWithDestinationNaming : EndpointConfigurationBuilder { public ReceivingEndpointWithDestinationNaming() => - EndpointSetup(b => b.Tracing().UseMessageDestinationInSpanNames = true); + EndpointSetup(b => b.Tracing().UseMessageTypeNamesInSpanNames = true); [Handler] public class MessageHandler(Context testContext) : IHandleMessages diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs index 7aed06a30a8..39b8e57885d 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs @@ -208,7 +208,7 @@ public class PublisherWithDestinationNaming : EndpointConfigurationBuilder public PublisherWithDestinationNaming() => EndpointSetup(b => { - b.Tracing().UseMessageDestinationInSpanNames = true; + b.Tracing().UseMessageTypeNamesInSpanNames = true; b.OnEndpointSubscribed((s, context) => { if (s.SubscriberEndpoint.Contains(Conventions.EndpointNamingConvention(typeof(SubscriberForPublisherWithDestinationNaming)))) diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_recoverability_action_occurs.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_recoverability_action_occurs.cs index ec458ed6ba7..6e24936ab53 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_recoverability_action_occurs.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_recoverability_action_occurs.cs @@ -59,7 +59,7 @@ await Scenario.Define() .CustomConfig(c => { c.Recoverability().Immediate(i => i.NumberOfRetries(1)).Delayed(i => i.NumberOfRetries(0)); - c.Tracing().UseMessageDestinationInSpanNames = true; + c.Tracing().UseMessageTypeNamesInSpanNames = true; }) .DoNotFailOnErrorMessages() .When(s => s.SendLocal(new FailingMessage()))) diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_messages.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_messages.cs index c08e660e883..a56a3caac7e 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_messages.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_messages.cs @@ -150,7 +150,7 @@ await Scenario.Define() public class TestEndpointWithDestinationNaming : EndpointConfigurationBuilder { public TestEndpointWithDestinationNaming() => - EndpointSetup(b => b.Tracing().UseMessageDestinationInSpanNames = true); + EndpointSetup(b => b.Tracing().UseMessageTypeNamesInSpanNames = true); [Handler] public class MessageHandler(Context testContext) : IHandleMessages diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_replies.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_replies.cs index c6c505d9236..7e79d6582c5 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_replies.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_replies.cs @@ -73,7 +73,7 @@ await Scenario.Define() public class TestEndpointWithDestinationNaming : EndpointConfigurationBuilder { public TestEndpointWithDestinationNaming() => - EndpointSetup(b => b.Tracing().UseMessageDestinationInSpanNames = true); + EndpointSetup(b => b.Tracing().UseMessageTypeNamesInSpanNames = true); [Handler] public class MessageHandler(Context testContext) : IHandleMessages, diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt index c7302ebe466..a58f2c75532 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt @@ -650,7 +650,7 @@ namespace NServiceBus public NServiceBus.TraceMode PublishTraceMode { get; set; } public NServiceBus.RecoverabilityInstrumentationOptions Recoverability { get; } public NServiceBus.TraceMode SendTraceMode { get; set; } - public bool UseMessageDestinationInSpanNames { get; set; } + public bool UseMessageTypeNamesInSpanNames { get; set; } } public sealed class KeyedServiceKey { diff --git a/src/NServiceBus.Core.Tests/Envelopes/EnvelopeUnwrapperTests.cs b/src/NServiceBus.Core.Tests/Envelopes/EnvelopeUnwrapperTests.cs index 2b52437cca8..b7136605694 100644 --- a/src/NServiceBus.Core.Tests/Envelopes/EnvelopeUnwrapperTests.cs +++ b/src/NServiceBus.Core.Tests/Envelopes/EnvelopeUnwrapperTests.cs @@ -15,7 +15,7 @@ public class EnvelopeUnwrapperTests ReadOnlyMemory originalBody; MessageContext messageContext; TestMeterFactory meterFactory; - IncomingPipelineMetrics incomingPipelineMetrics; + PipelineMetrics pipelineMetrics; List envelopeHandlers; [SetUp] @@ -29,7 +29,7 @@ public void Setup() originalBody = "payload"u8.ToArray().AsMemory(); messageContext = new MessageContext(nativeId, originalHeaders, originalBody, new TransportTransaction(), "receiveAddress", new ContextBag()); meterFactory = new TestMeterFactory(); - incomingPipelineMetrics = new IncomingPipelineMetrics(meterFactory, "queue", "disc", new MetersOptions()); + pipelineMetrics = new PipelineMetrics(meterFactory, "queue", "disc", new MetersOptions()); } [TearDown] @@ -152,7 +152,7 @@ public void ModifiedAndThrowingBodyWriterIsReset() Assert.That(result.Body.Span.SequenceEqual(firstBody.Span), Is.True); } - EnvelopeUnwrapper.IncomingMessageHandle RunTest() => new EnvelopeUnwrapper([.. envelopeHandlers], incomingPipelineMetrics).UnwrapEnvelope(messageContext); + EnvelopeUnwrapper.IncomingMessageHandle RunTest() => new EnvelopeUnwrapper([.. envelopeHandlers], pipelineMetrics).UnwrapEnvelope(messageContext); class ReturningHandler(Dictionary headersToReturn, ReadOnlyMemory bodyToReturn) : IEnvelopeHandler { diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/Helpers/TestingMetricListener.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/Helpers/TestingMetricListener.cs index f288d4f4476..783004cd7f2 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/Helpers/TestingMetricListener.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/Helpers/TestingMetricListener.cs @@ -34,17 +34,19 @@ class TestingMetricListener : IDisposable } }; + // Counters measure in long, the histograms in the incoming pipeline meter measure in double. Both callbacks + // have to be registered or the measurements (and therefore the tags) of one kind are never observed. A + // counter reports the measurement itself, a histogram reports how many times it recorded. meterListener.SetMeasurementEventCallback((Instrument instrument, long measurement, - ReadOnlySpan> t, - object _) => - { - TestContext.Out.WriteLine($"{instrument.Meter.Name}\\{instrument.Name}:{measurement}"); + ReadOnlySpan> tags, + object _) => RecordMeasurement(instrument, measurement, measurement, tags)); + + meterListener.SetMeasurementEventCallback((Instrument instrument, + double measurement, + ReadOnlySpan> tags, + object _) => RecordMeasurement(instrument, measurement, 1, tags)); - var tags = t.ToArray(); - ReportedMeters.AddOrUpdate(instrument.Name, measurement, (_, val) => val + measurement); - Tags.AddOrUpdate(instrument.Name, _ => tags, (_, _) => tags); - }); meterListener.Start(); } @@ -59,6 +61,15 @@ public static TestingMetricListener SetupMetricsListener(string sourceName) public void Dispose() => meterListener?.Dispose(); + void RecordMeasurement(Instrument instrument, T measurement, long reportedValue, ReadOnlySpan> measurementTags) + { + TestContext.Out.WriteLine($"{instrument.Meter.Name}\\{instrument.Name}:{measurement}"); + + var tags = measurementTags.ToArray(); + ReportedMeters.AddOrUpdate(instrument.Name, reportedValue, (_, val) => val + reportedValue); + Tags.AddOrUpdate(instrument.Name, _ => tags, (_, _) => tags); + } + public ConcurrentDictionary ReportedMeters { get; } = new(); public ConcurrentDictionary[]> Tags { get; } = new(); diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/InstrumentationOptionsTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/InstrumentationOptionsTests.cs index ded2e6a41c2..b01433e755d 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/InstrumentationOptionsTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/InstrumentationOptionsTests.cs @@ -13,7 +13,9 @@ public void Should_default_trace_connectors_to_current_behavior() using (Assert.EnterMultipleScope()) { Assert.That(options.SendTraceMode, Is.EqualTo(TraceMode.ContinueExisting), "sends continue the trace by default"); - Assert.That(options.PublishTraceMode, Is.EqualTo(TraceMode.StartNew), "publishes start a new linked trace by default"); + // Set by ApplyPreV11Defaults in obsolete_v11.cs. When that file is deleted in v11 this + // assertion fails: flip it to TraceMode.ContinueExisting, which is the initializer's value. + Assert.That(options.PublishTraceMode, Is.EqualTo(TraceMode.StartNew), "publishes start a new linked trace by default until v11"); } } } diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/MeterTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/MeterTests.cs index 83b1e16e1d3..77eb1764ca4 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/MeterTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/MeterTests.cs @@ -22,7 +22,7 @@ public void Verify_MeterAPI() using var meterFactory = new TestMeterFactory(); //The IncomingPipelineMeter constructor creates the meters, therefore a new instance before collecting the metrics. #pragma warning disable CA1806 - new IncomingPipelineMetrics(meterFactory, "queue", "disc", new MetersOptions()); + new PipelineMetrics(meterFactory, "queue", "disc", new MetersOptions()); #pragma warning restore CA1806 using var metricsListener = TestingMetricListener.SetupNServiceBusMetricsListener(); diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/TracingExtensionsTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/TracingExtensionsTests.cs deleted file mode 100644 index 4b6473de6f5..00000000000 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/TracingExtensionsTests.cs +++ /dev/null @@ -1,94 +0,0 @@ -namespace NServiceBus.Core.Tests.OpenTelemetry; - -using System; -using System.Collections.Immutable; -using System.Diagnostics; -using System.Linq; -using System.Threading.Tasks; -using NServiceBus.Pipeline; -using NUnit.Framework; - -[TestFixture] -public class TracingExtensionsTests -{ - [Test] - public async Task Invoke_should_invoke_pipeline_when_activity_null() - { - bool invokedPipeline = false; - var pipeline = new FakePipeline(() => - { - invokedPipeline = true; - return Task.CompletedTask; - }); - - await pipeline.Invoke(new FakeRootContext(), null, new ActivityFactory(new InstrumentationOptions())); - - Assert.That(invokedPipeline, Is.True); - } - - [Test] - public async Task Invoke_should_set_success_status_when_no_exception() - { - var pipeline = new FakePipeline(() => Task.CompletedTask); - using var activity = new Activity("test activity"); - activity.Start(); - - await pipeline.Invoke(new FakeRootContext(), activity, new ActivityFactory(new InstrumentationOptions())); - - Assert.That(activity.Status, Is.EqualTo(ActivityStatusCode.Ok)); - } - - [Test] - public void Invoke_should_set_error_status_and_tags_when_exception() - { - var exception = new Exception("test exception"); - var pipeline = new FakePipeline(() => throw exception); - using var activity = new Activity("test activity"); - activity.Start(); - - Assert.ThrowsAsync(() => pipeline.Invoke(new FakeRootContext(), activity, new ActivityFactory(new InstrumentationOptions()))); - - Assert.That(activity.Status, Is.EqualTo(ActivityStatusCode.Error)); - - var tags = activity.Tags.ToImmutableDictionary(); - using (Assert.EnterMultipleScope()) - { - Assert.That(tags["otel.status_code"], Is.EqualTo("ERROR")); - Assert.That(tags["otel.status_description"], Is.EqualTo(exception.Message)); - } - - var errorEvent = activity.Events.Single(); - Assert.That(errorEvent.Name, Is.EqualTo("exception")); - } - - [Test] - public void Invoke_should_set_error_status_without_exception_event_when_log_mode() - { - var exception = new Exception("test exception"); - var pipeline = new FakePipeline(() => throw exception); - using var activity = new Activity("test activity"); - activity.Start(); - - Assert.ThrowsAsync(() => pipeline.Invoke(new FakeRootContext(), activity, new ActivityFactory(new InstrumentationOptions { ExceptionRecordingMode = ExceptionRecordingMode.Logs }))); - - using (Assert.EnterMultipleScope()) - { - Assert.That(activity.Status, Is.EqualTo(ActivityStatusCode.Error)); - Assert.That(activity.Events, Is.Empty, "no exception event should be added when recording via the log instead"); - } - } - - class FakePipeline : IPipeline - { - readonly Func pipelineAction; - -#pragma warning disable PS0013 // A Func used as a method parameter with a Task, ValueTask, or ValueTask return type argument should have at least one CancellationToken parameter type argument unless it has a parameter type argument implementing ICancellableContext - public FakePipeline(Func pipelineAction) -#pragma warning restore PS0013 // A Func used as a method parameter with a Task, ValueTask, or ValueTask return type argument should have at least one CancellationToken parameter type argument unless it has a parameter type argument implementing ICancellableContext - { - this.pipelineAction = pipelineAction; - } - - public Task Invoke(IBehaviorContext context) => pipelineAction(); - } -} \ No newline at end of file diff --git a/src/NServiceBus.Core.Tests/Pipeline/Incoming/InvokeHandlerTerminatorTest.cs b/src/NServiceBus.Core.Tests/Pipeline/Incoming/InvokeHandlerTerminatorTest.cs index b794eb25413..5948f081060 100644 --- a/src/NServiceBus.Core.Tests/Pipeline/Incoming/InvokeHandlerTerminatorTest.cs +++ b/src/NServiceBus.Core.Tests/Pipeline/Incoming/InvokeHandlerTerminatorTest.cs @@ -11,7 +11,7 @@ [TestFixture] public class InvokeHandlerTerminatorTest { - readonly InvokeHandlerTerminator terminator = new(new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); + readonly InvokeHandlerTerminator terminator = new(new PipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); [Test] public async Task When_saga_found_and_handler_is_saga_should_invoke_handler() diff --git a/src/NServiceBus.Core.Tests/Pipeline/Incoming/SerializeMessageConnectorTests.cs b/src/NServiceBus.Core.Tests/Pipeline/Incoming/SerializeMessageConnectorTests.cs index 76e0efb2e00..dacfb4e7447 100644 --- a/src/NServiceBus.Core.Tests/Pipeline/Incoming/SerializeMessageConnectorTests.cs +++ b/src/NServiceBus.Core.Tests/Pipeline/Incoming/SerializeMessageConnectorTests.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.IO; using System.Threading.Tasks; +using NServiceBus.AcceptanceTests.Core.OpenTelemetry.Metrics; using NServiceBus.Core.Tests.OpenTelemetry; using NServiceBus.Pipeline; using NUnit.Framework; @@ -17,26 +18,54 @@ public class SerializeMessageConnectorTests [Test] public async Task Should_set_content_type_header() { - var registry = new MessageMetadataRegistry(); - registry.Initialize(new Conventions().IsMessageType, true); + var context = CreateContext(); + + await InvokeSerializer(context, queueName: "queue", discriminator: "disc"); + + Assert.That(context.Headers[Headers.ContentType], Is.EqualTo("myContentType")); + } + + [Test] + public async Task Should_prefer_queue_and_discriminator_from_the_incoming_pipeline_tags() + { + // A send from a message handler chains its context to the incoming pipeline, whose tags win over the + // endpoint configuration defaults. Seeding both tags from the configuration and leaving them off for + // send-only endpoints is covered by the When_serializing_outgoing_messages acceptance tests. + using var metricsListener = TestingMetricListener.SetupNServiceBusMetricsListener(); - registry.RegisterMessageTypes( - [ - typeof(MyMessage) - ]); + var context = CreateContext(); + var incomingTags = context.Extensions.GetOrCreate(); + incomingTags.Add("nservicebus.queue", "queue-from-incoming-pipeline"); + incomingTags.Add("nservicebus.discriminator", "disc-from-incoming-pipeline"); - var context = new TestableOutgoingLogicalMessageContext + await InvokeSerializer(context, queueName: "queue", discriminator: "disc"); + + using (Assert.EnterMultipleScope()) { - Message = new OutgoingLogicalMessage(typeof(MyMessage), new MyMessage()) - }; + Assert.That(metricsListener.AssertTagKeyExists(MessageSerializeTime, "nservicebus.queue"), Is.EqualTo("queue-from-incoming-pipeline")); + Assert.That(metricsListener.AssertTagKeyExists(MessageSerializeTime, "nservicebus.discriminator"), Is.EqualTo("disc-from-incoming-pipeline")); + } + } - var behavior = new SerializeMessageConnector(new FakeSerializer("myContentType"), registry, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); + static TestableOutgoingLogicalMessageContext CreateContext() => + new() { Message = new OutgoingLogicalMessage(typeof(MyMessage), new MyMessage()) }; - await behavior.Invoke(context, c => Task.CompletedTask); + static Task InvokeSerializer(TestableOutgoingLogicalMessageContext context, string queueName, string discriminator) + { + var registry = new MessageMetadataRegistry(); + registry.Initialize(new Conventions().IsMessageType, true); + registry.RegisterMessageTypes([typeof(MyMessage)]); - Assert.That(context.Headers[Headers.ContentType], Is.EqualTo("myContentType")); + var behavior = new SerializeMessageConnector( + new FakeSerializer("myContentType"), + registry, + new PipelineMetrics(new TestMeterFactory(), queueName, discriminator, new MetersOptions())); + + return behavior.Invoke(context, _ => Task.CompletedTask); } + const string MessageSerializeTime = "nservicebus.messaging.serialize_time"; + class FakeSerializer(string contentType) : IMessageSerializer { public void Serialize(object message, Stream stream) diff --git a/src/NServiceBus.Core.Tests/Pipeline/MainPipelineExecutorTests.cs b/src/NServiceBus.Core.Tests/Pipeline/MainPipelineExecutorTests.cs index 8befd4e48c9..9e56b4ee174 100644 --- a/src/NServiceBus.Core.Tests/Pipeline/MainPipelineExecutorTests.cs +++ b/src/NServiceBus.Core.Tests/Pipeline/MainPipelineExecutorTests.cs @@ -123,7 +123,7 @@ static MessageContext CreateMessageContext() => static MainPipelineExecutor CreateMainPipelineExecutor(ServiceProvider serviceProvider, IPipeline receivePipeline) { - var incomingPipelineMetrics = new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions()); + var incomingPipelineMetrics = new PipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions()); var executor = new MainPipelineExecutor( serviceProvider, new PipelineCache(serviceProvider, new PipelineModifications()), diff --git a/src/NServiceBus.Core.Tests/Pipeline/IncomingPipelineMetricTagsTests.cs b/src/NServiceBus.Core.Tests/Pipeline/PipelineMetricTagsTests.cs similarity index 90% rename from src/NServiceBus.Core.Tests/Pipeline/IncomingPipelineMetricTagsTests.cs rename to src/NServiceBus.Core.Tests/Pipeline/PipelineMetricTagsTests.cs index 1ce103a841c..b316643e361 100644 --- a/src/NServiceBus.Core.Tests/Pipeline/IncomingPipelineMetricTagsTests.cs +++ b/src/NServiceBus.Core.Tests/Pipeline/PipelineMetricTagsTests.cs @@ -14,7 +14,7 @@ namespace NServiceBus.Core.Tests.Pipeline.Incoming; using Unicast.Messages; [TestFixture] -public class IncomingPipelineMetricTagsTests +public class PipelineMetricTagsTests { [Test] public void Should_not_fail_when_handling_more_than_one_logical_message() @@ -36,11 +36,11 @@ public void Should_not_fail_when_handling_more_than_one_logical_message() }; var messageMapper = new MessageMapper(); - var behavior = new DeserializeMessageConnector(new MessageDeserializerResolver(new FakeSerializer(), []), new LogicalMessageFactory(registry, messageMapper), registry, messageMapper, false, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); + var behavior = new DeserializeMessageConnector(new MessageDeserializerResolver(new FakeSerializer(), []), new LogicalMessageFactory(registry, messageMapper), registry, messageMapper, false, new PipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); Assert.DoesNotThrowAsync(async () => await behavior.Invoke(context, c => { - c.IncomingMetricTags.Add("Same", "Same"); + c.PipelineMetricTags.Add("Same", "Same"); return Task.CompletedTask; })); } diff --git a/src/NServiceBus.Core.Tests/Pipeline/TestableMessageOperations.cs b/src/NServiceBus.Core.Tests/Pipeline/TestableMessageOperations.cs index 909b9fb796a..e4e76b40f14 100644 --- a/src/NServiceBus.Core.Tests/Pipeline/TestableMessageOperations.cs +++ b/src/NServiceBus.Core.Tests/Pipeline/TestableMessageOperations.cs @@ -5,7 +5,15 @@ using System.Threading.Tasks; using MessageInterfaces.MessageMapper.Reflection; -class TestableMessageOperations : MessageOperations +class TestableMessageOperations() : MessageOperations( + new MessageMapper(), + new Pipeline(), + new Pipeline(), + new Pipeline(), + new Pipeline(), + new Pipeline(), + new ActivityFactory(new InstrumentationOptions()) + ) { public Pipeline PublishPipeline => (Pipeline)publishPipeline; public Pipeline SendPipeline => (Pipeline)sendPipeline; @@ -13,10 +21,6 @@ class TestableMessageOperations : MessageOperations public Pipeline SubscribePipeline => (Pipeline)subscribePipeline; public Pipeline UnsubscribePipeline => (Pipeline)unsubscribePipeline; - public TestableMessageOperations() : base(new MessageMapper(), new Pipeline(), new Pipeline(), new Pipeline(), new Pipeline(), new Pipeline(), new ActivityFactory(new InstrumentationOptions())) - { - } - public class Pipeline : IPipeline where TContext : IBehaviorContext { public Action OnInvoke { get; set; } diff --git a/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs b/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs index e33a9ef34a2..ca48d520767 100644 --- a/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs +++ b/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs @@ -197,7 +197,7 @@ static TestableTransportReceiveContext CreateContext(FakeBatchPipeline pipeline, }; context.Extensions.Set(new FakePipelineCache(pipeline)); - context.Extensions.Set(new IncomingPipelineMetricTags()); + context.Extensions.Set(new PipelineMetricTags()); return context; } @@ -209,7 +209,7 @@ public void SetUp() fakeBatchPipeline = new FakeBatchPipeline(); fakeMeterFactory = new TestMeterFactory(); - behavior = new TransportReceiveToPhysicalMessageConnector(fakeOutbox, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions()), new InstrumentationOptions()); + behavior = new TransportReceiveToPhysicalMessageConnector(fakeOutbox, new PipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions()), new InstrumentationOptions()); } [TearDown] diff --git a/src/NServiceBus.Core.Tests/Unicast/LoadHandlersConnectorTests.cs b/src/NServiceBus.Core.Tests/Unicast/LoadHandlersConnectorTests.cs index 67cb7048465..291c25dac83 100644 --- a/src/NServiceBus.Core.Tests/Unicast/LoadHandlersConnectorTests.cs +++ b/src/NServiceBus.Core.Tests/Unicast/LoadHandlersConnectorTests.cs @@ -17,7 +17,7 @@ public class LoadHandlersConnectorTests [Test] public void Should_throw_when_there_are_no_registered_message_handlers() { - var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); + var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance, new PipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); var context = new TestableIncomingLogicalMessageContext(); @@ -30,7 +30,7 @@ public void Should_throw_when_there_are_no_registered_message_handlers() [Test] public void Should_throw_if_ambient_transaction_is_different_from_scope_used_by_transport() { - var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); + var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance, new PipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); var context = new TestableIncomingLogicalMessageContext(); @@ -50,7 +50,7 @@ public void Should_throw_if_ambient_transaction_is_different_from_scope_used_by_ [Test] public void Should_throw_if_ambient_transaction_suppressed_when_transport_uses_a_scope() { - var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); + var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance, new PipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); var context = new TestableIncomingLogicalMessageContext(); @@ -78,7 +78,7 @@ public void Should_not_throw_if_ambient_scope_is_same_as_transport_scope() context.Services.AddSingleton(); context.Extensions.Set(new NoOpOutboxTransaction()); - var behavior = new LoadHandlersConnector(messageHandlerRegistry, NoOpActivityFactory.Instance, new IncomingPipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); + var behavior = new LoadHandlersConnector(messageHandlerRegistry, NoOpActivityFactory.Instance, new PipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); using (new TransactionScope(TransactionScopeAsyncFlowOption.Enabled)) { diff --git a/src/NServiceBus.Core/Envelopes/EnvelopeComponent.cs b/src/NServiceBus.Core/Envelopes/EnvelopeComponent.cs index a0962296ae9..f6a2cb5d887 100644 --- a/src/NServiceBus.Core/Envelopes/EnvelopeComponent.cs +++ b/src/NServiceBus.Core/Envelopes/EnvelopeComponent.cs @@ -10,7 +10,7 @@ namespace NServiceBus; class EnvelopeComponent(EnvelopeComponent.Settings settings) { public EnvelopeUnwrapper CreateUnwrapper(IServiceProvider serviceProvider) - => new([.. settings.HandlerFactories.Select(factory => factory(serviceProvider))], serviceProvider.GetRequiredService()); + => new([.. settings.HandlerFactories.Select(factory => factory(serviceProvider))], serviceProvider.GetRequiredService()); public class Settings { diff --git a/src/NServiceBus.Core/Envelopes/EnvelopeUnwrapper.cs b/src/NServiceBus.Core/Envelopes/EnvelopeUnwrapper.cs index 3b3bd71202b..b5ef94ac326 100644 --- a/src/NServiceBus.Core/Envelopes/EnvelopeUnwrapper.cs +++ b/src/NServiceBus.Core/Envelopes/EnvelopeUnwrapper.cs @@ -6,7 +6,7 @@ namespace NServiceBus; using Logging; using Transport; -class EnvelopeUnwrapper(IEnvelopeHandler[] envelopeHandlers, IncomingPipelineMetrics metrics) +class EnvelopeUnwrapper(IEnvelopeHandler[] envelopeHandlers, PipelineMetrics metrics) { static IncomingMessage GetDefaultIncomingMessage(MessageContext messageContext) => new(messageContext.NativeMessageId, messageContext.Headers, messageContext.Body, messageContext.ReceiveProperties); diff --git a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs index 4930b5d2ffe..c1ebb7ad1ab 100644 --- a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs @@ -9,22 +9,19 @@ namespace NServiceBus; public partial class InstrumentationOptions { /// - /// Appends the destination to span names following the OTel messaging convention - /// {messaging.operation.name} {destination}, e.g. "process orders" or "send payments". - /// Disabled by default for backward compatibility. + /// Initializes the options with their defaults. /// - public bool UseMessageDestinationInSpanNames { get; set; } + public InstrumentationOptions() => ApplyPreV11Defaults(); + + // Implemented in obsolete_v11.cs. A partial void method without an implementation is removed + // by the compiler, so deleting that file makes the initializers below the final defaults. + partial void ApplyPreV11Defaults(); /// /// Controls instrumentation of the recoverability pipeline (retries and error handling). /// public RecoverabilityInstrumentationOptions Recoverability { get; } = new(); - /// - /// Controls meter instruments behaviors. - /// - public MetersOptions Meters { get; } = new(); - /// /// Controls instrumentation of explicitly delayed messages (SendOptions.DelayDeliveryWith /// / DoNotDeliverBefore and saga timeouts. @@ -33,14 +30,6 @@ public partial class InstrumentationOptions /// public DelayedDeliveryInstrumentationOptions DelayedDelivery { get; } = new(); - /// - /// Controls whether the "Start dispatching" and "Finished dispatching" activity events - /// are added to the incoming message span when outgoing messages are dispatched. - /// Enabled by default for backward compatibility. Disable to avoid the ingestion cost - /// of these events when they add no diagnostic value. - /// - public bool EmitMessageDispatchingEvents { get; set; } = true; - /// /// Controls how the receive-side processing span relates to the send span for messages sent by this endpoint. /// Defaults to : receivers continue the trace. @@ -51,17 +40,12 @@ public partial class InstrumentationOptions /// /// Controls how the receive-side processing span relates to the publish span for events published by this endpoint. - /// Defaults to : receivers start a new trace linked back to the publish span. + /// Defaults to : receivers continue the trace. + /// Until v11 the default is for backward compatibility, see obsolete_v11.cs. /// Can be overridden per message via /// or . /// - public TraceMode PublishTraceMode { get; set; } = TraceMode.StartNew; - - /// - /// Controls how exception details are recorded when an operation fails. - /// Defaults to : exceptions are recorded as an event on the activity. - /// - public ExceptionRecordingMode ExceptionRecordingMode { get; set; } = ExceptionRecordingMode.SpanAndLogs; + public TraceMode PublishTraceMode { get; set; } = TraceMode.ContinueExisting; } /// diff --git a/src/NServiceBus.Core/OpenTelemetry/MetersOptions.cs b/src/NServiceBus.Core/OpenTelemetry/MetersOptions.cs deleted file mode 100644 index 0838da8aea8..00000000000 --- a/src/NServiceBus.Core/OpenTelemetry/MetersOptions.cs +++ /dev/null @@ -1,17 +0,0 @@ -#nullable enable - -namespace NServiceBus; - -/// -/// Controls opt-in meter instruments behaviors. -/// Accessed via endpointConfiguration.Tracing().Meters. -/// -public class MetersOptions -{ - /// - /// Emits the legacy execution.result tag with values "success" or "failure" - /// on handler time, processing time, saga fetch time, deserialize time, and serialize time metrics. - /// Enabled by default for backwards compatibility. Disable to reduce tag cardinality. - /// - public bool EmitExecutionResultTags { get; set; } = true; -} diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs index 50b874bfac3..ec5f69f29fd 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs @@ -12,7 +12,7 @@ static class ActivityDisplayNames public const string ReplyMessage = "reply"; public const string Recoverability = "recover"; - // Operation-only prefixes used when UseMessageDestinationInSpanNames is enabled + // Operation-only prefixes used when UseMessageTypeNamesInSpanNames is enabled internal const string ProcessOperation = "process"; internal const string PublishOperation = "publish"; internal const string SendOperation = "send"; diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs index c8a958733fd..90d5fec31ee 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs @@ -95,7 +95,7 @@ sealed class ActivityFactory(InstrumentationOptions options) : IActivityFactory return activity; } - activity.DisplayName = Options.UseMessageDestinationInSpanNames + activity.DisplayName = Options.UseMessageTypeNamesInSpanNames ? $"{ActivityDisplayNames.ProcessOperation} {context.ReceiveAddress}" : ActivityDisplayNames.ProcessMessage; @@ -175,7 +175,7 @@ public void UpdateActivityFromRecoverabilityAction(Activity activity, Recoverabi activity.AddTag(ActivityTags.RecoverabilityAction, "immediate_retry"); activity.DisplayName = ActivityDisplayNames.ImmediateRetryOperation; - if (Options.UseMessageDestinationInSpanNames) + if (Options.UseMessageTypeNamesInSpanNames) { activity.DisplayName += $" {receiveAddress}"; } @@ -185,7 +185,7 @@ public void UpdateActivityFromRecoverabilityAction(Activity activity, Recoverabi activity.AddTag(ActivityTags.RecoverabilityAction, "delayed_retry"); activity.DisplayName = ActivityDisplayNames.DelayedRetryOperation; - if (Options.UseMessageDestinationInSpanNames) + if (Options.UseMessageTypeNamesInSpanNames) { activity.DisplayName += $" {receiveAddress}"; } @@ -194,7 +194,7 @@ public void UpdateActivityFromRecoverabilityAction(Activity activity, Recoverabi { activity.AddTag(ActivityTags.RecoverabilityAction, "move_to_error"); - activity.DisplayName = Options.UseMessageDestinationInSpanNames + activity.DisplayName = Options.UseMessageTypeNamesInSpanNames ? $"{ActivityDisplayNames.MoveToErrorOperation} {moveToError.ErrorQueue}" : $"{ActivityDisplayNames.MoveToErrorOperation} error"; } @@ -205,8 +205,13 @@ public void UpdateActivityFromRecoverabilityAction(Activity activity, Recoverabi } } - public void RecordError(Activity activity, Exception exception, ContextBag context) + public void RecordError(Activity? activity, Exception exception, ContextBag context) { + if (activity == null) + { + return; + } + activity.SetStatus(ActivityStatusCode.Error, exception.Message); activity.SetTag(ActivityTags.ErrorType, exception.GetType().FullName); diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs index bccdf87faf6..44943a4ce04 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs @@ -16,5 +16,5 @@ interface IActivityFactory Activity? StartHandlerActivity(MessageHandler messageHandler); Activity? StartRecoverabilityActivity(ErrorContext context); void UpdateActivityFromRecoverabilityAction(Activity activity, RecoverabilityAction recoverabilityAction, string receiveAddress); - void RecordError(Activity activity, Exception exception, ContextBag context); + void RecordError(Activity? activity, Exception exception, ContextBag context); } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs index 6e2e183c637..5a025496460 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs @@ -24,7 +24,7 @@ public void UpdateActivityFromRecoverabilityAction(Activity activity, Recoverabi { } - public void RecordError(Activity activity, Exception exception, ContextBag context) + public void RecordError(Activity? activity, Exception exception, ContextBag context) { } } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/TracingExtensions.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/TracingExtensions.cs deleted file mode 100644 index afdad2c908a..00000000000 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/TracingExtensions.cs +++ /dev/null @@ -1,32 +0,0 @@ -#nullable enable - -namespace NServiceBus; - -using System; -using System.Diagnostics; -using System.Threading.Tasks; -using Pipeline; - -static class TracingExtensions -{ - public static Task Invoke(this IPipeline pipeline, TContext context, Activity? activity, IActivityFactory activityFactory) where TContext : IBehaviorContext - { - return activity is null ? pipeline.Invoke(context) : TracePipelineStatus(pipeline, context, activity, activityFactory); - - static async Task TracePipelineStatus(IPipeline pipeline, TContext context, Activity activity, IActivityFactory activityFactory) - { -#pragma warning disable PS0019 // When catching System.Exception, cancellation needs to be properly accounted for - try - { - await pipeline.Invoke(context).ConfigureAwait(false); - activity.SetStatus(ActivityStatusCode.Ok); - } - catch (Exception ex) - { - activityFactory.RecordError(activity, ex, context.Extensions); - throw; - } -#pragma warning restore PS0019 // When catching System.Exception, cancellation needs to be properly accounted for - } - } -} \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs index 0e8dd5f4d4f..92e85862308 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs @@ -208,4 +208,56 @@ public static void SetLegacyStatusTags(Activity activity, Exception exception) } public static TagList EscapedTagList { get; } = new() { { "exception.escaped", true } }; -} \ No newline at end of file +} + +public partial class InstrumentationOptions +{ + // Publishes start a new linked trace by default in v10 for backward compatibility. In v11, + // delete this method: PublishTraceMode then defaults to TraceMode.ContinueExisting through + // its initializer in InstrumentationOptions.cs, and the empty partial declaration plus the + // constructor call there can be removed. + partial void ApplyPreV11Defaults() => PublishTraceMode = TraceMode.StartNew; + + /// + /// Appends the destination to span names following the OTel messaging convention + /// {messaging.operation.name} {destination}, e.g. "process orders" or "send payments". + /// Disabled by default for backward compatibility. + /// + /// + /// In v11, this property will be dropped and "true" will be the only behavior. + /// + public bool UseMessageTypeNamesInSpanNames { get; set; } + + /// + /// Controls whether the "Start dispatching" and "Finished dispatching" activity events + /// are added to the incoming message span when outgoing messages are dispatched. + /// Enabled by default for backward compatibility. Disable to avoid the ingestion cost + /// of these events when they add no diagnostic value. + /// + /// + /// In v11, this property will be dropped and "false" will be the only behavior. + /// + public bool EmitMessageDispatchingEvents { get; set; } = true; + + /// + /// Controls meter instruments behaviors. + /// + public MetersOptions Meters { get; } = new(); +} + +/// +/// Controls opt-in meter instruments behaviors. +/// Accessed via endpointConfiguration.Tracing().Meters. +/// +public class MetersOptions +{ + /// + /// Emits the legacy execution.result tag with values "success" or "failure" + /// on handler time, processing time, saga fetch time, deserialize time, and serialize time metrics. + /// Enabled by default for backwards compatibility. Disable to reduce tag cardinality. + /// + /// + /// In v11, this property will be dropped and "false" will be the only behavior. + /// + public bool EmitExecutionResultTags { get; set; } = true; +} diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v12.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v12.cs index 13e193f8957..4ad0abd625e 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v12.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v12.cs @@ -20,6 +20,12 @@ namespace NServiceBus; // OpenTelemetryFeature's constructor. public partial class InstrumentationOptions { + /// + /// Controls how exception details are recorded when an operation fails. + /// Defaults to : exceptions are recorded as an event on the activity. + /// + public ExceptionRecordingMode ExceptionRecordingMode { get; set; } = ExceptionRecordingMode.SpanAndLogs; + internal static void SetExceptionRecordingModeDefault(SettingsHolder settings) { var options = settings.GetOrCreate(); diff --git a/src/NServiceBus.Core/Pipeline/Incoming/DeserializeMessageConnector.cs b/src/NServiceBus.Core/Pipeline/Incoming/DeserializeMessageConnector.cs index 1afa8f9986c..aa5df44c0cc 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/DeserializeMessageConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/DeserializeMessageConnector.cs @@ -19,7 +19,7 @@ class DeserializeMessageConnector( MessageMetadataRegistry messageMetadataRegistry, IMessageMapper mapper, bool allowContentTypeInference, - IncomingPipelineMetrics incomingPipelineMetrics) + PipelineMetrics pipelineMetrics) : StageConnector { public override async Task Invoke(IIncomingPhysicalMessageContext context, Func stage) @@ -36,19 +36,18 @@ public override async Task Invoke(IIncomingPhysicalMessageContext context, Func< catch (Exception ex) #pragma warning restore PS0019 { - incomingPipelineMetrics.RecordDeserializeTime(context, Stopwatch.GetElapsedTime(deserializeStart), error: ex); + pipelineMetrics.RecordDeserializeTime(context, Stopwatch.GetElapsedTime(deserializeStart), error: ex); throw; } - incomingPipelineMetrics.RecordDeserializeTime(context, Stopwatch.GetElapsedTime(deserializeStart)); + pipelineMetrics.RecordDeserializeTime(context, Stopwatch.GetElapsedTime(deserializeStart)); bool first = true; foreach (var message in messages) { if (first) // ignore the legacy case in which a single message payload contained multiple messages { - var availableMetricTags = context.IncomingMetricTags; - availableMetricTags.Add(MeterTags.MessageType, message.MessageType.FullName!); + context.PipelineMetricTags.Add(MeterTags.MessageType, message.MessageType.FullName!); first = false; } await stage(this.CreateIncomingLogicalMessageContext(message, context)).ConfigureAwait(false); diff --git a/src/NServiceBus.Core/Pipeline/Incoming/InvokeHandlerTerminator.cs b/src/NServiceBus.Core/Pipeline/Incoming/InvokeHandlerTerminator.cs index e177dc8404d..f6a6ce1b2c9 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/InvokeHandlerTerminator.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/InvokeHandlerTerminator.cs @@ -7,7 +7,7 @@ namespace NServiceBus; using Pipeline; using Sagas; -class InvokeHandlerTerminator(IncomingPipelineMetrics messagingMetricsMeters) : PipelineTerminator +class InvokeHandlerTerminator(PipelineMetrics messagingMetricsMeters) : PipelineTerminator { protected override async Task Terminate(IInvokeHandlerContext context) { diff --git a/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs b/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs index bb140dcb96c..e1385d74fef 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs @@ -16,7 +16,7 @@ namespace NServiceBus; using Pipeline; using Unicast; -class LoadHandlersConnector(MessageHandlerRegistry messageHandlerRegistry, IActivityFactory activityFactory, IncomingPipelineMetrics incomingPipelineMetrics) : StageConnector +class LoadHandlersConnector(MessageHandlerRegistry messageHandlerRegistry, IActivityFactory activityFactory, PipelineMetrics pipelineMetrics) : StageConnector { public override async Task Invoke(IIncomingLogicalMessageContext context, Func stage) { @@ -43,8 +43,7 @@ public override async Task Invoke(IIncomingLogicalMessageContext context, Func x.HandlerType.FullName))); + context.PipelineMetricTags.Add(MeterTags.MessageHandlerTypes, string.Join(';', handlersToInvoke.Select(x => x.HandlerType.FullName))); foreach (var messageHandler in handlersToInvoke) { @@ -82,7 +81,7 @@ public override async Task Invoke(IIncomingLogicalMessageContext context, Func collected for the message currently being processed. Add to this /// collection to have the tags applied to the metrics emitted for that message. /// - public IMetricsTags MetricTags => context.Extensions.GetOrCreate(); + public IMetricsTags MetricTags => context.Extensions.GetOrCreate(); - internal IncomingPipelineMetricTags IncomingMetricTags => context.Extensions.GetOrCreate(); + internal PipelineMetricTags PipelineMetricTags => context.Extensions.GetOrCreate(); } extension(MessageContext context) { - internal IncomingPipelineMetricTags IncomingMetricTags => context.Extensions.GetOrCreate(); + internal PipelineMetricTags MetricTags => context.Extensions.GetOrCreate(); } } diff --git a/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetricTags.cs b/src/NServiceBus.Core/Pipeline/Incoming/PipelineMetricTags.cs similarity index 98% rename from src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetricTags.cs rename to src/NServiceBus.Core/Pipeline/Incoming/PipelineMetricTags.cs index f1f0f4d7a47..1e6f8a314a0 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetricTags.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/PipelineMetricTags.cs @@ -9,7 +9,7 @@ namespace NServiceBus; /// /// Captures possible metric tags that can be applied to a metric throughout the incoming processing pipeline. /// -sealed class IncomingPipelineMetricTags : IMetricsTags +sealed class PipelineMetricTags : IMetricsTags { readonly Dictionary> tags = []; readonly Dictionary>> instrumentTags = []; diff --git a/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs b/src/NServiceBus.Core/Pipeline/Incoming/PipelineMetrics.cs similarity index 74% rename from src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs rename to src/NServiceBus.Core/Pipeline/Incoming/PipelineMetrics.cs index 45ee4da5c5e..84c8aa2c7b0 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/IncomingPipelineMetrics.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/PipelineMetrics.cs @@ -3,13 +3,12 @@ namespace NServiceBus; using System; -using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.Metrics; using Transport; using Pipeline; -class IncomingPipelineMetrics +class PipelineMetrics { const string TotalProcessedSuccessfully = "nservicebus.messaging.successes"; const string TotalFetched = "nservicebus.messaging.fetches"; @@ -30,7 +29,8 @@ class IncomingPipelineMetrics const string OutboxStoreTime = "nservicebus.outbox.store_time"; const string CommitTime = "nservicebus.persistence.commit_time"; - public IncomingPipelineMetrics(IMeterFactory meterFactory, string queueName, string discriminator, MetersOptions metersOptions) + // queueName and discriminator are null for send-only endpoints, which have no receive queue to report. + public PipelineMetrics(IMeterFactory meterFactory, string? queueName, string? discriminator, MetersOptions metersOptions) { emitExecutionResultTags = metersOptions.EmitExecutionResultTags; var meter = meterFactory.Create("NServiceBus.Core.Pipeline.Incoming", "0.4.0"); @@ -71,14 +71,17 @@ public IncomingPipelineMetrics(IMeterFactory meterFactory, string queueName, str persistenceTime = meter.CreateHistogram(CommitTime, "s", "The time in seconds for completing the synchronized storage session."); - queueNameBase = queueName; - endpointDiscriminator = discriminator; - } + // queueName and discriminator can be null (e.g., for a send-only endpoint) + // in such cases we don't want to report these tags + if (queueName != null) + { + DefaultMetricTags.Add(MeterTags.QueueName, queueName); + } - public void AddDefaultIncomingPipelineMetricTags(IncomingPipelineMetricTags incomingPipelineMetricTags) - { - incomingPipelineMetricTags.Add(MeterTags.QueueName, queueNameBase); - incomingPipelineMetricTags.Add(MeterTags.EndpointDiscriminator, endpointDiscriminator); + if (discriminator != null) + { + DefaultMetricTags.Add(MeterTags.EndpointDiscriminator, discriminator); + } } public void RecordProcessingTime(ITransportReceiveContext context, TimeSpan elapsed) @@ -88,14 +91,15 @@ public void RecordProcessingTime(ITransportReceiveContext context, TimeSpan elap return; } - TagList tags; + var tags = DefaultMetricTags; + if (emitExecutionResultTags) { // Execution result is to be removed so we don't support overriding it by the user - tags.Add(new KeyValuePair(MeterTags.ExecutionResult, "success")); + tags.Add(MeterTags.ExecutionResult, "success"); } - context.IncomingMetricTags.ApplyTags(ref tags, [ + context.PipelineMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.MessageType, @@ -111,16 +115,17 @@ public void RecordCriticalTimeAndTotalProcessed(ITransportReceiveContext context { return; } - TagList tags; + + var tags = DefaultMetricTags; if (emitExecutionResultTags) { // Execution result is to be removed so we don't support overriding it by the user - tags.Add(new KeyValuePair(MeterTags.ExecutionResult, "success")); + tags.Add(MeterTags.ExecutionResult, "success"); } // totalProcessedSuccessfully and criticalTime always share the same tags in this method, so overrides are // looked up under criticalTime's instrument name. - context.IncomingMetricTags.ApplyTags(ref tags, [ + context.PipelineMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.MessageType, @@ -144,23 +149,23 @@ public void RecordCriticalTimeAndTotalProcessed(ITransportReceiveContext context } } - public void RecordMessageProcessingFailure(IncomingPipelineMetricTags incomingPipelineMetricTags, Exception error) + public void RecordMessageProcessingFailure(PipelineMetricTags pipelineMetricTags, Exception error) { if (!totalFailures.Enabled) { return; } - TagList tags; + var tags = DefaultMetricTags; if (emitExecutionResultTags) { // Execution result is to be removed so we don't support overriding it by the user - tags.Add(new KeyValuePair(MeterTags.ExecutionResult, "failure")); + tags.Add(MeterTags.ExecutionResult, "failure"); } - tags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); + tags.Add(MeterTags.ErrorType, error.GetType().FullName); - incomingPipelineMetricTags.ApplyTags(ref tags, [ + pipelineMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.MessageType, @@ -172,15 +177,15 @@ public void RecordMessageProcessingFailure(IncomingPipelineMetricTags incomingPi // the processing and critical time are intentionally not recorded in case of failure } - public void RecordFetchedMessage(IncomingPipelineMetricTags incomingPipelineMetricTags) + public void RecordFetchedMessage(PipelineMetricTags pipelineMetricTags) { if (!totalFetched.Enabled) { return; } - TagList tags; - incomingPipelineMetricTags.ApplyTags(ref tags, [ + var tags = DefaultMetricTags; + pipelineMetricTags.ApplyTags(ref tags, [ MeterTags.EndpointDiscriminator, MeterTags.QueueName, MeterTags.MessageType], @@ -196,8 +201,8 @@ public void RecordDeduplicatedMessage(ITransportReceiveContext context) return; } - TagList tags; - context.IncomingMetricTags.ApplyTags(ref tags, [ + var tags = DefaultMetricTags; + context.PipelineMetricTags.ApplyTags(ref tags, [ MeterTags.EndpointDiscriminator, MeterTags.QueueName, MeterTags.MessageType], @@ -213,16 +218,16 @@ public void RecordSuccessfulMessageHandlerTime(IInvokeHandlerContext context, Ti return; } - TagList tags; + var tags = DefaultMetricTags; if (emitExecutionResultTags) { // Execution result is to be removed so we don't support overriding it by the user - tags.Add(new KeyValuePair(MeterTags.ExecutionResult, "success")); + tags.Add(MeterTags.ExecutionResult, "success"); } - tags.Add(new KeyValuePair(MeterTags.MessageHandlerType, context.MessageHandler.HandlerType.FullName)); + tags.Add(MeterTags.MessageHandlerType, context.MessageHandler.HandlerType.FullName); - context.IncomingMetricTags.ApplyTags(ref tags, [ + context.PipelineMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.MessageType, @@ -238,17 +243,17 @@ public void RecordFailedMessageHandlerTime(IInvokeHandlerContext context, TimeSp return; } - TagList tags; + var tags = DefaultMetricTags; if (emitExecutionResultTags) { // Execution result is to be removed so we don't support overriding it by the user - tags.Add(new KeyValuePair(MeterTags.ExecutionResult, "failure")); + tags.Add(MeterTags.ExecutionResult, "failure"); } - tags.Add(new KeyValuePair(MeterTags.MessageHandlerType, context.MessageHandler.HandlerType.FullName)); - tags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); + tags.Add(MeterTags.MessageHandlerType, context.MessageHandler.HandlerType.FullName); + tags.Add(MeterTags.ErrorType, error.GetType().FullName); - context.IncomingMetricTags.ApplyTags(ref tags, [ + context.PipelineMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.MessageType, @@ -265,16 +270,17 @@ public void RecordImmediateRetry(IRecoverabilityContext context) return; } - TagList tags; - tags.Add(new KeyValuePair(MeterTags.ErrorType, context.Exception.GetType().FullName)); + var tags = DefaultMetricTags; + tags.Add(MeterTags.ErrorType, context.Exception.GetType().FullName); - context.IncomingMetricTags.ApplyTags(ref tags, [ + context.PipelineMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.MessageType, MeterTags.MessageHandlerType, MeterTags.ErrorType], - totalImmediateRetries.Name); + totalImmediateRetries.Name + ); totalImmediateRetries.Add(1, tags); } @@ -285,10 +291,10 @@ public void RecordDelayedRetry(IRecoverabilityContext context) return; } - TagList tags; - tags.Add(new KeyValuePair(MeterTags.ErrorType, context.Exception.GetType().FullName)); + var tags = DefaultMetricTags; + tags.Add(MeterTags.ErrorType, context.Exception.GetType().FullName); - context.IncomingMetricTags.ApplyTags(ref tags, [ + context.PipelineMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.MessageType, @@ -305,10 +311,10 @@ public void RecordSendToErrorQueue(IRecoverabilityContext context) return; } - TagList tags; - tags.Add(new KeyValuePair(MeterTags.ErrorType, context.Exception.GetType().FullName)); + var tags = DefaultMetricTags; + tags.Add(MeterTags.ErrorType, context.Exception.GetType().FullName); - context.IncomingMetricTags.ApplyTags(ref tags, [ + context.PipelineMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.MessageType, @@ -318,19 +324,19 @@ public void RecordSendToErrorQueue(IRecoverabilityContext context) totalSentToErrorQueue.Add(1, tags); } - public ActiveMessageScope TrackMessageProcessing(IncomingPipelineMetricTags incomingPipelineMetricTags, IncomingMessage message) + public ActiveMessageScope TrackMessageProcessing(PipelineMetricTags pipelineMetricTags, IncomingMessage message) { if (!activeMessages.Enabled) { return default; } - TagList tags; + var tags = DefaultMetricTags; if (message.Headers.TryGetValue(Headers.EnclosedMessageTypes, out var enclosedMessageTypes)) { - tags.Add(new KeyValuePair(MeterTags.EnclosedMessageTypes, enclosedMessageTypes)); + tags.Add(MeterTags.EnclosedMessageTypes, enclosedMessageTypes); } - incomingPipelineMetricTags.ApplyTags(ref tags, [ + pipelineMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.EnclosedMessageTypes], @@ -347,19 +353,19 @@ public void RecordSagaFetchTime(IInvokeHandlerContext context, TimeSpan elapsed, return; } - TagList tags; + var tags = DefaultMetricTags; if (emitExecutionResultTags) { // Execution result is to be removed so we don't support overriding it by the user - tags.Add(new KeyValuePair(MeterTags.ExecutionResult, error != null ? "failure" : "success")); + tags.Add(MeterTags.ExecutionResult, error != null ? "failure" : "success"); } - tags.Add(new KeyValuePair(MeterTags.SagaType, sagaType)); + tags.Add(MeterTags.SagaType, sagaType); if (error != null) { - tags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); + tags.Add(MeterTags.ErrorType, error.GetType().FullName); } - context.IncomingMetricTags.ApplyTags(ref tags, [ + context.PipelineMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.MessageType, @@ -376,22 +382,22 @@ public void RecordDeserializeTime(IIncomingPhysicalMessageContext context, TimeS return; } - TagList tags; + var tags = DefaultMetricTags; if (context.Message.Headers.TryGetValue(Headers.EnclosedMessageTypes, out var messageTypes)) { - tags.Add(new KeyValuePair(MeterTags.EnclosedMessageTypes, messageTypes)); + tags.Add(MeterTags.EnclosedMessageTypes, messageTypes); } if (error != null) { - tags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); + tags.Add(MeterTags.ErrorType, error.GetType().FullName); } if (emitExecutionResultTags) { // Execution result is to be removed so we don't support overriding it by the user - tags.Add(new KeyValuePair(MeterTags.ExecutionResult, error != null ? "failure" : "success")); + tags.Add(MeterTags.ExecutionResult, error != null ? "failure" : "success"); } - context.IncomingMetricTags.ApplyTags(ref tags, [ + context.PipelineMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.EnclosedMessageTypes, @@ -403,28 +409,29 @@ public void RecordDeserializeTime(IIncomingPhysicalMessageContext context, TimeS public void RecordSerializeTime(IOutgoingLogicalMessageContext context, TimeSpan elapsed, string? messageType, Exception? error = null) { - // No incoming pipeline context is available here (this fires from the outgoing send pipeline, which may - // run with no incoming message at all), so there's no IncomingPipelineMetricTags to route these through. if (!messageSerializeTime.Enabled) { return; } - TagList tags; + var tags = DefaultMetricTags; if (messageType != null) { - tags.Add(new KeyValuePair(MeterTags.MessageType, messageType)); // tag-bag-bypass: see comment above + tags.Add(MeterTags.MessageType, messageType); } if (error != null) { - tags.Add(new KeyValuePair(MeterTags.ErrorType, error.GetType().FullName)); // tag-bag-bypass: see comment above + tags.Add(MeterTags.ErrorType, error.GetType().FullName); } if (emitExecutionResultTags) { - tags.Add(new KeyValuePair(MeterTags.ExecutionResult, error != null ? "failure" : "success")); // tag-bag-bypass: see comment above + // Execution result is to be removed so we don't support overriding it by the user + tags.Add(MeterTags.ExecutionResult, error != null ? "failure" : "success"); } - context.IncomingMetricTags.ApplyTags(ref tags, [ + context.PipelineMetricTags.ApplyTags(ref tags, [ + MeterTags.QueueName, + MeterTags.EndpointDiscriminator, MeterTags.MessageType, MeterTags.ErrorType], messageSerializeTime.Name); @@ -439,8 +446,8 @@ public void RecordOutboxFetchTime(ITransportReceiveContext context, TimeSpan ela return; } - TagList tags; - context.IncomingMetricTags.ApplyTags(ref tags, [ + var tags = DefaultMetricTags; + context.PipelineMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator], outboxFetchTime.Name); @@ -455,8 +462,8 @@ public void RecordOutboxStoreTime(ITransportReceiveContext context, TimeSpan ela return; } - TagList tags; - context.IncomingMetricTags.ApplyTags(ref tags, [ + var tags = DefaultMetricTags; + context.PipelineMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator], outboxStoreTime.Name); @@ -471,8 +478,8 @@ public void RecordPersistenceTime(IIncomingLogicalMessageContext context, TimeSp return; } - TagList tags; - context.IncomingMetricTags.ApplyTags(ref tags, [ + var tags = DefaultMetricTags; + context.PipelineMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.MessageType, @@ -490,14 +497,16 @@ void RecordEnvelopeUnwrapping(MessageContext messageContext, IEnvelopeHandler ty { return; } - TagList tags; - tags.Add(new KeyValuePair(MeterTags.EnvelopeUnwrapperType, type.GetType().FullName)); + + var tags = DefaultMetricTags; + tags.Add(MeterTags.EnvelopeUnwrapperType, type.GetType().FullName); + if (exception != null) { - tags.Add(new KeyValuePair(MeterTags.ErrorType, exception.GetType().FullName)); + tags.Add(MeterTags.ErrorType, exception.GetType().FullName); } - messageContext.IncomingMetricTags.ApplyTags(ref tags, [ + messageContext.MetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, MeterTags.EndpointDiscriminator, MeterTags.EnvelopeUnwrapperType, @@ -531,7 +540,6 @@ public readonly struct ActiveMessageScope(UpDownCounter? counter, TagList readonly Histogram outboxStoreTime; readonly Histogram persistenceTime; - readonly string queueNameBase; - readonly string endpointDiscriminator; + readonly TagList DefaultMetricTags; readonly bool emitExecutionResultTags; } diff --git a/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs b/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs index e50a23835d1..76003cd78f6 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs @@ -15,7 +15,7 @@ namespace NServiceBus; class TransportReceiveToPhysicalMessageConnector( IOutboxStorage outboxStorage, - IncomingPipelineMetrics incomingPipelineMetrics, + PipelineMetrics pipelineMetrics, InstrumentationOptions instrumentationOptions) : IStageForkConnector { @@ -27,7 +27,7 @@ public async Task Invoke(ITransportReceiveContext context, Func(); await outboxTransaction.Commit(context.CancellationToken).ConfigureAwait(false); @@ -54,7 +54,7 @@ public async Task Invoke(ITransportReceiveContext context, Func(); } @@ -63,7 +63,7 @@ public async Task Invoke(ITransportReceiveContext context, Func receivePipelineNotification, IPipeline receivePipeline, IActivityFactory activityFactory, - IncomingPipelineMetrics incomingPipelineMetrics, + PipelineMetrics pipelineMetrics, EnvelopeUnwrapper envelopeUnwrapper) : IPipelineExecutor { public async Task Invoke(MessageContext messageContext, CancellationToken cancellationToken = default) { var pipelineStartedAt = DateTimeOffset.UtcNow; - using var activity = activityFactory.StartIncomingPipelineActivity(messageContext); - - var incomingPipelineMetricsTags = messageContext.IncomingMetricTags; - incomingPipelineMetrics.AddDefaultIncomingPipelineMetricTags(incomingPipelineMetricsTags); + using var activity = activityFactory.StartIncomingPipelineActivity(messageContext); var childScope = rootBuilder.CreateAsyncScope(); await using (childScope.ConfigureAwait(false)) @@ -36,7 +34,7 @@ public async Task Invoke(MessageContext messageContext, CancellationToken cancel IncomingMessage message = incomingMessageHandle; //This needs to happen after envelope unwrapping to ensure the proper value of the EnclosedMessageTypes header - using var activeMessageScope = incomingPipelineMetrics.TrackMessageProcessing(incomingPipelineMetricsTags, message); + using var activeMessageScope = pipelineMetrics.TrackMessageProcessing(messageContext.MetricTags, message); var transportReceiveContext = new TransportReceiveContext( childScope.ServiceProvider, @@ -54,12 +52,15 @@ public async Task Invoke(MessageContext messageContext, CancellationToken cancel try { - await receivePipeline.Invoke(transportReceiveContext, activity, activityFactory).ConfigureAwait(false); + + await receivePipeline.Invoke(transportReceiveContext).ConfigureAwait(false); + activity?.SetStatus(ActivityStatusCode.Ok); } #pragma warning disable PS0019 // Do not catch Exception without considering OperationCanceledException - enriching and rethrowing catch (Exception ex) #pragma warning restore PS0019 // Do not catch Exception without considering OperationCanceledException { + activityFactory.RecordError(activity, ex, transportReceiveContext.Extensions); ex.Data["Message ID"] = message.MessageId; if (message.NativeMessageId != message.MessageId) @@ -71,13 +72,13 @@ public async Task Invoke(MessageContext messageContext, CancellationToken cancel if (!ex.IsCausedBy(transportReceiveContext.CancellationToken)) { - incomingPipelineMetrics.RecordMessageProcessingFailure(incomingPipelineMetricsTags, ex); + pipelineMetrics.RecordMessageProcessingFailure(messageContext.MetricTags, ex); } throw; } finally { - incomingPipelineMetrics.RecordFetchedMessage(incomingPipelineMetricsTags); + pipelineMetrics.RecordFetchedMessage(messageContext.MetricTags); } var completedAt = DateTimeOffset.UtcNow; diff --git a/src/NServiceBus.Core/Pipeline/Outgoing/RoutingToDispatchConnector.cs b/src/NServiceBus.Core/Pipeline/Outgoing/RoutingToDispatchConnector.cs index 3243dd7b150..203e24fee8b 100644 --- a/src/NServiceBus.Core/Pipeline/Outgoing/RoutingToDispatchConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Outgoing/RoutingToDispatchConnector.cs @@ -70,7 +70,7 @@ public override Task Invoke(IRoutingContext context, Func 0 && operations[0].AddressTag is UnicastAddressTag unicastTag && outgoingMessage.Headers.TryGetValue(Headers.MessageIntent, out var intentStr) diff --git a/src/NServiceBus.Core/Pipeline/Outgoing/SerializeMessageConnector.cs b/src/NServiceBus.Core/Pipeline/Outgoing/SerializeMessageConnector.cs index f8fe0d459e8..d4352db1ea1 100644 --- a/src/NServiceBus.Core/Pipeline/Outgoing/SerializeMessageConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Outgoing/SerializeMessageConnector.cs @@ -13,11 +13,11 @@ namespace NServiceBus; class SerializeMessageConnector : StageConnector { - public SerializeMessageConnector(IMessageSerializer messageSerializer, MessageMetadataRegistry messageMetadataRegistry, IncomingPipelineMetrics incomingPipelineMetrics) + public SerializeMessageConnector(IMessageSerializer messageSerializer, MessageMetadataRegistry messageMetadataRegistry, PipelineMetrics pipelineMetrics) { this.messageSerializer = messageSerializer; this.messageMetadataRegistry = messageMetadataRegistry; - this.incomingPipelineMetrics = incomingPipelineMetrics; + this.pipelineMetrics = pipelineMetrics; } public override async Task Invoke(IOutgoingLogicalMessageContext context, Func stage) @@ -45,13 +45,13 @@ public override async Task Invoke(IOutgoingLogicalMessageContext context, Func(); } diff --git a/src/NServiceBus.Core/Pipeline/PipelineComponent.cs b/src/NServiceBus.Core/Pipeline/PipelineComponent.cs index 03326dc412b..cedb2d2ea7a 100644 --- a/src/NServiceBus.Core/Pipeline/PipelineComponent.cs +++ b/src/NServiceBus.Core/Pipeline/PipelineComponent.cs @@ -19,8 +19,14 @@ public static PipelineComponent Initialize(PipelineSettings settings, hostingConfiguration.Services.AddSingleton(sp => { var meterFactory = sp.GetRequiredService(); + + if (receiveConfiguration.IsSendOnlyEndpoint) + { + return new PipelineMetrics(meterFactory, null, null, metersOptions); + } + string discriminator = receiveConfiguration.InstanceSpecificQueueAddress?.Discriminator ?? ""; - return new IncomingPipelineMetrics(meterFactory, receiveConfiguration.LocalQueueAddress.BaseAddress, discriminator, metersOptions); + return new PipelineMetrics(meterFactory, receiveConfiguration.LocalQueueAddress.BaseAddress, discriminator, metersOptions); }); return new PipelineComponent(settings.modifications); diff --git a/src/NServiceBus.Core/Receiving/ReceiveComponent.cs b/src/NServiceBus.Core/Receiving/ReceiveComponent.cs index 5d859914cef..85213d1e5aa 100644 --- a/src/NServiceBus.Core/Receiving/ReceiveComponent.cs +++ b/src/NServiceBus.Core/Receiving/ReceiveComponent.cs @@ -67,12 +67,12 @@ public static ReceiveComponent Configure( pipelineSettings.Register("TransportReceiveToPhysicalMessageProcessingConnector", b => { var storage = b.GetService() ?? new NoOpOutboxStorage(); - return new TransportReceiveToPhysicalMessageConnector(storage, b.GetRequiredService(), hostingConfiguration.ActivityFactory.Options); + return new TransportReceiveToPhysicalMessageConnector(storage, b.GetRequiredService(), hostingConfiguration.ActivityFactory.Options); }, "Allows to abort processing the message"); - pipelineSettings.Register("LoadHandlersConnector", b => new LoadHandlersConnector(b.GetRequiredService(), hostingConfiguration.ActivityFactory, b.GetRequiredService()), "Gets all the handlers to invoke from the MessageHandler registry based on the message type."); + pipelineSettings.Register("LoadHandlersConnector", b => new LoadHandlersConnector(b.GetRequiredService(), hostingConfiguration.ActivityFactory, b.GetRequiredService()), "Gets all the handlers to invoke from the MessageHandler registry based on the message type."); - pipelineSettings.Register("InvokeHandlers", sp => new InvokeHandlerTerminator(sp.GetRequiredService()), "Calls the IHandleMessages.Handle(T)"); + pipelineSettings.Register("InvokeHandlers", sp => new InvokeHandlerTerminator(sp.GetRequiredService()), "Calls the IHandleMessages.Handle(T)"); var handlerDiagnostics = new Dictionary>(); @@ -172,7 +172,7 @@ public async Task Initialize( var receivePipeline = pipelineComponent.CreatePipeline(builder); - var pipelineMetrics = builder.GetRequiredService(); + var pipelineMetrics = builder.GetRequiredService(); var envelopeUnwrapper = envelopeComponent.CreateUnwrapper(builder); var mainPipelineExecutor = new MainPipelineExecutor(builder, pipelineCache, messageOperations, configuration.PipelineCompletedSubscribers, receivePipeline, activityFactory, pipelineMetrics, envelopeUnwrapper); diff --git a/src/NServiceBus.Core/Recoverability/RecoverabilityComponent.cs b/src/NServiceBus.Core/Recoverability/RecoverabilityComponent.cs index 972bcba47d2..0490fb8202a 100644 --- a/src/NServiceBus.Core/Recoverability/RecoverabilityComponent.cs +++ b/src/NServiceBus.Core/Recoverability/RecoverabilityComponent.cs @@ -58,7 +58,7 @@ public void Initialize( faultMetadataExtractor = CreateFaultMetadataExtractor(hostInformation, settings); - pipelineSettings.Register(sp => new RecoverabilityRoutingConnector(sp.GetRequiredService(), messageRetryNotification, messageFaultedNotification), "Executes the configured retry policy"); + pipelineSettings.Register(sp => new RecoverabilityRoutingConnector(sp.GetRequiredService(), messageRetryNotification, messageFaultedNotification), "Executes the configured retry policy"); hostingConfiguration.AddStartupDiagnosticsSection("Recoverability", new { diff --git a/src/NServiceBus.Core/Recoverability/RecoverabilityRoutingConnector.cs b/src/NServiceBus.Core/Recoverability/RecoverabilityRoutingConnector.cs index 62f10fb27c2..483c0e968e3 100644 --- a/src/NServiceBus.Core/Recoverability/RecoverabilityRoutingConnector.cs +++ b/src/NServiceBus.Core/Recoverability/RecoverabilityRoutingConnector.cs @@ -8,14 +8,14 @@ namespace NServiceBus; class RecoverabilityRoutingConnector : StageConnector { - readonly IncomingPipelineMetrics incomingPipelineMetrics; + readonly PipelineMetrics pipelineMetrics; public RecoverabilityRoutingConnector( - IncomingPipelineMetrics incomingPipelineMetrics, + PipelineMetrics pipelineMetrics, INotificationSubscriptions messageRetryNotification, INotificationSubscriptions messageFaultedNotification) { - this.incomingPipelineMetrics = incomingPipelineMetrics; + this.pipelineMetrics = pipelineMetrics; notifications = new CompositeNotification(); notifications.Register(messageRetryNotification); notifications.Register(messageFaultedNotification); @@ -35,15 +35,15 @@ public override async Task Invoke(IRecoverabilityContext context, Func { public async Task Invoke(IInvokeHandlerContext context, Func next) @@ -72,13 +72,13 @@ public async Task Invoke(IInvokeHandlerContext context, Func(); // Register the Saga related behaviors for incoming messages - context.Pipeline.Register("InvokeSaga", b => new SagaPersistenceBehavior(b.GetRequiredService(), sagaIdGenerator, sagaMetaModel, b, b.GetRequiredService()), "Invokes the saga logic"); + context.Pipeline.Register("InvokeSaga", b => new SagaPersistenceBehavior(b.GetRequiredService(), sagaIdGenerator, sagaMetaModel, b, b.GetRequiredService()), "Invokes the saga logic"); context.Pipeline.Register("AttachSagaDetailsToOutGoingMessage", new AttachSagaDetailsToOutGoingMessageBehavior(), "Makes sure that outgoing messages have saga info attached to them"); } } \ No newline at end of file diff --git a/src/NServiceBus.Core/Serialization/SerializationFeature.cs b/src/NServiceBus.Core/Serialization/SerializationFeature.cs index 41bec7b6d5a..ca9ee82a60b 100644 --- a/src/NServiceBus.Core/Serialization/SerializationFeature.cs +++ b/src/NServiceBus.Core/Serialization/SerializationFeature.cs @@ -48,8 +48,8 @@ protected override void Setup(FeatureConfigurationContext context) var allowMessageTypeInference = settings.IsMessageTypeInferenceEnabled(); var resolver = new MessageDeserializerResolver(mainSerializer, additionalDeserializers); var logicalMessageFactory = new LogicalMessageFactory(messageMetadataRegistry, mapper); - context.Pipeline.Register("DeserializeLogicalMessagesConnector", b => new DeserializeMessageConnector(resolver, logicalMessageFactory, messageMetadataRegistry, mapper, allowMessageTypeInference, b.GetRequiredService()), "Deserializes the physical message body into logical messages"); - context.Pipeline.Register("SerializeMessageConnector", b => new SerializeMessageConnector(mainSerializer, messageMetadataRegistry, b.GetRequiredService()), "Converts a logical message into a physical message"); + context.Pipeline.Register("DeserializeLogicalMessagesConnector", b => new DeserializeMessageConnector(resolver, logicalMessageFactory, messageMetadataRegistry, mapper, allowMessageTypeInference, b.GetRequiredService()), "Deserializes the physical message body into logical messages"); + context.Pipeline.Register("SerializeMessageConnector", b => new SerializeMessageConnector(mainSerializer, messageMetadataRegistry, b.GetRequiredService()), "Converts a logical message into a physical message"); context.Services.AddSingleton(mapper); context.Services.AddSingleton(mapper); diff --git a/src/NServiceBus.Core/Unicast/MessageOperations.cs b/src/NServiceBus.Core/Unicast/MessageOperations.cs index cfceeb4c684..dfe850c0c3c 100644 --- a/src/NServiceBus.Core/Unicast/MessageOperations.cs +++ b/src/NServiceBus.Core/Unicast/MessageOperations.cs @@ -2,39 +2,30 @@ namespace NServiceBus; using System; using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; using System.Threading.Tasks; using Extensibility; using MessageInterfaces; using Pipeline; using Transport; -class MessageOperations +class MessageOperations( + IMessageMapper messageMapper, + IPipeline publishPipeline, + IPipeline sendPipeline, + IPipeline replyPipeline, + IPipeline subscribePipeline, + IPipeline unsubscribePipeline, + IActivityFactory activityFactory) { - readonly IMessageMapper messageMapper; - protected readonly IPipeline publishPipeline; - protected readonly IPipeline sendPipeline; - protected readonly IPipeline replyPipeline; - protected readonly IPipeline subscribePipeline; - protected readonly IPipeline unsubscribePipeline; - protected readonly IActivityFactory activityFactory; - - public MessageOperations( - IMessageMapper messageMapper, - IPipeline publishPipeline, - IPipeline sendPipeline, - IPipeline replyPipeline, - IPipeline subscribePipeline, - IPipeline unsubscribePipeline, - IActivityFactory activityFactory) - { - this.messageMapper = messageMapper; - this.publishPipeline = publishPipeline; - this.sendPipeline = sendPipeline; - this.replyPipeline = replyPipeline; - this.subscribePipeline = subscribePipeline; - this.unsubscribePipeline = unsubscribePipeline; - this.activityFactory = activityFactory; - } + protected readonly IPipeline publishPipeline = publishPipeline; + protected readonly IPipeline sendPipeline = sendPipeline; + protected readonly IPipeline replyPipeline = replyPipeline; + protected readonly IPipeline subscribePipeline = subscribePipeline; + protected readonly IPipeline unsubscribePipeline = unsubscribePipeline; + readonly bool UseMessageTypeNamesInSpanNames = activityFactory.Options.UseMessageTypeNamesInSpanNames; + public Task Publish(IBehaviorContext context, Action messageConstructor, PublishOptions options) { @@ -65,13 +56,24 @@ async Task Publish(IBehaviorContext context, Type messageType, object message, P MergeDispatchProperties(publishContext, options.DispatchProperties); - var publishDisplayName = activityFactory.Options.UseMessageDestinationInSpanNames + var displayName = UseMessageTypeNamesInSpanNames ? $"{ActivityDisplayNames.PublishOperation} {messageType.Name}" : ActivityDisplayNames.PublishEvent; - using var activity = activityFactory.StartOutgoingPipelineActivity(ActivityNames.OutgoingEventActivityName, publishDisplayName, publishContext); + using var activity = activityFactory.StartOutgoingPipelineActivity(ActivityNames.OutgoingEventActivityName, displayName, publishContext); - await publishPipeline.Invoke(publishContext, activity, activityFactory).ConfigureAwait(false); +#pragma warning disable PS0019 // When catching System.Exception, cancellation needs to be properly accounted for - recording and rethrowing + try + { + await publishPipeline.Invoke(publishContext) + .ConfigureAwait(false); + activity?.SetStatus(ActivityStatusCode.Ok); + } + catch (Exception ex) + { + activityFactory.RecordError(activity, ex, context.Extensions); + throw; + } } public Task Subscribe(IBehaviorContext context, Type eventType, SubscribeOptions options) @@ -88,9 +90,23 @@ public async Task Subscribe(IBehaviorContext context, Type[] eventTypes, Subscri MergeDispatchProperties(subscribeContext, options.DispatchProperties); - using var activity = activityFactory.StartOutgoingPipelineActivity(ActivityNames.SubscribeActivityName, ActivityDisplayNames.SubscribeEvent, context); + var displayName = UseMessageTypeNamesInSpanNames + ? $"{ActivityDisplayNames.SubscribeEvent} {string.Join(' ', eventTypes.Select(x => x.Name))}" + : ActivityDisplayNames.SubscribeEvent; - await subscribePipeline.Invoke(subscribeContext, activity, activityFactory).ConfigureAwait(false); + using var activity = activityFactory.StartOutgoingPipelineActivity(ActivityNames.SubscribeActivityName, displayName, context); + + try + { + await subscribePipeline.Invoke(subscribeContext) + .ConfigureAwait(false); + activity?.SetStatus(ActivityStatusCode.Ok); + } + catch (Exception ex) + { + activityFactory.RecordError(activity, ex, context.Extensions); + throw; + } } public async Task Unsubscribe(IBehaviorContext context, Type eventType, UnsubscribeOptions options) @@ -98,13 +114,29 @@ public async Task Unsubscribe(IBehaviorContext context, Type eventType, Unsubscr var unsubscribeContext = new UnsubscribeContext( context, eventType, - options.Context); + options.Context + ); MergeDispatchProperties(unsubscribeContext, options.DispatchProperties); - using var activity = activityFactory.StartOutgoingPipelineActivity(ActivityNames.UnsubscribeActivityName, ActivityDisplayNames.UnsubscribeEvent, context); + var displayName = UseMessageTypeNamesInSpanNames + ? $"{ActivityDisplayNames.UnsubscribeEvent} {eventType.Name}" + : ActivityDisplayNames.UnsubscribeEvent; - await unsubscribePipeline.Invoke(unsubscribeContext, activity, activityFactory).ConfigureAwait(false); + using var activity = activityFactory.StartOutgoingPipelineActivity(ActivityNames.UnsubscribeActivityName, displayName, context); + +#pragma warning disable PS0019 // When catching System.Exception, cancellation needs to be properly accounted for - recording and rethrowing + try + { + await unsubscribePipeline.Invoke(unsubscribeContext) + .ConfigureAwait(false); + activity?.SetStatus(ActivityStatusCode.Ok); + } + catch (Exception ex) + { + activityFactory.RecordError(activity, ex, context.Extensions); + throw; + } } public Task Send(IBehaviorContext context, Action messageConstructor, SendOptions options) @@ -136,9 +168,23 @@ async Task SendMessage(IBehaviorContext context, Type messageType, object messag MergeDispatchProperties(outgoingContext, options.DispatchProperties); - using var activity = activityFactory.StartOutgoingPipelineActivity(ActivityNames.OutgoingMessageActivityName, ActivityDisplayNames.SendMessage, outgoingContext); + var displayName = UseMessageTypeNamesInSpanNames + ? $"{ActivityDisplayNames.SendMessage} {messageType.Name}" + : ActivityDisplayNames.SendMessage; - await sendPipeline.Invoke(outgoingContext, activity, activityFactory).ConfigureAwait(false); + using var activity = activityFactory.StartOutgoingPipelineActivity(ActivityNames.OutgoingMessageActivityName, displayName, outgoingContext); + + try + { + await sendPipeline.Invoke(outgoingContext) + .ConfigureAwait(false); + activity?.SetStatus(ActivityStatusCode.Ok); + } + catch (Exception ex) + { + activityFactory.RecordError(activity, ex, context.Extensions); + throw; + } } public Task Reply(IBehaviorContext context, object message, ReplyOptions options) @@ -170,9 +216,23 @@ async Task ReplyMessage(IBehaviorContext context, Type messageType, object messa MergeDispatchProperties(outgoingContext, options.DispatchProperties); - using var activity = activityFactory.StartOutgoingPipelineActivity(ActivityNames.OutgoingMessageActivityName, ActivityDisplayNames.ReplyMessage, outgoingContext); + var displayName = UseMessageTypeNamesInSpanNames + ? $"{ActivityDisplayNames.ReplyMessage} {messageType.Name}" + : ActivityDisplayNames.ReplyMessage; + + using var activity = activityFactory.StartOutgoingPipelineActivity(ActivityNames.OutgoingMessageActivityName, displayName, context); - await replyPipeline.Invoke(outgoingContext, activity, activityFactory).ConfigureAwait(false); + try + { + await replyPipeline.Invoke(outgoingContext) + .ConfigureAwait(false); + activity?.SetStatus(ActivityStatusCode.Ok); + } + catch (Exception ex) + { + activityFactory.RecordError(activity, ex, context.Extensions); + throw; + } } static void MergeDispatchProperties(ContextBag context, DispatchProperties dispatchProperties) From 47de8551a5990ed722f2afe2dacab3cb0f8abf3e Mon Sep 17 00:00:00 2001 From: Tomasz Masternak Date: Tue, 29 Sep 2026 13:53:27 +0200 Subject: [PATCH 24/37] Custom NServiceBus-specific OpenTelemetry headers (#7947) * refactor receiver activity creation * fix string comparison for StartNewTrace header and improve comments * Introduce `NServiceBus.TraceParent` header to prevent the clash with native header set by the SDK * remove redundant context propagation compatibility tests --- .../When_incoming_message_has_no_trace.cs | 1 + ...IApprovals.ApproveNServiceBus.approved.txt | 1 + ...fy_promotable_headers.ignored.approved.txt | 1 + .../OpenTelemetry/ActivityFactoryTests.cs | 18 +++++ .../ContextPropagationCompatibilityTests.cs | 3 +- .../ContextPropagationDefaultBehaviorTests.cs | 17 +++++ src/NServiceBus.Core/Headers.cs | 9 +++ .../OpenTelemetry/Tracing/ActivityFactory.cs | 67 +++++++++++-------- .../Tracing/ContextPropagation.cs | 17 +++-- 9 files changed, 100 insertions(+), 34 deletions(-) diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_incoming_message_has_no_trace.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_incoming_message_has_no_trace.cs index 69360955a0e..ecb290027bf 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_incoming_message_has_no_trace.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_incoming_message_has_no_trace.cs @@ -36,6 +36,7 @@ public override Task Invoke(IDispatchContext context, Func next) { foreach (TransportOperation transportOperation in context.Operations) { + transportOperation.Message.Headers.Remove(Headers.NServiceBusDiagnosticsTraceParent); transportOperation.Message.Headers.Remove(Headers.DiagnosticsTraceParent); transportOperation.Message.Headers.Remove(Headers.DiagnosticsTraceState); transportOperation.Message.Headers.Remove(Headers.DiagnosticsBaggage); diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt index a58f2c75532..04b4bf1c532 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt @@ -398,6 +398,7 @@ namespace NServiceBus public const string IsSagaTimeoutMessage = "NServiceBus.IsSagaTimeoutMessage"; public const string MessageId = "NServiceBus.MessageId"; public const string MessageIntent = "NServiceBus.MessageIntent"; + public const string NServiceBusDiagnosticsTraceParent = "NServiceBus.TraceParent"; public const string NServiceBusVersion = "NServiceBus.Version"; public const string NonDurableMessage = "NServiceBus.NonDurableMessage"; public const string OriginatingAddress = "NServiceBus.OriginatingAddress"; diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/ActivityDecoratorTests.Verify_promotable_headers.ignored.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/ActivityDecoratorTests.Verify_promotable_headers.ignored.approved.txt index 2366e3eaa9d..1f74f1e4add 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/ActivityDecoratorTests.Verify_promotable_headers.ignored.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/ActivityDecoratorTests.Verify_promotable_headers.ignored.approved.txt @@ -11,6 +11,7 @@ Headers.TimeSent Headers.SubscriptionMessageType Headers.SubscriberTransportAddress Headers.SubscriberEndpoint +Headers.NServiceBusDiagnosticsTraceParent Headers.DiagnosticsTraceParent Headers.DiagnosticsTraceState Headers.DiagnosticsBaggage diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs index 500740cc6ba..2a960c4256d 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs @@ -144,6 +144,24 @@ public void Should_attach_to_header_trace_when_no_activity_on_context_and_trace_ } } + [Test] + public void Should_prefer_nservicebus_trace_header_over_w3c_trace_header() + { + using var sendActivity = CreateCompletedActivity("send activity"); + using var transportActivity = CreateCompletedActivity("transport activity that overwrote the w3c header"); + + var messageHeaders = new Dictionary + { + { Headers.NServiceBusDiagnosticsTraceParent, sendActivity.Id! }, + { Headers.DiagnosticsTraceParent, transportActivity.Id! } + }; + + var activity = activityFactory.StartIncomingPipelineActivity(CreateMessageContext(messageHeaders)); + + Assert.That(activity, Is.Not.Null, "should create activity for receive pipeline"); + Assert.That(activity.ParentId, Is.EqualTo(sendActivity.Id), "should use the NServiceBus header, not the W3C one"); + } + [TestCase(ActivityIdFormat.W3C)] [TestCase(ActivityIdFormat.Hierarchical)] public void Should_attach_to_ambient_trace_when_no_activity_on_context_and_no_trace_header_and_ambient_activity(ActivityIdFormat ambientActivityIdFormat) diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationCompatibilityTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationCompatibilityTests.cs index 3dee5e145f4..697cebf27c8 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationCompatibilityTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationCompatibilityTests.cs @@ -23,6 +23,7 @@ public void ResetDistributedContextPropagator() } delegate void Writer(Activity activity, Dictionary headers); + delegate void Reader(Activity activity, IDictionary headers); static readonly Writer LegacyWrite = LegacyContextPropagation.PropagateContextToHeaders; @@ -76,7 +77,7 @@ public void New_sender_to_legacy_receiver_prepends_a_leading_space_but_keeps_the var received = Transmit(AllSpecialCharacters, NewWrite, LegacyRead); Assert.That(received, Is.EqualTo(" " + AllSpecialCharacters), - "ignoring the leading space, every special character round-trips correctly"); + "ignoring the leading space, every special character round-trips correctly"); } [Test] diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs index 0984daf14fd..83a8d3c0225 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs @@ -68,4 +68,21 @@ public void Default_round_trip_preserves_value_whitespace() // the DistributedContextPropagator (opt-in) would trim it. Assert.That(incoming.GetBaggageItem("key1"), Is.EqualTo(" leading-and-trailing ")); } + + [Test] + public void Default_writes_the_nservicebus_trace_header_alongside_the_w3c_header() + { + using var activity = new Activity(ActivityNames.OutgoingMessageActivityName); + activity.SetIdFormat(ActivityIdFormat.W3C); + activity.Start(); + + var headers = new Dictionary(); + ContextPropagation.PropagateContextToHeaders(activity, headers); + + using (Assert.EnterMultipleScope()) + { + Assert.That(headers[Headers.NServiceBusDiagnosticsTraceParent], Is.EqualTo(activity.Id)); + Assert.That(headers[Headers.DiagnosticsTraceParent], Is.EqualTo(activity.Id), "the W3C header stays for older receivers"); + } + } } diff --git a/src/NServiceBus.Core/Headers.cs b/src/NServiceBus.Core/Headers.cs index 46924320363..8736c214276 100644 --- a/src/NServiceBus.Core/Headers.cs +++ b/src/NServiceBus.Core/Headers.cs @@ -229,6 +229,15 @@ public static partial class Headers /// public const string TimeToBeReceived = "NServiceBus.TimeToBeReceived"; + /// + /// Carries the trace context of the NServiceBus send or publish operation, in the format of the W3C + /// traceparent header. It exists because transport SDKs that instrument themselves write their own + /// context to the W3C header on the same message, overwriting + /// the NServiceBus one. Receivers prefer this header and fall back to the W3C header, and senders + /// always write both, so endpoints on versions that only know the W3C header keep interoperating. + /// + public const string NServiceBusDiagnosticsTraceParent = "NServiceBus.TraceParent"; + /// /// Traceparent header according to the W3C spec: /// https://www.w3.org/TR/trace-context/#traceparent-header diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs index 90d5fec31ee..9b9bafa8018 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs @@ -24,46 +24,40 @@ sealed class ActivityFactory(InstrumentationOptions options) : IActivityFactory return null; } + var senderContextExists = TryParseSenderContext(headers, out ActivityContext senderContext); + Activity? activity; - var incomingTraceParentExists = headers.TryGetValue(Headers.DiagnosticsTraceParent, out var sendSpanId); - var activityContextCreatedFromIncomingTraceParent = ActivityContext.TryParse(sendSpanId, null, out var sendSpanContext); - if (extensions.TryGet(out var transportActivity)) // attach to transport span but link receive pipeline span to send pipeline span + if (extensions.TryGet(out var transportActivity)) { - ActivityLink[]? links = null; - if (incomingTraceParentExists && sendSpanId != transportActivity.Id) - { - if (activityContextCreatedFromIncomingTraceParent) - { - links = [new ActivityLink(sendSpanContext)]; - } - } - - activity = activitySource.CreateActivity(name: activityName, - ActivityKind.Consumer, transportActivity.Context, links: links, idFormat: ActivityIdFormat.W3C); + // Create a child span of the transport span and link to the NSB sender span + activity = activitySource.CreateActivity( + activityName, + ActivityKind.Consumer, + transportActivity.Context, + links: senderContextExists ? [new ActivityLink(senderContext)] : null); } - else if (incomingTraceParentExists && activityContextCreatedFromIncomingTraceParent) // otherwise directly create child from logical send + else if (senderContextExists) // otherwise directly create a child from a logical send { - var isStartNewTraceHeaderAvailable = headers.TryGetValue(Headers.StartNewTrace, out var shouldStartNewTrace); - if (isStartNewTraceHeaderAvailable && shouldStartNewTrace?.Equals(bool.TrueString) is true) + if (headers.TryGetValue(Headers.StartNewTrace, out var startNewTrace) && + string.Equals(startNewTrace, bool.TrueString, StringComparison.OrdinalIgnoreCase)) { - // create a new trace or root activity - ActivityLink[] links = [new(sendSpanContext)]; - //null the current activity so that the new one is created as root https://github.com/dotnet/runtime/issues/65528#issuecomment-2613486896 + // Create a brand-new trace and link the span to the NSB sender span. + // An activity without a parent context adopts Activity.Current as its parent when it + // starts, so Current has to be cleared. See: https://github.com/dotnet/runtime/issues/65528#issuecomment-2613486896 Activity.Current = null; - activity = activitySource.StartActivity(name: activityName, ActivityKind.Consumer, parentContext: default, tags: null, links: links); + activity = activitySource.CreateActivity(activityName, ActivityKind.Consumer, parentContext: default, links: [new ActivityLink(senderContext)]); } else { - // no new trace was requested, so start a child trace - ActivityContext.TryParse(sendSpanId, null, true, out var remoteParentActivityContext); - activity = activitySource.CreateActivity(name: activityName, ActivityKind.Consumer, remoteParentActivityContext); + // Create a span that is a child of the NSB sender span + activity = activitySource.CreateActivity(activityName, ActivityKind.Consumer, parentContext: senderContext); } } - else // otherwise start a new trace + else { - // This will set Activity.Current as parent if available - activity = activitySource.CreateActivity(name: activityName, ActivityKind.Consumer); + // Create a span that will be a child of Activity.Current if available + activity = activitySource.CreateActivity(activityName, ActivityKind.Consumer, parentContext: default); } if (activity is null) @@ -74,13 +68,30 @@ sealed class ActivityFactory(InstrumentationOptions options) : IActivityFactory ContextPropagation.PropagateContextFromHeaders(activity, headers); activity.SetIdFormat(ActivityIdFormat.W3C); - activity.AddTag(ActivityTags.NativeMessageId, nativeMessageId); + activity.AddTag(ActivityTags.NativeMessageId, nativeMessageId); ActivityDecorator.PromoteHeadersToTags(activity, headers); return activity; } + static bool TryParseSenderContext(Dictionary headers, out ActivityContext senderContext) + { + senderContext = default; + + // The NServiceBus-specific header takes precedence because a transport SDK may have + // overwritten the W3C header with its own context. The W3C header is the fallback so + // messages from endpoints on older versions, which only write that one, still continue + // the trace. A present but unparsable NServiceBus header does not fall back. + if (headers.TryGetValue(Headers.NServiceBusDiagnosticsTraceParent, out var senderSpanId) || + headers.TryGetValue(Headers.DiagnosticsTraceParent, out senderSpanId)) + { + return ActivityContext.TryParse(senderSpanId, null, isRemote: true, out senderContext); + } + + return false; + } + public Activity? StartIncomingPipelineActivity(MessageContext context) { var activity = CreateActivityFromIncomingMessage( diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs index dc0e568dd87..520ee67fdde 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs @@ -9,6 +9,18 @@ static class ContextPropagation { public static void PropagateContextToHeaders(Activity? activity, Dictionary headers) { + if (activity is null) + { + return; + } + + // Written on both propagation paths below. The W3C "traceparent" header is still written + // too, so endpoints on older versions keep reading the context from where they expect it. + if (activity.Id is not null) + { + headers[Headers.NServiceBusDiagnosticsTraceParent] = activity.Id; + } + // TODO: investigate if we need to improve the switch check for better performance // Removed in v11, see obsolete_v11.cs if (!LegacyContextPropagation.UseDistributedContextPropagator) @@ -19,11 +31,6 @@ public static void PropagateContextToHeaders(Activity? activity, Dictionary Date: Tue, 29 Sep 2026 14:33:49 +0200 Subject: [PATCH 25/37] Opt-in: parent the incoming span on an ambient transport SDK span (#7949) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ Opt-in switch to parent the incoming span on an ambient transport SDK span When a transport SDK (Azure Service Bus, RabbitMQ, SQS) has its own OpenTelemetry instrumentation, its receive span is Activity.Current when the incoming pipeline starts. With the `NServiceBus.Core.OpenTelemetry.UseTransportSpanAsParent` AppContext switch enabled the incoming message span becomes a child of that ambient span and links to the NServiceBus sender span, matching the existing behavior for a transport activity supplied via the ContextBag. The switch is off by default so the v10 trace shape (incoming span is a child of the sender span) is unchanged on a minor upgrade. The switch and the pre-v11 default test live in obsolete_v11.cs / TransportParentSpanDefaultBehaviorTests.cs and are deleted in v11; ActivityFactoryTests already runs with the v11 default enabled. * ♻️ Remove the unused transport Activity-from-ContextBag parenting path No transport supplies its receive Activity through the ContextBag; a transport SDK's receive span reaches the incoming pipeline as Activity.Current instead, which the UseTransportSpanAsParent switch now handles. Drop the ContextBag lookup, the unused ContextBag parameter and the four unit tests that exercised it. * ✅ Prove receive-side trace header backwards compatibility Cover receiving with only the W3C traceparent header (sender on a version that predates NServiceBus.TraceParent), with only the NServiceBus header, and that an invalid NServiceBus header does not fall back to the W3C one. * make names of the activity factory tests more descriptive --------- Co-authored-by: Tomasz Masternak --- .../OpenTelemetry/ActivityFactoryTests.cs | 103 ++++++------------ ...TransportParentSpanDefaultBehaviorTests.cs | 62 +++++++++++ .../OpenTelemetry/Tracing/ActivityFactory.cs | 53 +++++---- .../OpenTelemetry/Tracing/obsolete_v11.cs | 47 ++++++++ 4 files changed, 173 insertions(+), 92 deletions(-) create mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/TransportParentSpanDefaultBehaviorTests.cs diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs index 2a960c4256d..1ab193a3fbf 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs @@ -51,104 +51,69 @@ static MessageContext CreateMessageContext() => class StartIncomingActivity : ActivityFactoryTests { - [Test] - public void Should_attach_to_context_activity_when_activity_on_context() + // Until v11 the "transport span as parent" behavior is opt-in. This fixture runs with it + // enabled because that is the v11 default the tests below describe; the pre-v11 default + // is covered by TransportParentSpanDefaultBehaviorTests. In v11, delete this SetUp/TearDown + // pair together with obsolete_v11.cs. + [SetUp] + public void OptInToTransportSpanAsParent() { - using var contextActivity = CreateCompletedActivity("transport receive activity"); - - var contextBag = new ContextBag(); - contextBag.Set(contextActivity); - - var activity = activityFactory.StartIncomingPipelineActivity(CreateMessageContext(contextBag: contextBag)); - - Assert.That(activity, Is.Not.Null, "should create activity for receive pipeline"); - using (Assert.EnterMultipleScope()) - { - Assert.That(activity.ParentId, Is.EqualTo(contextActivity.Id), "should use context activity as parent"); - Assert.That(activity.Links.Count(), Is.EqualTo(0), "should not link to logical send span"); - } + AppContext.SetSwitch(TransportParentSpanSwitch.UseTransportSpanAsParentSwitchName, true); + TransportParentSpanSwitch.ResetUseTransportSpanAsParent(); } - [Test] - public void Should_attach_to_context_activity_when_activity_on_context_and_trace_message_header() + [TearDown] + public void ResetTransportSpanSwitch() { - using var contextActivity = CreateCompletedActivity("transport receive activity"); - using var sendActivity = CreateCompletedActivity("send activity"); - - var contextBag = new ContextBag(); - contextBag.Set(contextActivity); - - var messageHeaders = new Dictionary { { Headers.DiagnosticsTraceParent, sendActivity.Id! } }; - - var activity = activityFactory.StartIncomingPipelineActivity(CreateMessageContext(messageHeaders, contextBag)); - - Assert.That(activity, Is.Not.Null, "should create activity for receive pipeline"); - using (Assert.EnterMultipleScope()) - { - Assert.That(activity.ParentId, Is.EqualTo(contextActivity.Id), "should use context activity as parent"); - Assert.That(activity.Links.Count(), Is.EqualTo(1), "should link to logical send span"); - Assert.That(activity.Links.Single().Context.TraceId, Is.EqualTo(sendActivity.TraceId)); - Assert.That(activity.Links.Single().Context.SpanId, Is.EqualTo(sendActivity.SpanId)); - } + AppContext.SetSwitch(TransportParentSpanSwitch.UseTransportSpanAsParentSwitchName, false); + TransportParentSpanSwitch.ResetUseTransportSpanAsParent(); } - [Test] - public void Should_attach_to_context_activity_when_activity_on_context_and_ambient_activity() - { - using var contextActivity = CreateCompletedActivity("transport receive activity"); - var contextBag = new ContextBag(); - contextBag.Set(contextActivity); - - using var ambientActivity = ActivitySources.Main.StartActivity("ambient activity"); - Assert.That(Activity.Current, Is.EqualTo(ambientActivity)); - - var activity = activityFactory.StartIncomingPipelineActivity(CreateMessageContext(contextBag: contextBag)); - - Assert.That(activity, Is.Not.Null, "should create activity for receive pipeline"); - Assert.That(activity.ParentId, Is.EqualTo(contextActivity.Id), "should use context activity as parent"); - } - - [Test] - public void Should_start_new_trace_when_activity_on_context_uses_legacy_id_format() + [TestCase(Headers.NServiceBusDiagnosticsTraceParent)] + [TestCase(Headers.DiagnosticsTraceParent)] // for backwards compatibility + public void Should_attach_to_header_trace_when_available_and_no_ambient_activity(string headerName) { - using var contextActivity = CreateCompletedActivity("transport receive activity", ActivityIdFormat.Hierarchical); - Assert.That(contextActivity.IdFormat, Is.EqualTo(ActivityIdFormat.Hierarchical)); + var sendActivity = CreateCompletedActivity("send activity"); - var contextBag = new ContextBag(); - contextBag.Set(contextActivity); + var messageHeaders = new Dictionary { { headerName, sendActivity.Id! } }; - var activity = activityFactory.StartIncomingPipelineActivity(CreateMessageContext(contextBag: contextBag)); + var activity = activityFactory.StartIncomingPipelineActivity(CreateMessageContext(messageHeaders)); Assert.That(activity, Is.Not.Null, "should create activity for receive pipeline"); using (Assert.EnterMultipleScope()) { - Assert.That(activity.ParentId, Is.Null, "should create a new trace"); - Assert.That(activity.IdFormat, Is.EqualTo(ActivityIdFormat.W3C)); + Assert.That(activity.ParentId, Is.EqualTo(sendActivity.Id)); + Assert.That(activity.Links.Count(), Is.EqualTo(0), "should not link to logical send span"); } } [Test] - public void Should_attach_to_header_trace_when_no_activity_on_context_and_trace_header() + public void Should_attach_to_ambient_activity_when_available_and_link_to_header_trace() { - using var sendActivity = CreateCompletedActivity("send activity"); + var sendActivity = CreateCompletedActivity("send activity"); + + using var ambientActivity = new Activity("transport sdk receive activity"); + ambientActivity.Start(); - var messageHeaders = new Dictionary { { Headers.DiagnosticsTraceParent, sendActivity.Id! } }; + var messageHeaders = new Dictionary { { Headers.NServiceBusDiagnosticsTraceParent, sendActivity.Id! } }; var activity = activityFactory.StartIncomingPipelineActivity(CreateMessageContext(messageHeaders)); Assert.That(activity, Is.Not.Null, "should create activity for receive pipeline"); using (Assert.EnterMultipleScope()) { - Assert.That(activity.ParentId, Is.EqualTo(sendActivity.Id)); - Assert.That(activity.Links.Count(), Is.EqualTo(0), "should not link to logical send span"); + Assert.That(activity.ParentId, Is.EqualTo(ambientActivity.Id), "should use the ambient transport activity as parent"); + Assert.That(activity.Links.Count(), Is.EqualTo(1), "should link to logical send span"); + Assert.That(activity.Links.Single().Context.TraceId, Is.EqualTo(sendActivity.TraceId)); + Assert.That(activity.Links.Single().Context.SpanId, Is.EqualTo(sendActivity.SpanId)); } } [Test] public void Should_prefer_nservicebus_trace_header_over_w3c_trace_header() { - using var sendActivity = CreateCompletedActivity("send activity"); - using var transportActivity = CreateCompletedActivity("transport activity that overwrote the w3c header"); + var sendActivity = CreateCompletedActivity("send activity"); + var transportActivity = CreateCompletedActivity("transport activity that overwrote the w3c header"); var messageHeaders = new Dictionary { @@ -227,14 +192,14 @@ static Activity CreateCompletedActivity(string activityName, ActivityIdFormat id return activity; } - static MessageContext CreateMessageContext(Dictionary? messageHeaders = null, ContextBag? contextBag = null) => + static MessageContext CreateMessageContext(Dictionary? messageHeaders = null) => new( Guid.NewGuid().ToString(), messageHeaders ?? [], Array.Empty(), new TransportTransaction(), "receiver", - contextBag ?? new ContextBag()); + new ContextBag()); } class StartOutgoingPipelineActivity : ActivityFactoryTests diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/TransportParentSpanDefaultBehaviorTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/TransportParentSpanDefaultBehaviorTests.cs new file mode 100644 index 00000000000..ab459ccabae --- /dev/null +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/TransportParentSpanDefaultBehaviorTests.cs @@ -0,0 +1,62 @@ +#nullable enable + +namespace NServiceBus.Core.Tests.OpenTelemetry; + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using Helpers; +using NServiceBus.Extensibility; +using NServiceBus.Transport; +using NUnit.Framework; + +// Covers the pre-v11 default of TransportParentSpanSwitch: without the opt-in switch the incoming +// message span stays a child of the NServiceBus sender span even when a transport SDK span is +// ambient. In v11 that default is gone, so delete this file together with obsolete_v11.cs. +[TestFixture] +public class TransportParentSpanDefaultBehaviorTests +{ + readonly ActivityFactory activityFactory = new(new InstrumentationOptions()); + + TestingActivityListener nsbActivityListener; + + [SetUp] + public void SetUp() + { + nsbActivityListener = TestingActivityListener.SetupNServiceBusDiagnosticListener(); + AppContext.SetSwitch(TransportParentSpanSwitch.UseTransportSpanAsParentSwitchName, false); + TransportParentSpanSwitch.ResetUseTransportSpanAsParent(); + } + + [TearDown] + public void TearDown() + { + nsbActivityListener.Dispose(); + TransportParentSpanSwitch.ResetUseTransportSpanAsParent(); + } + + [Test] + public void Default_attaches_to_header_trace_when_even_id_ambient_activity_exists() + { + var sendActivity = new Activity("send activity"); + sendActivity.SetIdFormat(ActivityIdFormat.W3C); + sendActivity.Start(); + sendActivity.Stop(); + + using var ambientActivity = new Activity("transport sdk receive activity"); + ambientActivity.Start(); + + var messageHeaders = new Dictionary { { Headers.NServiceBusDiagnosticsTraceParent, sendActivity.Id! } }; + var messageContext = new MessageContext(Guid.NewGuid().ToString(), messageHeaders, Array.Empty(), new TransportTransaction(), "receiver", new ContextBag()); + + var activity = activityFactory.StartIncomingPipelineActivity(messageContext); + + Assert.That(activity, Is.Not.Null, "should create activity for receive pipeline"); + using (Assert.EnterMultipleScope()) + { + Assert.That(activity.ParentId, Is.EqualTo(sendActivity.Id), "should use the sender span as parent, ignoring the ambient activity"); + Assert.That(activity.Links.Count(), Is.EqualTo(0), "should not link to logical send span"); + } + } +} diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs index 9b9bafa8018..4669e372477 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs @@ -15,7 +15,7 @@ sealed class ActivityFactory(InstrumentationOptions options) : IActivityFactory { public InstrumentationOptions Options { get; } = options; - static Activity? CreateActivityFromIncomingMessage(ActivitySource activitySource, string activityName, Dictionary headers, string nativeMessageId, ContextBag extensions) + static Activity? CreateActivityFromIncomingMessage(ActivitySource activitySource, string activityName, Dictionary headers, string nativeMessageId) { // CreateActivity is a no-op if there are no listeners but we are doing a fast path check // here nonetheless to avoid having to parse headers, access the extension bag, etc. @@ -28,19 +28,12 @@ sealed class ActivityFactory(InstrumentationOptions options) : IActivityFactory Activity? activity; - if (extensions.TryGet(out var transportActivity)) + if (senderContextExists) // create a child from a logical send { - // Create a child span of the transport span and link to the NSB sender span - activity = activitySource.CreateActivity( - activityName, - ActivityKind.Consumer, - transportActivity.Context, - links: senderContextExists ? [new ActivityLink(senderContext)] : null); - } - else if (senderContextExists) // otherwise directly create a child from a logical send - { - if (headers.TryGetValue(Headers.StartNewTrace, out var startNewTrace) && - string.Equals(startNewTrace, bool.TrueString, StringComparison.OrdinalIgnoreCase)) + var startNewTrace = headers.TryGetValue(Headers.StartNewTrace, out var startNewTraceHeaderValue) + && string.Equals(startNewTraceHeaderValue, bool.TrueString, StringComparison.OrdinalIgnoreCase); + + if (startNewTrace) { // Create a brand-new trace and link the span to the NSB sender span. // An activity without a parent context adopts Activity.Current as its parent when it @@ -48,6 +41,12 @@ sealed class ActivityFactory(InstrumentationOptions options) : IActivityFactory Activity.Current = null; activity = activitySource.CreateActivity(activityName, ActivityKind.Consumer, parentContext: default, links: [new ActivityLink(senderContext)]); } + else if (TransportParentSpanSwitch.UseTransportSpanAsParent && Activity.Current != null) // remove the switch check in v11, see obsolete_v11.cs + { + // A transport SDK receive span is ambient: make it the parent (an activity without + // a parent context adopts Activity.Current when it starts) and link to the NSB sender span. + activity = activitySource.CreateActivity(activityName, ActivityKind.Consumer, parentContext: default, links: [new ActivityLink(senderContext)]); + } else { // Create a span that is a child of the NSB sender span @@ -80,13 +79,23 @@ static bool TryParseSenderContext(Dictionary headers, out Activi senderContext = default; // The NServiceBus-specific header takes precedence because a transport SDK may have - // overwritten the W3C header with its own context. The W3C header is the fallback so - // messages from endpoints on older versions, which only write that one, still continue - // the trace. A present but unparsable NServiceBus header does not fall back. - if (headers.TryGetValue(Headers.NServiceBusDiagnosticsTraceParent, out var senderSpanId) || - headers.TryGetValue(Headers.DiagnosticsTraceParent, out senderSpanId)) + // overwritten the W3C header with its own context. + if (headers.TryGetValue(Headers.NServiceBusDiagnosticsTraceParent, out var senderSpanId)) { - return ActivityContext.TryParse(senderSpanId, null, isRemote: true, out senderContext); + if (ActivityContext.TryParse(senderSpanId, null, isRemote: true, out senderContext)) + { + return true; + } + } + + // The W3C header is the fallback, so messages from endpoints on older versions, + // which only write that one, still continue the trace. + if (headers.TryGetValue(Headers.DiagnosticsTraceParent, out senderSpanId)) + { + if (ActivityContext.TryParse(senderSpanId, null, isRemote: true, out senderContext)) + { + return true; + } } return false; @@ -98,8 +107,7 @@ static bool TryParseSenderContext(Dictionary headers, out Activi ActivitySources.Main, ActivityNames.IncomingMessageActivityName, context.Headers, - context.NativeMessageId, - context.Extensions); + context.NativeMessageId); if (activity is null) { @@ -164,8 +172,7 @@ static bool TryParseSenderContext(Dictionary headers, out Activi ActivitySources.Recoverability, ActivityNames.RecoverabilityActivityName, context.Headers, - context.NativeMessageId, - context.Extensions); + context.NativeMessageId); if (activity is null) { diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs index 92e85862308..dd240e2d4a6 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs @@ -182,6 +182,53 @@ public static bool UseHandlerActivitySource } +// When a transport SDK (Azure Service Bus, RabbitMQ, SQS, ...) has its own OpenTelemetry +// instrumentation, an SDK "receive" span is the ambient Activity.Current while the incoming +// pipeline starts. The incoming message span should then be a child of that SDK span and only +// link to the NServiceBus sender span. Until v11 the sender span stays the parent by default so +// existing trace shapes don't change on a minor upgrade; the new shape is opt-in via an +// AppContext switch. +// +// In v11 the SDK span becomes the parent unconditionally: delete this class, remove the +// `TransportParentSpanSwitch.UseTransportSpanAsParent` check in +// ActivityFactory.CreateActivityFromIncomingMessage (keeping only the branch that links to the +// sender span), delete TransportParentSpanDefaultBehaviorTests.cs and remove the opt-in +// SetUp/TearDown pair in ActivityFactoryTests.StartIncomingActivity. +static class TransportParentSpanSwitch +{ + enum SwitchState : byte + { + Unchecked = 0, + Enabled = 1, + Disabled = 2 + } + + static SwitchState cachedUseTransportSpanAsParent; + + public const string UseTransportSpanAsParentSwitchName = "NServiceBus.Core.OpenTelemetry.UseTransportSpanAsParent"; + + public static bool UseTransportSpanAsParent + { + get + { + var state = cachedUseTransportSpanAsParent; + if (state != SwitchState.Unchecked) + { + return state == SwitchState.Enabled; + } + + state = AppContext.TryGetSwitch(UseTransportSpanAsParentSwitchName, out var isEnabled) && isEnabled + ? SwitchState.Enabled + : SwitchState.Disabled; + cachedUseTransportSpanAsParent = state; + + return state == SwitchState.Enabled; + } + } + + internal static void ResetUseTransportSpanAsParent() => cachedUseTransportSpanAsParent = SwitchState.Unchecked; +} + // This class bridges two independent legacy exception-tagging behaviors, both // scheduled for removal in v11: // From e0a6f3361134d7ea20444c2303b1de68158a80a4 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 1 Oct 2026 12:45:16 +0200 Subject: [PATCH 26/37] Propagate trace context only in the outgoing message pipeline (#7952) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * 🐛 Propagate trace context only in the outgoing message pipeline Move context propagation from RoutingToDispatchConnector to OutgoingPhysicalToRoutingConnector so it only runs for messages sent through the outgoing pipeline. The routing to dispatch stage is also used by the move to error and audit pipelines, which caused the current activity to be written into the headers of forwarded messages. * ✅ Add regression tests for trace context preservation in forwarding paths Messages moved to the error queue, delayed retries, audit copies and ForwardCurrentMessageTo enter the pipeline at the routing stage. Each test runs the real action through RoutingToDispatchConnector under an unrelated ambient activity and asserts the received traceparent is kept and no baggage leaks in. All four fail on the parent commit. * 🐛 Propagate trace context to ServiceControl retry acknowledgements The acknowledgement is a control message forked straight into the routing stage, so it no longer receives trace headers from the dispatch connector. Propagate explicitly, as the subscribe and unsubscribe terminators do, to keep it correlated to the processing of the retried message. * Update trace parent header in tests to the new nsb header * Fix header key for diagnostics trace parent --------- Co-authored-by: Tomasz Masternak --- .../TraceContextPreservationTests.cs | 144 ++++++++++++++++++ .../RetryAcknowledgementBehaviorTests.cs | 26 +++- .../OutgoingPhysicalToRoutingConnector.cs | 3 +- .../Outgoing/RoutingToDispatchConnector.cs | 5 - .../Retries/RetryAcknowledgementBehavior.cs | 6 + 5 files changed, 177 insertions(+), 7 deletions(-) create mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/TraceContextPreservationTests.cs diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/TraceContextPreservationTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/TraceContextPreservationTests.cs new file mode 100644 index 00000000000..e7e578a599d --- /dev/null +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/TraceContextPreservationTests.cs @@ -0,0 +1,144 @@ +namespace NServiceBus.Core.Tests.OpenTelemetry; + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Threading.Tasks; +using NServiceBus.Audit; +using NServiceBus.Pipeline; +using NServiceBus.Routing; +using NServiceBus.Transport; +using NUnit.Framework; +using Testing; + +/// +/// Messages forwarded from the incoming side (recoverability, audit, ForwardCurrentMessageTo) enter the pipeline at +/// the routing stage and never pass through the outgoing pipeline. They must keep the trace headers of the message +/// as it was received and must not be re-parented to whatever activity happens to be current at that time. +/// +[TestFixture] +public class TraceContextPreservationTests +{ + const string OriginalTraceParent = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"; + + [Test] + public async Task Should_preserve_trace_headers_when_moving_to_error_queue() + { + var recoverabilityContext = CreateRecoverabilityContext(); + var routingContext = new MoveToError("error-queue") + .GetRoutingContexts(recoverabilityContext) + .Single(); + + var headers = await DispatchWithAmbientActivity(routingContext); + + AssertTraceHeadersPreserved(headers); + } + + [Test] + public async Task Should_preserve_trace_headers_when_delayed_retrying() + { + var recoverabilityContext = CreateRecoverabilityContext(); + var routingContext = new DelayedRetry(TimeSpan.FromSeconds(1)) + .GetRoutingContexts(recoverabilityContext) + .Single(); + + var headers = await DispatchWithAmbientActivity(routingContext); + + AssertTraceHeadersPreserved(headers); + } + + [Test] + public async Task Should_preserve_trace_headers_when_auditing() + { + var auditContext = new TestableAuditContext + { + Message = new OutgoingMessage("message-id", CreateReceivedHeaders(), ReadOnlyMemory.Empty) + }; + var routingContext = RouteToAudit.Instance + .GetRoutingContexts(auditContext) + .Single(); + + var headers = await DispatchWithAmbientActivity(routingContext); + + AssertTraceHeadersPreserved(headers); + } + + [Test] + public async Task Should_preserve_trace_headers_when_forwarding_current_message() + { + var receivedHeaders = CreateReceivedHeaders(); + var incomingMessage = new IncomingMessage("message-id", receivedHeaders, ReadOnlyMemory.Empty); + var incomingContext = new TestableIncomingLogicalMessageContext(); + incomingContext.Extensions.Set(incomingMessage); + + Dictionary headers = null; + var routingPipelineCache = new RoutingPipelineCache(async routingContext => + headers = await DispatchWithAmbientActivity(routingContext)); + incomingContext.Extensions.Set(routingPipelineCache); + + await IncomingMessageOperations.ForwardCurrentMessageTo(incomingContext, "destination"); + + AssertTraceHeadersPreserved(headers); + // The forwarded message shares the header dictionary of the message being processed + AssertTraceHeadersPreserved(receivedHeaders); + } + + static TestableRecoverabilityContext CreateRecoverabilityContext() => + new() + { + MessageId = "message-id", + Headers = CreateReceivedHeaders() + }; + + static Dictionary CreateReceivedHeaders() => + new() + { + [Headers.NServiceBusDiagnosticsTraceParent] = OriginalTraceParent + }; + + static async Task> DispatchWithAmbientActivity(IRoutingContext routingContext) + { + using var ambientActivity = new Activity("ambient activity"); + ambientActivity.SetIdFormat(ActivityIdFormat.W3C); + ambientActivity.AddBaggage("ambient-key", "ambient-value"); + ambientActivity.Start(); + + var connector = new RoutingToDispatchConnector(NoOpActivityFactory.Instance); + TransportOperation transportOperation = null; + + await connector.Invoke(routingContext, dispatchContext => + { + transportOperation = dispatchContext.Operations.Single(); + return Task.CompletedTask; + }); + + return transportOperation.Message.Headers; + } + + static void AssertTraceHeadersPreserved(Dictionary headers) + { + using (Assert.EnterMultipleScope()) + { + Assert.That( + headers[Headers.NServiceBusDiagnosticsTraceParent], + Is.EqualTo(OriginalTraceParent), + "traceparent of the received message must be preserved"); + Assert.That( + headers, + Does.Not.ContainKey(Headers.DiagnosticsBaggage), + "baggage of the current activity must not leak into the forwarded message"); + } + } + + sealed class RoutingPipelineCache(Func routingPipeline) : IPipelineCache + { + public IPipeline Pipeline() where TContext : IBehaviorContext => + (IPipeline)(object)new RoutingPipeline(routingPipeline); + + sealed class RoutingPipeline(Func invoke) : IPipeline + { + public Task Invoke(IRoutingContext context) => invoke(context); + } + } +} diff --git a/src/NServiceBus.Core.Tests/ServicePlatform/Retries/RetryAcknowledgementBehaviorTests.cs b/src/NServiceBus.Core.Tests/ServicePlatform/Retries/RetryAcknowledgementBehaviorTests.cs index ee1ededda3f..7a1ecd929b9 100644 --- a/src/NServiceBus.Core.Tests/ServicePlatform/Retries/RetryAcknowledgementBehaviorTests.cs +++ b/src/NServiceBus.Core.Tests/ServicePlatform/Retries/RetryAcknowledgementBehaviorTests.cs @@ -2,6 +2,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Linq; using System.Threading.Tasks; using Microsoft.Extensions.DependencyInjection; @@ -50,6 +51,29 @@ public async Task Should_confirm_successful_retries_to_acknowledgement_queue() } } + [Test] + public async Task Should_propagate_current_trace_context_to_acknowledgement() + { + var routingPipeline = new RoutingPipeline(); + var behavior = new RetryAcknowledgementBehavior(); + + var context = SetupTestableContext(routingPipeline); + context.Message.Headers[RetryAcknowledgementBehavior.RetryUniqueMessageIdHeaderKey] = Guid.NewGuid().ToString("N"); + context.Message.Headers[RetryAcknowledgementBehavior.RetryConfirmationQueueHeaderKey] = "SomeQueue"; + + using var processingActivity = new Activity("processing activity"); + processingActivity.SetIdFormat(ActivityIdFormat.W3C); + processingActivity.Start(); + + await behavior.Invoke(context, _ => Task.CompletedTask); + + var outgoingMessage = routingPipeline.ForkInvocations.Single(); + Assert.That( + outgoingMessage.Message.Headers[Headers.NServiceBusDiagnosticsTraceParent], + Is.EqualTo(processingActivity.Id), + "the acknowledgement should be correlated to the processing of the retried message"); + } + [Test] public void Should_not_confirm_when_processing_fails() { @@ -134,4 +158,4 @@ public override Task Invoke(IRoutingContext context, Func next) return Task.CompletedTask; } } -} \ No newline at end of file +} diff --git a/src/NServiceBus.Core/Pipeline/Outgoing/OutgoingPhysicalToRoutingConnector.cs b/src/NServiceBus.Core/Pipeline/Outgoing/OutgoingPhysicalToRoutingConnector.cs index b84035eaf96..635dcc7061b 100644 --- a/src/NServiceBus.Core/Pipeline/Outgoing/OutgoingPhysicalToRoutingConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Outgoing/OutgoingPhysicalToRoutingConnector.cs @@ -3,6 +3,7 @@ namespace NServiceBus; using System; +using System.Diagnostics; using System.Threading.Tasks; using Pipeline; using Transport; @@ -11,8 +12,8 @@ class OutgoingPhysicalToRoutingConnector : StageConnector stage) { + ContextPropagation.PropagateContextToHeaders(Activity.Current, context.Headers); var message = new OutgoingMessage(context.MessageId, context.Headers, context.Body); - return stage(this.CreateRoutingContext(message, context.RoutingStrategies, context)); } } \ No newline at end of file diff --git a/src/NServiceBus.Core/Pipeline/Outgoing/RoutingToDispatchConnector.cs b/src/NServiceBus.Core/Pipeline/Outgoing/RoutingToDispatchConnector.cs index 203e24fee8b..633dc2bd25c 100644 --- a/src/NServiceBus.Core/Pipeline/Outgoing/RoutingToDispatchConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Outgoing/RoutingToDispatchConnector.cs @@ -3,7 +3,6 @@ namespace NServiceBus; using System; -using System.Diagnostics; using System.Text; using System.Threading.Tasks; using Extensibility; @@ -31,10 +30,6 @@ public override Task Invoke(IRoutingContext context, Func()); + + // The acknowledgement is a control message that bypasses the outgoing pipeline. Propagate the trace + // context explicitly so it stays correlated to the processing of the retried message. + ContextPropagation.PropagateContextToHeaders(Activity.Current, messageToDispatch.Headers); + var routingContext = new RoutingContext(messageToDispatch, new UnicastRoutingStrategy(acknowledgementQueue), context); await this.Fork(routingContext).ConfigureAwait(false); } From 8af7dbc7b9df8d77d397ae877d6be99baa651a83 Mon Sep 17 00:00:00 2001 From: Tomasz Masternak Date: Mon, 5 Oct 2026 13:13:25 +0200 Subject: [PATCH 27/37] rename `TryGetRecordingPipelineActivity` methods to `TryGetPipelineActivity` --- .../OpenTelemetry/ActivityExtensionsTests.cs | 6 +++--- .../OpenTelemetry/Tracing/ActivityExtensions.cs | 10 +++++----- .../Tracing/SubscribeDiagnosticsBehavior.cs | 2 +- .../Tracing/UnsubscribeDiagnosticsBehavior.cs | 2 +- ...ransportReceiveToPhysicalMessageConnector.cs | 17 +++++++++++++---- .../Outgoing/RoutingToDispatchConnector.cs | 2 +- 6 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityExtensionsTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityExtensionsTests.cs index a8880919f67..c4825b33064 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityExtensionsTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityExtensionsTests.cs @@ -16,7 +16,7 @@ public void TryGetRecordingPipelineActivity_should_return_false_when_key_not_fou var contextBag = new ContextBag(); using (Assert.EnterMultipleScope()) { - Assert.That(contextBag.TryGetRecordingOutgoingPipelineActivity(out var activity), Is.False); + Assert.That(contextBag.TryGetOutgoingPipelineActivity(out var activity), Is.False); Assert.That(activity, Is.Null); } } @@ -31,7 +31,7 @@ public void TryGetRecordingPipelineActivity_should_return_false_when_value_null( using (Assert.EnterMultipleScope()) { - Assert.That(contextBag.TryGetRecordingOutgoingPipelineActivity(out var activity), Is.False); + Assert.That(contextBag.TryGetOutgoingPipelineActivity(out var activity), Is.False); Assert.That(activity, Is.Null); } } @@ -50,7 +50,7 @@ public void TryGetRecordingPipelineActivity_should_return_false_when_not_recordi using (Assert.EnterMultipleScope()) { - Assert.That(contextBag.TryGetRecordingOutgoingPipelineActivity(out var activity), Is.True); + Assert.That(contextBag.TryGetOutgoingPipelineActivity(out var activity), Is.True); Assert.That(activity, Is.EqualTo(recordingActivity)); } } diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs index 3f784962a9b..023cec6aadb 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityExtensions.cs @@ -11,13 +11,13 @@ static class ActivityExtensions public const string OutgoingActivityKey = "NServiceBus.Diagnostics.Activity.Outgoing"; public const string IncomingActivityKey = "NServiceBus.Diagnostics.Activity.Incoming"; - public static bool TryGetRecordingOutgoingPipelineActivity(this ContextBag pipelineContext, [NotNullWhen(true)] out Activity? activity) - => pipelineContext.TryGetRecordingPipelineActivity(OutgoingActivityKey, out activity); + public static bool TryGetOutgoingPipelineActivity(this ContextBag pipelineContext, [NotNullWhen(true)] out Activity? activity) + => pipelineContext.TryGetPipelineActivity(OutgoingActivityKey, out activity); - public static bool TryGetRecordingIncomingPipelineActivity(this ContextBag pipelineContext, [NotNullWhen(true)] out Activity? activity) - => pipelineContext.TryGetRecordingPipelineActivity(IncomingActivityKey, out activity); + public static bool TryGetIncomingPipelineActivity(this ContextBag pipelineContext, [NotNullWhen(true)] out Activity? activity) + => pipelineContext.TryGetPipelineActivity(IncomingActivityKey, out activity); - static bool TryGetRecordingPipelineActivity(this ContextBag pipelineContext, string activityKey, [NotNullWhen(true)] out Activity? activity) + static bool TryGetPipelineActivity(this ContextBag pipelineContext, string activityKey, [NotNullWhen(true)] out Activity? activity) { if (Activity.Current is not null // Cheaper to check than searching the pipeline context to start with. If there is no ambient activity, there can't be an activity in the context. && pipelineContext.TryGet(activityKey, out activity) // Search activity in context bag diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/SubscribeDiagnosticsBehavior.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/SubscribeDiagnosticsBehavior.cs index f5f531b6324..dfa025c6099 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/SubscribeDiagnosticsBehavior.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/SubscribeDiagnosticsBehavior.cs @@ -10,7 +10,7 @@ class SubscribeDiagnosticsBehavior : IBehavior next) { - if (context.Extensions.TryGetRecordingOutgoingPipelineActivity(out var activity)) + if (context.Extensions.TryGetOutgoingPipelineActivity(out var activity)) { activity.SetTag(ActivityTags.EventTypes, string.Join(",", (object[])context.EventTypes)); } diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/UnsubscribeDiagnosticsBehavior.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/UnsubscribeDiagnosticsBehavior.cs index fd5f0936f24..93cc7433afe 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/UnsubscribeDiagnosticsBehavior.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/UnsubscribeDiagnosticsBehavior.cs @@ -10,7 +10,7 @@ class UnsubscribeDiagnosticsBehavior : IBehavior next) { - if (context.Extensions.TryGetRecordingOutgoingPipelineActivity(out var activity)) + if (context.Extensions.TryGetOutgoingPipelineActivity(out var activity)) { activity.SetTag(ActivityTags.EventTypes, context.EventType.FullName); } diff --git a/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs b/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs index 55d33debdd4..9c06a8c35a9 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs @@ -76,8 +76,8 @@ async Task InvokeWithOutbox(ITransportReceiveContext context, Func 0) { var batchDispatchContext = this.CreateBatchDispatchContext(operations, physicalMessageContext); - var dispatchActivity = WriteStartDispatchingEvent(physicalMessageContext, operations.Length); + + Activity? dispatchActivity = null; + if (!instrumentationOptions.EmitMessageDispatchingEvents) + { + physicalMessageContext.Extensions.TryGetIncomingPipelineActivity(out dispatchActivity); + } + + dispatchActivity?.AddEvent(new("Start dispatching", tags: new() { { "message-count", operations.Length } })); + await this.Fork(batchDispatchContext).ConfigureAwait(false); + dispatchActivity?.AddEvent(new("Finished dispatching")); } @@ -140,7 +149,7 @@ async Task InvokeWithoutOutbox(ITransportReceiveContext context, Func Date: Mon, 5 Oct 2026 14:11:30 +0200 Subject: [PATCH 28/37] =?UTF-8?q?=E2=9A=9C=EF=B8=8F=20Put=20the=20connecto?= =?UTF-8?q?r=20constructor=20arguments=20on=20separate=20lines?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/NServiceBus.Core/Receiving/ReceiveComponent.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/NServiceBus.Core/Receiving/ReceiveComponent.cs b/src/NServiceBus.Core/Receiving/ReceiveComponent.cs index a92b0465116..f25328126dc 100644 --- a/src/NServiceBus.Core/Receiving/ReceiveComponent.cs +++ b/src/NServiceBus.Core/Receiving/ReceiveComponent.cs @@ -69,7 +69,12 @@ public static ReceiveComponent Configure( pipelineSettings.Register("TransportReceiveToPhysicalMessageProcessingConnector", b => { var storage = b.GetService() ?? new NoOpOutboxStorage(); - return new TransportReceiveToPhysicalMessageConnector(storage, b.GetRequiredService(), hostingConfiguration.ActivityFactory.Options, b.GetRequiredService>()); + return new TransportReceiveToPhysicalMessageConnector( + storage, + b.GetRequiredService(), + hostingConfiguration.ActivityFactory.Options, + b.GetRequiredService>() + ); }, "Allows to abort processing the message"); pipelineSettings.Register("LoadHandlersConnector", sp => new LoadHandlersConnector(sp.GetRequiredService(), hostingConfiguration.ActivityFactory, sp.GetRequiredService()), "Gets all the handlers to invoke from the MessageHandler registry based on the message type."); @@ -195,7 +200,7 @@ public async Task Initialize( pipelineComponent, messageOperations, activityFactory - ); + ); await mainPump.Initialize( configuration.PushRuntimeSettings, From c43ada9662475dc8f0cd0176a03da096e362c902 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Mon, 5 Oct 2026 14:11:30 +0200 Subject: [PATCH 29/37] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Dispatch=20through?= =?UTF-8?q?=20one=20method=20in=20both=20outbox=20paths?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The inlined event gating in the with-outbox path had an inverted condition and duplicated the fork block. Both paths now call Dispatch, which forks directly when the dispatching events are off and goes through DispatchWithEvents when they are on. Removes the unused WriteStartDispatchingEvent helper. --- ...nsportReceiveToPhysicalMessageConnector.cs | 44 ++++++++----------- 1 file changed, 18 insertions(+), 26 deletions(-) diff --git a/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs b/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs index 9c06a8c35a9..8ba0e81163b 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs @@ -86,18 +86,7 @@ async Task InvokeWithOutbox(ITransportReceiveContext context, Func 0) { var batchDispatchContext = this.CreateBatchDispatchContext(operations, physicalMessageContext); - - Activity? dispatchActivity = null; - if (!instrumentationOptions.EmitMessageDispatchingEvents) - { - physicalMessageContext.Extensions.TryGetIncomingPipelineActivity(out dispatchActivity); - } - - dispatchActivity?.AddEvent(new("Start dispatching", tags: new() { { "message-count", operations.Length } })); - - await this.Fork(batchDispatchContext).ConfigureAwait(false); - - dispatchActivity?.AddEvent(new("Finished dispatching")); + await Dispatch(batchDispatchContext).ConfigureAwait(false); } await outboxStorage.SetAsDispatched(messageId, context.Extensions, context.CancellationToken).ConfigureAwait(false); @@ -133,29 +122,32 @@ async Task InvokeWithoutOutbox(ITransportReceiveContext context, Func 0) { var batchDispatchContext = this.CreateBatchDispatchContext(operations, physicalMessageContext); - var dispatchActivity = WriteStartDispatchingEvent(physicalMessageContext, operations.Length); - await this.Fork(batchDispatchContext).ConfigureAwait(false); - dispatchActivity?.AddEvent(new("Finished dispatching")); + await Dispatch(batchDispatchContext).ConfigureAwait(false); } pipelineMetrics.RecordCriticalTimeAndTotalProcessed(context); } - // Synchronous by design so the dispatch instrumentation lives in one place without adding an async state - // machine to either path; the Fork await stays in the callers. The activity is resolved from the physical - // message context; child context bags read through to their parent, so this sees the same activity as the - // root receive context. Returns the activity (or null) so the caller can emit the finished event after the fork. - // Returns null when the dispatching events are turned off so neither event is written. - Activity? WriteStartDispatchingEvent(IIncomingPhysicalMessageContext physicalMessageContext, int operationCount) + // Both paths dispatch through here so the event instrumentation lives in one place. The activity is resolved + // from the batch dispatch context; child context bags read through to their parent, so this sees the same + // activity as the root receive context. When the events are off (the pre-v11 default) this returns the Fork + // task directly and adds no state machine of its own. + Task Dispatch(IBatchDispatchContext batchDispatchContext) { - if (!instrumentationOptions.EmitMessageDispatchingEvents || - !physicalMessageContext.Extensions.TryGetIncomingPipelineActivity(out var activity)) + if (instrumentationOptions.EmitMessageDispatchingEvents && + batchDispatchContext.Extensions.TryGetIncomingPipelineActivity(out var activity)) { - return null; + return DispatchWithEvents(batchDispatchContext, activity); } - activity.AddEvent(new("Start dispatching", tags: new() { { "message-count", operationCount } })); - return activity; + return this.Fork(batchDispatchContext); + } + + async Task DispatchWithEvents(IBatchDispatchContext batchDispatchContext, Activity activity) + { + activity.AddEvent(new("Start dispatching", tags: new() { { "message-count", batchDispatchContext.Operations.Count } })); + await this.Fork(batchDispatchContext).ConfigureAwait(false); + activity.AddEvent(new("Finished dispatching")); } static void ConvertToPendingOperations(OutboxMessage deduplicationEntry, PendingTransportOperations pendingTransportOperations) From e96ef16bc0a9252f6f8252afcf43275dbf83da41 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Mon, 5 Oct 2026 14:23:35 +0200 Subject: [PATCH 30/37] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Resolve=20the=20Acti?= =?UTF-8?q?vityFactory=20logger=20through=20the=20service=20provider?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The factory is created before the container exists, so it cannot take an ILogger in its constructor. RecordError now receives the behavior context's service provider instead of the unused ContextBag and resolves ILogger on first use. The message is emitted through a LoggerMessage method. Removes the last LogManager usage added by the otel branch. --- ...gerUsage.ApproveLogManagerUsage.approved.txt | 1 - .../OpenTelemetry/Tracing/ActivityFactory.cs | 17 +++++++++++------ .../OpenTelemetry/Tracing/IActivityFactory.cs | 3 +-- .../Tracing/NoOpActivityFactory.cs | 3 +-- .../Pipeline/Incoming/LoadHandlersConnector.cs | 2 +- .../Pipeline/MainPipelineExecutor.cs | 2 +- .../Unicast/MessageOperations.cs | 10 +++++----- 7 files changed, 20 insertions(+), 18 deletions(-) diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/LogManagerUsage.ApproveLogManagerUsage.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/LogManagerUsage.ApproveLogManagerUsage.approved.txt index b46abffb5a1..fb3f7c59c76 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/LogManagerUsage.ApproveLogManagerUsage.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/LogManagerUsage.ApproveLogManagerUsage.approved.txt @@ -4,7 +4,6 @@ Microsoft.Extensions.Logging instead (see https://learn.microsoft.com/dotnet/cor Changes that make this list longer should only be approved when the type affects user-facing code or when DI is not available. The list should otherwise shrink over time. ----- -NServiceBus.ActivityFactory NServiceBus.AsyncFile NServiceBus.AuditConfigReader NServiceBus.AuditInvalidLicenseBehavior diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs index 4669e372477..dec4c1374f1 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs @@ -6,12 +6,12 @@ namespace NServiceBus; using System.Collections.Generic; using System.Diagnostics; using System.Threading.Tasks; -using Extensibility; -using Logging; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Pipeline; using Transport; -sealed class ActivityFactory(InstrumentationOptions options) : IActivityFactory +sealed partial class ActivityFactory(InstrumentationOptions options) : IActivityFactory { public InstrumentationOptions Options { get; } = options; @@ -223,7 +223,7 @@ public void UpdateActivityFromRecoverabilityAction(Activity activity, Recoverabi } } - public void RecordError(Activity? activity, Exception exception, ContextBag context) + public void RecordError(Activity? activity, Exception exception, IServiceProvider serviceProvider) { if (activity == null) { @@ -239,7 +239,9 @@ public void RecordError(Activity? activity, Exception exception, ContextBag cont { if (Options.ExceptionRecordingMode == ExceptionRecordingMode.Logs) { - Logger.Error($"An exception occurred while executing '{activity.DisplayName}'.", exception); + // The factory is created before the container exists, so the logger is resolved on first use. + logger ??= serviceProvider.GetRequiredService>(); + LogExceptionWhileExecuting(logger, exception, activity.DisplayName); } else { @@ -257,5 +259,8 @@ public void RecordError(Activity? activity, Exception exception, ContextBag cont const string ExceptionRecordedFlag = "otel.exception.recorded"; - static readonly ILog Logger = LogManager.GetLogger(); + ILogger? logger; + + [LoggerMessage(LogLevel.Error, "An exception occurred while executing '{DisplayName}'.")] + static partial void LogExceptionWhileExecuting(ILogger logger, Exception exception, string displayName); } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs index 44943a4ce04..edec8ea2b24 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/IActivityFactory.cs @@ -4,7 +4,6 @@ namespace NServiceBus; using System; using System.Diagnostics; -using Extensibility; using Pipeline; using Transport; @@ -16,5 +15,5 @@ interface IActivityFactory Activity? StartHandlerActivity(MessageHandler messageHandler); Activity? StartRecoverabilityActivity(ErrorContext context); void UpdateActivityFromRecoverabilityAction(Activity activity, RecoverabilityAction recoverabilityAction, string receiveAddress); - void RecordError(Activity? activity, Exception exception, ContextBag context); + void RecordError(Activity? activity, Exception exception, IServiceProvider serviceProvider); } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs index 5a025496460..b161531fa7b 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/NoOpActivityFactory.cs @@ -4,7 +4,6 @@ namespace NServiceBus; using System; using System.Diagnostics; -using Extensibility; using Pipeline; using Transport; @@ -24,7 +23,7 @@ public void UpdateActivityFromRecoverabilityAction(Activity activity, Recoverabi { } - public void RecordError(Activity? activity, Exception exception, ContextBag context) + public void RecordError(Activity? activity, Exception exception, IServiceProvider serviceProvider) { } } \ No newline at end of file diff --git a/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs b/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs index e1385d74fef..91d46f88ebc 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/LoadHandlersConnector.cs @@ -65,7 +65,7 @@ public override async Task Invoke(IIncomingLogicalMessageContext context, Func Date: Wed, 30 Sep 2026 18:03:00 +0200 Subject: [PATCH 31/37] =?UTF-8?q?=E2=9C=A8=20Scope=20incoming=20trace=20st?= =?UTF-8?q?ate=20and=20baggage=20propagation=20to=20NServiceBus=20context?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Trace state and baggage headers are only meaningful together with a trace parent. Apply them only when the message has NServiceBus trace context. Without it, NServiceBus has nothing to propagate. The incoming span then inherits what an ambient activity has through the parent chain. Baggage from the headers is always applied, also when a transport SDK receive span is the parent. None of the supported SDKs (Azure Service Bus, RabbitMQ, SQS) propagate baggage yet. For when they do, a key the adopted parent already has is skipped. Activity.Baggage reads through the parent chain, so adding it again would double the baggage on every hop. Trace state from the headers is not applied when the message starts a new trace. It describes the trace of the sender, which the new trace is detached from. Start the incoming activity before the headers are read. Activity.Start() sets Activity.Parent, so the skip can read the parent chain of the runtime instead of tracking the adopted parent separately. The id format is still set to W3C before the start. Split ContextPropagation.PropagateContextFromHeaders into a trace-state part and a baggage part on both propagator paths. The combined method stays for callers that need both. --- .../OpenTelemetry/ActivityFactoryTests.cs | 165 ++++++++++++++++++ .../ContextPropagationCompatibilityTests.cs | 25 ++- .../ContextPropagationDefaultBehaviorTests.cs | 25 ++- ...TransportParentSpanDefaultBehaviorTests.cs | 32 +++- .../OpenTelemetry/Tracing/ActivityFactory.cs | 46 +++-- .../Tracing/ContextPropagation.cs | 40 ++++- .../OpenTelemetry/Tracing/obsolete_v11.cs | 76 +++++--- 7 files changed, 355 insertions(+), 54 deletions(-) diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs index 1ab193a3fbf..03681340206 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs @@ -173,6 +173,171 @@ public void Should_start_new_trace_when_trace_header_contains_invalid_data() } } + [Test] + public void Should_propagate_header_trace_state_when_attached_to_header_trace() + { + var sendActivity = CreateCompletedActivity("send activity"); + + var messageHeaders = new Dictionary + { + { Headers.NServiceBusDiagnosticsTraceParent, sendActivity.Id! }, + { Headers.DiagnosticsTraceState, "vendor=value" } + }; + + var activity = activityFactory.StartIncomingPipelineActivity(CreateMessageContext(messageHeaders)); + + Assert.That(activity, Is.Not.Null, "should create activity for receive pipeline"); + Assert.That(activity.TraceStateString, Is.EqualTo("vendor=value")); + } + + [Test] + public void Should_propagate_header_trace_state_when_attached_to_ambient_activity() + { + var sendActivity = CreateCompletedActivity("send activity"); + + using var ambientActivity = new Activity("transport sdk receive activity"); + ambientActivity.Start(); + + var messageHeaders = new Dictionary + { + { Headers.NServiceBusDiagnosticsTraceParent, sendActivity.Id! }, + { Headers.DiagnosticsTraceState, "vendor=value" } + }; + + var activity = activityFactory.StartIncomingPipelineActivity(CreateMessageContext(messageHeaders)); + + Assert.That(activity, Is.Not.Null, "should create activity for receive pipeline"); + using (Assert.EnterMultipleScope()) + { + Assert.That(activity.Parent, Is.SameAs(ambientActivity)); + Assert.That(activity.TraceStateString, Is.EqualTo("vendor=value")); + } + } + + [Test] + public void Should_not_propagate_header_trace_state_when_starting_new_trace() + { + var sendActivity = CreateCompletedActivity("send activity"); + + var messageHeaders = new Dictionary + { + { Headers.NServiceBusDiagnosticsTraceParent, sendActivity.Id! }, + { Headers.StartNewTrace, bool.TrueString }, + { Headers.DiagnosticsTraceState, "vendor=value" } + }; + + var activity = activityFactory.StartIncomingPipelineActivity(CreateMessageContext(messageHeaders)); + + Assert.That(activity, Is.Not.Null, "should create activity for receive pipeline"); + using (Assert.EnterMultipleScope()) + { + Assert.That(activity.Parent, Is.Null, "a new trace has no parent"); + Assert.That(activity.TraceStateString, Is.Null, "trace state belongs to the trace it was recorded in"); + } + } + + [Test] + public void Should_return_started_activity() + { + var activity = activityFactory.StartIncomingPipelineActivity(CreateMessageContext()); + + Assert.That(activity, Is.Not.Null, "should create activity for receive pipeline"); + using (Assert.EnterMultipleScope()) + { + Assert.That(activity.Id, Is.Not.Null, "an id is only assigned to a started activity"); + Assert.That(Activity.Current, Is.SameAs(activity)); + } + } + + [Test] + public void Should_propagate_header_baggage_when_attached_to_ambient_activity() + { + var sendActivity = CreateCompletedActivity("send activity"); + + using var ambientActivity = new Activity("transport sdk receive activity"); + ambientActivity.Start(); + + var messageHeaders = new Dictionary + { + { Headers.NServiceBusDiagnosticsTraceParent, sendActivity.Id! }, + { Headers.DiagnosticsBaggage, "tenant=acme" } + }; + + var activity = activityFactory.StartIncomingPipelineActivity(CreateMessageContext(messageHeaders)); + + Assert.That(activity, Is.Not.Null, "should create activity for receive pipeline"); + using (Assert.EnterMultipleScope()) + { + Assert.That(activity.Parent, Is.SameAs(ambientActivity)); + Assert.That(activity.GetBaggageItem("tenant"), Is.EqualTo("acme"), "transport SDKs do not propagate baggage, so NServiceBus propagates it even with an ambient parent"); + } + } + + [Test] + public void Should_not_add_header_baggage_already_carried_by_ambient_parent() + { + var sendActivity = CreateCompletedActivity("send activity"); + + using var ambientActivity = new Activity("transport sdk receive activity that extracted baggage"); + ambientActivity.AddBaggage("tenant", "acme"); + ambientActivity.Start(); + + var messageHeaders = new Dictionary + { + { Headers.NServiceBusDiagnosticsTraceParent, sendActivity.Id! }, + { Headers.DiagnosticsBaggage, "tenant=acme,region=eu" } + }; + + var activity = activityFactory.StartIncomingPipelineActivity(CreateMessageContext(messageHeaders)); + + Assert.That(activity, Is.Not.Null, "should create activity for receive pipeline"); + using (Assert.EnterMultipleScope()) + { + Assert.That(activity.Parent, Is.SameAs(ambientActivity)); + Assert.That(activity.GetBaggageItem("tenant"), Is.EqualTo("acme")); + Assert.That(activity.GetBaggageItem("region"), Is.EqualTo("eu")); + Assert.That(activity.Baggage.Count(item => item.Key == "tenant"), Is.EqualTo(1), "a key inherited from the parent must not be added again or it doubles on every hop"); + } + } + + [Test] + public void Should_propagate_header_baggage_but_not_ambient_baggage_when_starting_new_trace() + { + var sendActivity = CreateCompletedActivity("send activity"); + + using var ambientActivity = new Activity("transport sdk receive activity"); + ambientActivity.AddBaggage("ambient-only", "value"); + ambientActivity.Start(); + + var messageHeaders = new Dictionary + { + { Headers.NServiceBusDiagnosticsTraceParent, sendActivity.Id! }, + { Headers.StartNewTrace, bool.TrueString }, + { Headers.DiagnosticsBaggage, "tenant=acme" } + }; + + var activity = activityFactory.StartIncomingPipelineActivity(CreateMessageContext(messageHeaders)); + + Assert.That(activity, Is.Not.Null, "should create activity for receive pipeline"); + using (Assert.EnterMultipleScope()) + { + Assert.That(activity.Parent, Is.Null, "a new trace has no parent"); + Assert.That(activity.GetBaggageItem("tenant"), Is.EqualTo("acme"), "baggage from the message is propagated regardless of the trace shape"); + Assert.That(activity.GetBaggageItem("ambient-only"), Is.Null, "the ambient activity is not part of the new trace"); + } + } + + [Test] + public void Should_ignore_baggage_header_when_no_trace_header() + { + var messageHeaders = new Dictionary { { Headers.DiagnosticsBaggage, "tenant=acme" } }; + + var activity = activityFactory.StartIncomingPipelineActivity(CreateMessageContext(messageHeaders)); + + Assert.That(activity, Is.Not.Null, "should create activity for receive pipeline"); + Assert.That(activity.GetBaggageItem("tenant"), Is.Null, "baggage is only meaningful together with a trace parent"); + } + [Test] public void Should_add_native_message_id_tag() { diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationCompatibilityTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationCompatibilityTests.cs index 697cebf27c8..ae5dad4ba34 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationCompatibilityTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationCompatibilityTests.cs @@ -1,8 +1,9 @@ -namespace NServiceBus.Core.Tests.OpenTelemetry; +namespace NServiceBus.Core.Tests.OpenTelemetry; using System; using System.Collections.Generic; using System.Diagnostics; +using System.Linq; using NUnit.Framework; [TestFixture] @@ -64,6 +65,28 @@ static string Receive(Dictionary headers, Reader read) static string Transmit(string value, Writer write, Reader read) => Receive(Send(value, write), read); + [Test] + public void New_receiver_skips_baggage_the_parent_already_carries() + { + using var parent = new Activity("transport sdk receive activity"); + parent.AddBaggage("tenant", "acme"); + parent.Start(); + + // Adopts the parent from Activity.Current; the propagation reads through that chain + using var incoming = new Activity(ActivityNames.IncomingMessageActivityName); + incoming.Start(); + var headers = new Dictionary { { Headers.DiagnosticsBaggage, "tenant=acme,region=eu" } }; + + ContextPropagation.PropagateBaggageFromHeaders(incoming, headers); + + using (Assert.EnterMultipleScope()) + { + Assert.That(incoming.Parent, Is.SameAs(parent)); + Assert.That(incoming.GetBaggageItem("region"), Is.EqualTo("eu")); + Assert.That(incoming.Baggage.Count(item => item.Key == "tenant"), Is.EqualTo(1), "should only be the inherited one"); + } + } + [Test] public void Legacy_sender_to_new_receiver_preserves_the_value() { diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs index 83a8d3c0225..4f496a5e660 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs @@ -1,8 +1,9 @@ -namespace NServiceBus.Core.Tests.OpenTelemetry; +namespace NServiceBus.Core.Tests.OpenTelemetry; using System; using System.Collections.Generic; using System.Diagnostics; +using System.Linq; using NUnit.Framework; [TestFixture] @@ -69,6 +70,28 @@ public void Default_round_trip_preserves_value_whitespace() Assert.That(incoming.GetBaggageItem("key1"), Is.EqualTo(" leading-and-trailing ")); } + [Test] + public void Default_skips_baggage_the_parent_already_carries() + { + using var parent = new Activity("transport sdk receive activity"); + parent.AddBaggage("tenant", "acme"); + parent.Start(); + + // Adopts the parent from Activity.Current; the propagation reads through that chain + using var incoming = new Activity(ActivityNames.IncomingMessageActivityName); + incoming.Start(); + var headers = new Dictionary { { Headers.DiagnosticsBaggage, "tenant=acme,region=eu" } }; + + ContextPropagation.PropagateBaggageFromHeaders(incoming, headers); + + using (Assert.EnterMultipleScope()) + { + Assert.That(incoming.Parent, Is.SameAs(parent)); + Assert.That(incoming.GetBaggageItem("region"), Is.EqualTo("eu")); + Assert.That(incoming.Baggage.Count(item => item.Key == "tenant"), Is.EqualTo(1), "should only be the inherited one"); + } + } + [Test] public void Default_writes_the_nservicebus_trace_header_alongside_the_w3c_header() { diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/TransportParentSpanDefaultBehaviorTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/TransportParentSpanDefaultBehaviorTests.cs index ab459ccabae..2b7b71bfad2 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/TransportParentSpanDefaultBehaviorTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/TransportParentSpanDefaultBehaviorTests.cs @@ -1,4 +1,4 @@ -#nullable enable +#nullable enable namespace NServiceBus.Core.Tests.OpenTelemetry; @@ -59,4 +59,34 @@ public void Default_attaches_to_header_trace_when_even_id_ambient_activity_exist Assert.That(activity.Links.Count(), Is.EqualTo(0), "should not link to logical send span"); } } + + [Test] + public void Default_propagates_header_baggage_but_not_ambient_baggage() + { + var sendActivity = new Activity("send activity"); + sendActivity.SetIdFormat(ActivityIdFormat.W3C); + sendActivity.Start(); + sendActivity.Stop(); + + using var ambientActivity = new Activity("transport sdk receive activity"); + ambientActivity.AddBaggage("ambient-only", "value"); + ambientActivity.Start(); + + var messageHeaders = new Dictionary + { + { Headers.NServiceBusDiagnosticsTraceParent, sendActivity.Id! }, + { Headers.DiagnosticsBaggage, "tenant=acme" } + }; + var messageContext = new MessageContext(Guid.NewGuid().ToString(), messageHeaders, Array.Empty(), new TransportTransaction(), "receiver", new ContextBag()); + + var activity = activityFactory.StartIncomingPipelineActivity(messageContext); + + Assert.That(activity, Is.Not.Null, "should create activity for receive pipeline"); + using (Assert.EnterMultipleScope()) + { + Assert.That(activity.Parent, Is.Null, "the sender span is the parent, so the ambient activity is not in the parent chain"); + Assert.That(activity.GetBaggageItem("tenant"), Is.EqualTo("acme"), "baggage from the message is propagated"); + Assert.That(activity.GetBaggageItem("ambient-only"), Is.Null, "baggage of an activity that is not the parent is not NServiceBus' concern"); + } + } } diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs index dec4c1374f1..5d797d72c5b 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs @@ -15,7 +15,7 @@ sealed partial class ActivityFactory(InstrumentationOptions options) : IActivity { public InstrumentationOptions Options { get; } = options; - static Activity? CreateActivityFromIncomingMessage(ActivitySource activitySource, string activityName, Dictionary headers, string nativeMessageId) + static Activity? StartActivityFromIncomingMessage(ActivitySource activitySource, string activityName, Dictionary headers, string nativeMessageId) { // CreateActivity is a no-op if there are no listeners but we are doing a fast path check // here nonetheless to avoid having to parse headers, access the extension bag, etc. @@ -25,13 +25,14 @@ sealed partial class ActivityFactory(InstrumentationOptions options) : IActivity } var senderContextExists = TryParseSenderContext(headers, out ActivityContext senderContext); + var startNewTrace = false; Activity? activity; if (senderContextExists) // create a child from a logical send { - var startNewTrace = headers.TryGetValue(Headers.StartNewTrace, out var startNewTraceHeaderValue) - && string.Equals(startNewTraceHeaderValue, bool.TrueString, StringComparison.OrdinalIgnoreCase); + startNewTrace = headers.TryGetValue(Headers.StartNewTrace, out var startNewTraceHeaderValue) + && string.Equals(startNewTraceHeaderValue, bool.TrueString, StringComparison.OrdinalIgnoreCase); if (startNewTrace) { @@ -55,22 +56,43 @@ sealed partial class ActivityFactory(InstrumentationOptions options) : IActivity } else { - // Create a span that will be a child of Activity.Current if available + // No NServiceBus trace context on the message, so there is nothing for NServiceBus to + // propagate from the headers: trace state and baggage are only meaningful together with + // a trace parent. The span adopts Activity.Current as parent, if available, and inherits + // whatever trace state and baggage that activity carries through the parent chain. activity = activitySource.CreateActivity(activityName, ActivityKind.Consumer, parentContext: default); } if (activity is null) { - return activity; + return null; } - ContextPropagation.PropagateContextFromHeaders(activity, headers); - + // The id format can only be set on an activity that has not started yet. It is forced to W3C + // so an ambient hierarchical activity does not leak its format into the trace headers. activity.SetIdFormat(ActivityIdFormat.W3C); - activity.AddTag(ActivityTags.NativeMessageId, nativeMessageId); ActivityDecorator.PromoteHeadersToTags(activity, headers); + // Start before reading the headers: Activity.Parent is only assigned by Start(), and the + // baggage propagation below skips keys the parent chain already carries. + activity.Start(); + + if (senderContextExists) + { + // The message carries NServiceBus trace context, so the trace state and baggage headers + // that travel with it are NServiceBus' responsibility. + if (!startNewTrace) + { + // Trace state belongs to the trace it was recorded in, so it is not carried into a new trace. + ContextPropagation.PropagateTraceStateFromHeaders(activity, headers); + } + + // Baggage is always propagated, also when a transport SDK span is the parent: none of the + // supported SDKs propagate baggage yet. + ContextPropagation.PropagateBaggageFromHeaders(activity, headers); + } + return activity; } @@ -103,7 +125,7 @@ static bool TryParseSenderContext(Dictionary headers, out Activi public Activity? StartIncomingPipelineActivity(MessageContext context) { - var activity = CreateActivityFromIncomingMessage( + var activity = StartActivityFromIncomingMessage( ActivitySources.Main, ActivityNames.IncomingMessageActivityName, context.Headers, @@ -118,8 +140,6 @@ static bool TryParseSenderContext(Dictionary headers, out Activi ? $"{ActivityDisplayNames.ProcessOperation} {context.ReceiveAddress}" : ActivityDisplayNames.ProcessMessage; - activity.Start(); - return activity; } @@ -168,7 +188,7 @@ static bool TryParseSenderContext(Dictionary headers, out Activi public Activity? StartRecoverabilityActivity(ErrorContext context) { - var activity = CreateActivityFromIncomingMessage( + var activity = StartActivityFromIncomingMessage( ActivitySources.Recoverability, ActivityNames.RecoverabilityActivityName, context.Headers, @@ -181,8 +201,6 @@ static bool TryParseSenderContext(Dictionary headers, out Activi activity.DisplayName = ActivityDisplayNames.Recoverability; - activity.Start(); - return activity; } diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs index 520ee67fdde..ffa1b15091f 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs @@ -30,33 +30,54 @@ public static void PropagateContextToHeaders(Activity? activity, Dictionary headers) { - // Removed in v11, see obsolete_v11.cs - if (!LegacyContextPropagation.UseDistributedContextPropagator) + if (activity is null) { - LegacyContextPropagation.PropagateContextFromHeaders(activity, headers); return; } - // The following part was intentionally not extracted to a separate class to prevent - // accidental leftovers when because that the legacy propagator will be removed in v11 - if (activity is null) + PropagateTraceStateFromHeaders(activity, headers); + PropagateBaggageFromHeaders(activity, headers); + } + + public static void PropagateTraceStateFromHeaders(Activity activity, IDictionary headers) + { + // Removed in v11, see obsolete_v11.cs + if (!LegacyContextPropagation.UseDistributedContextPropagator) { + LegacyContextPropagation.PropagateTraceStateFromHeaders(activity, headers); return; } + // The following part was intentionally not extracted to a separate class to prevent + // accidental leftovers when because that the legacy propagator will be removed in v11 DistributedContextPropagator.Current.ExtractTraceIdAndState(headers, Getter, out _, out var traceState); if (traceState is not null) { activity.TraceStateString = traceState; } + } + + // The activity must be started. Baggage the parent chain already carries (for example because a + // transport SDK extracted it from the same message onto its receive span) is not added again from + // the headers. Otherwise every hop would put each key on the wire twice. + public static void PropagateBaggageFromHeaders(Activity activity, IDictionary headers) + { + // Removed in v11, see obsolete_v11.cs + if (!LegacyContextPropagation.UseDistributedContextPropagator) + { + LegacyContextPropagation.PropagateBaggageFromHeaders(activity, headers, activity.Parent); + return; + } + // The following part was intentionally not extracted to a separate class to prevent + // accidental leftovers when because that the legacy propagator will be removed in v11 var baggage = DistributedContextPropagator.Current.ExtractBaggage(headers, Getter); if (baggage is null) @@ -66,7 +87,10 @@ public static void PropagateContextFromHeaders(Activity? activity, IDictionary headers) + { if (headers.TryGetValue(Headers.DiagnosticsTraceState, out var traceState)) { activity.TraceStateString = traceState; } + } + + // parent: the activity whose baggage chain is checked so keys it already carries are not added again, + // see ContextPropagation.PropagateBaggageFromHeaders. + public static void PropagateBaggageFromHeaders(Activity activity, IDictionary headers, Activity? parent = null) + { + if (!headers.TryGetValue(Headers.DiagnosticsBaggage, out var baggageValue)) + { + return; + } - if (headers.TryGetValue(Headers.DiagnosticsBaggage, out var baggageValue)) + var baggageSpan = baggageValue.AsSpan(); + // HINT: Iterate in reverse order because Activity baggage is LIFO + while (!baggageSpan.IsEmpty) { - var baggageSpan = baggageValue.AsSpan(); - // HINT: Iterate in reverse order because Activity baggage is LIFO - while (!baggageSpan.IsEmpty) + var lastComma = baggageSpan.LastIndexOf(','); + ReadOnlySpan baggageItem; + + if (lastComma >= 0) { - var lastComma = baggageSpan.LastIndexOf(','); - ReadOnlySpan baggageItem; - - if (lastComma >= 0) - { - baggageItem = baggageSpan[(lastComma + 1)..]; - baggageSpan = baggageSpan[..lastComma]; - } - else - { - baggageItem = baggageSpan; - baggageSpan = []; - } - - var firstEquals = baggageItem.IndexOf('='); - if (firstEquals < 0 || firstEquals >= baggageItem.Length) - { - continue; - } - - var key = baggageItem[..firstEquals].Trim(); - var value = baggageItem[(firstEquals + 1)..]; - activity.AddBaggage(key.ToString(), Uri.UnescapeDataString(value)); + baggageItem = baggageSpan[(lastComma + 1)..]; + baggageSpan = baggageSpan[..lastComma]; } + else + { + baggageItem = baggageSpan; + baggageSpan = []; + } + + var firstEquals = baggageItem.IndexOf('='); + if (firstEquals < 0 || firstEquals >= baggageItem.Length) + { + continue; + } + + var key = baggageItem[..firstEquals].Trim().ToString(); + if (parent?.GetBaggageItem(key) is not null) + { + continue; + } + + var value = baggageItem[(firstEquals + 1)..]; + activity.AddBaggage(key, Uri.UnescapeDataString(value)); } } } From af8ebb78d3527807f3f3802aa3c194c63b1bea9b Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Fri, 2 Oct 2026 12:57:53 +0200 Subject: [PATCH 32/37] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Rename=20TransportPa?= =?UTF-8?q?rentSpanSwitch=20to=20TransportParentActivitySwitch?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The switch selects the parent Activity of the incoming span. Rename it and update all references, tests and comments to the Activity terminology. --- .../OpenTelemetry/ActivityFactoryTests.cs | 8 ++++---- .../TransportParentSpanDefaultBehaviorTests.cs | 8 ++++---- .../OpenTelemetry/Tracing/ActivityFactory.cs | 2 +- .../OpenTelemetry/Tracing/obsolete_v11.cs | 18 +++++++++--------- 4 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs index 03681340206..b2c983d4278 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs @@ -58,15 +58,15 @@ class StartIncomingActivity : ActivityFactoryTests [SetUp] public void OptInToTransportSpanAsParent() { - AppContext.SetSwitch(TransportParentSpanSwitch.UseTransportSpanAsParentSwitchName, true); - TransportParentSpanSwitch.ResetUseTransportSpanAsParent(); + AppContext.SetSwitch(TransportParentActivitySwitch.UseTransportActivityAsParentSwitchName, true); + TransportParentActivitySwitch.ResetUseTransportActivityAsParent(); } [TearDown] public void ResetTransportSpanSwitch() { - AppContext.SetSwitch(TransportParentSpanSwitch.UseTransportSpanAsParentSwitchName, false); - TransportParentSpanSwitch.ResetUseTransportSpanAsParent(); + AppContext.SetSwitch(TransportParentActivitySwitch.UseTransportActivityAsParentSwitchName, false); + TransportParentActivitySwitch.ResetUseTransportActivityAsParent(); } [TestCase(Headers.NServiceBusDiagnosticsTraceParent)] diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/TransportParentSpanDefaultBehaviorTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/TransportParentSpanDefaultBehaviorTests.cs index 2b7b71bfad2..0064600331e 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/TransportParentSpanDefaultBehaviorTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/TransportParentSpanDefaultBehaviorTests.cs @@ -11,7 +11,7 @@ namespace NServiceBus.Core.Tests.OpenTelemetry; using NServiceBus.Transport; using NUnit.Framework; -// Covers the pre-v11 default of TransportParentSpanSwitch: without the opt-in switch the incoming +// Covers the pre-v11 default of TransportParentActivitySwitch: without the opt-in switch the incoming // message span stays a child of the NServiceBus sender span even when a transport SDK span is // ambient. In v11 that default is gone, so delete this file together with obsolete_v11.cs. [TestFixture] @@ -25,15 +25,15 @@ public class TransportParentSpanDefaultBehaviorTests public void SetUp() { nsbActivityListener = TestingActivityListener.SetupNServiceBusDiagnosticListener(); - AppContext.SetSwitch(TransportParentSpanSwitch.UseTransportSpanAsParentSwitchName, false); - TransportParentSpanSwitch.ResetUseTransportSpanAsParent(); + AppContext.SetSwitch(TransportParentActivitySwitch.UseTransportActivityAsParentSwitchName, false); + TransportParentActivitySwitch.ResetUseTransportActivityAsParent(); } [TearDown] public void TearDown() { nsbActivityListener.Dispose(); - TransportParentSpanSwitch.ResetUseTransportSpanAsParent(); + TransportParentActivitySwitch.ResetUseTransportActivityAsParent(); } [Test] diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs index 5d797d72c5b..687dcd15067 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs @@ -42,7 +42,7 @@ sealed partial class ActivityFactory(InstrumentationOptions options) : IActivity Activity.Current = null; activity = activitySource.CreateActivity(activityName, ActivityKind.Consumer, parentContext: default, links: [new ActivityLink(senderContext)]); } - else if (TransportParentSpanSwitch.UseTransportSpanAsParent && Activity.Current != null) // remove the switch check in v11, see obsolete_v11.cs + else if (TransportParentActivitySwitch.UseTransportActivityAsParent && Activity.Current != null) // remove the switch check in v11, see obsolete_v11.cs { // A transport SDK receive span is ambient: make it the parent (an activity without // a parent context adopts Activity.Current when it starts) and link to the NSB sender span. diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs index 7e2f3d4be25..4836a5834db 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs @@ -208,11 +208,11 @@ public static bool UseHandlerActivitySource // AppContext switch. // // In v11 the SDK span becomes the parent unconditionally: delete this class, remove the -// `TransportParentSpanSwitch.UseTransportSpanAsParent` check in +// `TransportParentActivitySwitch.UseTransportSpanAsParent` check in // ActivityFactory.CreateActivityFromIncomingMessage (keeping only the branch that links to the // sender span), delete TransportParentSpanDefaultBehaviorTests.cs and remove the opt-in // SetUp/TearDown pair in ActivityFactoryTests.StartIncomingActivity. -static class TransportParentSpanSwitch +static class TransportParentActivitySwitch { enum SwitchState : byte { @@ -221,30 +221,30 @@ enum SwitchState : byte Disabled = 2 } - static SwitchState cachedUseTransportSpanAsParent; + static SwitchState cachedUseTransportActivityAsParent; - public const string UseTransportSpanAsParentSwitchName = "NServiceBus.Core.OpenTelemetry.UseTransportSpanAsParent"; + public const string UseTransportActivityAsParentSwitchName = "NServiceBus.Core.OpenTelemetry.UseTransportActivityAsParent"; - public static bool UseTransportSpanAsParent + public static bool UseTransportActivityAsParent { get { - var state = cachedUseTransportSpanAsParent; + var state = cachedUseTransportActivityAsParent; if (state != SwitchState.Unchecked) { return state == SwitchState.Enabled; } - state = AppContext.TryGetSwitch(UseTransportSpanAsParentSwitchName, out var isEnabled) && isEnabled + state = AppContext.TryGetSwitch(UseTransportActivityAsParentSwitchName, out var isEnabled) && isEnabled ? SwitchState.Enabled : SwitchState.Disabled; - cachedUseTransportSpanAsParent = state; + cachedUseTransportActivityAsParent = state; return state == SwitchState.Enabled; } } - internal static void ResetUseTransportSpanAsParent() => cachedUseTransportSpanAsParent = SwitchState.Unchecked; + internal static void ResetUseTransportActivityAsParent() => cachedUseTransportActivityAsParent = SwitchState.Unchecked; } // This class bridges two independent legacy exception-tagging behaviors, both From 522c367e0e9b7c5cc42d391ce0adc2977276125e Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Thu, 1 Oct 2026 15:20:51 +0200 Subject: [PATCH 33/37] =?UTF-8?q?=F0=9F=93=9D=20ADR:=20receive-side=20trac?= =?UTF-8?q?e=20state=20and=20baggage=20propagation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Records why NServiceBus applies trace state and baggage from the message headers on receive only when its own trace context is present, also under an ambient transport SDK span, and why keys the adopted parent already has are skipped. Captures the verified SDK behavior and the Activity.Parent mechanics the decision rests on, and the alternatives that were rejected. --- ...ide-trace-state-and-baggage-propagation.md | 177 ++++++++++++++++++ 1 file changed, 177 insertions(+) create mode 100644 docs/decisions/2026-10-01-receive-side-trace-state-and-baggage-propagation.md diff --git a/docs/decisions/2026-10-01-receive-side-trace-state-and-baggage-propagation.md b/docs/decisions/2026-10-01-receive-side-trace-state-and-baggage-propagation.md new file mode 100644 index 00000000000..64dcd34a22c --- /dev/null +++ b/docs/decisions/2026-10-01-receive-side-trace-state-and-baggage-propagation.md @@ -0,0 +1,177 @@ +# NServiceBus propagates trace state and baggage on the receive side + +**Date:** 2026-10-01 +**Pull request:** [#7954](https://github.com/Particular/NServiceBus/pull/7954) +**Related:** + +- [#7947](https://github.com/Particular/NServiceBus/pull/7947) NServiceBus-specific trace parent header +- [#7949](https://github.com/Particular/NServiceBus/pull/7949) ambient transport SDK span as parent +- [#7952](https://github.com/Particular/NServiceBus/pull/7952) propagation only in the outgoing pipeline + +## Context + +NServiceBus writes OpenTelemetry context to each outgoing message as headers. The headers are +`NServiceBus.TraceParent` and the W3C headers `traceparent`, `tracestate` and `baggage`. On receive, +`ActivityFactory` creates the incoming span in one of three shapes. Then it reads the headers back. + +```mermaid +flowchart TD + A[Incoming message] --> B{NServiceBus trace parent header present?} + B -- no --> C[Span adopts Activity.Current as parent if any
Nothing read from headers] + B -- yes --> D{StartNewTrace header?} + D -- yes --> E[New trace, link to sender span
Activity.Current cleared] + D -- no --> F{Ambient SDK span and
UseTransportSpanAsParent?} + F -- yes --> G[Child of ambient SDK span
link to sender span] + F -- no --> H[Child of sender span
v10 default] + E --> K[Propagate baggage from headers
tracestate stays with the old trace] + G --> J[Propagate tracestate from headers
Propagate baggage, skip keys the SDK span already has] + H --> I[Propagate tracestate and baggage from headers] +``` + +This record answers two questions: + +- Who owns baggage on the receive side, now that transport SDKs have their own OpenTelemetry + instrumentation? +- What happens when those SDKs start to propagate baggage? + +Three sets of facts shaped the answer. + +### How the .NET runtime reads baggage + +`Activity.Baggage`, `Activity.GetBaggageItem` and `Activity.TraceStateString` all walk the +`Activity.Parent` chain. `Activity.Start()` sets `Parent` only in one case. The activity has no explicit +parent context, and `Activity.Current` is set at that moment. + +An activity created from an `ActivityContext` gets the trace id and the parent span id. The trace tree is +correct. But `Parent` stays null. Such a span inherits no baggage, also when the context belongs to the +ambient `Activity.Current`. A small program against .NET 10 confirmed this: same trace id, same parent +span id, `Parent` null, baggage count zero. + +So only two branches inherit the baggage of the ambient span. These are the branches that pass a default +parent context: the ambient SDK span as parent, and no sender context. The v10 default branch and the +start-new-trace branch do not inherit it. The runtime does not close that gap. + +`Start()` sets `Parent`. So a check against the parent chain must run on a started activity. `SetIdFormat` +works only before `Start()`. On a started activity the runtime ignores it. The runtime throws and catches +an `InvalidOperationException` internally. + +### What the transport SDKs do today + +Verified against the SDK sources on 2026-09-30: + +| SDK | Injects baggage on send | Extracts baggage on receive | +|---|---|---| +| Azure.Messaging.ServiceBus (`Azure.Core` `MessagingClientDiagnostics`) | no, only `Diagnostic-Id`, `traceparent`, `tracestate` | no | +| RabbitMQ.Client 7 (`RabbitMQActivitySource`) | yes, with `DistributedContextPropagator.Current.Inject`. It overwrites existing keys. | no, `DefaultContextExtractor` reads trace id and state only | +| AWS SDK for SQS | no tracing in the SDK. The OpenTelemetry contrib instrumentation uses `Baggage.Current`. That store is separate from `Activity.Baggage`. | same | +| SQL Server, MSMQ, Azure Storage Queues, Learning transport | no SDK tracing | no SDK tracing | + +No supported SDK delivers baggage end to end. The one SDK that writes baggage does not read it. The header +that NServiceBus reads is always a superset of what an SDK receive span can carry. RabbitMQ basic headers +and Azure Service Bus application properties map one to one to NServiceBus headers. + +### Constraints of the DistributedContextPropagator path + +.NET 10 changed `DistributedContextPropagator.CreateDefaultPropagator()`. It now returns the W3C +propagator. The baggage header on that path is `baggage`, not `Correlation-Context`. `Inject` writes +`traceparent`, `tracestate` and `baggage` in one call. The runtime has W3C, pre-W3C, pass-through and +no-output propagators. It has no option to suppress baggage alone. To disable baggage on the outgoing +side, NServiceBus has two options. It can filter header names in the setter. Or it can write trace +context by hand again. + +## Decision + +1. **Trace state and baggage from the headers are propagated only when the message has NServiceBus trace + context.** NServiceBus trace context is a parseable `NServiceBus.TraceParent` or `traceparent` header. + The W3C specifications define `tracestate` and `baggage` as companions of `traceparent`. Without + `traceparent`, NServiceBus has nothing of its own to propagate. The span adopts `Activity.Current` as + parent if one exists. It inherits the trace state and baggage of that activity through the parent chain. + +2. **NServiceBus always propagates the header baggage.** This includes the case where an ambient transport + SDK span is the parent. No supported SDK propagates baggage. NServiceBus is the middleware, so it + carries the baggage. + +3. **When the ambient SDK span is the parent, NServiceBus skips a header key that the span already has.** + This prepares for the future. When an SDK extracts baggage to its receive span, the parent gets the same + items. Without the skip, the child would get them too. Outgoing serialization would write both. The next hop + would double them again. The incoming activity is started before the headers are read. So the skip + reads through the `Activity.Parent` chain of the runtime. NServiceBus does not track the adopted parent + separately. + +4. **Trace state from the headers is not propagated when the message starts a new trace.** `tracestate` + carries vendor data about the trace of the sender, such as sampling decisions. A new trace has no + relation to that data. The ambient-parent branch and the child-of-sender branch propagate trace state as + before. + +5. **Baggage is never copied from an ambient activity that does not become the parent.** Baggage on such an + activity has one of two sources. It is from the same message, and the header already covers it. Or it is + process-local context, and NServiceBus deliberately did not parent on it. + +6. **Outgoing propagation does not change.** NServiceBus writes `baggage` together with its trace headers. + +Mechanically, `ActivityFactory` creates the incoming activity, forces the W3C id format, adds the tags and +starts it. Only then does it read the headers. `ContextPropagation.PropagateContextFromHeaders` is split +into `PropagateTraceStateFromHeaders(activity, headers)` and `PropagateBaggageFromHeaders(activity, +headers)`. The split applies to both propagator paths, including the legacy path in `obsolete_v11.cs`. +Both methods expect a started activity. The combined method stays for callers that need both. + +## Consequences + +- Handlers can read message baggage with `Activity.Current.GetBaggageItem` on every transport. The + transport SDK does not affect this. The v10 default branch did this before. The ambient-parent branch + now guarantees it too. +- A `baggage` header on a message without a NServiceBus trace header is ignored. Accepted. A producer + that wants its baggage honored must also send `traceparent`. The W3C specification requires this + anyway. No NServiceBus version sent one header without the other. +- When the ambient SDK span is the parent, a key can exist on the span and in the header. The value + on the span wins, because the header item is skipped. Accepted. Both values come from the same message, + so they are the same in practice. The alternative doubles baggage per hop. +- Under the v10 default, the sender span is the parent. Process-local baggage that code around the + message pump adds is then not visible to handlers. Accepted. This is consistent with not parenting on + that activity. In v11 the ambient span becomes the parent, and the runtime inherits that baggage. +- Duplicate keys inside one `baggage` header are still added as before. The skip looks only at the parent + chain, not at the activity itself. Legacy parsing does not change. +- Trace state needs no equal treatment. `TraceStateString` also reads through the parent chain. But it is + a single value. The value of the child hides the value of the parent, so nothing accumulates. +- A message that starts a new trace drops the `tracestate` of the sender. Accepted. The value describes a + trace that the new trace is deliberately detached from. The sender span is still reachable through the + link. +- `ActivityFactory` starts the incoming activity before it returns it. The caller sets the display name + after that. `ActivityListener.ActivityStarted` callbacks see the operation name and the tags, but not + the display name. Exporters read the activity when it stops, so this is cosmetic. +- Cost: one `GetBaggageItem` lookup per header item. This applies only when the activity has a parent. +- Follow-up: document the receive-side behavior on the public OpenTelemetry page on docs.particular.net. +- Open: whether `InstrumentationOptions` needs an opt-out for baggage propagation. This record does not + decide it. +- Open: when an SDK starts to extract baggage, verify the skip again against the real span. + +## Alternative approaches + +### Make NServiceBus baggage propagation opt-in and rely on the transport SDKs + +The proposal: default on in the next minor, default off in v11. The assumption: the native SDK propagates +baggage. Rejected for four reasons: + +- The assumption does not hold for any supported transport. See the table above. +- The SDK propagates the ambient context at physical dispatch, not the logical send context. These differ + under the outbox, batched dispatch, the messaging bridge and ServiceControl retries. #7947 introduced + `NServiceBus.TraceParent` for the same reason. +- Default off fails silently. Traces stay intact. Only downstream logic that reads baggage breaks. +- `DistributedContextPropagator.Inject` has no option to drop baggage alone. + +### Copy the baggage of the ambient activity to the incoming span when it is not the parent + +This was implemented first in this change, then reverted. Each source of ambient baggage on receive is one +of two things. It is the same message, where the header is a superset. Or it is process-local context that +NServiceBus chose not to parent on. The copy added allocations per message and a precedence rule for a +case that cannot occur. + +### Add header baggage unconditionally, also under an ambient parent + +This is the simplest reading of "NServiceBus always propagates baggage". Rejected. It doubles baggage on +each hop as soon as an SDK extracts baggage to the parent span. There is no error, and the headers grow. + +### Skip all header baggage when the ambient parent has any baggage + +Rejected. Unrelated baggage that host code adds would suppress the baggage of the message. The per-key +check costs the same and does not depend on the source of the baggage of the parent. From 3034d431f1cc0b565e5afdbc780a6bb2fe333a8b Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Fri, 2 Oct 2026 17:22:41 +0200 Subject: [PATCH 34/37] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20Rename=20obsolete=5F?= =?UTF-8?q?v12.cs=20to=20obsoletes-v11.cs?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Matches the root obsoletes-v10.cs convention where obsoletes-vN.cs holds code that is obsolete in vN and removed in vN+1. The ExceptionRecordingMode environment variable scaffold is removed in v12, so it belongs in obsoletes-v11.cs. --- .../{OpenTelemetry/Tracing/obsolete_v12.cs => obsoletes-v11.cs} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename src/NServiceBus.Core/{OpenTelemetry/Tracing/obsolete_v12.cs => obsoletes-v11.cs} (100%) diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v12.cs b/src/NServiceBus.Core/obsoletes-v11.cs similarity index 100% rename from src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v12.cs rename to src/NServiceBus.Core/obsoletes-v11.cs From 4ad89b13557ecc85c918c73b472d763ba72494f5 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Fri, 2 Oct 2026 17:22:41 +0200 Subject: [PATCH 35/37] =?UTF-8?q?=E2=9C=A8=20Single=20AppContext=20switch?= =?UTF-8?q?=20for=20the=20OpenTelemetry=20v11=20behaviors?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Several OpenTelemetry behaviors become the only behavior in v11 with no option left to opt in or out. Until then they are gated behind one AppContext switch, NServiceBus.Core.OpenTelemetry.UseV11Behavior, so an endpoint adopts the v11 behavior as a whole instead of per feature: - DistributedContextPropagator-based trace context and baggage propagation - handler spans from the NServiceBus.Core.Handler ActivitySource - ambient transport SDK receive span as parent of the incoming span - destination and message type in span names - no Start/Finished dispatching events on the incoming span - no execution.result metric tag - no legacy otel.status_code/otel.status_description tags and no exception.escaped attribute Replaces the UseDistributedContextPropagator, UseHandlerActivitySource and UseTransportActivityAsParent switches and removes the unreleased UseMessageTypeNamesInSpanNames, EmitMessageDispatchingEvents and Meters.EmitExecutionResultTags options. PublishTraceMode keeps its public option in v11 and is not governed by the switch. All v11 scaffolding now lives in the root obsoletes-v10.cs, replacing OpenTelemetry/Tracing/obsolete_v11.cs. --- ...ide-trace-state-and-baggage-propagation.md | 2 +- .../OpenTelemetryAcceptanceTest.cs | 2 +- .../When_outgoing_activity_has_baggage.cs | 2 +- .../When_processing_incoming_message.cs | 21 +- ...g_message_with_default_activity_sources.cs | 2 +- .../Traces/When_publishing_messages.cs | 20 +- .../When_recoverability_action_occurs.cs | 9 +- .../Traces/When_sending_messages.cs | 21 +- .../Traces/When_sending_replies.cs | 24 +- .../NServiceBus.AcceptanceTests.csproj | 1 + .../obsoletes-v10.cs | 40 +++ ...IApprovals.ApproveNServiceBus.approved.txt | 8 - .../Envelopes/EnvelopeUnwrapperTests.cs | 2 +- .../OpenTelemetry/ActivityFactoryTests.cs | 23 +- .../ContextPropagationCompatibilityTests.cs | 15 +- .../ContextPropagationDefaultBehaviorTests.cs | 11 +- .../HandlerActivitySourceTests.cs | 27 +- .../InstrumentationOptionsTests.cs | 2 +- .../OpenTelemetry/MeterTests.cs | 2 +- .../PipelineMetricsExecutionResultTagTests.cs | 43 +++ ...TransportParentSpanDefaultBehaviorTests.cs | 7 +- .../Incoming/InvokeHandlerTerminatorTest.cs | 2 +- .../SerializeMessageConnectorTests.cs | 2 +- .../Pipeline/MainPipelineExecutorTests.cs | 2 +- .../Pipeline/PipelineMetricTagsTests.cs | 2 +- ...tReceiveToPhysicalMessageConnectorTests.cs | 27 +- .../Unicast/LoadHandlersConnectorTests.cs | 8 +- src/NServiceBus.Core.Tests/obsoletes-v10.cs | 29 ++ src/NServiceBus.Core/EndpointCreator.cs | 2 +- .../OpenTelemetry/InstrumentationOptions.cs | 4 +- .../Tracing/ActivityDisplayNames.cs | 2 +- .../OpenTelemetry/Tracing/ActivityFactory.cs | 21 +- .../Tracing/ContextPropagation.cs | 12 +- .../OpenTelemetry/Tracing/obsolete_v11.cs | 328 ------------------ .../Pipeline/Incoming/PipelineMetrics.cs | 4 +- ...nsportReceiveToPhysicalMessageConnector.cs | 3 +- .../Outgoing/RoutingToDispatchConnector.cs | 2 +- .../Pipeline/PipelineComponent.cs | 7 +- .../Receiving/ReceiveComponent.cs | 3 +- .../Unicast/MessageOperations.cs | 12 +- src/NServiceBus.Core/obsoletes-v10.cs | 206 +++++++++++ 41 files changed, 436 insertions(+), 526 deletions(-) create mode 100644 src/NServiceBus.AcceptanceTests/obsoletes-v10.cs create mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/PipelineMetricsExecutionResultTagTests.cs create mode 100644 src/NServiceBus.Core.Tests/obsoletes-v10.cs delete mode 100644 src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs diff --git a/docs/decisions/2026-10-01-receive-side-trace-state-and-baggage-propagation.md b/docs/decisions/2026-10-01-receive-side-trace-state-and-baggage-propagation.md index 64dcd34a22c..6a3b0d9253d 100644 --- a/docs/decisions/2026-10-01-receive-side-trace-state-and-baggage-propagation.md +++ b/docs/decisions/2026-10-01-receive-side-trace-state-and-baggage-propagation.md @@ -112,7 +112,7 @@ context by hand again. Mechanically, `ActivityFactory` creates the incoming activity, forces the W3C id format, adds the tags and starts it. Only then does it read the headers. `ContextPropagation.PropagateContextFromHeaders` is split into `PropagateTraceStateFromHeaders(activity, headers)` and `PropagateBaggageFromHeaders(activity, -headers)`. The split applies to both propagator paths, including the legacy path in `obsolete_v11.cs`. +headers)`. The split applies to both propagator paths, including the legacy path in `obsoletes-v10.cs`. Both methods expect a started activity. The combined method stays for callers that need both. ## Consequences diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/OpenTelemetryAcceptanceTest.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/OpenTelemetryAcceptanceTest.cs index a2748753077..f67d0ab06bd 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/OpenTelemetryAcceptanceTest.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/OpenTelemetryAcceptanceTest.cs @@ -9,7 +9,7 @@ public abstract class OpenTelemetryAcceptanceTest : NServiceBusAcceptanceTest protected TestingActivityListener NServiceBusActivityListener { get; private set; } [SetUp] - public void Setup() => NServiceBusActivityListener = TestingActivityListener.SetupDiagnosticListener("NServiceBus.Core", "NServiceBus.Core.Recoverability"); + public void Setup() => NServiceBusActivityListener = TestingActivityListener.SetupDiagnosticListener("NServiceBus.Core", "NServiceBus.Core.Handler", "NServiceBus.Core.Recoverability"); [TearDown] public void Cleanup() diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_outgoing_activity_has_baggage.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_outgoing_activity_has_baggage.cs index b6a2310ad25..33cfd28674b 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_outgoing_activity_has_baggage.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_outgoing_activity_has_baggage.cs @@ -35,7 +35,7 @@ public async Task Should_propagate_baggage_to_headers() // Default (backwards-compatible) propagation produces the legacy comma-separated, percent-encoded format. // The W3C OWS format ("key3 = , key2 = value2, key1 = value1") is produced only when the - // NServiceBus.Core.OpenTelemetry.UseDistributedContextPropagator AppContext switch is enabled (default in v11). + // NServiceBus.Core.OpenTelemetry.UseV11Behavior AppContext switch is enabled (the only behavior in v11). Assert.That(context.BaggageHeader, Is.EqualTo("key3=,key2=value2,key1=value1")); } diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_incoming_message.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_incoming_message.cs index 86fc87529a4..32d60e7e036 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_incoming_message.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_incoming_message.cs @@ -81,10 +81,11 @@ public Task Handle(IncomingMessage message, IMessageHandlerContext context) } [Test] - public async Task Should_use_receive_address_in_span_name_when_opted_in() + [OpenTelemetryV11Defaults] + public async Task Should_include_receive_address_in_span_name() { await Scenario.Define() - .WithEndpoint(e => e + .WithEndpoint(e => e .When(s => s.SendLocal(new IncomingMessage()))) .Run(); @@ -96,21 +97,5 @@ await Scenario.Define() Assert.That(incomingActivity.DisplayName, Is.Not.EqualTo("process message")); } - public class ReceivingEndpointWithDestinationNaming : EndpointConfigurationBuilder - { - public ReceivingEndpointWithDestinationNaming() => - EndpointSetup(b => b.Tracing().UseMessageTypeNamesInSpanNames = true); - - [Handler] - public class MessageHandler(Context testContext) : IHandleMessages - { - public Task Handle(IncomingMessage message, IMessageHandlerContext context) - { - testContext.MarkAsCompleted(); - return Task.CompletedTask; - } - } - } - public class IncomingMessage : IMessage; } \ No newline at end of file diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_message_with_default_activity_sources.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_message_with_default_activity_sources.cs index a233aa178b1..4b6c631b89b 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_message_with_default_activity_sources.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_message_with_default_activity_sources.cs @@ -10,7 +10,7 @@ public class When_processing_message_with_default_activity_sources : OpenTelemet { // Until v11, handler spans are emitted from the "NServiceBus.Core" ActivitySource by default // for backwards compatibility. The dedicated "NServiceBus.Core.Handler" source is opt-in via - // the NServiceBus.Core.OpenTelemetry.UseHandlerActivitySource AppContext switch (default in v11). + // the NServiceBus.Core.OpenTelemetry.UseV11Behavior AppContext switch (the only behavior in v11). [Test] public async Task Should_emit_handler_span_from_main_source() { diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs index 39b8e57885d..c0359fd053f 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_publishing_messages.cs @@ -180,12 +180,13 @@ public Task Handle(ThisIsAnEvent @event, IMessageHandlerContext context) } [Test] - public async Task Should_use_event_type_in_span_name_when_opted_in() + [OpenTelemetryV11Defaults] + public async Task Should_include_event_type_in_span_name() { await Scenario.Define() - .WithEndpoint(b => b + .WithEndpoint(b => b .When(ctx => ctx.SomeEventSubscribed, s => s.Publish())) - .WithEndpoint(b => b.When((session, ctx) => + .WithEndpoint(b => b.When((session, ctx) => { if (ctx.HasNativePubSubSupport) { @@ -203,15 +204,14 @@ await Scenario.Define() Assert.That(publishedMessage.DisplayName, Is.EqualTo("publish ThisIsAnEvent")); } - public class PublisherWithDestinationNaming : EndpointConfigurationBuilder + public class PublisherForV11Naming : EndpointConfigurationBuilder { - public PublisherWithDestinationNaming() => + public PublisherForV11Naming() => EndpointSetup(b => { - b.Tracing().UseMessageTypeNamesInSpanNames = true; b.OnEndpointSubscribed((s, context) => { - if (s.SubscriberEndpoint.Contains(Conventions.EndpointNamingConvention(typeof(SubscriberForPublisherWithDestinationNaming)))) + if (s.SubscriberEndpoint.Contains(Conventions.EndpointNamingConvention(typeof(SubscriberForPublisherForV11Naming)))) { if (s.MessageType == typeof(ThisIsAnEvent).AssemblyQualifiedName) { @@ -222,13 +222,13 @@ public PublisherWithDestinationNaming() => }); } - public class SubscriberForPublisherWithDestinationNaming : EndpointConfigurationBuilder + public class SubscriberForPublisherForV11Naming : EndpointConfigurationBuilder { - public SubscriberForPublisherWithDestinationNaming() => + public SubscriberForPublisherForV11Naming() => EndpointSetup(c => { }, metadata => { - metadata.RegisterPublisherFor(); + metadata.RegisterPublisherFor(); }); [Handler] diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_recoverability_action_occurs.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_recoverability_action_occurs.cs index 6e24936ab53..09e3b64de80 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_recoverability_action_occurs.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_recoverability_action_occurs.cs @@ -52,15 +52,12 @@ await Scenario.Define() } [Test] - public async Task Should_include_destination_in_display_name_when_opted_in() + [OpenTelemetryV11Defaults] + public async Task Should_include_destination_in_display_name() { await Scenario.Define() .WithEndpoint(b => b - .CustomConfig(c => - { - c.Recoverability().Immediate(i => i.NumberOfRetries(1)).Delayed(i => i.NumberOfRetries(0)); - c.Tracing().UseMessageTypeNamesInSpanNames = true; - }) + .CustomConfig(c => c.Recoverability().Immediate(i => i.NumberOfRetries(1)).Delayed(i => i.NumberOfRetries(0))) .DoNotFailOnErrorMessages() .When(s => s.SendLocal(new FailingMessage()))) .Done(_ => ActionTags().Contains("move_to_error")) diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_messages.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_messages.cs index a56a3caac7e..61499ddee54 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_messages.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_messages.cs @@ -132,10 +132,11 @@ public Task Handle(OutgoingMessage message, IMessageHandlerContext context) } [Test] - public async Task Should_use_destination_in_send_span_name_when_opted_in() + [OpenTelemetryV11Defaults] + public async Task Should_include_destination_in_send_span_name() { await Scenario.Define() - .WithEndpoint(b => b + .WithEndpoint(b => b .When(s => s.SendLocal(new OutgoingMessage()))) .Run(); @@ -147,22 +148,6 @@ await Scenario.Define() Assert.That(sentMessage.DisplayName, Is.Not.EqualTo("send message")); } - public class TestEndpointWithDestinationNaming : EndpointConfigurationBuilder - { - public TestEndpointWithDestinationNaming() => - EndpointSetup(b => b.Tracing().UseMessageTypeNamesInSpanNames = true); - - [Handler] - public class MessageHandler(Context testContext) : IHandleMessages - { - public Task Handle(OutgoingMessage message, IMessageHandlerContext context) - { - testContext.MarkAsCompleted(); - return Task.CompletedTask; - } - } - } - [Test] public async Task Should_create_new_linked_trace_on_receive_when_endpoint_defaults_to_span_link() { diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_replies.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_replies.cs index 7e79d6582c5..8077fbb2027 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_replies.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_sending_replies.cs @@ -56,10 +56,11 @@ public Task Handle(OutgoingReply message, IMessageHandlerContext context) } [Test] - public async Task Should_use_destination_in_reply_span_name_when_opted_in() + [OpenTelemetryV11Defaults] + public async Task Should_include_destination_in_reply_span_name() { await Scenario.Define() - .WithEndpoint(b => b + .WithEndpoint(b => b .When(s => s.SendLocal(new IncomingMessage()))) .Run(); @@ -70,25 +71,6 @@ await Scenario.Define() Assert.That(replyMessage.DisplayName, Does.StartWith("reply ")); } - public class TestEndpointWithDestinationNaming : EndpointConfigurationBuilder - { - public TestEndpointWithDestinationNaming() => - EndpointSetup(b => b.Tracing().UseMessageTypeNamesInSpanNames = true); - - [Handler] - public class MessageHandler(Context testContext) : IHandleMessages, - IHandleMessages - { - public Task Handle(IncomingMessage message, IMessageHandlerContext context) => context.Reply(new OutgoingReply()); - - public Task Handle(OutgoingReply message, IMessageHandlerContext context) - { - testContext.MarkAsCompleted(); - return Task.CompletedTask; - } - } - } - public class IncomingMessage : IMessage; public class OutgoingReply : IMessage; diff --git a/src/NServiceBus.AcceptanceTests/NServiceBus.AcceptanceTests.csproj b/src/NServiceBus.AcceptanceTests/NServiceBus.AcceptanceTests.csproj index 2f335d4f9e5..82995eed81c 100644 --- a/src/NServiceBus.AcceptanceTests/NServiceBus.AcceptanceTests.csproj +++ b/src/NServiceBus.AcceptanceTests/NServiceBus.AcceptanceTests.csproj @@ -48,6 +48,7 @@ + diff --git a/src/NServiceBus.AcceptanceTests/obsoletes-v10.cs b/src/NServiceBus.AcceptanceTests/obsoletes-v10.cs new file mode 100644 index 00000000000..1b6fd71d6eb --- /dev/null +++ b/src/NServiceBus.AcceptanceTests/obsoletes-v10.cs @@ -0,0 +1,40 @@ +namespace NServiceBus.AcceptanceTests; + +using System; +using System.Reflection; +using NUnit.Framework; +using NUnit.Framework.Interfaces; + +// ============================================================================= +// EVERYTHING IN THIS FILE IS TEMPORARY AND WILL BE REMOVED IN v11, together with +// the V11BehaviorSwitch block in NServiceBus.Core/obsoletes-v10.cs. The file is +// excluded from the shipped acceptance test sources in the project file. +// ============================================================================= + +// Runs the test with the NServiceBus.Core.OpenTelemetry.UseV11Behavior AppContext switch enabled, i.e. with +// the OpenTelemetry defaults of v11. The switch value is cached process-wide, so the cache is reset through +// the internal V11BehaviorSwitch.ResetUseV11Behavior method. Only safe in [NonParallelizable] fixtures such +// as OpenTelemetryAcceptanceTest. In v11 these defaults are the only behavior: delete this attribute and +// remove it from the tests that use it. +[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class, Inherited = true)] +public sealed class OpenTelemetryV11DefaultsAttribute : Attribute, ITestAction +{ + const string SwitchName = "NServiceBus.Core.OpenTelemetry.UseV11Behavior"; + + static readonly MethodInfo ResetSwitch = typeof(EndpointConfiguration).Assembly + .GetType("NServiceBus.V11BehaviorSwitch", throwOnError: true)! + .GetMethod("ResetUseV11Behavior", BindingFlags.Static | BindingFlags.NonPublic) + ?? throw new InvalidOperationException("NServiceBus.V11BehaviorSwitch.ResetUseV11Behavior not found"); + + public ActionTargets Targets => ActionTargets.Test; + + public void BeforeTest(ITest test) => Set(true); + + public void AfterTest(ITest test) => Set(false); + + static void Set(bool enabled) + { + AppContext.SetSwitch(SwitchName, enabled); + ResetSwitch.Invoke(null, null); + } +} diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt index 5a9feef8a4a..25091c74f63 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/APIApprovals.ApproveNServiceBus.approved.txt @@ -691,13 +691,10 @@ namespace NServiceBus { public InstrumentationOptions() { } public NServiceBus.DelayedDeliveryInstrumentationOptions DelayedDelivery { get; } - public bool EmitMessageDispatchingEvents { get; set; } public NServiceBus.ExceptionRecordingMode ExceptionRecordingMode { get; set; } - public NServiceBus.MetersOptions Meters { get; } public NServiceBus.TraceMode PublishTraceMode { get; set; } public NServiceBus.RecoverabilityInstrumentationOptions Recoverability { get; } public NServiceBus.TraceMode SendTraceMode { get; set; } - public bool UseMessageTypeNamesInSpanNames { get; set; } } public sealed class KeyedServiceKey { @@ -857,11 +854,6 @@ namespace NServiceBus [System.Diagnostics.CodeAnalysis.RequiresUnreferencedCode(@"AddMessageType relies on an NServiceBus source-generated interceptor to register the statically known message hierarchy without reflection when trimming is enabled. If this warning is reported, the interceptor was not used for this call; enable or restore the NServiceBus analyzer/source-generator tooling and use a statically known message type.")] public static void AddMessageType<[System.Diagnostics.CodeAnalysis.DynamicallyAccessedMembers(System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.None | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicParameterlessConstructor | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.NonPublicConstructors | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.PublicProperties | System.Diagnostics.CodeAnalysis.DynamicallyAccessedMemberTypes.Interfaces)] TMessage>(this NServiceBus.EndpointConfiguration config) { } } - public class MetersOptions - { - public MetersOptions() { } - public bool EmitExecutionResultTags { get; set; } - } public static class MetricTagsExtensions { extension(NServiceBus.Pipeline.IBehaviorContext context) diff --git a/src/NServiceBus.Core.Tests/Envelopes/EnvelopeUnwrapperTests.cs b/src/NServiceBus.Core.Tests/Envelopes/EnvelopeUnwrapperTests.cs index b7136605694..d5bd75d9546 100644 --- a/src/NServiceBus.Core.Tests/Envelopes/EnvelopeUnwrapperTests.cs +++ b/src/NServiceBus.Core.Tests/Envelopes/EnvelopeUnwrapperTests.cs @@ -29,7 +29,7 @@ public void Setup() originalBody = "payload"u8.ToArray().AsMemory(); messageContext = new MessageContext(nativeId, originalHeaders, originalBody, new TransportTransaction(), "receiveAddress", new ContextBag()); meterFactory = new TestMeterFactory(); - pipelineMetrics = new PipelineMetrics(meterFactory, "queue", "disc", new MetersOptions()); + pipelineMetrics = new PipelineMetrics(meterFactory, "queue", "disc"); } [TearDown] diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs index b2c983d4278..6c10d823757 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityFactoryTests.cs @@ -49,26 +49,13 @@ static MessageContext CreateMessageContext() => new(Guid.NewGuid().ToString(), [], Array.Empty(), new TransportTransaction(), "receiver", new ContextBag()); } + // Until v11 the "transport span as parent" behavior is opt-in. This fixture runs with the v11 + // defaults because that is what the tests below describe; the pre-v11 default is covered by + // TransportParentSpanDefaultBehaviorTests. In v11, remove the attribute together with the + // V11BehaviorSwitch block in obsoletes-v10.cs. + [OpenTelemetryV11Defaults] class StartIncomingActivity : ActivityFactoryTests { - // Until v11 the "transport span as parent" behavior is opt-in. This fixture runs with it - // enabled because that is the v11 default the tests below describe; the pre-v11 default - // is covered by TransportParentSpanDefaultBehaviorTests. In v11, delete this SetUp/TearDown - // pair together with obsolete_v11.cs. - [SetUp] - public void OptInToTransportSpanAsParent() - { - AppContext.SetSwitch(TransportParentActivitySwitch.UseTransportActivityAsParentSwitchName, true); - TransportParentActivitySwitch.ResetUseTransportActivityAsParent(); - } - - [TearDown] - public void ResetTransportSpanSwitch() - { - AppContext.SetSwitch(TransportParentActivitySwitch.UseTransportActivityAsParentSwitchName, false); - TransportParentActivitySwitch.ResetUseTransportActivityAsParent(); - } - [TestCase(Headers.NServiceBusDiagnosticsTraceParent)] [TestCase(Headers.DiagnosticsTraceParent)] // for backwards compatibility public void Should_attach_to_header_trace_when_available_and_no_ambient_activity(string headerName) diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationCompatibilityTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationCompatibilityTests.cs index ae5dad4ba34..950a847047c 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationCompatibilityTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationCompatibilityTests.cs @@ -7,22 +7,9 @@ using NUnit.Framework; [TestFixture] +[OpenTelemetryV11Defaults] public class ContextPropagationCompatibilityTests { - [SetUp] - public void EnableDistributedContextPropagator() - { - AppContext.SetSwitch(LegacyContextPropagation.UseDistributedContextPropagatorSwitchName, true); - LegacyContextPropagation.ResetUseDistributedContextPropagator(); - } - - [TearDown] - public void ResetDistributedContextPropagator() - { - AppContext.SetSwitch(LegacyContextPropagation.UseDistributedContextPropagatorSwitchName, false); - LegacyContextPropagation.ResetUseDistributedContextPropagator(); - } - delegate void Writer(Activity activity, Dictionary headers); delegate void Reader(Activity activity, IDictionary headers); diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs index 4f496a5e660..af9b6897042 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ContextPropagationDefaultBehaviorTests.cs @@ -9,15 +9,8 @@ [TestFixture] public class ContextPropagationDefaultBehaviorTests { - // Without the opt-in switch, the endpoint default must remain the backwards-compatible - // legacy propagator (percent-encoded, comma-separated, whitespace preserved). - [SetUp] - public void EnsureDefault() - { - AppContext.SetSwitch(LegacyContextPropagation.UseDistributedContextPropagatorSwitchName, false); - LegacyContextPropagation.ResetUseDistributedContextPropagator(); - } - + // Without the OpenTelemetryV11Defaults attribute, the endpoint default must remain the + // backwards-compatible legacy propagator (percent-encoded, comma-separated, whitespace preserved). [Test] public void Default_uses_legacy_percent_encoded_baggage_format() { diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/HandlerActivitySourceTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/HandlerActivitySourceTests.cs index 8af728a9c8a..2d87714a419 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/HandlerActivitySourceTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/HandlerActivitySourceTests.cs @@ -21,21 +21,10 @@ public class HandlerActivitySourceTests public void SetUp() => mainListener = TestingActivityListener.SetupNServiceBusDiagnosticListener(); [TearDown] - public void TearDown() - { - mainListener.Dispose(); - AppContext.SetSwitch(HandlerActivitySourceSwitch.UseHandlerActivitySourceSwitchName, false); - HandlerActivitySourceSwitch.ResetUseHandlerActivitySource(); - } - - static void OptIn() - { - AppContext.SetSwitch(HandlerActivitySourceSwitch.UseHandlerActivitySourceSwitchName, true); - HandlerActivitySourceSwitch.ResetUseHandlerActivitySource(); - } + public void TearDown() => mainListener.Dispose(); [Test] - public void Default_emits_handler_activity_from_main_source() + public void Should_emit_handler_activity_from_main_source() { using var ambientActivity = new Activity("ambient activity"); ambientActivity.Start(); @@ -47,9 +36,9 @@ public void Default_emits_handler_activity_from_main_source() } [Test] - public void Opt_in_emits_handler_activity_from_handler_source() + [OpenTelemetryV11Defaults] + public void Should_emit_handler_activity_from_handler_source() { - OptIn(); using var handlerListener = TestingActivityListener.SetupDiagnosticListener("NServiceBus.Core.Handler"); using var ambientActivity = new Activity("ambient activity"); @@ -62,9 +51,9 @@ public void Opt_in_emits_handler_activity_from_handler_source() } [Test] - public void Opt_in_preserves_display_name_and_handler_type_tag() + [OpenTelemetryV11Defaults] + public void Should_preserve_display_name_and_handler_type_tag() { - OptIn(); using var handlerListener = TestingActivityListener.SetupDiagnosticListener("NServiceBus.Core.Handler"); using var ambientActivity = new Activity("ambient activity"); @@ -80,9 +69,9 @@ public void Opt_in_preserves_display_name_and_handler_type_tag() } [Test] - public void Opt_in_without_handler_source_listener_does_not_create_handler_activity() + [OpenTelemetryV11Defaults] + public void Should_not_create_handler_activity_without_handler_source_listener() { - OptIn(); using var ambientActivity = new Activity("ambient activity"); ambientActivity.Start(); diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/InstrumentationOptionsTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/InstrumentationOptionsTests.cs index b01433e755d..2b1847401bb 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/InstrumentationOptionsTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/InstrumentationOptionsTests.cs @@ -13,7 +13,7 @@ public void Should_default_trace_connectors_to_current_behavior() using (Assert.EnterMultipleScope()) { Assert.That(options.SendTraceMode, Is.EqualTo(TraceMode.ContinueExisting), "sends continue the trace by default"); - // Set by ApplyPreV11Defaults in obsolete_v11.cs. When that file is deleted in v11 this + // Set by ApplyPreV11Defaults in obsoletes-v10.cs. When that file is deleted in v11 this // assertion fails: flip it to TraceMode.ContinueExisting, which is the initializer's value. Assert.That(options.PublishTraceMode, Is.EqualTo(TraceMode.StartNew), "publishes start a new linked trace by default until v11"); } diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/MeterTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/MeterTests.cs index 77eb1764ca4..14bd2817b1f 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/MeterTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/MeterTests.cs @@ -22,7 +22,7 @@ public void Verify_MeterAPI() using var meterFactory = new TestMeterFactory(); //The IncomingPipelineMeter constructor creates the meters, therefore a new instance before collecting the metrics. #pragma warning disable CA1806 - new PipelineMetrics(meterFactory, "queue", "disc", new MetersOptions()); + new PipelineMetrics(meterFactory, "queue", "disc"); #pragma warning restore CA1806 using var metricsListener = TestingMetricListener.SetupNServiceBusMetricsListener(); diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/PipelineMetricsExecutionResultTagTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/PipelineMetricsExecutionResultTagTests.cs new file mode 100644 index 00000000000..0c676641ebd --- /dev/null +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/PipelineMetricsExecutionResultTagTests.cs @@ -0,0 +1,43 @@ +namespace NServiceBus.Core.Tests.OpenTelemetry; + +using System; +using System.Collections.Generic; +using System.Linq; +using AcceptanceTests.Core.OpenTelemetry.Metrics; +using NServiceBus.Pipeline; +using NUnit.Framework; + +// In v11 the execution.result tag is gone: delete this file together with the V11BehaviorSwitch block in +// obsoletes-v10.cs. +[TestFixture] +public class PipelineMetricsExecutionResultTagTests +{ + [Test] + public void Should_emit_execution_result_tag() + { + var tags = RecordFailure(); + + Assert.That(tags.Select(t => t.Key), Does.Contain("execution.result")); + } + + [Test] + [OpenTelemetryV11Defaults] + public void Should_not_emit_execution_result_tag() + { + var tags = RecordFailure(); + + Assert.That(tags.Select(t => t.Key), Does.Not.Contain("execution.result")); + } + + static KeyValuePair[] RecordFailure() + { + using var meterFactory = new TestMeterFactory(); + using var listener = TestingMetricListener.SetupNServiceBusMetricsListener(); + + var metrics = new PipelineMetrics(meterFactory, "queue", "disc"); + metrics.RecordMessageProcessingFailure(new PipelineMetricTags(), new InvalidOperationException("boom")); + + Assert.That(listener.Tags, Has.Count.EqualTo(1), "exactly one metric is expected to be recorded"); + return listener.Tags.Values.Single(); + } +} diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/TransportParentSpanDefaultBehaviorTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/TransportParentSpanDefaultBehaviorTests.cs index 0064600331e..81bad803712 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/TransportParentSpanDefaultBehaviorTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/TransportParentSpanDefaultBehaviorTests.cs @@ -11,9 +11,9 @@ namespace NServiceBus.Core.Tests.OpenTelemetry; using NServiceBus.Transport; using NUnit.Framework; -// Covers the pre-v11 default of TransportParentActivitySwitch: without the opt-in switch the incoming +// Covers the pre-v11 default of V11BehaviorSwitch: without the OpenTelemetryV11Defaults attribute the incoming // message span stays a child of the NServiceBus sender span even when a transport SDK span is -// ambient. In v11 that default is gone, so delete this file together with obsolete_v11.cs. +// ambient. In v11 that default is gone, so delete this file together with the V11BehaviorSwitch block in obsoletes-v10.cs. [TestFixture] public class TransportParentSpanDefaultBehaviorTests { @@ -25,15 +25,12 @@ public class TransportParentSpanDefaultBehaviorTests public void SetUp() { nsbActivityListener = TestingActivityListener.SetupNServiceBusDiagnosticListener(); - AppContext.SetSwitch(TransportParentActivitySwitch.UseTransportActivityAsParentSwitchName, false); - TransportParentActivitySwitch.ResetUseTransportActivityAsParent(); } [TearDown] public void TearDown() { nsbActivityListener.Dispose(); - TransportParentActivitySwitch.ResetUseTransportActivityAsParent(); } [Test] diff --git a/src/NServiceBus.Core.Tests/Pipeline/Incoming/InvokeHandlerTerminatorTest.cs b/src/NServiceBus.Core.Tests/Pipeline/Incoming/InvokeHandlerTerminatorTest.cs index 5948f081060..2b74cf4a74d 100644 --- a/src/NServiceBus.Core.Tests/Pipeline/Incoming/InvokeHandlerTerminatorTest.cs +++ b/src/NServiceBus.Core.Tests/Pipeline/Incoming/InvokeHandlerTerminatorTest.cs @@ -11,7 +11,7 @@ [TestFixture] public class InvokeHandlerTerminatorTest { - readonly InvokeHandlerTerminator terminator = new(new PipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); + readonly InvokeHandlerTerminator terminator = new(new PipelineMetrics(new TestMeterFactory(), "queue", "disc")); [Test] public async Task When_saga_found_and_handler_is_saga_should_invoke_handler() diff --git a/src/NServiceBus.Core.Tests/Pipeline/Incoming/SerializeMessageConnectorTests.cs b/src/NServiceBus.Core.Tests/Pipeline/Incoming/SerializeMessageConnectorTests.cs index dacfb4e7447..9aeddeaec52 100644 --- a/src/NServiceBus.Core.Tests/Pipeline/Incoming/SerializeMessageConnectorTests.cs +++ b/src/NServiceBus.Core.Tests/Pipeline/Incoming/SerializeMessageConnectorTests.cs @@ -59,7 +59,7 @@ static Task InvokeSerializer(TestableOutgoingLogicalMessageContext context, stri var behavior = new SerializeMessageConnector( new FakeSerializer("myContentType"), registry, - new PipelineMetrics(new TestMeterFactory(), queueName, discriminator, new MetersOptions())); + new PipelineMetrics(new TestMeterFactory(), queueName, discriminator)); return behavior.Invoke(context, _ => Task.CompletedTask); } diff --git a/src/NServiceBus.Core.Tests/Pipeline/MainPipelineExecutorTests.cs b/src/NServiceBus.Core.Tests/Pipeline/MainPipelineExecutorTests.cs index 9e56b4ee174..f78b9f7261b 100644 --- a/src/NServiceBus.Core.Tests/Pipeline/MainPipelineExecutorTests.cs +++ b/src/NServiceBus.Core.Tests/Pipeline/MainPipelineExecutorTests.cs @@ -123,7 +123,7 @@ static MessageContext CreateMessageContext() => static MainPipelineExecutor CreateMainPipelineExecutor(ServiceProvider serviceProvider, IPipeline receivePipeline) { - var incomingPipelineMetrics = new PipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions()); + var incomingPipelineMetrics = new PipelineMetrics(new TestMeterFactory(), "queue", "disc"); var executor = new MainPipelineExecutor( serviceProvider, new PipelineCache(serviceProvider, new PipelineModifications()), diff --git a/src/NServiceBus.Core.Tests/Pipeline/PipelineMetricTagsTests.cs b/src/NServiceBus.Core.Tests/Pipeline/PipelineMetricTagsTests.cs index b316643e361..aa3353a319c 100644 --- a/src/NServiceBus.Core.Tests/Pipeline/PipelineMetricTagsTests.cs +++ b/src/NServiceBus.Core.Tests/Pipeline/PipelineMetricTagsTests.cs @@ -36,7 +36,7 @@ public void Should_not_fail_when_handling_more_than_one_logical_message() }; var messageMapper = new MessageMapper(); - var behavior = new DeserializeMessageConnector(new MessageDeserializerResolver(new FakeSerializer(), []), new LogicalMessageFactory(registry, messageMapper), registry, messageMapper, false, new PipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); + var behavior = new DeserializeMessageConnector(new MessageDeserializerResolver(new FakeSerializer(), []), new LogicalMessageFactory(registry, messageMapper), registry, messageMapper, false, new PipelineMetrics(new TestMeterFactory(), "queue", "disc")); Assert.DoesNotThrowAsync(async () => await behavior.Invoke(context, c => { diff --git a/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs b/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs index 1f5eae8ffc0..82d939f1946 100644 --- a/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs +++ b/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs @@ -172,6 +172,29 @@ await Invoke(context, c => Assert.That(pipelineActivity.Events.Count(e => e.Name == "Finished dispatching"), Is.EqualTo(1)); } + // In v11 the dispatching events are gone: delete this test and + // Should_add_batch_dispatch_events_when_sending_batched_messages together with the + // V11BehaviorSwitch block in obsoletes-v10.cs. + [Test] + [OpenTelemetryV11Defaults] + public async Task Should_not_add_batch_dispatch_events_when_sending_batched_messages() + { + var context = CreateContext(fakeBatchPipeline, Guid.NewGuid().ToString()); + + using var pipelineActivity = new Activity("test activity"); + pipelineActivity.Start(); + context.Extensions.SetIncomingPipelineActivity(pipelineActivity); + + await Invoke(context, c => + { + var batchedSends = c.Extensions.Get(); + batchedSends.Add(new TransportOperation(new OutgoingMessage(Guid.NewGuid().ToString(), [], Array.Empty()), new UnicastAddressTag("destination"))); + return Task.CompletedTask; + }); + + Assert.That(pipelineActivity.Events, Is.Empty); + } + [Test] public async Task Should_not_add_batch_dispatch_events_when_no_batched_messages() { @@ -216,7 +239,7 @@ await Invoke(context, c => public async Task Should_still_dispatch_when_outbox_is_disabled() { var noOpBehavior = new TransportReceiveToPhysicalMessageConnector( - new NoOpOutboxStorage(), new PipelineMetrics(fakeMeterFactory, "queue", "disc", new MetersOptions()), new InstrumentationOptions(), NullLogger.Instance); + new NoOpOutboxStorage(), new PipelineMetrics(fakeMeterFactory, "queue", "disc"), NullLogger.Instance); var context = CreateContext(fakeBatchPipeline, "id"); @@ -267,7 +290,7 @@ public void SetUp() fakeBatchPipeline = new FakeBatchPipeline(); fakeMeterFactory = new TestMeterFactory(); - behavior = new TransportReceiveToPhysicalMessageConnector(fakeOutbox, new PipelineMetrics(fakeMeterFactory, "queue", "disc", new MetersOptions()), new InstrumentationOptions(), NullLogger.Instance); + behavior = new TransportReceiveToPhysicalMessageConnector(fakeOutbox, new PipelineMetrics(fakeMeterFactory, "queue", "disc"), NullLogger.Instance); } [TearDown] diff --git a/src/NServiceBus.Core.Tests/Unicast/LoadHandlersConnectorTests.cs b/src/NServiceBus.Core.Tests/Unicast/LoadHandlersConnectorTests.cs index 291c25dac83..7a4f512495c 100644 --- a/src/NServiceBus.Core.Tests/Unicast/LoadHandlersConnectorTests.cs +++ b/src/NServiceBus.Core.Tests/Unicast/LoadHandlersConnectorTests.cs @@ -17,7 +17,7 @@ public class LoadHandlersConnectorTests [Test] public void Should_throw_when_there_are_no_registered_message_handlers() { - var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance, new PipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); + var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance, new PipelineMetrics(new TestMeterFactory(), "queue", "disc")); var context = new TestableIncomingLogicalMessageContext(); @@ -30,7 +30,7 @@ public void Should_throw_when_there_are_no_registered_message_handlers() [Test] public void Should_throw_if_ambient_transaction_is_different_from_scope_used_by_transport() { - var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance, new PipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); + var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance, new PipelineMetrics(new TestMeterFactory(), "queue", "disc")); var context = new TestableIncomingLogicalMessageContext(); @@ -50,7 +50,7 @@ public void Should_throw_if_ambient_transaction_is_different_from_scope_used_by_ [Test] public void Should_throw_if_ambient_transaction_suppressed_when_transport_uses_a_scope() { - var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance, new PipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); + var behavior = new LoadHandlersConnector(new MessageHandlerRegistry(), NoOpActivityFactory.Instance, new PipelineMetrics(new TestMeterFactory(), "queue", "disc")); var context = new TestableIncomingLogicalMessageContext(); @@ -78,7 +78,7 @@ public void Should_not_throw_if_ambient_scope_is_same_as_transport_scope() context.Services.AddSingleton(); context.Extensions.Set(new NoOpOutboxTransaction()); - var behavior = new LoadHandlersConnector(messageHandlerRegistry, NoOpActivityFactory.Instance, new PipelineMetrics(new TestMeterFactory(), "queue", "disc", new MetersOptions())); + var behavior = new LoadHandlersConnector(messageHandlerRegistry, NoOpActivityFactory.Instance, new PipelineMetrics(new TestMeterFactory(), "queue", "disc")); using (new TransactionScope(TransactionScopeAsyncFlowOption.Enabled)) { diff --git a/src/NServiceBus.Core.Tests/obsoletes-v10.cs b/src/NServiceBus.Core.Tests/obsoletes-v10.cs new file mode 100644 index 00000000000..f1b823e10e3 --- /dev/null +++ b/src/NServiceBus.Core.Tests/obsoletes-v10.cs @@ -0,0 +1,29 @@ +namespace NServiceBus.Core.Tests; + +using System; +using NUnit.Framework; +using NUnit.Framework.Interfaces; + +// ============================================================================= +// EVERYTHING IN THIS FILE IS TEMPORARY AND WILL BE REMOVED IN v11, together with +// the V11BehaviorSwitch block in NServiceBus.Core/obsoletes-v10.cs. +// ============================================================================= + +// Runs the test with the NServiceBus.Core.OpenTelemetry.UseV11Behavior AppContext switch enabled, i.e. with +// the OpenTelemetry defaults of v11. In v11 these defaults are the only behavior: delete this attribute and +// remove it from the tests that use it. +[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class, Inherited = true)] +public sealed class OpenTelemetryV11DefaultsAttribute : Attribute, ITestAction +{ + public ActionTargets Targets => ActionTargets.Test; + + public void BeforeTest(ITest test) => Set(true); + + public void AfterTest(ITest test) => Set(false); + + static void Set(bool enabled) + { + AppContext.SetSwitch(V11BehaviorSwitch.UseV11BehaviorSwitchName, enabled); + V11BehaviorSwitch.ResetUseV11Behavior(); + } +} diff --git a/src/NServiceBus.Core/EndpointCreator.cs b/src/NServiceBus.Core/EndpointCreator.cs index 9cbd8239550..8ba13b52343 100644 --- a/src/NServiceBus.Core/EndpointCreator.cs +++ b/src/NServiceBus.Core/EndpointCreator.cs @@ -156,7 +156,7 @@ void Configure() pipelineSettings); receiveComponent.AddManifest(hostingConfiguration, settings); - pipelineComponent = PipelineComponent.Initialize(pipelineSettings, hostingConfiguration, receiveConfiguration, hostingConfiguration.ActivityFactory.Options.Meters); + pipelineComponent = PipelineComponent.Initialize(pipelineSettings, hostingConfiguration, receiveConfiguration); // The settings can only be locked after initializing the feature component since it uses the settings to store & share feature state. // As well as all the other components have been initialized diff --git a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs index c1ebb7ad1ab..33080ad7218 100644 --- a/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs +++ b/src/NServiceBus.Core/OpenTelemetry/InstrumentationOptions.cs @@ -13,7 +13,7 @@ public partial class InstrumentationOptions /// public InstrumentationOptions() => ApplyPreV11Defaults(); - // Implemented in obsolete_v11.cs. A partial void method without an implementation is removed + // Implemented in obsoletes-v10.cs. A partial void method without an implementation is removed // by the compiler, so deleting that file makes the initializers below the final defaults. partial void ApplyPreV11Defaults(); @@ -41,7 +41,7 @@ public partial class InstrumentationOptions /// /// Controls how the receive-side processing span relates to the publish span for events published by this endpoint. /// Defaults to : receivers continue the trace. - /// Until v11 the default is for backward compatibility, see obsolete_v11.cs. + /// Until v11 the default is for backward compatibility, see obsoletes-v10.cs. /// Can be overridden per message via /// or . /// diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs index ec5f69f29fd..418698e2754 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs @@ -12,7 +12,7 @@ static class ActivityDisplayNames public const string ReplyMessage = "reply"; public const string Recoverability = "recover"; - // Operation-only prefixes used when UseMessageTypeNamesInSpanNames is enabled + // Operation-only prefixes used when V11BehaviorSwitch.UseV11Behavior is enabled (the only naming in v11, see obsoletes-v10.cs) internal const string ProcessOperation = "process"; internal const string PublishOperation = "publish"; internal const string SendOperation = "send"; diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs index 687dcd15067..5e0cbfabe30 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs @@ -42,7 +42,7 @@ sealed partial class ActivityFactory(InstrumentationOptions options) : IActivity Activity.Current = null; activity = activitySource.CreateActivity(activityName, ActivityKind.Consumer, parentContext: default, links: [new ActivityLink(senderContext)]); } - else if (TransportParentActivitySwitch.UseTransportActivityAsParent && Activity.Current != null) // remove the switch check in v11, see obsolete_v11.cs + else if (V11BehaviorSwitch.UseV11Behavior && Activity.Current != null) // remove the switch check in v11, see obsoletes-v10.cs { // A transport SDK receive span is ambient: make it the parent (an activity without // a parent context adopts Activity.Current when it starts) and link to the NSB sender span. @@ -136,7 +136,7 @@ static bool TryParseSenderContext(Dictionary headers, out Activi return activity; } - activity.DisplayName = Options.UseMessageTypeNamesInSpanNames + activity.DisplayName = V11BehaviorSwitch.UseV11Behavior ? $"{ActivityDisplayNames.ProcessOperation} {context.ReceiveAddress}" : ActivityDisplayNames.ProcessMessage; @@ -170,7 +170,8 @@ static bool TryParseSenderContext(Dictionary headers, out Activi // Until v11 the dedicated handler source is opt-in; existing configurations only // subscribe to the main source and must keep receiving handler spans from it. - var source = HandlerActivitySourceSwitch.UseHandlerActivitySource + // Remove the switch check in v11, see obsoletes-v10.cs. + var source = V11BehaviorSwitch.UseV11Behavior ? ActivitySources.Handler : ActivitySources.Main; @@ -211,7 +212,7 @@ public void UpdateActivityFromRecoverabilityAction(Activity activity, Recoverabi activity.AddTag(ActivityTags.RecoverabilityAction, "immediate_retry"); activity.DisplayName = ActivityDisplayNames.ImmediateRetryOperation; - if (Options.UseMessageTypeNamesInSpanNames) + if (V11BehaviorSwitch.UseV11Behavior) { activity.DisplayName += $" {receiveAddress}"; } @@ -221,7 +222,7 @@ public void UpdateActivityFromRecoverabilityAction(Activity activity, Recoverabi activity.AddTag(ActivityTags.RecoverabilityAction, "delayed_retry"); activity.DisplayName = ActivityDisplayNames.DelayedRetryOperation; - if (Options.UseMessageTypeNamesInSpanNames) + if (V11BehaviorSwitch.UseV11Behavior) { activity.DisplayName += $" {receiveAddress}"; } @@ -230,7 +231,7 @@ public void UpdateActivityFromRecoverabilityAction(Activity activity, Recoverabi { activity.AddTag(ActivityTags.RecoverabilityAction, "move_to_error"); - activity.DisplayName = Options.UseMessageTypeNamesInSpanNames + activity.DisplayName = V11BehaviorSwitch.UseV11Behavior ? $"{ActivityDisplayNames.MoveToErrorOperation} {moveToError.ErrorQueue}" : $"{ActivityDisplayNames.MoveToErrorOperation} error"; } @@ -251,7 +252,11 @@ public void RecordError(Activity? activity, Exception exception, IServiceProvide activity.SetStatus(ActivityStatusCode.Error, exception.Message); activity.SetTag(ActivityTags.ErrorType, exception.GetType().FullName); - LegacyExceptionTags.SetLegacyStatusTags(activity, exception); + // Removed in v11, see obsoletes-v10.cs + if (!V11BehaviorSwitch.UseV11Behavior) + { + LegacyExceptionTags.SetLegacyStatusTags(activity, exception); + } if (!exception.Data.Contains(ExceptionRecordedFlag)) { @@ -263,7 +268,7 @@ public void RecordError(Activity? activity, Exception exception, IServiceProvide } else { - activity.AddException(exception, LegacyExceptionTags.EscapedTagList); + activity.AddException(exception, V11BehaviorSwitch.UseV11Behavior ? default : LegacyExceptionTags.EscapedTagList); // drop the tag list in v11, see obsoletes-v10.cs } exception.Data[ExceptionRecordedFlag] = true; diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs index ffa1b15091f..083b3961bc8 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ContextPropagation.cs @@ -22,8 +22,8 @@ public static void PropagateContextToHeaders(Activity? activity, Dictionary headers) { - // Removed in v11, see obsolete_v11.cs - if (!LegacyContextPropagation.UseDistributedContextPropagator) + // Removed in v11, see obsoletes-v10.cs + if (!V11BehaviorSwitch.UseV11Behavior) { LegacyContextPropagation.PropagateTraceStateFromHeaders(activity, headers); return; @@ -69,8 +69,8 @@ public static void PropagateTraceStateFromHeaders(Activity activity, IDictionary // the headers. Otherwise every hop would put each key on the wire twice. public static void PropagateBaggageFromHeaders(Activity activity, IDictionary headers) { - // Removed in v11, see obsolete_v11.cs - if (!LegacyContextPropagation.UseDistributedContextPropagator) + // Removed in v11, see obsoletes-v10.cs + if (!V11BehaviorSwitch.UseV11Behavior) { LegacyContextPropagation.PropagateBaggageFromHeaders(activity, headers, activity.Parent); return; diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs deleted file mode 100644 index 4836a5834db..00000000000 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/obsolete_v11.cs +++ /dev/null @@ -1,328 +0,0 @@ -#nullable enable - -namespace NServiceBus; - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.Linq; -using Particular.Obsoletes; - -// ============================================================================= -// EVERYTHING IN THIS FILE IS TEMPORARY AND WILL BE REMOVED IN v11. -// -// In v10.3, switching to System.Diagnostics.DistributedContextPropagator for -// OpenTelemetry trace-context/baggage propagation changes the baggage wire -// format (W3C OWS encoding + whitespace trimming) and is therefore breaking on -// rolling upgrades. To stay backwards compatible it is opt-in via an AppContext -// switch and the legacy propagator below remains the default. -// -// In v11 the new propagator becomes the default: delete this entire file and -// remove the `if (!LegacyContextPropagation.UseDistributedContextPropagator)` -// delegation blocks in ContextPropagation.cs. -// ============================================================================= -static class LegacyContextPropagation -{ - enum SwitchState : byte - { - Unchecked = 0, - Enabled = 1, - Disabled = 2 - } - - static SwitchState cachedUseDistributedContextPropagator; - - [PreObsolete("https://github.com/Particular/NServiceBus/issues/7825", - Note = "In v11, DistributedContextPropagator-based context propagation becomes the default and this switch will be removed together with the legacy propagator in obsolete_v11.cs.", - ReplacementTypeOrMember = "ContextPropagation")] - public const string UseDistributedContextPropagatorSwitchName = "NServiceBus.Core.OpenTelemetry.UseDistributedContextPropagator"; - - [PreObsolete("https://github.com/Particular/NServiceBus/issues/7825", - Note = "In v11, DistributedContextPropagator-based context propagation becomes the default and this switch will be removed together with the legacy propagator in obsolete_v11.cs.", - ReplacementTypeOrMember = "ContextPropagation")] - public static bool UseDistributedContextPropagator - { - get - { - var state = cachedUseDistributedContextPropagator; - if (state != SwitchState.Unchecked) - { - return state == SwitchState.Enabled; - } - - state = AppContext.TryGetSwitch(UseDistributedContextPropagatorSwitchName, out var isEnabled) && isEnabled - ? SwitchState.Enabled - : SwitchState.Disabled; - cachedUseDistributedContextPropagator = state; - - return state == SwitchState.Enabled; - } - } - - internal static void ResetUseDistributedContextPropagator() => cachedUseDistributedContextPropagator = SwitchState.Unchecked; - - public static void PropagateContextToHeaders(Activity? activity, Dictionary headers) - { - if (activity is null) - { - return; - } - - if (activity.Id is not null) - { - headers[Headers.DiagnosticsTraceParent] = activity.Id; - } - - if (activity.TraceStateString is not null) - { - headers[Headers.DiagnosticsTraceState] = activity.TraceStateString; - } - - var baggage = string.Join(",", activity.Baggage.Select(item => $"{item.Key}={Uri.EscapeDataString(item.Value ?? string.Empty)}")); - if (!string.IsNullOrEmpty(baggage)) - { - headers[Headers.DiagnosticsBaggage] = baggage; - } - } - - public static void PropagateContextFromHeaders(Activity? activity, IDictionary headers) - { - if (activity is null) - { - return; - } - - PropagateTraceStateFromHeaders(activity, headers); - PropagateBaggageFromHeaders(activity, headers); - } - - public static void PropagateTraceStateFromHeaders(Activity activity, IDictionary headers) - { - if (headers.TryGetValue(Headers.DiagnosticsTraceState, out var traceState)) - { - activity.TraceStateString = traceState; - } - } - - // parent: the activity whose baggage chain is checked so keys it already carries are not added again, - // see ContextPropagation.PropagateBaggageFromHeaders. - public static void PropagateBaggageFromHeaders(Activity activity, IDictionary headers, Activity? parent = null) - { - if (!headers.TryGetValue(Headers.DiagnosticsBaggage, out var baggageValue)) - { - return; - } - - var baggageSpan = baggageValue.AsSpan(); - // HINT: Iterate in reverse order because Activity baggage is LIFO - while (!baggageSpan.IsEmpty) - { - var lastComma = baggageSpan.LastIndexOf(','); - ReadOnlySpan baggageItem; - - if (lastComma >= 0) - { - baggageItem = baggageSpan[(lastComma + 1)..]; - baggageSpan = baggageSpan[..lastComma]; - } - else - { - baggageItem = baggageSpan; - baggageSpan = []; - } - - var firstEquals = baggageItem.IndexOf('='); - if (firstEquals < 0 || firstEquals >= baggageItem.Length) - { - continue; - } - - var key = baggageItem[..firstEquals].Trim().ToString(); - if (parent?.GetBaggageItem(key) is not null) - { - continue; - } - - var value = baggageItem[(firstEquals + 1)..]; - activity.AddBaggage(key, Uri.UnescapeDataString(value)); - } - } -} - -// Handler spans move from the "NServiceBus.Core" ActivitySource to the dedicated -// "NServiceBus.Core.Handler" source so they can be filtered/sampled independently -// (https://github.com/Particular/NServiceBus/issues/7284). Existing OpenTelemetry -// configurations only subscribe to "NServiceBus.Core" and would silently lose handler -// spans, so the new source is opt-in via an AppContext switch until v11. -// -// In v11 the dedicated source becomes the default: delete this class and remove the -// source selection in ActivityFactory.StartHandlerActivity so it always uses -// ActivitySources.Handler. -static class HandlerActivitySourceSwitch -{ - enum SwitchState : byte - { - Unchecked = 0, - Enabled = 1, - Disabled = 2 - } - - static SwitchState cachedUseHandlerActivitySource; - - [PreObsolete("https://github.com/Particular/NServiceBus/issues/7284", - Note = "In v11, handler spans are always emitted from the NServiceBus.Core.Handler ActivitySource and this switch will be removed.", - ReplacementTypeOrMember = "ActivitySources.Handler")] - public const string UseHandlerActivitySourceSwitchName = "NServiceBus.Core.OpenTelemetry.UseHandlerActivitySource"; - - [PreObsolete("https://github.com/Particular/NServiceBus/issues/7284", - Note = "In v11, handler spans are always emitted from the NServiceBus.Core.Handler ActivitySource and this switch will be removed.", - ReplacementTypeOrMember = "ActivitySources.Handler")] - public static bool UseHandlerActivitySource - { - get - { - var state = cachedUseHandlerActivitySource; - if (state != SwitchState.Unchecked) - { - return state == SwitchState.Enabled; - } - - state = AppContext.TryGetSwitch(UseHandlerActivitySourceSwitchName, out var isEnabled) && isEnabled - ? SwitchState.Enabled - : SwitchState.Disabled; - cachedUseHandlerActivitySource = state; - - return state == SwitchState.Enabled; - } - } - - internal static void ResetUseHandlerActivitySource() => cachedUseHandlerActivitySource = SwitchState.Unchecked; -} - - -// When a transport SDK (Azure Service Bus, RabbitMQ, SQS, ...) has its own OpenTelemetry -// instrumentation, an SDK "receive" span is the ambient Activity.Current while the incoming -// pipeline starts. The incoming message span should then be a child of that SDK span and only -// link to the NServiceBus sender span. Until v11 the sender span stays the parent by default so -// existing trace shapes don't change on a minor upgrade; the new shape is opt-in via an -// AppContext switch. -// -// In v11 the SDK span becomes the parent unconditionally: delete this class, remove the -// `TransportParentActivitySwitch.UseTransportSpanAsParent` check in -// ActivityFactory.CreateActivityFromIncomingMessage (keeping only the branch that links to the -// sender span), delete TransportParentSpanDefaultBehaviorTests.cs and remove the opt-in -// SetUp/TearDown pair in ActivityFactoryTests.StartIncomingActivity. -static class TransportParentActivitySwitch -{ - enum SwitchState : byte - { - Unchecked = 0, - Enabled = 1, - Disabled = 2 - } - - static SwitchState cachedUseTransportActivityAsParent; - - public const string UseTransportActivityAsParentSwitchName = "NServiceBus.Core.OpenTelemetry.UseTransportActivityAsParent"; - - public static bool UseTransportActivityAsParent - { - get - { - var state = cachedUseTransportActivityAsParent; - if (state != SwitchState.Unchecked) - { - return state == SwitchState.Enabled; - } - - state = AppContext.TryGetSwitch(UseTransportActivityAsParentSwitchName, out var isEnabled) && isEnabled - ? SwitchState.Enabled - : SwitchState.Disabled; - cachedUseTransportActivityAsParent = state; - - return state == SwitchState.Enabled; - } - } - - internal static void ResetUseTransportActivityAsParent() => cachedUseTransportActivityAsParent = SwitchState.Unchecked; -} - -// This class bridges two independent legacy exception-tagging behaviors, both -// scheduled for removal in v11: -// -// - SetLegacyStatusTags sets the "otel.status_code"/"otel.status_description" -// tags, which predate native support for Activity.SetStatus/Activity.Status -// and are now redundant with it. Kept only for consumers still reading the -// tags directly instead of Activity.Status. -// - EscapedTagList carries "exception.escaped", an attribute the OTel semantic -// conventions have marked Deprecated: -// https://opentelemetry.io/docs/specs/semconv/exceptions/exceptions-logs/ -// It's added to the exception event for backward compatibility with -// existing consumers of that attribute. -// -// In v11, delete this entire class, remove the -// `LegacyExceptionTags.SetLegacyStatusTags(activity, exception);` call in -// ActivityFactory.RecordError, and stop passing EscapedTagList to -// activity.AddException in the same method. -static class LegacyExceptionTags -{ - public static void SetLegacyStatusTags(Activity activity, Exception exception) - { - activity.SetTag("otel.status_code", "ERROR"); - activity.SetTag("otel.status_description", exception.Message); - } - - public static TagList EscapedTagList { get; } = new() { { "exception.escaped", true } }; -} - -public partial class InstrumentationOptions -{ - // Publishes start a new linked trace by default in v10 for backward compatibility. In v11, - // delete this method: PublishTraceMode then defaults to TraceMode.ContinueExisting through - // its initializer in InstrumentationOptions.cs, and the empty partial declaration plus the - // constructor call there can be removed. - partial void ApplyPreV11Defaults() => PublishTraceMode = TraceMode.StartNew; - - /// - /// Appends the destination to span names following the OTel messaging convention - /// {messaging.operation.name} {destination}, e.g. "process orders" or "send payments". - /// Disabled by default for backward compatibility. - /// - /// - /// In v11, this property will be dropped and "true" will be the only behavior. - /// - public bool UseMessageTypeNamesInSpanNames { get; set; } - - /// - /// Controls whether the "Start dispatching" and "Finished dispatching" activity events - /// are added to the incoming message span when outgoing messages are dispatched. - /// Enabled by default for backward compatibility. Disable to avoid the ingestion cost - /// of these events when they add no diagnostic value. - /// - /// - /// In v11, this property will be dropped and "false" will be the only behavior. - /// - public bool EmitMessageDispatchingEvents { get; set; } = true; - - /// - /// Controls meter instruments behaviors. - /// - public MetersOptions Meters { get; } = new(); -} - -/// -/// Controls opt-in meter instruments behaviors. -/// Accessed via endpointConfiguration.Tracing().Meters. -/// -public class MetersOptions -{ - /// - /// Emits the legacy execution.result tag with values "success" or "failure" - /// on handler time, processing time, saga fetch time, deserialize time, and serialize time metrics. - /// Enabled by default for backwards compatibility. Disable to reduce tag cardinality. - /// - /// - /// In v11, this property will be dropped and "false" will be the only behavior. - /// - public bool EmitExecutionResultTags { get; set; } = true; -} diff --git a/src/NServiceBus.Core/Pipeline/Incoming/PipelineMetrics.cs b/src/NServiceBus.Core/Pipeline/Incoming/PipelineMetrics.cs index 84c8aa2c7b0..04a9a284d9e 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/PipelineMetrics.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/PipelineMetrics.cs @@ -30,9 +30,9 @@ class PipelineMetrics const string CommitTime = "nservicebus.persistence.commit_time"; // queueName and discriminator are null for send-only endpoints, which have no receive queue to report. - public PipelineMetrics(IMeterFactory meterFactory, string? queueName, string? discriminator, MetersOptions metersOptions) + public PipelineMetrics(IMeterFactory meterFactory, string? queueName, string? discriminator) { - emitExecutionResultTags = metersOptions.EmitExecutionResultTags; + emitExecutionResultTags = !V11BehaviorSwitch.UseV11Behavior; // removed in v11 together with the tag, see obsoletes-v10.cs var meter = meterFactory.Create("NServiceBus.Core.Pipeline.Incoming", "0.4.0"); totalProcessedSuccessfully = meter.CreateCounter(TotalProcessedSuccessfully, description: "Total number of messages processed successfully by the endpoint."); diff --git a/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs b/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs index 8ba0e81163b..d9d45700d2a 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs @@ -17,7 +17,6 @@ namespace NServiceBus; partial class TransportReceiveToPhysicalMessageConnector( IOutboxStorage outboxStorage, PipelineMetrics pipelineMetrics, - InstrumentationOptions instrumentationOptions, ILogger logger) : IStageForkConnector { @@ -134,7 +133,7 @@ async Task InvokeWithoutOutbox(ITransportReceiveContext context, Func 0 && operations[0].AddressTag is UnicastAddressTag unicastTag && outgoingMessage.Headers.TryGetValue(Headers.MessageIntent, out var intentStr) diff --git a/src/NServiceBus.Core/Pipeline/PipelineComponent.cs b/src/NServiceBus.Core/Pipeline/PipelineComponent.cs index cedb2d2ea7a..75e01b3a046 100644 --- a/src/NServiceBus.Core/Pipeline/PipelineComponent.cs +++ b/src/NServiceBus.Core/Pipeline/PipelineComponent.cs @@ -12,8 +12,7 @@ sealed class PipelineComponent PipelineComponent(PipelineModifications modifications) => this.modifications = modifications; public static PipelineComponent Initialize(PipelineSettings settings, - HostingComponent.Configuration hostingConfiguration, ReceiveComponent.Configuration receiveConfiguration, - MetersOptions metersOptions) + HostingComponent.Configuration hostingConfiguration, ReceiveComponent.Configuration receiveConfiguration) { // make the PipelineMetrics available to the Pipeline hostingConfiguration.Services.AddSingleton(sp => @@ -22,11 +21,11 @@ public static PipelineComponent Initialize(PipelineSettings settings, if (receiveConfiguration.IsSendOnlyEndpoint) { - return new PipelineMetrics(meterFactory, null, null, metersOptions); + return new PipelineMetrics(meterFactory, null, null); } string discriminator = receiveConfiguration.InstanceSpecificQueueAddress?.Discriminator ?? ""; - return new PipelineMetrics(meterFactory, receiveConfiguration.LocalQueueAddress.BaseAddress, discriminator, metersOptions); + return new PipelineMetrics(meterFactory, receiveConfiguration.LocalQueueAddress.BaseAddress, discriminator); }); return new PipelineComponent(settings.modifications); diff --git a/src/NServiceBus.Core/Receiving/ReceiveComponent.cs b/src/NServiceBus.Core/Receiving/ReceiveComponent.cs index f25328126dc..887d5ff1494 100644 --- a/src/NServiceBus.Core/Receiving/ReceiveComponent.cs +++ b/src/NServiceBus.Core/Receiving/ReceiveComponent.cs @@ -66,13 +66,12 @@ public static ReceiveComponent Configure( return transport.Receivers[MainReceiverId].Subscriptions; }); - pipelineSettings.Register("TransportReceiveToPhysicalMessageProcessingConnector", b => + pipelineSettings.Register("TransportReceiveToPhysicalMessageProcessingConnector", static b => { var storage = b.GetService() ?? new NoOpOutboxStorage(); return new TransportReceiveToPhysicalMessageConnector( storage, b.GetRequiredService(), - hostingConfiguration.ActivityFactory.Options, b.GetRequiredService>() ); }, "Allows to abort processing the message"); diff --git a/src/NServiceBus.Core/Unicast/MessageOperations.cs b/src/NServiceBus.Core/Unicast/MessageOperations.cs index c3a8e40d784..1bd1af04ca1 100644 --- a/src/NServiceBus.Core/Unicast/MessageOperations.cs +++ b/src/NServiceBus.Core/Unicast/MessageOperations.cs @@ -25,7 +25,7 @@ class MessageOperations( protected readonly IPipeline replyPipeline = replyPipeline; protected readonly IPipeline subscribePipeline = subscribePipeline; protected readonly IPipeline unsubscribePipeline = unsubscribePipeline; - readonly bool UseMessageTypeNamesInSpanNames = activityFactory.Options.UseMessageTypeNamesInSpanNames; + readonly bool useMessageTypeNamesInSpanNames = V11BehaviorSwitch.UseV11Behavior; // removed in v11, see obsoletes-v10.cs public Task Publish<[DynamicallyAccessedMembers(DynamicMemberTypeAccess.Message)] T>(IBehaviorContext context, T message, PublishOptions options) @@ -69,7 +69,7 @@ async Task PublishMessage(IBehaviorContext context, [DynamicallyAccessedMembers( MergeDispatchProperties(publishContext, options.DispatchProperties); - var displayName = UseMessageTypeNamesInSpanNames + var displayName = useMessageTypeNamesInSpanNames ? $"{ActivityDisplayNames.PublishOperation} {messageType.Name}" : ActivityDisplayNames.PublishEvent; @@ -103,7 +103,7 @@ public async Task Subscribe(IBehaviorContext context, Type[] eventTypes, Subscri MergeDispatchProperties(subscribeContext, options.DispatchProperties); - var displayName = UseMessageTypeNamesInSpanNames + var displayName = useMessageTypeNamesInSpanNames ? $"{ActivityDisplayNames.SubscribeEvent} {string.Join(' ', eventTypes.Select(x => x.Name))}" : ActivityDisplayNames.SubscribeEvent; @@ -132,7 +132,7 @@ public async Task Unsubscribe(IBehaviorContext context, Type eventType, Unsubscr MergeDispatchProperties(unsubscribeContext, options.DispatchProperties); - var displayName = UseMessageTypeNamesInSpanNames + var displayName = useMessageTypeNamesInSpanNames ? $"{ActivityDisplayNames.UnsubscribeEvent} {eventType.Name}" : ActivityDisplayNames.UnsubscribeEvent; @@ -193,7 +193,7 @@ async Task SendMessage(IBehaviorContext context, [DynamicallyAccessedMembers(Dyn MergeDispatchProperties(outgoingContext, options.DispatchProperties); - var displayName = UseMessageTypeNamesInSpanNames + var displayName = useMessageTypeNamesInSpanNames ? $"{ActivityDisplayNames.SendMessage} {messageType.Name}" : ActivityDisplayNames.SendMessage; @@ -253,7 +253,7 @@ async Task ReplyMessage(IBehaviorContext context, [DynamicallyAccessedMembers(Dy MergeDispatchProperties(outgoingContext, options.DispatchProperties); - var displayName = UseMessageTypeNamesInSpanNames + var displayName = useMessageTypeNamesInSpanNames ? $"{ActivityDisplayNames.ReplyMessage} {messageType.Name}" : ActivityDisplayNames.ReplyMessage; diff --git a/src/NServiceBus.Core/obsoletes-v10.cs b/src/NServiceBus.Core/obsoletes-v10.cs index 208b3c2cade..d5042055337 100644 --- a/src/NServiceBus.Core/obsoletes-v10.cs +++ b/src/NServiceBus.Core/obsoletes-v10.cs @@ -621,5 +621,211 @@ public MessageMetadataRegistry(Func isMessageType, bool allowDynamic } } +namespace NServiceBus +{ +#nullable enable + using System; + using System.Collections.Generic; + using System.Diagnostics; + using System.Linq; + + // ============================================================================= + // OPENTELEMETRY V11 BEHAVIOR OPT-IN. EVERYTHING IN THIS BLOCK IS REMOVED IN v11. + // + // Several OpenTelemetry behaviors become the one and only behavior in v11, with no + // option left to opt in or out. Until then an endpoint can adopt them as a whole on + // v10 through a single AppContext switch: + // + // AppContext.SetSwitch("NServiceBus.Core.OpenTelemetry.UseV11Behavior", true); + // + // or in runtimeconfig.template.json / the project file: + // + // + // + // + // + // With the switch enabled: + // - Trace context and baggage are propagated through System.Diagnostics.DistributedContextPropagator + // (W3C baggage encoding) instead of LegacyContextPropagation below. The baggage wire format changes + // (W3C OWS encoding + whitespace trimming), which is breaking on rolling upgrades. + // https://github.com/Particular/NServiceBus/issues/7825 + // - Handler spans are emitted from the dedicated "NServiceBus.Core.Handler" ActivitySource instead of + // "NServiceBus.Core" so they can be filtered/sampled independently. Existing OpenTelemetry + // configurations that only subscribe to "NServiceBus.Core" would silently lose handler spans. + // https://github.com/Particular/NServiceBus/issues/7284 + // - When a transport SDK (Azure Service Bus, RabbitMQ, SQS, ...) has its own OpenTelemetry + // instrumentation, its ambient "receive" span becomes the parent of the incoming message span and + // the NServiceBus sender span is linked instead of being the parent. This changes trace shapes. + // - Span names follow the OTel messaging convention `{messaging.operation.name} {destination}`, + // e.g. "process orders", "send Payment", "publish OrderPlaced", "immediate retry orders". + // - The "Start dispatching"/"Finished dispatching" events are no longer added to the incoming span + // when outgoing messages are dispatched. + // - The legacy `execution.result` tag ("success"/"failure") is no longer emitted on the handler time, + // processing time, saga fetch time, deserialize time, and serialize time metrics. + // - The legacy `otel.status_code`/`otel.status_description` tags (redundant with Activity.Status) and + // the deprecated `exception.escaped` exception event attribute are no longer set on failures. + // https://opentelemetry.io/docs/specs/semconv/exceptions/exceptions-logs/ + // + // In v11: delete this entire namespace block, search the code base for `V11BehaviorSwitch` and keep + // only the branch each check guards for the enabled case. ActivityFactory, ContextPropagation, + // MessageOperations, RoutingToDispatchConnector, TransportReceiveToPhysicalMessageConnector and + // PipelineMetrics are the production call sites. Delete the pre-v11 default tests + // (ContextPropagationDefaultBehaviorTests, LegacyContextPropagationTests, TransportParentSpanDefaultBehaviorTests, + // the "Default_..." tests in HandlerActivitySourceTests) and the switch SetUp/TearDown pairs in the + // remaining unit and acceptance tests, together with the OpenTelemetryV11Defaults attribute in the + // obsoletes-v10.cs of NServiceBus.Core.Tests and NServiceBus.AcceptanceTests. + // ============================================================================= + static class V11BehaviorSwitch + { + enum SwitchState : byte + { + Unchecked = 0, + Enabled = 1, + Disabled = 2 + } + + static SwitchState cachedUseV11Behavior; + + public const string UseV11BehaviorSwitchName = "NServiceBus.Core.OpenTelemetry.UseV11Behavior"; + + public static bool UseV11Behavior + { + get + { + var state = cachedUseV11Behavior; + if (state != SwitchState.Unchecked) + { + return state == SwitchState.Enabled; + } + + state = AppContext.TryGetSwitch(UseV11BehaviorSwitchName, out var isEnabled) && isEnabled + ? SwitchState.Enabled + : SwitchState.Disabled; + cachedUseV11Behavior = state; + + return state == SwitchState.Enabled; + } + } + + internal static void ResetUseV11Behavior() => cachedUseV11Behavior = SwitchState.Unchecked; + } + + // The pre-v11 trace-context/baggage propagator. ContextPropagation.cs delegates here while + // V11BehaviorSwitch.UseV11Behavior is off. + static class LegacyContextPropagation + { + public static void PropagateContextToHeaders(Activity? activity, Dictionary headers) + { + if (activity is null) + { + return; + } + + if (activity.Id is not null) + { + headers[Headers.DiagnosticsTraceParent] = activity.Id; + } + + if (activity.TraceStateString is not null) + { + headers[Headers.DiagnosticsTraceState] = activity.TraceStateString; + } + + var baggage = string.Join(",", activity.Baggage.Select(item => $"{item.Key}={Uri.EscapeDataString(item.Value ?? string.Empty)}")); + if (!string.IsNullOrEmpty(baggage)) + { + headers[Headers.DiagnosticsBaggage] = baggage; + } + } + + public static void PropagateContextFromHeaders(Activity? activity, IDictionary headers) + { + if (activity is null) + { + return; + } + + PropagateTraceStateFromHeaders(activity, headers); + PropagateBaggageFromHeaders(activity, headers); + } + + public static void PropagateTraceStateFromHeaders(Activity activity, IDictionary headers) + { + if (headers.TryGetValue(Headers.DiagnosticsTraceState, out var traceState)) + { + activity.TraceStateString = traceState; + } + } + + // parent: the activity whose baggage chain is checked so keys it already carries are not added again, + // see ContextPropagation.PropagateBaggageFromHeaders. + public static void PropagateBaggageFromHeaders(Activity activity, IDictionary headers, Activity? parent = null) + { + if (!headers.TryGetValue(Headers.DiagnosticsBaggage, out var baggageValue)) + { + return; + } + + var baggageSpan = baggageValue.AsSpan(); + // HINT: Iterate in reverse order because Activity baggage is LIFO + while (!baggageSpan.IsEmpty) + { + var lastComma = baggageSpan.LastIndexOf(','); + ReadOnlySpan baggageItem; + + if (lastComma >= 0) + { + baggageItem = baggageSpan[(lastComma + 1)..]; + baggageSpan = baggageSpan[..lastComma]; + } + else + { + baggageItem = baggageSpan; + baggageSpan = []; + } + + var firstEquals = baggageItem.IndexOf('='); + if (firstEquals < 0 || firstEquals >= baggageItem.Length) + { + continue; + } + + var key = baggageItem[..firstEquals].Trim().ToString(); + if (parent?.GetBaggageItem(key) is not null) + { + continue; + } + + var value = baggageItem[(firstEquals + 1)..]; + activity.AddBaggage(key, Uri.UnescapeDataString(value)); + } + } + } + + // The pre-v11 exception tagging. ActivityFactory.RecordError applies these while + // V11BehaviorSwitch.UseV11Behavior is off. + static class LegacyExceptionTags + { + public static void SetLegacyStatusTags(Activity activity, Exception exception) + { + activity.SetTag("otel.status_code", "ERROR"); + activity.SetTag("otel.status_description", exception.Message); + } + + public static TagList EscapedTagList { get; } = new() { { "exception.escaped", true } }; + } + + public partial class InstrumentationOptions + { + // Publishes start a new linked trace by default in v10 for backward compatibility. This one keeps a + // public option (PublishTraceMode) in v11 and is therefore not governed by V11BehaviorSwitch. In v11, + // delete this method: PublishTraceMode then defaults to TraceMode.ContinueExisting through its + // initializer in InstrumentationOptions.cs, and the empty partial declaration plus the constructor + // call there can be removed. + partial void ApplyPreV11Defaults() => PublishTraceMode = TraceMode.StartNew; + } +#nullable restore +} + #pragma warning restore CS8632 // The annotation for nullable reference types should only be used in code within a '#nullable' annotations context. #pragma warning restore CS1591 // Missing XML comment for publicly visible type or member \ No newline at end of file From 2d77c9dae1832e21b513f7196dd0aa77b25011a3 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Tue, 6 Oct 2026 13:36:03 +0200 Subject: [PATCH 36/37] OpenTelemetry v11 behaviors: source version, outbox tag name, event_types format (#7959) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ Report ActivitySource version 1.0.0 when the OpenTelemetry v11 behaviors are enabled * ✨ Snake_case name for the outbox deduplication span tag with the OpenTelemetry v11 behaviors * ♻️ Move the pre-v11 outbox span tag and batch dispatch event tests into obsoletes-v10 The fixture is now partial. The tests that go away in v11 live in NServiceBus.Core.Tests/obsoletes-v10.cs next to the OpenTelemetryV11Defaults attribute, in the same layout as NServiceBus.Core/obsoletes-v10.cs. * ✨ Tag event_types and enclosed_message_types as arrays with the OpenTelemetry v11 behaviors The OpenTelemetry naming rules ask for an array value when an attribute holds several values. With the switch enabled the subscribe, unsubscribe and message spans carry arrays of full type names instead of delimited strings. Array-valued tags are only visible through Activity.TagObjects, not Activity.Tags. * Name the remaining OpenTelemetry literals and group the pre-v11 code (#7960) * 📝 Link the exception signal opt-in values to the OpenTelemetry spec * ♻️ Name the recoverability action tag values and the legacy move-to-error span name * ♻️ Compare the message intent through the MessageIntent enum when naming send spans * ♻️ Replace the switch-guarded execution.result blocks with one legacy helper * ♻️ Move the pre-v11 dispatching events into obsoletes-v10 * ⚜️ Reuse repeated telemetry names as fixture constants in the OpenTelemetry tests --- .../When_message_is_processed_successfully.cs | 71 ++++++----- .../Metrics/When_message_processing_fails.cs | 42 ++++--- .../Metrics/When_retrying_messages.cs | 22 ++-- .../Traces/When_incoming_message_has_trace.cs | 10 +- .../When_processing_incoming_message.cs | 16 +++ .../OpenTelemetry/Traces/When_subscribing.cs | 19 +++ ...TagsTests.Verify_ActivityTags.approved.txt | 21 +--- .../OpenTelemetry/ActivityDecoratorTests.cs | 25 ++++ .../OpenTelemetry/ActivityTagsTests.cs | 9 +- .../HandlerActivitySourceTests.cs | 8 +- .../SubscribeDiagnosticsBehaviorTests.cs | 49 ++++++++ .../UnsubscribeDiagnosticsBehaviorTests.cs | 47 +++++++ ...tReceiveToPhysicalMessageConnectorTests.cs | 58 +-------- src/NServiceBus.Core.Tests/obsoletes-v10.cs | 118 +++++++++++++++--- .../Tracing/ActivityDecorator.cs | 11 +- .../Tracing/ActivityDisplayNames.cs | 1 + .../OpenTelemetry/Tracing/ActivityFactory.cs | 10 +- .../OpenTelemetry/Tracing/ActivitySources.cs | 29 +++-- .../Tracing/ActivityTagValues.cs | 12 ++ .../OpenTelemetry/Tracing/ActivityTags.cs | 1 + .../Tracing/SubscribeDiagnosticsBehavior.cs | 13 +- .../Tracing/UnsubscribeDiagnosticsBehavior.cs | 11 +- .../Pipeline/Incoming/PipelineMetrics.cs | 50 ++------ ...nsportReceiveToPhysicalMessageConnector.cs | 11 +- .../Outgoing/RoutingToDispatchConnector.cs | 4 +- src/NServiceBus.Core/obsoletes-v10.cs | 51 +++++++- src/NServiceBus.Core/obsoletes-v11.cs | 1 + 27 files changed, 493 insertions(+), 227 deletions(-) create mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/SubscribeDiagnosticsBehaviorTests.cs create mode 100644 src/NServiceBus.Core.Tests/OpenTelemetry/UnsubscribeDiagnosticsBehaviorTests.cs create mode 100644 src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityTagValues.cs diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_message_is_processed_successfully.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_message_is_processed_successfully.cs index d798721b267..f83b9a767df 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_message_is_processed_successfully.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_message_is_processed_successfully.cs @@ -11,6 +11,15 @@ public class When_message_is_processed_successfully : OpenTelemetryAcceptanceTest { + const string SuccessesMetric = "nservicebus.messaging.successes"; + const string FetchesMetric = "nservicebus.messaging.fetches"; + const string CriticalTimeMetric = "nservicebus.messaging.critical_time"; + const string ProcessingTimeMetric = "nservicebus.messaging.processing_time"; + const string HandlerTimeMetric = "nservicebus.messaging.handler_time"; + const string QueueTag = "nservicebus.queue"; + const string DiscriminatorTag = "nservicebus.discriminator"; + const string MessageTypeTag = "nservicebus.message_type"; + [Test] public async Task Should_report_successful_message_metric() { @@ -27,51 +36,51 @@ public async Task Should_report_successful_message_metric() })) .Run(); - metricsListener.AssertMetric("nservicebus.messaging.successes", 5); - metricsListener.AssertMetric("nservicebus.messaging.fetches", 5); + metricsListener.AssertMetric(SuccessesMetric, 5); + metricsListener.AssertMetric(FetchesMetric, 5); metricsListener.AssertMetric("nservicebus.messaging.failures", 0); - metricsListener.AssertMetric("nservicebus.messaging.critical_time", 5); - metricsListener.AssertMetric("nservicebus.messaging.processing_time", 5); - metricsListener.AssertMetric("nservicebus.messaging.handler_time", 5); + metricsListener.AssertMetric(CriticalTimeMetric, 5); + metricsListener.AssertMetric(ProcessingTimeMetric, 5); + metricsListener.AssertMetric(HandlerTimeMetric, 5); - metricsListener.AssertTags("nservicebus.messaging.fetches", + metricsListener.AssertTags(FetchesMetric, new Dictionary { - ["nservicebus.queue"] = Conventions.EndpointNamingConvention(typeof(EndpointWithMetrics)), - ["nservicebus.discriminator"] = "disc", - ["nservicebus.message_type"] = typeof(OutgoingMessage).FullName + [QueueTag] = Conventions.EndpointNamingConvention(typeof(EndpointWithMetrics)), + [DiscriminatorTag] = "disc", + [MessageTypeTag] = typeof(OutgoingMessage).FullName }); - metricsListener.AssertTags("nservicebus.messaging.successes", + metricsListener.AssertTags(SuccessesMetric, new Dictionary { - ["nservicebus.queue"] = Conventions.EndpointNamingConvention(typeof(EndpointWithMetrics)), - ["nservicebus.discriminator"] = "disc", - ["nservicebus.message_type"] = typeof(OutgoingMessage).FullName + [QueueTag] = Conventions.EndpointNamingConvention(typeof(EndpointWithMetrics)), + [DiscriminatorTag] = "disc", + [MessageTypeTag] = typeof(OutgoingMessage).FullName }); - metricsListener.AssertTags("nservicebus.messaging.critical_time", + metricsListener.AssertTags(CriticalTimeMetric, new Dictionary { - ["nservicebus.queue"] = Conventions.EndpointNamingConvention(typeof(EndpointWithMetrics)), - ["nservicebus.discriminator"] = "disc", - ["nservicebus.message_type"] = typeof(OutgoingMessage).FullName + [QueueTag] = Conventions.EndpointNamingConvention(typeof(EndpointWithMetrics)), + [DiscriminatorTag] = "disc", + [MessageTypeTag] = typeof(OutgoingMessage).FullName }); - metricsListener.AssertTags("nservicebus.messaging.processing_time", + metricsListener.AssertTags(ProcessingTimeMetric, new Dictionary { - ["nservicebus.queue"] = Conventions.EndpointNamingConvention(typeof(EndpointWithMetrics)), - ["nservicebus.discriminator"] = "disc", - ["nservicebus.message_type"] = typeof(OutgoingMessage).FullName + [QueueTag] = Conventions.EndpointNamingConvention(typeof(EndpointWithMetrics)), + [DiscriminatorTag] = "disc", + [MessageTypeTag] = typeof(OutgoingMessage).FullName }); - metricsListener.AssertTags("nservicebus.messaging.handler_time", + metricsListener.AssertTags(HandlerTimeMetric, new Dictionary { - ["nservicebus.queue"] = Conventions.EndpointNamingConvention(typeof(EndpointWithMetrics)), - ["nservicebus.discriminator"] = "disc", - ["nservicebus.message_type"] = typeof(OutgoingMessage).FullName, + [QueueTag] = Conventions.EndpointNamingConvention(typeof(EndpointWithMetrics)), + [DiscriminatorTag] = "disc", + [MessageTypeTag] = typeof(OutgoingMessage).FullName, ["nservicebus.message_handler_type"] = typeof(EndpointWithMetrics.MessageHandler).FullName, ["execution.result"] = "success" }); @@ -93,18 +102,18 @@ public async Task Should_only_tag_most_concrete_type_on_metric() })) .Run(); - metricsListener.AssertMetric("nservicebus.messaging.successes", 5); - metricsListener.AssertMetric("nservicebus.messaging.fetches", 5); + metricsListener.AssertMetric(SuccessesMetric, 5); + metricsListener.AssertMetric(FetchesMetric, 5); metricsListener.AssertMetric("nservicebus.messaging.failures", 0); var successEndpoint = - metricsListener.AssertTagKeyExists("nservicebus.messaging.successes", "nservicebus.queue"); + metricsListener.AssertTagKeyExists(SuccessesMetric, QueueTag); var successType = - metricsListener.AssertTagKeyExists("nservicebus.messaging.successes", "nservicebus.message_type"); + metricsListener.AssertTagKeyExists(SuccessesMetric, MessageTypeTag); var successHandlerType = - metricsListener.AssertTagKeyExists("nservicebus.messaging.successes", "nservicebus.message_handler_types"); + metricsListener.AssertTagKeyExists(SuccessesMetric, "nservicebus.message_handler_types"); - var fetchedEndpoint = metricsListener.AssertTagKeyExists("nservicebus.messaging.fetches", "nservicebus.queue"); + var fetchedEndpoint = metricsListener.AssertTagKeyExists(FetchesMetric, QueueTag); using (Assert.EnterMultipleScope()) { diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_message_processing_fails.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_message_processing_fails.cs index 9c1aadacc89..fa982f05e43 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_message_processing_fails.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_message_processing_fails.cs @@ -9,6 +9,14 @@ public class When_message_processing_fails : OpenTelemetryAcceptanceTest { + const string FetchesMetric = "nservicebus.messaging.fetches"; + const string FailuresMetric = "nservicebus.messaging.failures"; + const string HandlerTimeMetric = "nservicebus.messaging.handler_time"; + const string QueueTag = "nservicebus.queue"; + const string DiscriminatorTag = "nservicebus.discriminator"; + const string MessageTypeTag = "nservicebus.message_type"; + const string ErrorTypeTag = "error.type"; + [Test] public async Task Should_report_failing_message_metrics() { @@ -20,38 +28,38 @@ public async Task Should_report_failing_message_metrics() .When(s => s.SendLocal(new FailingMessage()))) .Run(); - metricsListener.AssertMetric("nservicebus.messaging.fetches", 1); - metricsListener.AssertMetric("nservicebus.messaging.failures", 1); + metricsListener.AssertMetric(FetchesMetric, 1); + metricsListener.AssertMetric(FailuresMetric, 1); metricsListener.AssertMetric("nservicebus.messaging.successes", 0); metricsListener.AssertMetric("nservicebus.messaging.critical_time", 0); metricsListener.AssertMetric("nservicebus.messaging.processing_time", 0); - metricsListener.AssertMetric("nservicebus.messaging.handler_time", 1); + metricsListener.AssertMetric(HandlerTimeMetric, 1); - metricsListener.AssertTags("nservicebus.messaging.fetches", + metricsListener.AssertTags(FetchesMetric, new Dictionary { - ["nservicebus.queue"] = Conventions.EndpointNamingConvention(typeof(FailingEndpoint)), - ["nservicebus.discriminator"] = "disc", - ["nservicebus.message_type"] = typeof(FailingMessage).FullName + [QueueTag] = Conventions.EndpointNamingConvention(typeof(FailingEndpoint)), + [DiscriminatorTag] = "disc", + [MessageTypeTag] = typeof(FailingMessage).FullName }); - metricsListener.AssertTags("nservicebus.messaging.failures", + metricsListener.AssertTags(FailuresMetric, new Dictionary { - ["nservicebus.queue"] = Conventions.EndpointNamingConvention(typeof(FailingEndpoint)), - ["nservicebus.discriminator"] = "disc", - ["nservicebus.message_type"] = typeof(FailingMessage).FullName, - ["error.type"] = typeof(SimulatedException).FullName, + [QueueTag] = Conventions.EndpointNamingConvention(typeof(FailingEndpoint)), + [DiscriminatorTag] = "disc", + [MessageTypeTag] = typeof(FailingMessage).FullName, + [ErrorTypeTag] = typeof(SimulatedException).FullName, }); - metricsListener.AssertTags("nservicebus.messaging.handler_time", + metricsListener.AssertTags(HandlerTimeMetric, new Dictionary { - ["nservicebus.queue"] = Conventions.EndpointNamingConvention(typeof(FailingEndpoint)), - ["nservicebus.discriminator"] = "disc", - ["nservicebus.message_type"] = typeof(FailingMessage).FullName, + [QueueTag] = Conventions.EndpointNamingConvention(typeof(FailingEndpoint)), + [DiscriminatorTag] = "disc", + [MessageTypeTag] = typeof(FailingMessage).FullName, ["execution.result"] = "failure", - ["error.type"] = typeof(SimulatedException).FullName + [ErrorTypeTag] = typeof(SimulatedException).FullName }); } diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_retrying_messages.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_retrying_messages.cs index 0664a40a70e..808aff5c388 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_retrying_messages.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Metrics/When_retrying_messages.cs @@ -8,6 +8,10 @@ public class When_retrying_messages : OpenTelemetryAcceptanceTest { + const string ImmediateRetriesMetric = "nservicebus.recoverability.immediate"; + const string DelayedRetriesMetric = "nservicebus.recoverability.delayed"; + const string ErrorQueueMetric = "nservicebus.recoverability.error"; + [Test] public async Task Should_increment_immediate_meter() { @@ -20,9 +24,9 @@ await Scenario.Define() .When(s => s.SendLocal(new FailingMessage()))) .Run(); - metricsListener.AssertMetric("nservicebus.recoverability.immediate", 1); - metricsListener.AssertMetric("nservicebus.recoverability.delayed", 0); - metricsListener.AssertMetric("nservicebus.recoverability.error", 0); + metricsListener.AssertMetric(ImmediateRetriesMetric, 1); + metricsListener.AssertMetric(DelayedRetriesMetric, 0); + metricsListener.AssertMetric(ErrorQueueMetric, 0); } [Test] @@ -43,9 +47,9 @@ await Scenario.Define() .When(s => s.SendLocal(new FailingMessage()))) .Run(); - metricsListener.AssertMetric("nservicebus.recoverability.immediate", 0); - metricsListener.AssertMetric("nservicebus.recoverability.delayed", 1); - metricsListener.AssertMetric("nservicebus.recoverability.error", 0); + metricsListener.AssertMetric(ImmediateRetriesMetric, 0); + metricsListener.AssertMetric(DelayedRetriesMetric, 1); + metricsListener.AssertMetric(ErrorQueueMetric, 0); } [Test] @@ -66,9 +70,9 @@ await Scenario.Define() .Run(); }, Throws.Exception); - metricsListener.AssertMetric("nservicebus.recoverability.immediate", 0); - metricsListener.AssertMetric("nservicebus.recoverability.delayed", 0); - metricsListener.AssertMetric("nservicebus.recoverability.error", 1); + metricsListener.AssertMetric(ImmediateRetriesMetric, 0); + metricsListener.AssertMetric(DelayedRetriesMetric, 0); + metricsListener.AssertMetric(ErrorQueueMetric, 1); } public class Context : ScenarioContext diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_incoming_message_has_trace.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_incoming_message_has_trace.cs index fcfa4bd264e..099ab50cde9 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_incoming_message_has_trace.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_incoming_message_has_trace.cs @@ -9,6 +9,8 @@ public class When_incoming_message_has_trace : OpenTelemetryAcceptanceTest // assuming W3C trace! { + const string MessageIdTag = "nservicebus.message_id"; + [Test] public async Task Should_correlate_trace_from_send() { @@ -40,10 +42,10 @@ public async Task Should_correlate_trace_from_send() Assert.That(receiveReply.ParentId, Is.EqualTo(sendReply.Id), "second incoming message is correlated to the second send operation"); Assert.That(receiveReply.RootId, Is.EqualTo(sendRequest.RootId), "first send operation is the root activity"); - Assert.That(sendRequest.Tags.ToImmutableDictionary()["nservicebus.message_id"], Is.EqualTo(context.IncomingMessageId)); - Assert.That(receiveRequest.Tags.ToImmutableDictionary()["nservicebus.message_id"], Is.EqualTo(context.IncomingMessageId)); - Assert.That(sendReply.Tags.ToImmutableDictionary()["nservicebus.message_id"], Is.EqualTo(context.ReplyMessageId)); - Assert.That(receiveReply.Tags.ToImmutableDictionary()["nservicebus.message_id"], Is.EqualTo(context.ReplyMessageId)); + Assert.That(sendRequest.Tags.ToImmutableDictionary()[MessageIdTag], Is.EqualTo(context.IncomingMessageId)); + Assert.That(receiveRequest.Tags.ToImmutableDictionary()[MessageIdTag], Is.EqualTo(context.IncomingMessageId)); + Assert.That(sendReply.Tags.ToImmutableDictionary()[MessageIdTag], Is.EqualTo(context.ReplyMessageId)); + Assert.That(receiveReply.Tags.ToImmutableDictionary()[MessageIdTag], Is.EqualTo(context.ReplyMessageId)); } } diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_incoming_message.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_incoming_message.cs index 32d60e7e036..567d360fbf7 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_incoming_message.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_processing_incoming_message.cs @@ -97,5 +97,21 @@ await Scenario.Define() Assert.That(incomingActivity.DisplayName, Is.Not.EqualTo("process message")); } + [Test] + [OpenTelemetryV11Defaults] + public async Task Should_tag_enclosed_message_types_as_array() + { + var context = await Scenario.Define() + .WithEndpoint(e => e + .When(s => s.SendLocal(new IncomingMessage()))) + .Run(); + + var incomingActivity = NServiceBusActivityListener.CompletedActivities.GetReceiveMessageActivities().Single(); + + // Array-valued tags are only visible through TagObjects, not Tags. + var enclosedMessageTypes = incomingActivity.TagObjects.ToImmutableDictionary()["nservicebus.enclosed_message_types"]; + Assert.That(enclosedMessageTypes, Is.EqualTo(context.ReceivedHeaders[Headers.EnclosedMessageTypes].Split(';'))); + } + public class IncomingMessage : IMessage; } \ No newline at end of file diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_subscribing.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_subscribing.cs index ea93a2f1098..174b468d264 100644 --- a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_subscribing.cs +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/Traces/When_subscribing.cs @@ -64,6 +64,25 @@ public async Task Should_create_subscription_span_when_native_pubsub() Assert.That(subscriptionReceiveActivity, Is.Empty, "native pubsub should not produce a message"); } + [Test] + [OpenTelemetryV11Defaults] + public async Task Should_tag_event_types_as_array() + { + Requires.MessageDrivenPubSub(); + + await Scenario.Define() + .WithEndpoint(e => e + .When(s => s.Subscribe())) + .WithEndpoint() + .Run(); + + var subscribeActivity = NServiceBusActivityListener.CompletedActivities.Single(a => a.OperationName == "NServiceBus.Diagnostics.Subscribe"); + + // Array-valued tags are only visible through TagObjects, not Tags. + var eventTypes = subscribeActivity.TagObjects.ToImmutableDictionary()["nservicebus.event_types"]; + Assert.That(eventTypes, Is.EqualTo(new[] { typeof(DemoEvent).FullName })); + } + class Context : ScenarioContext; class SubscribingEndpoint : EndpointConfigurationBuilder diff --git a/src/NServiceBus.Core.Tests/ApprovalFiles/ActivityTagsTests.Verify_ActivityTags.approved.txt b/src/NServiceBus.Core.Tests/ApprovalFiles/ActivityTagsTests.Verify_ActivityTags.approved.txt index c012daa39d0..e93fea165f5 100644 --- a/src/NServiceBus.Core.Tests/ApprovalFiles/ActivityTagsTests.Verify_ActivityTags.approved.txt +++ b/src/NServiceBus.Core.Tests/ApprovalFiles/ActivityTagsTests.Verify_ActivityTags.approved.txt @@ -39,20 +39,11 @@ "EventTypes => nservicebus.event_types", "CancelledTask => nservicebus.cancelled", "ErrorType => error.type", - "RecoverabilityAction => nservicebus.recoverability_action" + "RecoverabilityAction => nservicebus.recoverability_action", + "OutboxDeduplicatedMessage => nservicebus.outbox.deduplicated_message" ], - "ActivitySourceVersions": [ - { - "Name": "Main", - "Version": "0.1.0" - }, - { - "Name": "Handler", - "Version": "0.1.0" - }, - { - "Name": "Recoverability", - "Version": "0.1.0" - } - ] + "ActivitySourceVersion": { + "PreV11Version": "0.1.0", + "Version": "1.0.0" + } } \ No newline at end of file diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityDecoratorTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityDecoratorTests.cs index c9c08d0ec64..a71ddcc2880 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityDecoratorTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityDecoratorTests.cs @@ -59,4 +59,29 @@ public void PromoteHeadersToTags_should_promote_promotable_headers_to_tags() Assert.That(tags["nservicebus.control_message"], Is.EqualTo(headers[Headers.ControlMessageHeader])); } } + + [Test] + public void PromoteHeadersToTags_should_promote_enclosed_message_types_as_string() + { + var activity = new Activity("test"); + var headers = new Dictionary { { Headers.EnclosedMessageTypes, string.Join(';', EnclosedMessageTypes) } }; + + ActivityDecorator.PromoteHeadersToTags(activity, headers); + + Assert.That(activity.TagObjects.ToImmutableDictionary()["nservicebus.enclosed_message_types"], Is.EqualTo("Ns.First;Ns.Second")); + } + + [Test] + [OpenTelemetryV11Defaults] + public void PromoteHeadersToTags_should_promote_enclosed_message_types_as_array() + { + var activity = new Activity("test"); + var headers = new Dictionary { { Headers.EnclosedMessageTypes, string.Join(';', EnclosedMessageTypes) } }; + + ActivityDecorator.PromoteHeadersToTags(activity, headers); + + Assert.That(activity.TagObjects.ToImmutableDictionary()["nservicebus.enclosed_message_types"], Is.EqualTo(EnclosedMessageTypes)); + } + + static readonly string[] EnclosedMessageTypes = ["Ns.First", "Ns.Second"]; } \ No newline at end of file diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityTagsTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityTagsTests.cs index a8adbd49f86..6231825097e 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityTagsTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/ActivityTagsTests.cs @@ -20,11 +20,12 @@ public void Verify_ActivityTags() { Note = "Changes to activity tags should result in ActivitySource version updates", Tags = activityTags, - ActivitySourceVersions = new[] + // All sources share one version. The constants are listed instead of ActivitySource.Version because the + // sources pick their version from the V11 switch once per process, see ActivitySources.cs. + ActivitySourceVersion = new { - new { Name = nameof(ActivitySources.Main), ActivitySources.Main.Version }, - new { Name = nameof(ActivitySources.Handler), ActivitySources.Handler.Version }, - new { Name = nameof(ActivitySources.Recoverability), ActivitySources.Recoverability.Version } + ActivitySources.PreV11Version, + ActivitySources.Version } }); } diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/HandlerActivitySourceTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/HandlerActivitySourceTests.cs index 2d87714a419..a8f34524e03 100644 --- a/src/NServiceBus.Core.Tests/OpenTelemetry/HandlerActivitySourceTests.cs +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/HandlerActivitySourceTests.cs @@ -13,6 +13,8 @@ namespace NServiceBus.Core.Tests.OpenTelemetry; [TestFixture] public class HandlerActivitySourceTests { + const string HandlerSourceName = "NServiceBus.Core.Handler"; + readonly ActivityFactory activityFactory = new(new InstrumentationOptions()); TestingActivityListener mainListener; @@ -39,7 +41,7 @@ public void Should_emit_handler_activity_from_main_source() [OpenTelemetryV11Defaults] public void Should_emit_handler_activity_from_handler_source() { - using var handlerListener = TestingActivityListener.SetupDiagnosticListener("NServiceBus.Core.Handler"); + using var handlerListener = TestingActivityListener.SetupDiagnosticListener(HandlerSourceName); using var ambientActivity = new Activity("ambient activity"); ambientActivity.Start(); @@ -47,14 +49,14 @@ public void Should_emit_handler_activity_from_handler_source() var activity = activityFactory.StartHandlerActivity(new MessageHandler { HandlerType = typeof(HandlerActivitySourceTests) }); Assert.That(activity, Is.Not.Null); - Assert.That(activity!.Source.Name, Is.EqualTo("NServiceBus.Core.Handler")); + Assert.That(activity!.Source.Name, Is.EqualTo(HandlerSourceName)); } [Test] [OpenTelemetryV11Defaults] public void Should_preserve_display_name_and_handler_type_tag() { - using var handlerListener = TestingActivityListener.SetupDiagnosticListener("NServiceBus.Core.Handler"); + using var handlerListener = TestingActivityListener.SetupDiagnosticListener(HandlerSourceName); using var ambientActivity = new Activity("ambient activity"); ambientActivity.Start(); diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/SubscribeDiagnosticsBehaviorTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/SubscribeDiagnosticsBehaviorTests.cs new file mode 100644 index 00000000000..3ef69ec42c9 --- /dev/null +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/SubscribeDiagnosticsBehaviorTests.cs @@ -0,0 +1,49 @@ +namespace NServiceBus.Core.Tests.OpenTelemetry; + +using System.Collections.Immutable; +using System.Diagnostics; +using System.Threading.Tasks; +using NUnit.Framework; +using Testing; + +[TestFixture] +public class SubscribeDiagnosticsBehaviorTests +{ + const string EventTypesTag = "nservicebus.event_types"; + + [Test] + public async Task Should_tag_event_types_as_comma_separated_string() + { + var context = CreateContext(out var activity); + + await new SubscribeDiagnosticsBehavior().Invoke(context, _ => Task.CompletedTask); + + Assert.That(activity.TagObjects.ToImmutableDictionary()[EventTypesTag], Is.EqualTo($"{typeof(FirstEvent)},{typeof(SecondEvent)}")); + } + + [Test] + [OpenTelemetryV11Defaults] + public async Task Should_tag_event_types_as_array_of_full_names() + { + var context = CreateContext(out var activity); + + await new SubscribeDiagnosticsBehavior().Invoke(context, _ => Task.CompletedTask); + + Assert.That(activity.TagObjects.ToImmutableDictionary()[EventTypesTag], Is.EqualTo(new[] { typeof(FirstEvent).FullName, typeof(SecondEvent).FullName })); + } + + static TestableSubscribeContext CreateContext(out Activity activity) + { + var context = new TestableSubscribeContext { EventTypes = [typeof(FirstEvent), typeof(SecondEvent)] }; + + activity = new Activity("subscribe"); + activity.Start(); + context.Extensions.SetOutgoingPipelineActivity(activity); + + return context; + } + + class FirstEvent; + + class SecondEvent; +} diff --git a/src/NServiceBus.Core.Tests/OpenTelemetry/UnsubscribeDiagnosticsBehaviorTests.cs b/src/NServiceBus.Core.Tests/OpenTelemetry/UnsubscribeDiagnosticsBehaviorTests.cs new file mode 100644 index 00000000000..c5502e5b8b5 --- /dev/null +++ b/src/NServiceBus.Core.Tests/OpenTelemetry/UnsubscribeDiagnosticsBehaviorTests.cs @@ -0,0 +1,47 @@ +namespace NServiceBus.Core.Tests.OpenTelemetry; + +using System.Collections.Immutable; +using System.Diagnostics; +using System.Threading.Tasks; +using NUnit.Framework; +using Testing; + +[TestFixture] +public class UnsubscribeDiagnosticsBehaviorTests +{ + const string EventTypesTag = "nservicebus.event_types"; + + [Test] + public async Task Should_tag_event_type_as_string() + { + var context = CreateContext(out var activity); + + await new UnsubscribeDiagnosticsBehavior().Invoke(context, _ => Task.CompletedTask); + + Assert.That(activity.TagObjects.ToImmutableDictionary()[EventTypesTag], Is.EqualTo(typeof(DemoEvent).FullName)); + } + + [Test] + [OpenTelemetryV11Defaults] + public async Task Should_tag_event_type_as_single_element_array() + { + var context = CreateContext(out var activity); + + await new UnsubscribeDiagnosticsBehavior().Invoke(context, _ => Task.CompletedTask); + + Assert.That(activity.TagObjects.ToImmutableDictionary()[EventTypesTag], Is.EqualTo(new[] { typeof(DemoEvent).FullName })); + } + + static TestableUnsubscribeContext CreateContext(out Activity activity) + { + var context = new TestableUnsubscribeContext { EventType = typeof(DemoEvent) }; + + activity = new Activity("unsubscribe"); + activity.Start(); + context.Extensions.SetOutgoingPipelineActivity(activity); + + return context; + } + + class DemoEvent; +} diff --git a/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs b/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs index 82d939f1946..3f50e5a2ce2 100644 --- a/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs +++ b/src/NServiceBus.Core.Tests/Reliability/Outbox/TransportReceiveToPhysicalMessageConnectorTests.cs @@ -20,7 +20,7 @@ namespace NServiceBus.Core.Tests.Reliability.Outbox; using TransportOperation = Transport.TransportOperation; [TestFixture] -public class TransportReceiveToPhysicalMessageConnectorTests +public partial class TransportReceiveToPhysicalMessageConnectorTests { [Test] public async Task Should_honor_stored_delivery_constraints() @@ -116,8 +116,11 @@ public async Task Should_honor_stored_pubsub_routing() } } + // In v11 the snake_case name is the only one: delete Should_add_outbox_span_tag_when_deduplicating in + // obsoletes-v10.cs and the attribute on this one. [Test] - public async Task Should_add_outbox_span_tag_when_deduplicating() + [OpenTelemetryV11Defaults] + public async Task Should_add_snake_case_outbox_span_tag_when_deduplicating() { string messageId = Guid.NewGuid().ToString(); fakeOutbox.ExistingMessage = new OutboxMessage(messageId, Array.Empty()); @@ -129,7 +132,7 @@ public async Task Should_add_outbox_span_tag_when_deduplicating() await Invoke(context); - Assert.That(pipelineActivity.TagObjects.ToImmutableDictionary()["nservicebus.outbox.deduplicate-message"], Is.EqualTo(true)); + Assert.That(pipelineActivity.TagObjects.ToImmutableDictionary()["nservicebus.outbox.deduplicated_message"], Is.EqualTo(true)); } [Test] @@ -146,55 +149,6 @@ public async Task Should_report_deduplicated_message_metric_when_deduplicating() metricsListener.AssertMetric("nservicebus.outbox.duplicates", 1); } - [Test] - public async Task Should_add_batch_dispatch_events_when_sending_batched_messages() - { - var context = CreateContext(fakeBatchPipeline, Guid.NewGuid().ToString()); - - using var pipelineActivity = new Activity("test activity"); - pipelineActivity.Start(); - context.Extensions.SetIncomingPipelineActivity(pipelineActivity); - - await Invoke(context, c => - { - var batchedSends = c.Extensions.Get(); - batchedSends.AddRange([ - new TransportOperation(new OutgoingMessage(Guid.NewGuid().ToString(), [], Array.Empty()), new UnicastAddressTag("destination")), - new TransportOperation(new OutgoingMessage(Guid.NewGuid().ToString(), [], Array.Empty()), new UnicastAddressTag("destination")), - new TransportOperation(new OutgoingMessage(Guid.NewGuid().ToString(), [], Array.Empty()), new UnicastAddressTag("destination")) - ]); - return Task.CompletedTask; - }); - - var startDispatcherActivityEvents = pipelineActivity.Events.Where(e => e.Name == "Start dispatching").ToArray(); - Assert.That(startDispatcherActivityEvents, Has.Length.EqualTo(1)); - Assert.That(startDispatcherActivityEvents.Single().Tags.ToImmutableDictionary()["message-count"], Is.EqualTo(3)); - Assert.That(pipelineActivity.Events.Count(e => e.Name == "Finished dispatching"), Is.EqualTo(1)); - } - - // In v11 the dispatching events are gone: delete this test and - // Should_add_batch_dispatch_events_when_sending_batched_messages together with the - // V11BehaviorSwitch block in obsoletes-v10.cs. - [Test] - [OpenTelemetryV11Defaults] - public async Task Should_not_add_batch_dispatch_events_when_sending_batched_messages() - { - var context = CreateContext(fakeBatchPipeline, Guid.NewGuid().ToString()); - - using var pipelineActivity = new Activity("test activity"); - pipelineActivity.Start(); - context.Extensions.SetIncomingPipelineActivity(pipelineActivity); - - await Invoke(context, c => - { - var batchedSends = c.Extensions.Get(); - batchedSends.Add(new TransportOperation(new OutgoingMessage(Guid.NewGuid().ToString(), [], Array.Empty()), new UnicastAddressTag("destination"))); - return Task.CompletedTask; - }); - - Assert.That(pipelineActivity.Events, Is.Empty); - } - [Test] public async Task Should_not_add_batch_dispatch_events_when_no_batched_messages() { diff --git a/src/NServiceBus.Core.Tests/obsoletes-v10.cs b/src/NServiceBus.Core.Tests/obsoletes-v10.cs index f1b823e10e3..9ebe04e2ec8 100644 --- a/src/NServiceBus.Core.Tests/obsoletes-v10.cs +++ b/src/NServiceBus.Core.Tests/obsoletes-v10.cs @@ -1,29 +1,113 @@ -namespace NServiceBus.Core.Tests; - -using System; -using NUnit.Framework; -using NUnit.Framework.Interfaces; - // ============================================================================= // EVERYTHING IN THIS FILE IS TEMPORARY AND WILL BE REMOVED IN v11, together with // the V11BehaviorSwitch block in NServiceBus.Core/obsoletes-v10.cs. // ============================================================================= -// Runs the test with the NServiceBus.Core.OpenTelemetry.UseV11Behavior AppContext switch enabled, i.e. with -// the OpenTelemetry defaults of v11. In v11 these defaults are the only behavior: delete this attribute and -// remove it from the tests that use it. -[AttributeUsage(AttributeTargets.Method | AttributeTargets.Class, Inherited = true)] -public sealed class OpenTelemetryV11DefaultsAttribute : Attribute, ITestAction +namespace NServiceBus.Core.Tests { - public ActionTargets Targets => ActionTargets.Test; + using System; + using NUnit.Framework; + using NUnit.Framework.Interfaces; + + // Runs the test with the NServiceBus.Core.OpenTelemetry.UseV11Behavior AppContext switch enabled, i.e. with + // the OpenTelemetry defaults of v11. In v11 these defaults are the only behavior: delete this attribute and + // remove it from the tests that use it. + [AttributeUsage(AttributeTargets.Method | AttributeTargets.Class, Inherited = true)] + public sealed class OpenTelemetryV11DefaultsAttribute : Attribute, ITestAction + { + public ActionTargets Targets => ActionTargets.Test; + + public void BeforeTest(ITest test) => Set(true); - public void BeforeTest(ITest test) => Set(true); + public void AfterTest(ITest test) => Set(false); - public void AfterTest(ITest test) => Set(false); + static void Set(bool enabled) + { + AppContext.SetSwitch(V11BehaviorSwitch.UseV11BehaviorSwitchName, enabled); + V11BehaviorSwitch.ResetUseV11Behavior(); + } + } +} - static void Set(bool enabled) +namespace NServiceBus.Core.Tests.Reliability.Outbox +{ + using System; + using System.Collections.Immutable; + using System.Diagnostics; + using System.Threading.Tasks; + using System.Linq; + using NServiceBus.Outbox; + using NServiceBus.Routing; + using NServiceBus.Transport; + using NUnit.Framework; + using TransportOperation = NServiceBus.Transport.TransportOperation; + + public partial class TransportReceiveToPhysicalMessageConnectorTests { - AppContext.SetSwitch(V11BehaviorSwitch.UseV11BehaviorSwitchName, enabled); - V11BehaviorSwitch.ResetUseV11Behavior(); + // Covers the pre-v11 name of the outbox deduplication span tag. In v11 the snake_case name is the only one: + // delete this test. Should_add_snake_case_outbox_span_tag_when_deduplicating covers the v11 name. + [Test] + public async Task Should_add_outbox_span_tag_when_deduplicating() + { + string messageId = Guid.NewGuid().ToString(); + fakeOutbox.ExistingMessage = new OutboxMessage(messageId, Array.Empty()); + var context = CreateContext(fakeBatchPipeline, messageId); + + using var pipelineActivity = new Activity("test activity"); + pipelineActivity.Start(); + context.Extensions.SetIncomingPipelineActivity(pipelineActivity); + + await Invoke(context); + + Assert.That(pipelineActivity.TagObjects.ToImmutableDictionary()["nservicebus.outbox.deduplicate-message"], Is.EqualTo(true)); + } + + [Test] + public async Task Should_add_batch_dispatch_events_when_sending_batched_messages() + { + var context = CreateContext(fakeBatchPipeline, Guid.NewGuid().ToString()); + + using var pipelineActivity = new Activity("test activity"); + pipelineActivity.Start(); + context.Extensions.SetIncomingPipelineActivity(pipelineActivity); + + await Invoke(context, c => + { + var batchedSends = c.Extensions.Get(); + batchedSends.AddRange([ + new TransportOperation(new OutgoingMessage(Guid.NewGuid().ToString(), [], Array.Empty()), new UnicastAddressTag("destination")), + new TransportOperation(new OutgoingMessage(Guid.NewGuid().ToString(), [], Array.Empty()), new UnicastAddressTag("destination")), + new TransportOperation(new OutgoingMessage(Guid.NewGuid().ToString(), [], Array.Empty()), new UnicastAddressTag("destination")) + ]); + return Task.CompletedTask; + }); + + var startDispatcherActivityEvents = pipelineActivity.Events.Where(e => e.Name == "Start dispatching").ToArray(); + Assert.That(startDispatcherActivityEvents, Has.Length.EqualTo(1)); + Assert.That(startDispatcherActivityEvents.Single().Tags.ToImmutableDictionary()["message-count"], Is.EqualTo(3)); + Assert.That(pipelineActivity.Events.Count(e => e.Name == "Finished dispatching"), Is.EqualTo(1)); + } + + // In v11 the dispatching events are gone: delete this test together with the one above and the + // V11BehaviorSwitch block in NServiceBus.Core/obsoletes-v10.cs. + [Test] + [OpenTelemetryV11Defaults] + public async Task Should_not_add_batch_dispatch_events_when_sending_batched_messages() + { + var context = CreateContext(fakeBatchPipeline, Guid.NewGuid().ToString()); + + using var pipelineActivity = new Activity("test activity"); + pipelineActivity.Start(); + context.Extensions.SetIncomingPipelineActivity(pipelineActivity); + + await Invoke(context, c => + { + var batchedSends = c.Extensions.Get(); + batchedSends.Add(new TransportOperation(new OutgoingMessage(Guid.NewGuid().ToString(), [], Array.Empty()), new UnicastAddressTag("destination"))); + return Task.CompletedTask; + }); + + Assert.That(pipelineActivity.Events, Is.Empty); + } } } diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDecorator.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDecorator.cs index bd1a802565c..f7cd1437597 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDecorator.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDecorator.cs @@ -13,11 +13,20 @@ public static void PromoteHeadersToTags(Activity activity, Dictionary + V11BehaviorSwitch.UseV11Behavior && headerKey == Headers.EnclosedMessageTypes + ? headerValue.Split(EnclosedMessageTypesSeparator) + : headerValue; + + const char EnclosedMessageTypesSeparator = ';'; + internal static readonly Dictionary HeaderMapping = new() { { Headers.SagaId, ActivityTags.SagaId }, diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs index 418698e2754..6b59933517b 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityDisplayNames.cs @@ -11,6 +11,7 @@ static class ActivityDisplayNames public const string SendMessage = "send message"; public const string ReplyMessage = "reply"; public const string Recoverability = "recover"; + public const string MoveToError = "move to error"; // Operation-only prefixes used when V11BehaviorSwitch.UseV11Behavior is enabled (the only naming in v11, see obsoletes-v10.cs) internal const string ProcessOperation = "process"; diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs index 5e0cbfabe30..96acb634741 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityFactory.cs @@ -209,7 +209,7 @@ public void UpdateActivityFromRecoverabilityAction(Activity activity, Recoverabi { if (recoverabilityAction is ImmediateRetry) { - activity.AddTag(ActivityTags.RecoverabilityAction, "immediate_retry"); + activity.AddTag(ActivityTags.RecoverabilityAction, ActivityTagValues.ImmediateRetry); activity.DisplayName = ActivityDisplayNames.ImmediateRetryOperation; if (V11BehaviorSwitch.UseV11Behavior) @@ -219,7 +219,7 @@ public void UpdateActivityFromRecoverabilityAction(Activity activity, Recoverabi } else if (recoverabilityAction is DelayedRetry) { - activity.AddTag(ActivityTags.RecoverabilityAction, "delayed_retry"); + activity.AddTag(ActivityTags.RecoverabilityAction, ActivityTagValues.DelayedRetry); activity.DisplayName = ActivityDisplayNames.DelayedRetryOperation; if (V11BehaviorSwitch.UseV11Behavior) @@ -229,15 +229,15 @@ public void UpdateActivityFromRecoverabilityAction(Activity activity, Recoverabi } else if (recoverabilityAction is MoveToError moveToError) { - activity.AddTag(ActivityTags.RecoverabilityAction, "move_to_error"); + activity.AddTag(ActivityTags.RecoverabilityAction, ActivityTagValues.MoveToError); activity.DisplayName = V11BehaviorSwitch.UseV11Behavior ? $"{ActivityDisplayNames.MoveToErrorOperation} {moveToError.ErrorQueue}" - : $"{ActivityDisplayNames.MoveToErrorOperation} error"; + : ActivityDisplayNames.MoveToError; } else if (recoverabilityAction is Discard) { - activity.AddTag(ActivityTags.RecoverabilityAction, "discard"); + activity.AddTag(ActivityTags.RecoverabilityAction, ActivityTagValues.Discard); activity.DisplayName = ActivityDisplayNames.DiscardOperation; } } diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivitySources.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivitySources.cs index b3d86aeaf63..2ea753987c6 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivitySources.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivitySources.cs @@ -1,4 +1,4 @@ -#nullable enable +#nullable enable namespace NServiceBus; @@ -6,15 +6,18 @@ namespace NServiceBus; static class ActivitySources { - public static readonly ActivitySource Main = - new("NServiceBus.Core", - "0.1.0"); - - public static readonly ActivitySource Handler = - new("NServiceBus.Core.Handler", - "0.1.0"); - - public static readonly ActivitySource Recoverability = - new("NServiceBus.Core.Recoverability", - "0.1.0"); -} \ No newline at end of file + public const string Version = "1.0.0"; + + // The pre-v11 sources stay on 0.1.0 so existing dashboards and tests can tell the two tag and span-name + // sets apart. Removed in v11 together with the switch, see obsoletes-v10.cs. + internal const string PreV11Version = "0.1.0"; + + // Static fields initialize in textual order, so this has to precede the sources below. + static readonly string EffectiveVersion = V11BehaviorSwitch.UseV11Behavior ? Version : PreV11Version; + + public static readonly ActivitySource Main = new("NServiceBus.Core", EffectiveVersion); + + public static readonly ActivitySource Handler = new("NServiceBus.Core.Handler", EffectiveVersion); + + public static readonly ActivitySource Recoverability = new("NServiceBus.Core.Recoverability", EffectiveVersion); +} diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityTagValues.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityTagValues.cs new file mode 100644 index 00000000000..ec3c4562cad --- /dev/null +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityTagValues.cs @@ -0,0 +1,12 @@ +#nullable enable + +namespace NServiceBus; + +// Values written to the ActivityTags.RecoverabilityAction tag. +static class ActivityTagValues +{ + public const string ImmediateRetry = "immediate_retry"; + public const string DelayedRetry = "delayed_retry"; + public const string MoveToError = "move_to_error"; + public const string Discard = "discard"; +} diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityTags.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityTags.cs index ff03de38003..810b4313685 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityTags.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/ActivityTags.cs @@ -43,4 +43,5 @@ static class ActivityTags public const string CancelledTask = "nservicebus.cancelled"; public const string ErrorType = "error.type"; public const string RecoverabilityAction = "nservicebus.recoverability_action"; + public const string OutboxDeduplicatedMessage = "nservicebus.outbox.deduplicated_message"; } \ No newline at end of file diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/SubscribeDiagnosticsBehavior.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/SubscribeDiagnosticsBehavior.cs index dfa025c6099..cc8a0c9d9fe 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/SubscribeDiagnosticsBehavior.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/SubscribeDiagnosticsBehavior.cs @@ -3,6 +3,7 @@ namespace NServiceBus; using System; +using System.Linq; using System.Threading.Tasks; using Pipeline; @@ -12,7 +13,17 @@ public Task Invoke(ISubscribeContext context, Func next { if (context.Extensions.TryGetOutgoingPipelineActivity(out var activity)) { - activity.SetTag(ActivityTags.EventTypes, string.Join(",", (object[])context.EventTypes)); + // An array of full type names: the OpenTelemetry naming rules ask for an array when an attribute holds + // several values. An array-valued tag is only visible through Activity.TagObjects, not Activity.Tags. + // Keep only the v11 branch in v11, see obsoletes-v10.cs. + if (V11BehaviorSwitch.UseV11Behavior) + { + activity.SetTag(ActivityTags.EventTypes, context.EventTypes.Select(static type => type.FullName).ToArray()); + } + else + { + activity.SetTag(ActivityTags.EventTypes, string.Join(",", (object[])context.EventTypes)); + } } return next(context); diff --git a/src/NServiceBus.Core/OpenTelemetry/Tracing/UnsubscribeDiagnosticsBehavior.cs b/src/NServiceBus.Core/OpenTelemetry/Tracing/UnsubscribeDiagnosticsBehavior.cs index 93cc7433afe..0ef69ed1711 100644 --- a/src/NServiceBus.Core/OpenTelemetry/Tracing/UnsubscribeDiagnosticsBehavior.cs +++ b/src/NServiceBus.Core/OpenTelemetry/Tracing/UnsubscribeDiagnosticsBehavior.cs @@ -12,7 +12,16 @@ public Task Invoke(IUnsubscribeContext context, Func { if (context.Extensions.TryGetOutgoingPipelineActivity(out var activity)) { - activity.SetTag(ActivityTags.EventTypes, context.EventType.FullName); + // A single-element array, so the tag has the same shape as on the subscribe span. + // Keep only the v11 branch in v11, see obsoletes-v10.cs. + if (V11BehaviorSwitch.UseV11Behavior) + { + activity.SetTag(ActivityTags.EventTypes, new[] { context.EventType.FullName }); + } + else + { + activity.SetTag(ActivityTags.EventTypes, context.EventType.FullName); + } } return next(context); diff --git a/src/NServiceBus.Core/Pipeline/Incoming/PipelineMetrics.cs b/src/NServiceBus.Core/Pipeline/Incoming/PipelineMetrics.cs index 04a9a284d9e..8203d9e1eeb 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/PipelineMetrics.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/PipelineMetrics.cs @@ -32,7 +32,6 @@ class PipelineMetrics // queueName and discriminator are null for send-only endpoints, which have no receive queue to report. public PipelineMetrics(IMeterFactory meterFactory, string? queueName, string? discriminator) { - emitExecutionResultTags = !V11BehaviorSwitch.UseV11Behavior; // removed in v11 together with the tag, see obsoletes-v10.cs var meter = meterFactory.Create("NServiceBus.Core.Pipeline.Incoming", "0.4.0"); totalProcessedSuccessfully = meter.CreateCounter(TotalProcessedSuccessfully, description: "Total number of messages processed successfully by the endpoint."); @@ -93,11 +92,7 @@ public void RecordProcessingTime(ITransportReceiveContext context, TimeSpan elap var tags = DefaultMetricTags; - if (emitExecutionResultTags) - { - // Execution result is to be removed so we don't support overriding it by the user - tags.Add(MeterTags.ExecutionResult, "success"); - } + LegacyExecutionResultTag.Add(ref tags); // removed in v11, see obsoletes-v10.cs context.PipelineMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, @@ -117,11 +112,7 @@ public void RecordCriticalTimeAndTotalProcessed(ITransportReceiveContext context } var tags = DefaultMetricTags; - if (emitExecutionResultTags) - { - // Execution result is to be removed so we don't support overriding it by the user - tags.Add(MeterTags.ExecutionResult, "success"); - } + LegacyExecutionResultTag.Add(ref tags); // removed in v11, see obsoletes-v10.cs // totalProcessedSuccessfully and criticalTime always share the same tags in this method, so overrides are // looked up under criticalTime's instrument name. @@ -157,11 +148,7 @@ public void RecordMessageProcessingFailure(PipelineMetricTags pipelineMetricTags } var tags = DefaultMetricTags; - if (emitExecutionResultTags) - { - // Execution result is to be removed so we don't support overriding it by the user - tags.Add(MeterTags.ExecutionResult, "failure"); - } + LegacyExecutionResultTag.Add(ref tags, error); // removed in v11, see obsoletes-v10.cs tags.Add(MeterTags.ErrorType, error.GetType().FullName); @@ -219,11 +206,7 @@ public void RecordSuccessfulMessageHandlerTime(IInvokeHandlerContext context, Ti } var tags = DefaultMetricTags; - if (emitExecutionResultTags) - { - // Execution result is to be removed so we don't support overriding it by the user - tags.Add(MeterTags.ExecutionResult, "success"); - } + LegacyExecutionResultTag.Add(ref tags); // removed in v11, see obsoletes-v10.cs tags.Add(MeterTags.MessageHandlerType, context.MessageHandler.HandlerType.FullName); @@ -244,11 +227,7 @@ public void RecordFailedMessageHandlerTime(IInvokeHandlerContext context, TimeSp } var tags = DefaultMetricTags; - if (emitExecutionResultTags) - { - // Execution result is to be removed so we don't support overriding it by the user - tags.Add(MeterTags.ExecutionResult, "failure"); - } + LegacyExecutionResultTag.Add(ref tags, error); // removed in v11, see obsoletes-v10.cs tags.Add(MeterTags.MessageHandlerType, context.MessageHandler.HandlerType.FullName); tags.Add(MeterTags.ErrorType, error.GetType().FullName); @@ -354,11 +333,7 @@ public void RecordSagaFetchTime(IInvokeHandlerContext context, TimeSpan elapsed, } var tags = DefaultMetricTags; - if (emitExecutionResultTags) - { - // Execution result is to be removed so we don't support overriding it by the user - tags.Add(MeterTags.ExecutionResult, error != null ? "failure" : "success"); - } + LegacyExecutionResultTag.Add(ref tags, error); // removed in v11, see obsoletes-v10.cs tags.Add(MeterTags.SagaType, sagaType); if (error != null) { @@ -391,11 +366,7 @@ public void RecordDeserializeTime(IIncomingPhysicalMessageContext context, TimeS { tags.Add(MeterTags.ErrorType, error.GetType().FullName); } - if (emitExecutionResultTags) - { - // Execution result is to be removed so we don't support overriding it by the user - tags.Add(MeterTags.ExecutionResult, error != null ? "failure" : "success"); - } + LegacyExecutionResultTag.Add(ref tags, error); // removed in v11, see obsoletes-v10.cs context.PipelineMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, @@ -423,11 +394,7 @@ public void RecordSerializeTime(IOutgoingLogicalMessageContext context, TimeSpan { tags.Add(MeterTags.ErrorType, error.GetType().FullName); } - if (emitExecutionResultTags) - { - // Execution result is to be removed so we don't support overriding it by the user - tags.Add(MeterTags.ExecutionResult, error != null ? "failure" : "success"); - } + LegacyExecutionResultTag.Add(ref tags, error); // removed in v11, see obsoletes-v10.cs context.PipelineMetricTags.ApplyTags(ref tags, [ MeterTags.QueueName, @@ -541,5 +508,4 @@ public readonly struct ActiveMessageScope(UpDownCounter? counter, TagList readonly Histogram persistenceTime; readonly TagList DefaultMetricTags; - readonly bool emitExecutionResultTags; } diff --git a/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs b/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs index d9d45700d2a..81dd69f9770 100644 --- a/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs +++ b/src/NServiceBus.Core/Pipeline/Incoming/TransportReceiveToPhysicalMessageConnector.cs @@ -76,7 +76,7 @@ async Task InvokeWithOutbox(ITransportReceiveContext context, Func 0 && operations[0].AddressTag is UnicastAddressTag unicastTag - && outgoingMessage.Headers.TryGetValue(Headers.MessageIntent, out var intentStr) - && intentStr is "Send" or "Reply") + && outgoingMessage.Headers.TryGetValue(Headers.MessageIntent, out var intentString) + && intentString is nameof(MessageIntent.Send) or nameof(MessageIntent.Reply)) { activity.DisplayName = $"{activity.DisplayName} {unicastTag.Destination}"; } diff --git a/src/NServiceBus.Core/obsoletes-v10.cs b/src/NServiceBus.Core/obsoletes-v10.cs index d5042055337..ce220e0bcce 100644 --- a/src/NServiceBus.Core/obsoletes-v10.cs +++ b/src/NServiceBus.Core/obsoletes-v10.cs @@ -628,6 +628,8 @@ namespace NServiceBus using System.Collections.Generic; using System.Diagnostics; using System.Linq; + using System.Threading.Tasks; + using NServiceBus.Pipeline; // ============================================================================= // OPENTELEMETRY V11 BEHAVIOR OPT-IN. EVERYTHING IN THIS BLOCK IS REMOVED IN v11. @@ -665,9 +667,19 @@ namespace NServiceBus // - The legacy `otel.status_code`/`otel.status_description` tags (redundant with Activity.Status) and // the deprecated `exception.escaped` exception event attribute are no longer set on failures. // https://opentelemetry.io/docs/specs/semconv/exceptions/exceptions-logs/ + // - The ActivitySources report version 1.0.0 instead of 0.1.0, so a consumer can tell the two tag and + // span-name sets apart. + // - The outbox deduplication span tag is named `nservicebus.outbox.deduplicated_message` instead of + // `nservicebus.outbox.deduplicate-message`, following the OpenTelemetry attribute naming rules + // (snake_case within a dot-delimited component, no hyphens). + // https://opentelemetry.io/docs/specs/semconv/general/naming/ + // - The `nservicebus.event_types` tag on subscribe and unsubscribe spans and the + // `nservicebus.enclosed_message_types` tag on message spans are arrays of full type names instead of + // delimited strings. The OpenTelemetry naming rules ask for an array when an attribute holds several + // values. Array-valued tags are only visible through Activity.TagObjects, not Activity.Tags. // // In v11: delete this entire namespace block, search the code base for `V11BehaviorSwitch` and keep - // only the branch each check guards for the enabled case. ActivityFactory, ContextPropagation, + // only the branch each check guards for the enabled case. ActivityFactory, ActivitySources, ContextPropagation, // MessageOperations, RoutingToDispatchConnector, TransportReceiveToPhysicalMessageConnector and // PipelineMetrics are the production call sites. Delete the pre-v11 default tests // (ContextPropagationDefaultBehaviorTests, LegacyContextPropagationTests, TransportParentSpanDefaultBehaviorTests, @@ -815,6 +827,43 @@ public static void SetLegacyStatusTags(Activity activity, Exception exception) public static TagList EscapedTagList { get; } = new() { { "exception.escaped", true } }; } + // The pre-v11 names of span tags that were renamed to follow the OpenTelemetry naming rules. + // TransportReceiveToPhysicalMessageConnector writes these while V11BehaviorSwitch.UseV11Behavior is off. + static class LegacyActivityTags + { + public const string OutboxDeduplicateMessage = "nservicebus.outbox.deduplicate-message"; + } + + // The pre-v11 execution.result metric tag. PipelineMetrics applies it while + // V11BehaviorSwitch.UseV11Behavior is off. + static class LegacyExecutionResultTag + { + // The tag is going away, so a user cannot override it through IMetricsTags. + public static void Add(ref TagList tags, Exception? error = null) + { + if (!V11BehaviorSwitch.UseV11Behavior) + { + tags.Add(MeterTags.ExecutionResult, error is null ? "success" : "failure"); + } + } + } + + // The pre-v11 "Start dispatching"/"Finished dispatching" events on the incoming span. + // TransportReceiveToPhysicalMessageConnector dispatches through here while V11BehaviorSwitch.UseV11Behavior is off. + static class LegacyDispatchEvents + { + public static async Task DispatchWithEvents(TransportReceiveToPhysicalMessageConnector connector, IBatchDispatchContext batchDispatchContext, Activity activity) + { + activity.AddEvent(new(StartDispatching, tags: new() { { MessageCount, batchDispatchContext.Operations.Count } })); + await connector.Fork(batchDispatchContext).ConfigureAwait(false); + activity.AddEvent(new(FinishedDispatching)); + } + + const string StartDispatching = "Start dispatching"; + const string FinishedDispatching = "Finished dispatching"; + const string MessageCount = "message-count"; + } + public partial class InstrumentationOptions { // Publishes start a new linked trace by default in v10 for backward compatibility. This one keeps a diff --git a/src/NServiceBus.Core/obsoletes-v11.cs b/src/NServiceBus.Core/obsoletes-v11.cs index 4ad0abd625e..c6cd15d5d2d 100644 --- a/src/NServiceBus.Core/obsoletes-v11.cs +++ b/src/NServiceBus.Core/obsoletes-v11.cs @@ -33,6 +33,7 @@ internal static void SetExceptionRecordingModeDefault(SettingsHolder settings) var environment = settings.Get(); var variableValue = environment.GetEnvironmentVariable(ExceptionSignalOptInEnvironmentVariableKey); + // Recognized values per https://opentelemetry.io/docs/specs/semconv/exceptions/exceptions-logs/#recording-an-exception options.ExceptionRecordingMode = variableValue switch { "logs" => ExceptionRecordingMode.Logs, From 0177a55bbb14484cca6996aa6ee2a6c4821e7071 Mon Sep 17 00:00:00 2001 From: Ramon Smits Date: Tue, 6 Oct 2026 13:46:38 +0200 Subject: [PATCH 37/37] Write OpenTelemetry instrumentation options to startup diagnostics (#7963) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * ✨ Write OpenTelemetry instrumentation options to startup diagnostics * exclude instrumentaiton diagnostic file write tests from nonparallelizable attribute tests --------- Co-authored-by: Tomasz Masternak --- ...int_starts_with_instrumentation_options.cs | 67 +++++++++++++++++++ .../ConventionEnforcementTests.cs | 7 +- .../OpenTelemetryDiagnostics.cs | 28 ++++++++ .../StartupDiagnosticsJsonContext.cs | 1 + .../OpenTelemetry/OpenTelemetryFeature.cs | 17 +++++ 5 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/When_endpoint_starts_with_instrumentation_options.cs create mode 100644 src/NServiceBus.Core/Hosting/StartupDiagnostics/OpenTelemetryDiagnostics.cs diff --git a/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/When_endpoint_starts_with_instrumentation_options.cs b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/When_endpoint_starts_with_instrumentation_options.cs new file mode 100644 index 00000000000..9274ccb07ea --- /dev/null +++ b/src/NServiceBus.AcceptanceTests/Core/OpenTelemetry/When_endpoint_starts_with_instrumentation_options.cs @@ -0,0 +1,67 @@ +namespace NServiceBus.AcceptanceTests.Core.OpenTelemetry; + +using System.IO; +using System.Text.Json; +using System.Threading.Tasks; +using AcceptanceTesting; +using AcceptanceTesting.Customization; +using EndpointTemplates; +using NUnit.Framework; + +public class When_endpoint_starts_with_instrumentation_options : NServiceBusAcceptanceTest +{ + static string basePath = Path.Combine(TestContext.CurrentContext.TestDirectory, TestContext.CurrentContext.Test.ID); + + [Test] + public async Task Should_write_instrumentation_options_to_startup_diagnostics() + { + // TestContext.CurrentContext.Test.ID is stable across test runs, + // therefore we need to clear existing diagnostics file to avoid asserting on a stale file + if (Directory.Exists(basePath)) + { + Directory.Delete(basePath, true); + } + + await Scenario.Define() + .WithEndpoint() + .Done(c => c.EndpointsStarted) + .Run(); + + var endpointName = Conventions.EndpointNamingConvention(typeof(MyEndpoint)); + var pathToFile = Path.Combine(basePath, $"{endpointName}-configuration.txt"); + + using var document = JsonDocument.Parse(await File.ReadAllTextAsync(pathToFile)); + var section = document.RootElement.GetProperty("OpenTelemetry"); + + Assert.Multiple(() => + { + Assert.That(section.GetProperty("SendTraceMode").GetString(), Is.EqualTo("StartNew")); + Assert.That(section.GetProperty("PublishTraceMode").GetString(), Is.EqualTo("ContinueExisting")); + Assert.That(section.GetProperty("ExceptionRecordingMode").GetString(), Is.EqualTo("Logs")); + Assert.That(section.GetProperty("Recoverability").GetProperty("DelayedRetryTraceMode").GetString(), Is.EqualTo("ContinueExisting")); + Assert.That(section.GetProperty("DelayedDelivery").GetProperty("SendOperationTraceMode").GetString(), Is.EqualTo("ContinueExisting")); + Assert.That(section.GetProperty("DelayedDelivery").GetProperty("SagaTimeoutTraceMode").GetString(), Is.EqualTo("ContinueExisting")); + Assert.That(section.GetProperty("UseV11Behavior").ValueKind, Is.EqualTo(JsonValueKind.False).Or.EqualTo(JsonValueKind.True)); + }); + } + + class Context : ScenarioContext; + + public class MyEndpoint : EndpointConfigurationBuilder + { + public MyEndpoint() => + EndpointSetup(c => + { + c.SetDiagnosticsPath(basePath); + + // Flip every option away from its default so the test proves the configured values are written + var tracing = c.Tracing(); + tracing.SendTraceMode = TraceMode.StartNew; + tracing.PublishTraceMode = TraceMode.ContinueExisting; + tracing.ExceptionRecordingMode = ExceptionRecordingMode.Logs; + tracing.Recoverability.DelayedRetryTraceMode = TraceMode.ContinueExisting; + tracing.DelayedDelivery.SendOperationTraceMode = TraceMode.ContinueExisting; + tracing.DelayedDelivery.SagaTimeoutTraceMode = TraceMode.ContinueExisting; + }).EnableStartupDiagnostics(); + } +} diff --git a/src/NServiceBus.AcceptanceTests/Core/SelfVerification/ConventionEnforcementTests.cs b/src/NServiceBus.AcceptanceTests/Core/SelfVerification/ConventionEnforcementTests.cs index 68477fa9496..76591f5d998 100644 --- a/src/NServiceBus.AcceptanceTests/Core/SelfVerification/ConventionEnforcementTests.cs +++ b/src/NServiceBus.AcceptanceTests/Core/SelfVerification/ConventionEnforcementTests.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Reflection; using NUnit.Framework; +using OpenTelemetry; [TestFixture] public class ConventionEnforcementTests : NServiceBusAcceptanceTest @@ -51,7 +52,11 @@ public void Ensure_all_diagnostics_tests_are_run_sequentially() diagnosticTests.Where(t => t.GetCustomAttribute() == null); Assert.That(diagnosticTests, Is.Not.Empty); - Assert.That(diagnosticTestsWithoutNonParallelizableAttribute, Is.Empty, string.Join(",", diagnosticTests)); + + var parallelizeTests = new[] { typeof(When_endpoint_starts_with_instrumentation_options) }; + Assert.That(diagnosticTestsWithoutNonParallelizableAttribute, + Is.EquivalentTo(parallelizeTests), + string.Join(",", diagnosticTests)); } [Test] diff --git a/src/NServiceBus.Core/Hosting/StartupDiagnostics/OpenTelemetryDiagnostics.cs b/src/NServiceBus.Core/Hosting/StartupDiagnostics/OpenTelemetryDiagnostics.cs new file mode 100644 index 00000000000..a360f6edf0b --- /dev/null +++ b/src/NServiceBus.Core/Hosting/StartupDiagnostics/OpenTelemetryDiagnostics.cs @@ -0,0 +1,28 @@ +#nullable enable + +namespace NServiceBus; + +// Mirrors the shape of InstrumentationOptions (endpointConfiguration.Tracing()) so the +// diagnostics output reads like the configuration API. +sealed class OpenTelemetryDiagnostics +{ + public required string SendTraceMode { get; init; } + public required string PublishTraceMode { get; init; } + public required string ExceptionRecordingMode { get; init; } + public required RecoverabilityInstrumentationDiagnostics Recoverability { get; init; } + public required DelayedDeliveryInstrumentationDiagnostics DelayedDelivery { get; init; } + + // Remove together with V11BehaviorSwitch in obsoletes-v10.cs. + public required bool UseV11Behavior { get; init; } +} + +sealed class RecoverabilityInstrumentationDiagnostics +{ + public required string DelayedRetryTraceMode { get; init; } +} + +sealed class DelayedDeliveryInstrumentationDiagnostics +{ + public required string SendOperationTraceMode { get; init; } + public required string SagaTimeoutTraceMode { get; init; } +} diff --git a/src/NServiceBus.Core/Hosting/StartupDiagnostics/StartupDiagnosticsJsonContext.cs b/src/NServiceBus.Core/Hosting/StartupDiagnostics/StartupDiagnosticsJsonContext.cs index 6979171c2b5..a22092eba33 100644 --- a/src/NServiceBus.Core/Hosting/StartupDiagnostics/StartupDiagnosticsJsonContext.cs +++ b/src/NServiceBus.Core/Hosting/StartupDiagnostics/StartupDiagnosticsJsonContext.cs @@ -23,6 +23,7 @@ namespace NServiceBus; [JsonSerializable(typeof(SatelliteDiagnostics))] [JsonSerializable(typeof(RecoverabilityDiagnostics))] [JsonSerializable(typeof(AuditDiagnostics))] +[JsonSerializable(typeof(OpenTelemetryDiagnostics))] [JsonSerializable(typeof(Dictionary))] [JsonSerializable(typeof(PersistenceDiagnosticsEntry))] [JsonSerializable(typeof(LicensingDiagnostics))] diff --git a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs index 7ae9288c9a7..bfaeac1e966 100644 --- a/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs +++ b/src/NServiceBus.Core/OpenTelemetry/OpenTelemetryFeature.cs @@ -26,5 +26,22 @@ protected override void Setup(FeatureConfigurationContext context) new PopulateRecoverabilityTraceMetadataBehavior(instrumentationOptions), "Populates the recoverability metadata" ); + + context.Settings.AddStartupDiagnosticsSection("OpenTelemetry", new OpenTelemetryDiagnostics + { + SendTraceMode = instrumentationOptions.SendTraceMode.ToString(), + PublishTraceMode = instrumentationOptions.PublishTraceMode.ToString(), + ExceptionRecordingMode = instrumentationOptions.ExceptionRecordingMode.ToString(), + Recoverability = new RecoverabilityInstrumentationDiagnostics + { + DelayedRetryTraceMode = instrumentationOptions.Recoverability.DelayedRetryTraceMode.ToString() + }, + DelayedDelivery = new DelayedDeliveryInstrumentationDiagnostics + { + SendOperationTraceMode = instrumentationOptions.DelayedDelivery.SendOperationTraceMode.ToString(), + SagaTimeoutTraceMode = instrumentationOptions.DelayedDelivery.SagaTimeoutTraceMode.ToString() + }, + UseV11Behavior = V11BehaviorSwitch.UseV11Behavior + }, StartupDiagnosticsJsonContext.Default.OpenTelemetryDiagnostics); } } \ No newline at end of file