From feca33dd1277c4a51e820e293c377135acba375a Mon Sep 17 00:00:00 2001 From: taha-rafique Date: Wed, 30 Sep 2026 13:13:48 +0100 Subject: [PATCH 1/8] Add outbox metrics --- docs/adr/0084-outbox-metrics.md | 612 ++++++++++++++++++ docs/adr/index.md | 3 +- specs/.current-spec | 2 +- specs/0037-outbox-metrics/.adr-list | 1 + .../.requirements-approved | 0 specs/0037-outbox-metrics/README.md | 60 ++ specs/0037-outbox-metrics/requirements.md | 293 +++++++++ 7 files changed, 969 insertions(+), 2 deletions(-) create mode 100644 docs/adr/0084-outbox-metrics.md create mode 100644 specs/0037-outbox-metrics/.adr-list create mode 100644 specs/0037-outbox-metrics/.requirements-approved create mode 100644 specs/0037-outbox-metrics/README.md create mode 100644 specs/0037-outbox-metrics/requirements.md diff --git a/docs/adr/0084-outbox-metrics.md b/docs/adr/0084-outbox-metrics.md new file mode 100644 index 0000000000..6a2a7b45e8 --- /dev/null +++ b/docs/adr/0084-outbox-metrics.md @@ -0,0 +1,612 @@ +--- +id: 0084-outbox-metrics +title: "Outbox Metrics" +status: Proposed +author: + - "taha-rafique" +created: 2026-09-28 +summary: "Outbox metrics are recorded by direct OpenTelemetry instruments (paramore.brighter.outbox.added.messages and .cleared.messages as Counter, .publish.duration as a Histogram in seconds with explicit long-tail buckets) behind a new IAmABrighterOutboxMeter/OutboxMeter role owned by OutboxProducerMediator rather than derived from traces, with the clear kind carried as an OutboxClearSource parameter on the private dispatch methods and confirmation-path creation time resolved from a bounded, TTL-evicted in-mediator pending-clear ConcurrentDictionary." +tags: + - "observability" + - "metrics" + - "otel" + - "outbox" +--- + +# 0084. Outbox Metrics + +Date: 2026-09-28 + +## Status + +Proposed + +## Context + +**Parent Requirement**: [specs/0037-outbox-metrics/requirements.md](../../specs/0037-outbox-metrics/requirements.md) + +**Scope**: This ADR focuses specifically on the complete design of Outbox metrics: the instrument set (names, types, units, attributes and their cardinality bound), the recording mechanism, the meter role and its implementation, how the clear kind reaches the point of recording, and how creation time reaches the asynchronous confirmation callback. It is the only ADR for spec 0037. + +### The problem + +Today the Outbox's only observability signal is trace spans and span events. `BrighterTracer.WriteOutboxEvent` (`src/Paramore.Brighter/Observability/BrighterTracer.cs:951`) opens with: + +```csharp +if (span == null) return; +``` + +and the add path passes `requestContext.Span`, which is frequently null. Combined with production tail/ratio sampling, this means the add path leaves no aggregate record at all. Spans also cannot answer the rate/ratio questions an operator needs ("what fraction of my volume is the sweeper rescuing?", "what is p99 publish lag?"). Incident PI-36093 (NOFR events emitted after significant Outbox lag, causing `IgnoredOrders`) was invisible for exactly this reason, and RLOI-4186 is blocked on having chartable, alertable signals. + +### Forces + +1. **FR-9 forbids any dependency on tracing or sampling.** Recording must happen when no `IAmABrighterTracer` is registered, when `requestContext.Span` and `Activity.Current` are null, when the sampler drops the trace, and when `InstrumentationOptions` is `None`. +2. **The two clear kinds converge (C-5).** `ClearOutbox`/`ClearOutboxAsync` (explicit) and `ClearOutstandingFromOutboxAsync` → `BackgroundDispatchUsingAsync` (sweeper) both reach the same private `Dispatch` (`OutboxProducerMediator.cs:1035`), `BulkDispatchAsync` (`:1092`) and `DispatchAsync` (`:1175`). FR-4's sweeper attribution is impossible at the point of recording unless the initiating context is carried down. +3. **Confirmation-based clears are asynchronous and data-poor (C-4).** For a producer implementing `ISupportPublishConfirmation`, `MarkDispatched` happens later, on a broker/threadpool thread, inside `HandleAsyncPublishConfirmation` (`:860`) or the `ConfigurePublisherCallbackMaybe` delegate (`:952`). The callback receives a `PublishConfirmationResult(bool Success, Id MessageId, RoutingKey? Topic, ActivityContext? PublishSpanContext)` — no `Message`, so no `Header.TimeStamp`. +4. **Bulk dispatch marks dispatched by id (C-8).** `BulkDispatchAsync` iterates `batch.Ids()` (`IEnumerable`, `MessageBatch.Ids()`), while the `Message` objects are one level up in `var messages = topicBatch.ToArray()` (`:1126`). +5. **Zero cost when nobody is listening (NFR-1), no behavioural change (NFR-2), faults isolated (NFR-3), thread safe (NFR-4).** The mediator is documented as a process singleton (`OutboxProducerMediator.cs:45`) whose sweeper dispatch (`BackgroundDispatchUsingAsync`, holding `_backgroundClearSemaphore`) runs concurrently with request-thread adds and with confirmation callbacks on producer threads. +6. **All TFMs (NFR-5).** `Paramore.Brighter` targets `netstandard2.0;net8.0;net9.0;net10.0` (`src/Directory.Build.props:43`). +7. **No public API break, no Outbox-implementation change, no schema change (C-11, C-12).** + +### Constraint that shapes the mechanism choice + +`BrighterMetricsFromTracesProcessor` is a `BaseProcessor` whose `OnEnd` reads `activity.GetTagItem(BrighterSemanticConventions.InstrumentationDomain)` on **ended, sampled** activities. Nothing it can observe exists on the Outbox add path (an `ActivityEvent` on a possibly-null span is not an `Activity`), and every span it could observe on the clear path is subject to the sampler. It is therefore structurally incapable of satisfying FR-9 for the Outbox. + +## Decision + +Record Outbox metrics **directly on instruments** created on a `Meter` named `BrighterSemanticConventions.MeterName` (`"Paramore.Brighter"`, `BrighterSemanticConventions.cs:97`), behind a new role `IAmABrighterOutboxMeter`, called from `OutboxProducerMediator` at the exact points where the Outbox store confirms the write or the dispatch. + +### Architecture Overview + +``` + Application OutboxProducerMediator OutboxMeter + ─────────── ────────────────────── ─────────── + AddToOutbox(Async) ──────────► write succeeds ─────────► RecordAdded ──────► added.messages (Counter) + EndBatchAddToOutbox(Async) ──► write succeeds, per msg ► RecordAdded ──────► │ + │ + ClearOutbox(Async) ──────────► Dispatch / DispatchAsync (clearSource: Explicit) │ + OutboxSweeper.SweepAsync ────► ClearOutstandingFromOutboxAsync │ + └► BackgroundDispatchUsingAsync │ + ├► DispatchAsync (clearSource: Sweeper) │ + └► BulkDispatchAsync (clearSource: Sweeper) │ + │ │ + non-confirmation producer │ MarkDispatched succeeded ▼ + └────────────► RecordCleared ──► cleared.messages (Counter) + │ publish.duration (Histogram) + confirmation producer │ + send ──► _pendingClears[id] = {TimeStamp, Topic, clearSource} + broker ──► OnMessagePublished(result) + MarkDispatched succeeded + _pendingClears.TryRemove(id) ────┘ +``` + +The meter is a **collaborator of the mediator**, not of the trace pipeline. `BrighterMetricsFromTracesProcessor` is untouched and continues to own the messaging and db signals it already derives (ADR 0022 remains in force). + +### 1. Instrument names, types and units (proposal, C-7) + +No OpenTelemetry semantic convention covers an outbox. The nearest existing convention, `messaging.client.sent.messages`, is defined as "number of messages producer **attempted to send to the broker**" and is already emitted by `MessagingMeter` (`MessagingMeter.cs:54`); reusing it for Outbox adds would conflate store writes with broker sends and double-count. `db.client.operation.duration` describes the duration of one store call, not a wait between two events. We therefore mint `paramore.brighter.*` names, as FR-10 and `BrighterSemanticConventions` prescribe for the no-convention case. + +| Instrument | Type | Unit (UCUM) | Description | +|---|---|---|---| +| `paramore.brighter.outbox.added.messages` | `Counter` | `{message}` | Number of messages written to the Outbox. | +| `paramore.brighter.outbox.cleared.messages` | `Counter` | `{message}` | Number of messages recorded as dispatched from the Outbox. | +| `paramore.brighter.outbox.publish.duration` | `Histogram` | `s` | Time a message waited between being created for the Outbox and being recorded as dispatched. | + +Notes and deliberate deviations, to be confirmed in `#interest-oss-brighter`: + +- **`.duration`, not `.latency`.** The requirements' illustrative name was `paramore.brighter.outbox.publish.latency`. OpenTelemetry metric naming guidance uses `duration` for timings (`messaging.client.operation.duration`, `db.client.operation.duration`, `messaging.process.duration` — all three already in `MessagingMeter`/`DbMeter`). `...publish.latency` is an acceptable fallback if maintainers prefer the spec's wording; nothing else in this design depends on the choice. +- **`Counter`, not `Counter`.** `MessagingMeter` uses `Counter`. The difference is cosmetic — the OTel SDK aggregates both into a 64-bit sum — but `long` is the OTel .NET recommendation for monotonic counts and removes any question about a long-lived publisher's cumulative total. Reverting to `Counter` for house consistency is an acceptable alternative. +- **Instrument names live as `public const` on `BrighterSemanticConventions`** (`OutboxAddedMessages`, `OutboxClearedMessages`, `OutboxPublishDuration`), unlike `MessagingMeter`'s inline literals, because RLOI-4186 needs a stable, referenceable identifier and AC-19 tests assert against constants. + +### 2. Attribute keys and value domains + +| Key | Constant | Value domain | Applies to | +|---|---|---|---| +| `messaging.destination.name` | `BrighterSemanticConventions.MessagingDestination` (existing) | `message.Header.Topic.Value`, or `RoutingKey.Empty.Value` (`""`) when the confirmation callback cannot supply a topic | all three | +| `paramore.brighter.outbox.clear_source` | **new** `BrighterSemanticConventions.OutboxClearSource` | `"explicit"`, `"sweeper"`, `"unknown"` | cleared counter, duration histogram | + +`messaging.destination.name` is the OTel messaging convention key and is already in `MessagingMeter`'s allowed-tag sets, so it satisfies FR-10's "OpenTelemetry semantic-convention keys where one exists". `snake_case` in `paramore.brighter.outbox.clear_source` matches the existing `paramore.brighter.outbox.shared_transaction`. + +**No `error.type` / outcome dimension.** FR-8 means only successes are counted, so an outcome dimension would be constant. Failures remain observable via `messaging.client.sent.messages` + `error.type`, logs and circuit-breaker trips (OOS-3). + +**Service-resource attributes are not duplicated as data-point attributes.** `MessagingMeter`/`DbMeter` append `meterProvider.GetServiceAttributes()` (`MeterProviderExtensions.cs:37`) because they are constructed *inside* a `MeterProvider`'s service collection. The mediator is not — `BrighterTracerBuilderExtensions` has to probe `services.Any(sd => sd.ServiceType == typeof(IAmABrighterMessagingMeter))` precisely because those registrations are scoped to the builder's own container, which the application container (and hence the mediator) does not see when the host uses `Sdk.CreateMeterProviderBuilder()`. For a directly instrumented metric the SDK attaches the `Resource` to every exported data point anyway, so `service.*` is already present downstream. `OutboxMeter` therefore takes `MeterProvider?` and appends `GetServiceAttributes()` **only** when a host explicitly supplies one, preserving parity for anyone who wants it. + +**Cardinality arithmetic (AC-20, NFR-6).** With `D` configured destinations: + +- added counter: `D` series. +- cleared counter: `≤ (D + 1) × 3` series (the `+1` is the empty-destination fallback; the `× 3` is the enumerated clear source). In normal operation `"unknown"` and `""` never appear, so `D × 2 = 6` for `D = 3`, within AC-20's `3 × (dimension values)` bound. +- duration histogram: the same `≤ (D + 1) × 3` series, each with 19 buckets + `+Inf`. + +Total upper bound `≤ 7 × (D + 1)` series, independent of message count. No dimension derives from per-message data. + +### 3. Histogram bucket strategy + +Publish lag spans milliseconds (inline dispatch) to minutes (sweeper rescue after a restart — AC-14 expects a 300 s sample). The OTel SDK default explicit boundaries are `[0, 5, 10, 25, 50, 75, 100, 250, 500, 750, 1000, 2500, 5000, 7500, 10000]`, unit-agnostic numbers: with a **seconds** instrument, every healthy sample collapses into the single `(0, 5]` bucket, destroying p50/p95 resolution exactly where operators need it. + +We therefore supply explicit boundaries via instrument advice: + +```csharp +private static readonly double[] s_publishDurationBoundaries = +[ + 0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, + 2.5, 5, 7.5, 10, 30, 60, 300, 600 +]; +``` + +The first fourteen are the OTel-recommended duration boundaries (as used for `db.client.operation.duration` and `messaging.client.operation.duration`); `30, 60, 300, 600` extend the tail for sweeper rescue. AC-14's 300 s sample lands on the `300` boundary; anything beyond 600 s lands in `+Inf`. + +**Per-TFM availability — verified.** `System.Diagnostics.Metrics.InstrumentAdvice` and `Meter.CreateHistogram(name, unit, description, tags, advice)` were introduced in .NET 9, but `Paramore.Brighter` resolves `System.Diagnostics.DiagnosticSource` **10.0.11** on every target (pinned in `Directory.Packages.props:154`, resolved transitively via `OpenTelemetry` 1.18.0), and the package's `lib/netstandard2.0` build contains `InstrumentAdvice` and `HistogramBucketBoundaries`. So **no `#if` is required** and no TFM loses the buckets. Honouring advice requires OTel SDK ≥ 1.9.0; the repo pins 1.18.0. + +Two consequences to accept: + +- `Paramore.Brighter.csproj` gains an explicit `` (version already centrally pinned) so this becomes an intentional dependency rather than a transitive accident that a later `OpenTelemetry` bump could downgrade. +- If maintainers reject raising that floor, the fallback is `#if NET9_0_OR_GREATER` around the advice with no advice on `netstandard2.0`/`net8.0`, and operator documentation of the equivalent view: `.AddView("paramore.brighter.outbox.publish.duration", new ExplicitBucketHistogramConfiguration { Boundaries = [...] })`. This is a documented degradation, not a silent one. + +### 4. Unit of the latency sample: seconds + +The instrument records **seconds** (`unit: "s"`), consistent with every existing Brighter duration instrument and with OTel's preference for base units. The requirements state examples in milliseconds; the acceptance criteria assert on the *sample value*, so they are satisfied by the seconds-equivalent: + +| AC | Requirement wording | Assertion in this design | +|---|---|---| +| AC-13 | one sample of 250 ms | one sample of `0.25` | +| AC-14 | one sample of 300,000 ms | one sample of `300` | +| AC-15 | one sample of 0 | one sample of `0` | + +Tests assert via `TimeSpan` on the spy double (`TimeSpan.FromMilliseconds(250)`), so the millisecond wording of the spec is preserved at the test level; only the exported instrument is in seconds. + +### 5. Key Components + +#### 5.1 `IAmABrighterOutboxMeter` — a new role + +```csharp +namespace Paramore.Brighter.Observability; + +public interface IAmABrighterOutboxMeter +{ + /// Record one message written to the Outbox. + void AddMessageAdded(RoutingKey destination); + + /// Record one message recorded as dispatched from the Outbox. + void AddMessageCleared(RoutingKey destination, OutboxClearSource clearSource); + + /// Record how long a cleared message waited in the Outbox. + void RecordPublishDuration(TimeSpan wait, RoutingKey destination, OutboxClearSource clearSource); + + /// False when no MeterListener is attached to any instrument. + bool Enabled { get; } +} +``` + +**Responsibilities.** *Knowing*: the instrument identities (name, type, unit, description, bucket boundaries) and any service-resource attributes it was given. *Doing*: turning a `(destination, clearSource, value)` triple into one instrument recording. *Deciding*: nothing beyond `Enabled` — whether an event is countable is the mediator's decision, because only the mediator knows whether the store accepted the write. Stereotype: service provider. + +**Contract.** `destination` may be `RoutingKey.Empty`; `clearSource` is always one of the three enumerated values; `wait` is expected non-negative (callers clamp per FR-7a) and an implementation receiving a negative value must record `0` rather than throw. **No member may throw** — see §7 for why the mediator nonetheless guards the call site. + +**Why a new role rather than extending `IAmABrighterMessagingMeter`.** Every member of that interface takes an `Activity` (`IAmABrighterMessagingMeter.cs:37-55`); its single responsibility is *translating an ended Brighter activity into OTel messaging metrics*, and its only collaborator is `BrighterMetricsFromTracesProcessor`. FR-9 makes Outbox recording activity-free, so adding activity-free members would give the type two incompatible collaboration contracts, and would force the existing `SpyMessagingMeter` to grow members it cannot meaningfully implement — editing a double that AC-23 wants left alone. Different collaborator, different data, different trigger ⇒ different role. + +#### 5.2 `OutboxMeter` + +```csharp +public sealed class OutboxMeter(MeterProvider? meterProvider = null) : IAmABrighterOutboxMeter, IDisposable +``` + +- Creates `new Meter(BrighterSemanticConventions.MeterName)`, held in a readonly field and disposed in `Dispose()`. +- `_serviceAttributes = meterProvider?.GetServiceAttributes() ?? []`. +- `Enabled => _addedMessages.Enabled || _clearedMessages.Enabled || _publishDuration.Enabled` — the same disjunction as `MessagingMeter.Enabled` (`MessagingMeter.cs:186`). +- Builds tags with `System.Diagnostics.TagList` (available on all TFMs, verified in the `netstandard2.0` lib), whose eight inline slots hold destination + clear source + up to four service attributes with **zero heap allocation**, and calls the `Add(T, in TagList)` / `Record(T, in TagList)` overloads. This is why **no `#if NET8_0_OR_GREATER` is needed**: unlike `MessagingMeter`/`DbMeter`, which filter an `Activity`'s tags through a `FrozenSet` allow-list, `OutboxMeter` constructs its tags directly and needs no allow-list at all. (NFR-5 is satisfied by not using any TFM-gated API, rather than by conditional compilation.) +- Each method body is wrapped in `try`/`catch` logging through a source-generated `Log` partial (ADR 0026), honouring the "must not throw" contract. + +#### 5.3 `OutboxClearSource` + +```csharp +namespace Paramore.Brighter.Observability; + +public enum OutboxClearSource { Explicit, Sweeper, Unknown } + +public static class OutboxClearSourceExtensions +{ + public static string ToClearSourceName(this OutboxClearSource clearSource) => clearSource switch + { + OutboxClearSource.Explicit => "explicit", + OutboxClearSource.Sweeper => "sweeper", + _ => "unknown" + }; +} +``` + +This mirrors `MessagingSystemExtensions.ToMessagingSystemName` / `DbSystemExtensions` exactly, except that the default arm returns `"unknown"` instead of throwing `ArgumentOutOfRangeException`: on a metrics path an unmapped value must degrade, not throw (NFR-3). + +Regarding ADR 0019: a closed two-and-a-half-state enumeration whose only job is to select one of three interned strings is what an `enum` is for; a record struct would add a type without necessity. The enum is a domain type, not a primitive, so it is not the primitive obsession ADR 0019 targets. + +Placed in `Paramore.Brighter.Observability` for cohesion with `MessagingSystem`/`DbSystem` (enums-as-attribute-domains). It could arguably live in the `Paramore.Brighter` root as a dispatch concept; observability is the only consumer, so cohesion wins. + +#### 5.4 Meter ownership and lifetime + +`OutboxProducerMediator`'s constructor gains a trailing optional parameter: + +```csharp +IAmABrighterOutboxMeter? outboxMeter = null +``` + +with `_outboxMeter = outboxMeter ?? new OutboxMeter();` and `_ownsOutboxMeter = outboxMeter is null;`. This reuses the `_ownsRegistry` / `_ownsTransformerFactories` pattern already in this class and satisfies ADR 0069 directly: `Dispose(bool)` calls `DisposeQuietly(_outboxMeter)` only when `_ownsOutboxMeter`, and additionally calls `_pendingClears.Clear()`. + +Mediator-owned-by-default is what makes FR-11/AC-21 work with **no DI at all**: instruments are published globally by `System.Diagnostics.Metrics`, and a `MeterProvider` that has called `AddMeter("Paramore.Brighter")` — which is exactly what `AddBrighterInstrumentation()` does (`BrighterMetricsBuilderExtensions.cs:41`) — attaches its listener to instruments on any `Meter` of that name, including ones created after the provider was built. No new registration call, no new package reference, no flag. + +Two call-site consequences, both verified: + +- `ServiceCollectionExtensions.BuildOutBoxProducerMediator` (`:760`) uses `Activator.CreateInstance` with explicit positional arguments and its own comment notes that "Activator.CreateInstance does not apply constructor default values". It **must** be updated to pass a new trailing argument `OutboxMeter(serviceProvider)`, a private helper returning `serviceProvider.GetService()` in the style of the existing `Tracer(serviceProvider)` / `OutboxCircuitBreaker(serviceProvider)` (`:926`, `:931`). This lets a host that wires OTel through DI share one meter instance; when nothing is registered it returns null and the mediator owns its own. +- `ControlBusSenderFactory.cs:61` and `ControlBusReceiverBuilder.cs:177` use `new OutboxProducerMediator(...)` with C# optional-argument binding and need no change. + +This is not a public-interface change: `IAmAnOutboxProducerMediator` and the `IAmAnOutbox*` hierarchy are untouched, and adding a trailing optional constructor parameter is source-compatible (C-11). + +### 6. Clear-kind plumbing + +`OutboxClearSource` travels as an explicit parameter on the private dispatch methods: + +```csharp +private void Dispatch( + IEnumerable posts, + RequestContext requestContext, + OutboxClearSource clearSource, + Dictionary? args = null) + +private async Task DispatchAsync( + IEnumerable posts, + RequestContext requestContext, + OutboxClearSource clearSource, + bool continueOnCapturedContext, + CancellationToken cancellationToken) + +private async Task BulkDispatchAsync( + IEnumerable posts, + RequestContext requestContext, + OutboxClearSource clearSource, + bool continueOnCapturedContext, + CancellationToken cancellationToken) +``` + +There are exactly four internal call sites, all inside `OutboxProducerMediator.cs`, and no callers outside it: + +| Line | Current call | Becomes | +|---|---|---| +| 398 (`ClearOutbox`) | `Dispatch([message], requestContext, args)` | `Dispatch([message], requestContext, OutboxClearSource.Explicit, args)` | +| 467 (`ClearOutboxAsync`) | `DispatchAsync([message], requestContext, continueOnCapturedContext, cancellationToken)` | `DispatchAsync([message], requestContext, OutboxClearSource.Explicit, continueOnCapturedContext, cancellationToken)` | +| 749 (`BackgroundDispatchUsingAsync`) | `BulkDispatchAsync(messages, requestContext, false, cancellationToken)` | `BulkDispatchAsync(messages, requestContext, OutboxClearSource.Sweeper, false, cancellationToken)` | +| 753 (`BackgroundDispatchUsingAsync`) | `DispatchAsync(messages, requestContext, false, cancellationToken)` | `DispatchAsync(messages, requestContext, OutboxClearSource.Sweeper, false, cancellationToken)` | + +`BackgroundDispatchUsingAsync`, `ClearOutbox`, `ClearOutboxAsync`, `ClearOutstandingFromOutboxAsync` and `OutboxSweeper` keep their signatures unchanged: the sweeper kind is already implied by entering through `ClearOutstandingFromOutboxAsync`, so `OutboxSweeper.cs` needs no edit at all. + +**Why a parameter.** It is compile-checked (a new dispatch path cannot forget it), introduces no shared mutable state (so no thread-safety question — NFR-4), changes no public API (C-11), and changes no dispatch behaviour (NFR-2). + +**Rejected: stamping `RequestContext.Bag`.** Implicit, unvalidated, and unsafe on a shared context: `RequestContext` is copied and mutated across the clear paths (`requestContext.CreateCopy()` in both confirmation handlers), and a stale bag entry would silently mis-attribute a later clear. **Rejected: a separate public entry point per clear kind.** Duplicates the dispatch logic and grows the public surface. + +### 7. Exactly where each recording sits, with guard and fault isolation + +Two private helpers in the mediator carry the guard, the FR-7a clamp and the fault isolation: + +```csharp +private void RecordAdded(Message message) +{ + if (!_outboxMeter.Enabled) return; // NFR-1: before any tag work + try { _outboxMeter.AddMessageAdded(message.Header.Topic); } + catch (Exception ex) { Log.OutboxMetricsFault(s_logger, ex); } +} + +private void RecordCleared( + RoutingKey destination, + OutboxClearSource clearSource, + DateTimeOffset? createdAt, + DateTimeOffset dispatchedAt) +{ + if (!_outboxMeter.Enabled) return; + try + { + _outboxMeter.AddMessageCleared(destination, clearSource); + + if (createdAt is { } created) // FR-7b: null => no sample + { + var wait = dispatchedAt - created; + _outboxMeter.RecordPublishDuration( + wait < TimeSpan.Zero ? TimeSpan.Zero : wait, destination, clearSource); // FR-7a + } + } + catch (Exception ex) { Log.OutboxMetricsFault(s_logger, ex); } +} +``` + +**Where the try/catch lives: both, deliberately.** The **call-site** catch is load-bearing for AC-22, whose double's record methods throw — a catch inside `OutboxMeter` cannot protect the mediator from a third-party or test implementation of `IAmABrighterOutboxMeter`. The **implementation-side** catch in `OutboxMeter` makes the "must not throw" contract true for the shipped type regardless of caller, including on the confirmation thread. Neither alone is sufficient; the cost is one extra `try` region on a path that already has one. + +**The FR-7a clamp lives at the call site, not in the meter**, so that a spy double observes the post-policy value and AC-15 can assert "exactly one sample of 0" against the double rather than only through an exporter. `OutboxMeter` also clamps defensively. + +Recording call sites, method by method: + +| Path | File:line today | Placement | +|---|---|---| +| **FR-1 async single add** | `AddToOutboxAsync` `:278-291` | after the `if (!written) throw new ChannelFailureException(...)` guard: `RecordAdded(message)`. Nothing recorded when `batchId != null` (early return at `:267`) or when the write failed (AC-3). | +| **FR-1 sync single add** | `AddToOutbox` `:321-329` | same, after the `!written` throw. | +| **FR-2 async batch** | `EndBatchAddToOutboxAsync` `:659-672` | after the `!written` throw, `foreach (var message in batch) RecordAdded(message);`. `batch` is the live `List` returned by `BeginBatchAddToOutbox` → `GetBatchOrThrow` and is the exact list handed to `_asyncOutbox.AddAsync`, so the mediator already knows the batch membership at flush time — no new state is needed for AC-2. | +| **FR-2 sync batch** | `EndBatchAddToOutbox` `:631-641` | same. | +| **FR-3/FR-5 sync dispatch** | `Dispatch` `:1066-1077` | non-confirmation branch: capture the MarkDispatched result and instant, then record. | +| **FR-3/FR-5 async dispatch** | `DispatchAsync` `:1211-1221` | as above. | +| **FR-3/FR-5 bulk** | `BulkDispatchAsync` `:1141-1154` | per id in `batch.Ids()`, after that id's MarkDispatched succeeded. | +| **FR-3/FR-5 async confirmation** | `HandleAsyncPublishConfirmation` `:879-884` | after MarkDispatchedAsync succeeded, using the pending-clear entry. | +| **FR-3/FR-5 sync confirmation** | `ConfigurePublisherCallbackMaybe` delegate `:976-978` | as above. | + +**Capturing the dispatch instant without changing behaviour.** FR-5 requires the *same* instant passed to the clearing `MarkDispatched`. Today `_timeProvider.GetUtcNow()` is evaluated **inside** the lambda handed to `ExecuteWithResiliencePipeline(Async)`, so under a retrying pipeline each attempt gets a fresh instant. Hoisting it outside would change the stored `MarkDispatched` timestamp under retry — a real, if small, behavioural change (NFR-2). We therefore capture it from inside the lambda: + +```csharp +var dispatchedAt = default(DateTimeOffset); +var marked = await ExecuteWithResiliencePipelineAsync( + async _ => + { + dispatchedAt = _timeProvider.GetUtcNow(); + await _asyncOutbox.MarkDispatchedAsync( + message.Id, requestContext, dispatchedAt, cancellationToken: cancellationToken); + }, + requestContext, cancellationToken: cancellationToken); + +if (marked) + RecordCleared(message.Header.Topic, clearSource, message.Header.TimeStamp, dispatchedAt); +``` + +`dispatchedAt` then holds exactly the instant used by the successful attempt. `ExecuteWithResiliencePipeline(Async)` already returns `bool` (`:1380`, `:1405`) — `true` only when the delegate completed without throwing — so "MarkDispatched succeeded" is observable with no new plumbing, satisfying FR-8's "metric follows the store, not the send" and AC-11. The additional captured local adds one field to an already-allocated closure display class; no new allocation, nothing computed when disabled beyond the assignment itself (NFR-1, AC-18). + +**Recordings sit outside every tracing guard.** None of them reads `_tracer`, `requestContext.Span` or `_instrumentationOptions`, and none is nested inside a `_tracer?.` or `instrumentationOptions.HasFlag(...)` block. That is the whole of FR-9/AC-17. + +**AC-12 (empty sweep).** `messages.Length == 0` means the `foreach` bodies in `DispatchAsync`/`BulkDispatchAsync` never execute, so there are no recordings at all — not even zero-valued ones. + +### 8. The bulk dispatch path (C-8, AC-7) + +Within one `topicBatch` the grouping key is `(WireTopic, LookupTopic)` (`:1108`), so **the destination is constant for the batch**: `topicBatch.Key.WireTopic`. Only `Header.TimeStamp` needs a per-message lookup, resolved from the `messages` array already materialised at `:1126`: + +```csharp +var messages = topicBatch.ToArray(); + +Dictionary? createdAtById = null; +if (_outboxMeter.Enabled) // NFR-1: not built when disabled +{ + createdAtById = new Dictionary(messages.Length, StringComparer.Ordinal); + foreach (var m in messages) + createdAtById[m.Id.Value] = m.Header.TimeStamp; // indexer, not Add: a duplicate id must not throw +} +``` + +and inside `foreach (var successfulMessage in batch.Ids())`, after that id's MarkDispatched returned `true`: + +```csharp +RecordCleared( + topicBatch.Key.WireTopic, + clearSource, + createdAtById is not null && createdAtById.TryGetValue(successfulMessage.Value, out var created) + ? created + : null, + dispatchedAt); +``` + +One cleared unit and one latency sample **per message**, not per batch (AC-7). Keying on `message.Id.Value` (string, `Ordinal`) matches the existing `producerSpans.TryAdd(message.Id.Value, span)` convention in these same methods. + +### 9. Confirmation-path creation time: the pending-clear map + +For a producer implementing `ISupportPublishConfirmation`, the callback gets a `PublishConfirmationResult` with no `Message`. The mediator therefore remembers the small amount it needs at send time. + +**State.** + +```csharp +private readonly ConcurrentDictionary _pendingClears = new(StringComparer.Ordinal); + +private readonly record struct PendingClear( + DateTimeOffset CreatedAt, + RoutingKey Destination, + OutboxClearSource ClearSource); + +private const int MAX_PENDING_CLEARS = 10_000; +private static readonly TimeSpan s_pendingClearTtl = TimeSpan.FromMinutes(5); +private static readonly TimeSpan s_pendingClearPurgeInterval = TimeSpan.FromMinutes(1); +private DateTimeOffset _nextPendingClearPurgeAt; +``` + +`ConcurrentDictionary` is the right collection because the confirmation callback arrives on a broker/threadpool thread while the sweeper's `BackgroundDispatchUsingAsync` and request-thread explicit clears may be writing: it gives lock-free reads, striped-lock writes, an atomic read-and-remove (`TryRemove(key, out value)`), and safe enumeration for the purge. No caller has to serialise access (NFR-4), and nothing about it reintroduces the serialisation ADR 0032 removed from the explicit clear path. `PendingClear` is a `readonly record struct` per ADR 0019, stored by value — no per-message heap allocation. + +**Population — only when the meter is enabled (NFR-1, AC-18).** In `Dispatch`, `DispatchAsync` and `BulkDispatchAsync`, in the `producer is ISupportPublishConfirmation` branch, **before** the send (the broker confirmation can arrive before `SendAsync` returns): + +```csharp +if (_outboxMeter.Enabled) + TrackPendingClear(message, clearSource); +``` + +```csharp +private void TrackPendingClear(Message message, OutboxClearSource clearSource) +{ + try + { + var now = _timeProvider.GetUtcNow(); + + if (now >= _nextPendingClearPurgeAt) + { + _nextPendingClearPurgeAt = now + s_pendingClearPurgeInterval; + foreach (var entry in _pendingClears) // snapshot enumeration; never throws + { + if (now - entry.Value.CreatedAt > s_pendingClearTtl) + _pendingClears.TryRemove(entry.Key, out _); + } + } + + if (_pendingClears.Count >= MAX_PENDING_CLEARS) return; // hard bound: drop-newest + + // Indexer, not TryAdd: a re-dispatch of the same id must refresh the clear source. + _pendingClears[message.Id.Value] = new PendingClear( + message.Header.TimeStamp, message.Header.Topic, clearSource); + } + catch (Exception ex) { Log.OutboxMetricsFault(s_logger, ex); } +} +``` + +In `DispatchAsync`'s confirmation branch, when `sent == false` (already detected at `:1223` for the `TripTopic` call) the entry is removed immediately, so a failed send does not occupy the map for the full TTL. + +**The eviction rule and its bound.** Two mechanisms, deliberately layered: + +- *Primary — age-based, amortised, driven by the existing `_timeProvider`.* At most once per minute (checked on insert) entries older than 5 minutes are removed. The scan is O(n) with n bounded below, and runs on the thread that was already about to send. The purge clock is `_timeProvider`, so `FakeTimeProvider` makes it fully deterministic in tests (NFR-7). We deliberately do **not** hang eviction off `OutstandingMessagesCheck`: that method exists to count outstanding messages, runs on a `Task.Run` background thread, is rate-limited by `_maxOutStandingCheckInterval`, and is skipped entirely when `_outBox` is null — piggy-backing on it would both muddy its responsibility and leave async-only configurations unpurged. +- *Backstop — hard cap.* Never more than 10,000 entries. At roughly 80 bytes per entry plus `ConcurrentDictionary` overhead this bounds the map at well under 2 MB regardless of broker behaviour. + +**The honest trade-off.** Both mechanisms lose latency samples rather than memory. A confirmation that arrives after the 5-minute TTL, or a send made while the map is at its cap, hits FR-7b: the **cleared unit is still recorded** (with `clear_source = "unknown"` and the destination from `result.Topic`), the **latency sample is skipped**. Latency-sample count can therefore be lower than cleared count — the divergence FR-7b requires the ADR to document. A 5-minute TTL is generous relative to RabbitMQ publisher confirms and Kafka acks (sub-second to seconds); a confirmation that takes longer than five minutes has almost certainly been lost. + +**Consumption in the callback.** In both `HandleAsyncPublishConfirmation` and the `ConfigurePublisherCallbackMaybe` delegate, after MarkDispatched returned `true`: + +```csharp +var resolved = _outboxMeter.Enabled + && !Id.IsNullOrEmpty(result.MessageId) + && _pendingClears.TryRemove(result.MessageId.Value, out var pending); + +RecordCleared( + resolved ? pending.Destination : result.Topic ?? RoutingKey.Empty, + resolved ? pending.ClearSource : OutboxClearSource.Unknown, + resolved ? pending.CreatedAt : null, + dispatchedAt); +``` + +- `result.MessageId == Id.Empty` (documented on `PublishConfirmationResult.MessageId` as "the producer could not determine the id from the broker response") makes lookup impossible ⇒ FR-7b/AC-16: no latency sample, cleared unit still recorded because MarkDispatched succeeded, no exception escapes the callback. +- The entry is **not** removed when MarkDispatched fails, so a later sweeper re-dispatch can still resolve a creation time. +- `result.Topic` is documented nullable; the fallback is `RoutingKey.Empty` (attribute value `""`), which is what the `+1` in §2's cardinality arithmetic accounts for. + +**Known gap, stated plainly.** Read-and-remove holds at most one pending entry per message id. If a confirmation-producer message is dispatched twice concurrently (the FR-3/AC-24 scenario: an explicit clear whose confirmation is late, then a sweeper re-dispatch), the **first** confirmation consumes the entry and the **second** records a cleared unit with `clear_source = "unknown"` and no latency sample. So AC-24's literal expectation — cleared units 2 (1 explicit, 1 sweeper) and latency samples 2 — is met on the non-confirmation dispatch paths, where each dispatch computes latency from its own in-scope `Message` and the map is not involved, and **not** on confirmation transports. The AC-24 test must therefore use a non-confirmation producer, and the confirmation variant is a documented FR-7b divergence. The escape hatch, if maintainers require parity, is to add an outstanding-send count to `PendingClear` and decrement-then-remove; that is deferred as unjustified complexity for a race that only occurs when the sweeper's `minimumAge` is shorter than a broker's confirmation latency. + +### 10. Registration (FR-11) + +No change to `BrighterMetricsBuilderExtensions.AddBrighterInstrumentation()` is required or made: it already calls `builder.AddMeter(BrighterSemanticConventions.MeterName)`, which is all the instruments need. ADR 0039 (`0039-opentelemetry-builder-extension`) delegates to this same per-signal extension, so its single-call `AddBrighterInstrumentation()` on `OpenTelemetryBuilder` picks Outbox metrics up for free. No new package, no flag, no second registration call. + +### 11. Testing strategy + +**Landing directory**: `tests/Paramore.Brighter.Core.Tests/Observability/Metrics/Outbox/`, alongside the existing `Metrics/` tests (`When_ending_a_send_span_should_still_record_a_client_operation.cs`), following the `When_...` file-naming convention. + +**Doubles** in `tests/Paramore.Brighter.Core.Tests/Observability/TestDoubles/`, in the style of `SpyMessagingMeter`/`DisabledDbMeter`: + +- `SpyOutboxMeter : IAmABrighterOutboxMeter` — `Enabled => true`; appends to `List<(RoutingKey Destination, OutboxClearSource Source)> Added/Cleared` and `List<(TimeSpan Wait, RoutingKey, OutboxClearSource)> Durations`. All three lists are the assertion surface for AC-1/2/4/5/6/7/8/13/14/15/16/24. Backed by a `lock` for AC-25's concurrent adds. +- `DisabledOutboxMeter` — `Enabled => false` plus a `RecordCallCount` that must remain `0` (AC-18), paired with an outbox double whose operation counters prove no extra store read. +- `ThrowingOutboxMeter` — `Enabled => true`, every record method throws (AC-22). + +**Determinism (NFR-7).** `FakeTimeProvider` (`Microsoft.Extensions.TimeProvider.Testing`, already referenced by `Paramore.Brighter.Core.Tests.csproj:25` and used e.g. in `tests/.../Archiving/When_Archiving_Old_Messages_From_The_Outbox.cs`) is injected as the mediator's `timeProvider`, giving the dispatch instant; `MessageHeader.TimeStamp` is `{ get; init; }` (`MessageHeader.cs:321`), so creation time is set explicitly. No `Thread.Sleep`, no wall clock. AC-14 constructs two independent mediators over one shared `InMemoryOutbox`, disposes the first, then advances the second's `FakeTimeProvider` by five minutes before sweeping. + +**AC-21 (in-memory exporter through `AddBrighterInstrumentation()`)** requires two changes to the test project only — no `src` change: + +- add `` to `Paramore.Brighter.Core.Tests.csproj` (no test project currently references it; `OpenTelemetry.Exporter.InMemory` is already referenced at line 28). +- build `Sdk.CreateMeterProviderBuilder().AddBrighterInstrumentation().AddInMemoryExporter(metrics).Build()`, add and clear one message, `ForceFlush()`, and assert the three instrument names appear with `Meter.Name == "Paramore.Brighter"`. + +Two complementary tests, because instrument identity is process-wide: + +- a **presence** test using the mediator's own default `OutboxMeter`, asserting the three instruments appear and are attributed to the right destination and clear source; +- a **value** test injecting a freshly constructed `OutboxMeter` into the mediator (still collected by name, so still exercising the registration path) and disposed with the test, so exact totals are isolated. + +`tests/Paramore.Brighter.Core.Tests/xunit.runner.json` already sets `"parallelizeTestCollections": false`, which removes cross-test interference on the shared meter name; the value test's explicit `Dispose` removes it across runs. + +**AC-23** is satisfied by running the existing Outbox, sweeper, circuit-breaker and observability suites unmodified; no existing assertion is touched, and `SpyMessagingMeter`/`DisabledDbMeter` are not edited. + +### 12. Documentation obligation (C-1, C-6) + +Three places, all part of this change: + +1. **`docs/guides/outbox-metrics.md`** (new, following the `docs/guides/box-provisioning-*.md` convention) — the instrument table, the attribute domains, and two explicit operator warnings: + - *What the duration actually measures.* The basis is `message.Header.TimeStamp`, stamped during message construction, so the value **includes message mapping/transform time** immediately preceding the Outbox write and is **subject to clock skew** between the host that created the message and the host that dispatched it (negative results are recorded as 0 per FR-7a). It is therefore *not* a pure store-to-broker duration, and must not be alerted on as one. + - *The circuit breaker can hide backlog (C-6).* `IAmAnOutboxCircuitBreaker.TrippedTopics` is passed to `OutstandingMessagesAsync` (`OutboxProducerMediator.cs:738`), so a tripped topic's messages are never selected by the sweeper. With no outstanding-count gauge (OOS-1), the alertable signature of a stalled topic is **`added.messages` for a destination continuing to rise while `cleared.messages` for that destination goes flat**. This is the alert RLOI-4186 should build; see ADR 0028. + - *Latency samples may be fewer than cleared units* on confirmation transports (FR-7b), with the three causes: empty `MessageId`, TTL eviction, cap drop. +2. **XML doc comments** on `IAmABrighterOutboxMeter.RecordPublishDuration`, `BrighterSemanticConventions.OutboxPublishDuration` and `OutboxClearSource.Unknown`, repeating the measurement basis and the `"unknown"` cause so it is visible at the call site (NFR-8). +3. **The Brighter documentation site** lives in a separate repository; updating it is a follow-up on the upstream PR, not a deliverable in this repo. + +### 13. FR/NFR → design element + +| Requirement | Design element | +|---|---| +| FR-1 single add | `RecordAdded` after the `!written` throw in `AddToOutboxAsync` / `AddToOutbox` (§7) | +| FR-2 batched add | per-message `RecordAdded` over the `batch` list in `EndBatchAddToOutbox(Async)` after the `!written` throw (§7) | +| FR-3 cleared count | `RecordCleared` in all four paths: `Dispatch`, `DispatchAsync`, `BulkDispatchAsync`, both confirmation handlers (§7, §8, §9) | +| FR-4 sweeper vs explicit | `OutboxClearSource` parameter on the three private dispatch methods; `paramore.brighter.outbox.clear_source` attribute (§2, §6) | +| FR-5 latency per cleared message | `RecordPublishDuration` in `RecordCleared`, using the closure-captured `dispatchedAt` and `Header.TimeStamp` (§4, §7) | +| FR-6 survives restart / cross-process | basis is the persisted `Header.TimeStamp`; no in-process state on the non-confirmation paths (§7) | +| FR-7a negative → 0 | clamp in `RecordCleared` at the call site (§7) | +| FR-7b no creation time → skip sample | `createdAt` is `DateTimeOffset?`; null on `Id.Empty`, TTL eviction, cap drop (§9) | +| FR-8 failures not counted | recording gated on `ExecuteWithResiliencePipeline(Async)` returning `true` for the **MarkDispatched** call; no recording on `!written`, `!sent`, failed confirmation (§7) | +| FR-9 independent of tracing | direct instruments; no recording reads `_tracer`, `requestContext.Span` or `_instrumentationOptions` (§7) | +| FR-10 safe, bounded attributes | exactly two attribute keys, both from `BrighterSemanticConventions`; no id, body, partition key, correlation id or bag value (§2) | +| FR-11 existing registration path | instruments on `new Meter(BrighterSemanticConventions.MeterName)`; `AddBrighterInstrumentation()` unchanged (§5.4, §10) | +| NFR-1 zero cost when disabled | `if (!_outboxMeter.Enabled) return;` first statement of both helpers; `TrackPendingClear` and the bulk `createdAtById` dictionary gated on `Enabled` (§7, §8, §9) | +| NFR-2 no behavioural change | signature changes confined to private methods; dispatch instant captured *inside* the resilience lambda rather than hoisted; no change to producer lookup, `MarkDispatched` args, ordering, exceptions, logs or spans (§6, §7) | +| NFR-3 faults isolated | try/catch at every call site **and** inside `OutboxMeter`; `ToClearSourceName` degrades instead of throwing (§5.2, §5.3, §7) | +| NFR-4 thread safety | instruments are thread-safe by contract; `ConcurrentDictionary` for the pending map; no new lock, no reintroduced serialisation (ADR 0032) (§9) | +| NFR-5 TFM compatibility | no TFM-gated API; `TagList` and `InstrumentAdvice` verified present in the `netstandard2.0` build of DiagnosticSource 10.0.11, with a documented `#if` + View fallback (§3, §5.2) | +| NFR-6 bounded cardinality | `≤ 7 × (D + 1)` series; arithmetic in §2 | +| NFR-7 test determinism | `FakeTimeProvider` for dispatch and purge; `MessageHeader.TimeStamp` `init` for creation (§11) | +| NFR-8 upstream acceptability | MIT headers, XML docs, `Add*`/`Record*`/`Enabled` naming and `MessagingSystemExtensions`-shaped mapper, source-generated logging (ADR 0026), no interface break (§5, §12) | + +## Consequences + +### Positive + +- Added volume, cleared volume, sweeper-versus-explicit ratio and publish-lag percentiles become available to every application that already calls `AddBrighterInstrumentation()`, with no new opt-in — unblocking RLOI-4186. +- Metrics survive the PI-36093 failure mode exactly: the latency basis is the persisted header timestamp, so a sweeper in a different process after a restart reports the full wait. +- Completely independent of sampling and of whether a tracer exists, which is the whole point of the spec. +- Zero heap allocation on the recording path (`TagList` inline slots, `readonly record struct` pending entries) and a single `bool` read when no listener is attached. +- The clear-kind parameter is compile-checked, so a future dispatch path cannot silently lose attribution. +- Ownership follows the `_ownsRegistry`/`_ownsTransformerFactories` precedent already in the class, so ADR 0069's disposal cascade extends naturally. +- No Outbox implementation, schema, public interface or existing test assertion changes. + +### Negative + +- **Latency samples can be fewer than cleared units.** On confirmation transports, an empty `MessageId`, a TTL eviction or a cap drop yields a cleared unit with `clear_source = "unknown"` and no sample. Dashboards must not assume `count(duration) == sum(cleared)`. +- **A third `clear_source` value exists.** `"unknown"` is bounded but is an admission that the confirmation path cannot always be attributed; AC-6's ratio should be computed as `sweeper / (sweeper + explicit + unknown)`. +- **AC-24 is not literally satisfiable on confirmation transports** (§9). This is a design limitation of read-and-remove, recorded openly rather than papered over. +- **New in-mediator state.** The pending-clear map is the only new mutable state; it is bounded and disposed, but it is state that did not exist and that a reviewer must reason about. +- **One extra field of closure capture and one extra `bool` branch** on the dispatch path even when metrics are off. +- **An explicit `System.Diagnostics.DiagnosticSource` reference** raises `Paramore.Brighter`'s direct dependency surface, in exchange for the bucket boundaries not depending on a transitive resolution. +- **Two mediators in one process produce duplicate instrument identity.** The mediator is documented as a singleton, but `ControlBusSenderFactory` and `ControlBusReceiverBuilder` each construct a second one; with both owning their own `OutboxMeter`, the OTel SDK sees two identical instruments on the meter name and will report a duplicate. Mitigation: inject a shared `IAmABrighterOutboxMeter` (the DI path does exactly this when one is registered). This is a pre-existing structural quirk of the control bus that the design exposes rather than creates. +- **The header-timestamp basis is imprecise** (mapping/transform time included, clock skew clamped to 0) and will be misread by someone despite §12. + +### Risks and Mitigations + +| Risk | Mitigation | +|---|---| +| Names/types rejected by maintainers (C-7) | Everything in §1–§2 is presented as a proposal; the design depends on none of the specific strings. Instrument names and the attribute key are single `const`s. | +| `InstrumentAdvice` unavailable if the DiagnosticSource floor is not raised | Documented `#if NET9_0_OR_GREATER` fallback plus the equivalent operator-side `AddView` (§3). | +| Bucket boundaries wrong for a given workload | Boundaries are only *advice*; a host can override with a View without a Brighter change. | +| A broker that never confirms grows the map | Hard cap + age eviction, both driven by `_timeProvider` (§9). | +| The purge scan runs on a request thread | Amortised to once per minute over at most 10,000 struct entries, and only when the meter is enabled. | +| Static/shared instrument identity makes tests flaky | `parallelizeTestCollections: false`; the value-asserting AC-21 test injects and disposes its own `OutboxMeter`. | +| `Activator.CreateInstance` in `BuildOutBoxProducerMediator` breaks on the new parameter | Identified explicitly (§5.4); the call site is updated in the same change, and the existing comment there already warns about exactly this. | +| An operator alerts on `publish.duration` as a broker latency | §12's documentation, repeated in XML docs at the point of use. | + +## Alternatives Considered + +**1. Extend `BrighterMetricsFromTracesProcessor` (ADR 0022) to derive Outbox metrics from spans.** *Rejected — cannot satisfy FR-9.* The processor's `OnEnd` only ever sees ended, sampled activities, and the Outbox add path writes an `ActivityEvent` onto a possibly-null `requestContext.Span` via `BrighterTracer.WriteOutboxEvent`, which begins `if (span == null) return;`. In the sampled environments the spec cares about there would be nothing to derive from. ADR 0022 is **not** superseded: it remains correct and in force for the messaging and db signals it already derives; this ADR adds a second, complementary mechanism for a signal that cannot be derived from traces. + +**2. Add a creation-time field to `PublishConfirmationResult` (ADR 0063).** *Rejected.* It is a positional `sealed record`, so a new parameter is a source-breaking change for every external `ISupportPublishConfirmation` implementer, and it would require **every** producer (Kafka, RMQ sync and async, and any third-party transport) to be edited to populate it correctly — including in the failure paths where the producer may not even know the message. That is a large, cross-transport, breaking change for a value the mediator already has in hand at send time. + +**3. Skip latency entirely for confirmation producers (lean on FR-7b).** *Rejected.* The cheapest option, and it would satisfy the letter of FR-7b, but it omits the metric on precisely Kafka and RabbitMQ — the transports PI-36093 ran on. The pending-clear map costs one bounded dictionary; the value is the motivating use case. + +**4. Carry the clear kind in `RequestContext.Bag`.** *Rejected.* Implicit and unvalidated, with a genuine correctness hazard: `RequestContext` is copied and re-parented across the clear and confirmation paths (`requestContext.CreateCopy()` at `:877` and `:974`), and a stale bag entry on a shared context would mis-attribute a later clear with no compiler help and no test that would necessarily catch it. + +**5. A separate public entry point per clear kind** (e.g. `ClearOutboxAsSweeperAsync`). *Rejected.* Duplicates the dispatch logic that ADR 0032 and the circuit-breaker work have already made delicate, and grows the public surface against C-11. + +**6. Add Outbox members to `IAmABrighterMessagingMeter`.** *Rejected.* Every existing member takes an `Activity`; Outbox recording is activity-free by FR-9. Merging them gives one type two unrelated collaboration contracts and forces `SpyMessagingMeter` to grow members it cannot implement, touching a double that AC-23 wants left alone. + +**7. Register `IAmABrighterOutboxMeter` in DI only, mirroring `MessagingMeter`/`DbMeter`.** *Rejected as the sole mechanism.* `AddBrighterInstrumentation()` on a `MeterProviderBuilder` writes its registrations into the builder's own service collection; with `Sdk.CreateMeterProviderBuilder()` (AC-21's exact scenario) the application container never sees them, so the mediator would be left with no instruments at all. DI injection is kept as an *override* so hosts that wire OTel through `AddOpenTelemetry()` can share one instance and get `GetServiceAttributes()` parity. + +**8. A process-wide static `OutboxMeter.Default`.** *Rejected.* It would remove the duplicate-instrument risk from the control-bus case, but at the cost of process-global mutable state that is never disposed, cumulative counters that leak across tests, and no way to give different hosts different service attributes. Mediator ownership with the existing `_owns*` pattern is testable, disposable and consistent with ADR 0069. + +**9. An outstanding/backlog gauge, archiver metrics, or publish-failure counters.** *Out of scope* (OOS-1, OOS-2, OOS-3). The gauge needs either a timer poll or an observable callback that queries the store, raising cost, multi-instance double-counting and API questions that deserve their own spec. §12 documents the added-minus-cleared approximation and the tripped-topic signature instead. + +**10. Adding an outstanding-send counter to `PendingClear`** so N concurrent dispatches of one id yield N latency samples. *Rejected for now* as unjustified complexity for a race that only arises when the sweeper's `minimumAge` is shorter than a broker's confirmation latency; recorded in §9 as the escape hatch if maintainers require AC-24 parity on confirmation transports. + +## References + +- Requirements: [specs/0037-outbox-metrics/requirements.md](../../specs/0037-outbox-metrics/requirements.md) +- Related ADRs: + - [ADR 0010 — Brighter OpenTelemetry Semantic Conventions](0010-brighter-semantic-conventions.md) *(Accepted)* — the attribute-key and naming conventions this ADR extends; `messaging.destination.name` is reused unchanged and the new key takes the `paramore.brighter.*` prefix it prescribes. + - [ADR 0022 — Generate Metrics from Traces](0022-generate-metrics-from-traces.md) *(Proposed)* — the trace-derived mechanism this ADR departs from for the Outbox, and only for the Outbox. Not superseded. + - [ADR 0063 — Failed Delivery Context for Confirmation-Based Producers](0063-failed-delivery-context.md) *(Accepted)* — supplies `PublishConfirmationResult`, which this ADR consumes (`MessageId`, `Topic`) but deliberately does not extend. + - [ADR 0028 — Support Circuit Breaking for Topics in Outbox Sweeper](0028-support-circuit-breaking-of-topics.md) *(Proposed)* — the tripped-topic behaviour that hides backlog; §12 states the resulting alert signature for RLOI-4186. + - [ADR 0032 — Remove Semaphore from Explicit Clear](0032-remove-explicit-clear-lock.md) *(Accepted)* — the explicit clear path must not be re-serialised; the pending-clear map uses `ConcurrentDictionary` and adds no lock. + - [ADR 0039 — OpenTelemetry Builder Extension](0039-opentelemetry-builder-extension.md) *(Proposed)* — delegates to the same `Extensions.Diagnostics` per-signal extension, so Outbox metrics arrive through it with no extra work. + - [ADR 0019 — Avoid Primitive Obsession](0019-avoid-primitive-obsession.md) *(Accepted)* — `RoutingKey` on the meter interface; `PendingClear` as a `readonly record struct`. + - [ADR 0026 — Use Source Generated Logging](0026-use-source-generated-logging.md) *(Accepted)* — the new `Log.OutboxMetricsFault` entries are source-generated. + - [ADR 0069 — Ownership and disposal cascade for mapper/transform factories](0069-factory-registry-ownership-and-disposal-cascade.md) *(Accepted)* — `_ownsOutboxMeter` follows its "a component that did not create a disposable does not dispose it" rule. +- External references: + - [OpenTelemetry Semantic Conventions — Messaging metrics](https://github.com/open-telemetry/semantic-conventions/blob/main/docs/messaging/messaging-metrics.md) + - [OpenTelemetry Metric naming guidelines](https://github.com/open-telemetry/semantic-conventions/blob/main/docs/general/naming.md) + - [.NET `InstrumentAdvice` / histogram bucket boundaries](https://learn.microsoft.com/dotnet/api/system.diagnostics.metrics.instrumentadvice-1) + - Jira: RLOI-4185 (this work), RLOI-4186 (blocked consumer), PI-36093 (motivating incident) diff --git a/docs/adr/index.md b/docs/adr/index.md index aa61991076..84fe619c56 100644 --- a/docs/adr/index.md +++ b/docs/adr/index.md @@ -133,5 +133,6 @@ Identity is the `id` (filename stem), not the leading number — numbers may rep | `0081-report-persistent-channel-failures-in-health-checks` | Report persistent channel failures in health checks | Proposed | observability, message-pump, resilience, api-design | Track consecutive receive failures in message pumps and report Degraded once a configurable threshold is reached. Successful receives, including empty queues, reset the count; an optional capability preserves custom pump and performer compatibility. | | `0081-timed-outbox-sweeper-health-metrics` | TimedOutboxSweeper health metrics | Accepted | observability, metrics, outbox, otel | TimedOutboxSweeper records three instruments directly through a new IAmABrighterSweeperMeter role (no-op by default, registered by AddBrighterInstrumentation): tick lag, sweep duration, and sweeps by outcome (completed, failed, lock_unavailable), so a stalled or late sweeper is visible even when traces are sampled out. | | `0083-coordinate-dispatcher-startup-and-shutdown` | Coordinate Dispatcher startup and shutdown | Proposed | service-activator, concurrency | Track accepted consumer operations so shutdown cannot overtake channel creation or task registration. | +| `0084-outbox-metrics` | Outbox Metrics | Proposed | observability, metrics, otel, outbox | Outbox metrics are recorded by direct OpenTelemetry instruments (paramore.brighter.outbox.added.messages and .cleared.messages as Counter, .publish.duration as a Histogram in seconds with explicit long-tail buckets) behind a new IAmABrighterOutboxMeter/OutboxMeter role owned by OutboxProducerMediator rather than derived from traces, with the clear kind carried as an OutboxClearSource parameter on the private dispatch methods and confirmation-path creation time resolved from a bounded, TTL-evicted in-mediator pending-clear ConcurrentDictionary. | -_120 ADRs indexed._ +_121 ADRs indexed._ diff --git a/specs/.current-spec b/specs/.current-spec index 706eb7f60a..39029aa525 100644 --- a/specs/.current-spec +++ b/specs/.current-spec @@ -1 +1 @@ -0037-show-me +0037-outbox-metrics \ No newline at end of file diff --git a/specs/0037-outbox-metrics/.adr-list b/specs/0037-outbox-metrics/.adr-list new file mode 100644 index 0000000000..f856f9ae60 --- /dev/null +++ b/specs/0037-outbox-metrics/.adr-list @@ -0,0 +1 @@ +0084-outbox-metrics.md diff --git a/specs/0037-outbox-metrics/.requirements-approved b/specs/0037-outbox-metrics/.requirements-approved new file mode 100644 index 0000000000..e69de29bb2 diff --git a/specs/0037-outbox-metrics/README.md b/specs/0037-outbox-metrics/README.md new file mode 100644 index 0000000000..dc0b25bc49 --- /dev/null +++ b/specs/0037-outbox-metrics/README.md @@ -0,0 +1,60 @@ +# Specification: Outbox Metrics + +**Feature Name**: Outbox Metrics +**Spec ID**: 0037 +**Created**: 2026-09-18 +**Status**: Requirements Draft + +## Overview + +Brighter emits trace spans when messages are written to an Outbox, when an Outbox is cleared, and +when the sweeper checks for outstanding messages. Those spans are the only observability signal for +the Outbox, and because they are not sampled in production there is effectively no visibility into +Outbox behaviour: how many messages are written, how many are dispatched inline versus left for the +sweeper to pick up, and how long a message waits between being written to the Outbox and actually +reaching the broker. + +This spec adds OpenTelemetry metrics for the Outbox so that an application using a Brighter Outbox +can answer those questions from metrics rather than from sampled traces. + +## Workflow Status + +- [x] Requirements defined +- [x] Requirements approved +- [x] ADR created +- [ ] ADR approved +- [ ] Adversarial review +- [ ] Tasks approved +- [ ] Implementation complete +- [ ] Tests passing +- [ ] PR submitted + +## Files + +- `requirements.md` — user requirements and problem statement +- [`docs/adr/0084-outbox-metrics.md`](../../docs/adr/0084-outbox-metrics.md) — the design decision + +## Scope + +### Affected (provisional, pending design) + +- `src/Paramore.Brighter/Observability/` — meters, semantic conventions, metrics-from-traces processor +- `src/Paramore.Brighter/OutboxProducerMediator.cs` — the add / clear / dispatch paths +- `src/Paramore.Brighter/OutboxSweeper.cs` +- `src/Paramore.Brighter.Extensions.Diagnostics/` — registration of any new meter + +### Not affected (provisional, pending design) + +- Individual Outbox implementations (`Paramore.Brighter.Outbox.*`) — instrumentation is expected to + sit above the Outbox abstraction in the mediator. + +## Related Issues + +- Jira: RLOI-4185 — Add outbox metrics to Brighter +- Jira: RLOI-4186 — [OrderDispatchWorker] Improve outbox observability (blocked by this) +- Jira: PI-36093 — production incident that motivated the ticket (events emitted after significant lag) + +## Dependencies + +- Metric names and instrument types need agreement with the Brighter maintainers + (`#interest-oss-brighter`). Until that lands, the ADR records the proposal and the open questions. diff --git a/specs/0037-outbox-metrics/requirements.md b/specs/0037-outbox-metrics/requirements.md new file mode 100644 index 0000000000..50dab3ee80 --- /dev/null +++ b/specs/0037-outbox-metrics/requirements.md @@ -0,0 +1,293 @@ +# Requirements + +> **Note**: This document captures user requirements and needs. Technical design decisions and implementation details should be documented in an Architecture Decision Record (ADR) in `docs/adr/`. + +**Linked Issue**: RLOI-4185 + +## Glossary + +These terms are used with precise meaning throughout this document. Where a term names existing code, the file is given so the term cannot be re-interpreted. + +- **Outbox add ("added")**: a message is *added* when Brighter has successfully written it to the Outbox store. Concretely, when `OutboxProducerMediator.AddToOutbox` / `AddToOutboxAsync` (`src/Paramore.Brighter/OutboxProducerMediator.cs`) complete without throwing `ChannelFailureException`, or when `EndBatchAddToOutbox` / `EndBatchAddToOutboxAsync` complete without throwing (in which case every message in that batch is added). A message buffered by `AddToOutbox(..., batchId: "x")` is **not** added until the matching `EndBatchAddToOutbox*` succeeds. +- **Dispatch attempt**: one call by the mediator to a producer's `Send` / `SendAsync` for a message (or to the bulk producer's send for a batch), made from `Dispatch`, `DispatchAsync`, or `BulkDispatchAsync`. +- **Cleared ("published")**: a message is *cleared* at the instant Brighter records it as dispatched in the Outbox — i.e. the instant a `MarkDispatched` / `MarkDispatchedAsync` call for that message id succeeds. This single rule covers every transport: + - **Non-confirmation producer** (producer does not implement `ISupportPublishConfirmation`): the send returned success (`sent == true`) and the mediator then marks the message dispatched inline, in `Dispatch`, `DispatchAsync` or `BulkDispatchAsync`. + - **Confirmation-based producer** (implements publish confirmation): the send call returning is *not* "cleared". The message is cleared only when the broker confirmation arrives reporting success and `HandleAsyncPublishConfirmation` / the `ConfigurePublisherCallbackMaybe` delegate marks it dispatched. A confirmation reporting failure never clears the message. + + "Cleared" and "published" are used interchangeably in this document and always mean the above. "Published" does **not** mean "consumed" or "processed by the downstream service". +- **Explicit clear**: a clear initiated by application code calling `ClearOutbox(Id[] posts, ...)` or `ClearOutboxAsync(IEnumerable posts, ...)` for named message ids. +- **Sweeper clear**: a clear initiated by `OutboxSweeper.SweepAsync()` (`src/Paramore.Brighter/OutboxSweeper.cs`) -> `ClearOutstandingFromOutboxAsync(amountToClear, minimumAge, useBulk, ...)` -> `BackgroundDispatchUsingAsync`, which selects messages via `OutstandingMessagesAsync` rather than by id. Both clear kinds converge on the same `DispatchAsync` / `BulkDispatchAsync` methods, so the two are **not** distinguishable at the dispatch point today. +- **Creation time**: `message.Header.TimeStamp` (`src/Paramore.Brighter/MessageHeader.cs`, `DateTimeOffset`, defaulting to `DateTimeOffset.UtcNow` at header construction). It is set on the add path, persisted by the Outbox stores, and read back with the message, including by a different process after a restart. +- **Add-to-publish latency ("latency")**: `dispatch time - creation time`, where *dispatch time* is the same `_timeProvider.GetUtcNow()` instant that is passed to the `MarkDispatched` call which clears the message. This basis is a decided requirement, not an option (see C-1). +- **Meter enabled / disabled**: Brighter's existing meters expose `bool Enabled` (`IAmABrighterMessagingMeter`, `IAmABrighterDbMeter` in `src/Paramore.Brighter/Observability/`), which is false when no OpenTelemetry `MeterListener` is attached to any of the meter's instruments — i.e. when the host application has not called `AddBrighterInstrumentation()` on a `MeterProviderBuilder` (`src/Paramore.Brighter.Extensions.Diagnostics/BrighterMetricsBuilderExtensions.cs`). +- **Brighter meter**: the meter named by `BrighterSemanticConventions.MeterName` (`"Paramore.Brighter"`), the only meter name registered by `AddBrighterInstrumentation()`. +- **Operator**: a person running a Brighter application in production (SRE / on-call engineer) who consumes metrics via a dashboard or alert, and who cannot read traces because Brighter's Outbox spans are unsampled in that environment. + +## Problem Statement + +As an **operator of a service that publishes through a Brighter Outbox**, I need **metrics for how many messages enter the Outbox, how many leave it, how many of those were rescued by the sweeper rather than dispatched inline, and how long messages wait between being written and being published**, so that **I can see, dashboard and alert on Outbox publish lag and Outbox backlog growth instead of discovering them from downstream business failures.** + +Today the only Outbox observability signals are trace spans and span events: `BrighterTracer.WriteOutboxEvent(BoxDbOperation.Add, ...)` on the add path, per-message clear spans on the explicit clear path, and one clear span plus a `BoxDbOperation.OutStandingMessages` event on the sweeper path. These are inadequate for three reasons: + +1. **They are unsampled in production.** Operators run tail/ratio sampling, so Outbox spans are effectively absent. There is no aggregate signal at all. +2. **The add-path event is silently dropped when there is no ambient span.** `BrighterTracer.WriteOutboxEvent` writes an `ActivityEvent` onto `requestContext.Span` rather than creating its own span, so adds made outside a sampled trace leave no record whatsoever. +3. **Spans cannot answer rate/ratio questions.** "What fraction of my messages are being left to the sweeper?" and "what is my p99 add-to-publish latency?" are aggregate questions that require counters and a histogram. + +This gap caused a production incident (Jira PI-36093, UK): NOFR events were emitted after significant lag, so `IgnoredOrders` were raised before Partners / OrderPad / JetConnect could "Accept" the orders. The Outbox publish lag that caused it was invisible — there was no metric to dashboard, no metric to alert on, and no way to confirm after the fact how long messages had sat in the Outbox. The consuming service work (Jira RLOI-4186, "[OrderDispatchWorker] Improve outbox observability") is blocked on this spec, because it intends to build dashboards and alerts directly on these metrics. + +Aggravating factor: the Outbox circuit breaker (`IAmAnOutboxCircuitBreaker`) can trip a topic so that `OutstandingMessagesAsync` skips its messages entirely. When that happens the Outbox quietly stops draining for that topic, and today nothing in metrics moves. + +## Proposed Solution + +Emit OpenTelemetry **metrics** for Outbox activity from the `Paramore.Brighter` meter, so that an operator of any Brighter application that already calls `AddBrighterInstrumentation()` gains four capabilities with no further opt-in: + +1. **Added volume** — the number of messages written to the Outbox, per destination. +2. **Cleared volume** — the number of messages recorded as dispatched from the Outbox, per destination. +3. **Sweeper attribution** — the ability to tell, from the cleared volume, which clears were sweeper clears and which were explicit clears, so "how much is my sweeper rescuing?" becomes a ratio an operator can chart and alert on. +4. **Add-to-publish latency distribution** — a distribution of `dispatch time - message.Header.TimeStamp` per cleared message, so p50/p95/p99 publish lag is chartable, including for messages dispatched by a sweeper in a different process instance or after a restart. + +Because the latency basis is the *persisted message creation time* rather than an in-process stopwatch, the measurement survives the exact scenario that caused PI-36093: the process that wrote the message is gone and a later sweeper publishes it. + +These metrics must be independent of tracing and sampling: they must be recorded on the add and clear code paths regardless of whether an ambient `Activity` exists, whether a tracer is registered, or what the sampler decides. That is a behavioural requirement on the design, not a suggestion — the existing trace-derived pipeline (`BrighterMetricsFromTracesProcessor`, which reads ended Brighter activities keyed off `instrumentation.domain`) is only acceptable as a mechanism to the extent it satisfies it. + +Exact metric names, instrument types, units and attribute keys are **not** fixed by this document. They are an ADR proposal to be agreed with the Brighter maintainers in `#interest-oss-brighter` before the upstream PR (see C-7). Where this document shows a name, it is illustrative and marked as ADR-owned. + +## Requirements + +Requirements are stated as capabilities an operator must gain, plus the constraints those capabilities place on the design. Illustrative names appear as `[ADR-owned: paramore.brighter.outbox....]` and are **not** requirements. + +### Functional Requirements + +**FR-1 — Count messages added to the Outbox (single add).** +When a message is *added* via `AddToOutbox` (sync Outbox) or `AddToOutboxAsync` (async Outbox) without a `batchId`, Brighter MUST record exactly one "added" unit for that message. The recording MUST identify the message's destination (`message.Header.Topic`). No unit may be recorded when the add fails (see FR-8). `[ADR-owned: paramore.brighter.outbox.added.messages]` +*Example*: one `AddToOutboxAsync` call for a message with `Header.Topic == "order.created"` that returns without throwing results in exactly one added unit attributed to destination `order.created`. A second identical call results in a cumulative total of 2. + +**FR-2 — Count messages added to the Outbox (batched add).** +When messages are buffered via `AddToOutbox(..., batchId)` / `AddToOutboxAsync(..., batchId)` and later written by a successful `EndBatchAddToOutbox` / `EndBatchAddToOutboxAsync`, Brighter MUST record one added unit **per message in the batch**, at the point the batch write succeeds — not at the point each message was buffered. +*Example A*: `StartBatchAddToOutbox()` then three `AddToOutboxAsync(..., batchId)` calls (topics `a`, `a`, `b`) then a successful `EndBatchAddToOutboxAsync` results in 3 added units: 2 attributed to destination `a`, 1 to destination `b`. +*Example B*: the same three buffered calls with **no** `EndBatchAddToOutbox*` call result in 0 added units, because no message was written to the store. +*Example C*: the same three buffered calls where `EndBatchAddToOutboxAsync` throws result in 0 added units (see FR-8). + +**FR-3 — Count messages cleared from the Outbox.** +Whenever a message becomes *cleared* (per the Glossary: a `MarkDispatched` / `MarkDispatchedAsync` call for that message id succeeds), Brighter MUST record exactly one "cleared" unit, identifying the destination. This MUST hold for all four dispatch paths: `Dispatch` (sync), `DispatchAsync`, `BulkDispatchAsync`, and the publish-confirmation callbacks (`HandleAsyncPublishConfirmation`, the `ConfigurePublisherCallbackMaybe` delegate). `[ADR-owned: paramore.brighter.outbox.cleared.messages]` +The counter counts **clear events, not distinct messages**: if the same message id is cleared twice (for example a sweeper re-dispatches a message whose broker confirmation arrived late), two units are recorded. This is intended and consistent with Brighter's at-least-once delivery. +*Example A (non-confirmation, single)*: `ClearOutboxAsync([id1, id2])` where both sends succeed and both `MarkDispatchedAsync` calls succeed results in 2 cleared units. +*Example B (bulk)*: `BulkDispatchAsync` sends one batch of 4 messages for topic `t` with a non-confirmation producer and all 4 are marked dispatched, resulting in 4 cleared units attributed to destination `t` — not 1 unit for the batch. +*Example C (confirmation-based)*: `DispatchAsync` sends a message through a producer implementing publish confirmation. At the moment the send returns, 0 cleared units have been recorded. When the confirmation later fires reporting success and `MarkDispatchedAsync` succeeds, exactly 1 cleared unit is recorded. +*Example D (confirmation failure)*: the same producer fires a confirmation reporting failure. 0 cleared units are recorded; the message stays un-dispatched for the sweeper. + +**FR-4 — Attribute cleared messages to sweeper clears versus explicit clears.** +The cleared volume of FR-3 MUST be decomposable by an operator into **sweeper clears** and **explicit clears**, using a bounded, low-cardinality dimension with exactly those two distinguishable states (the ADR may name a third state only if it also bounds it). It MUST be possible to build the ratio "sweeper clears / all clears" from the emitted metric alone, without joining to traces or logs. +Because both clear paths converge on the same `DispatchAsync` / `BulkDispatchAsync` methods (C-5), satisfying this requires the initiating context to be carried to the point of recording. Doing so MUST NOT change dispatch behaviour, ordering, producer lookup, `MarkDispatched` arguments, or circuit-breaker behaviour. +*Example A*: `OutboxSweeper.SweepAsync()` finds and successfully dispatches 5 messages on topic `t`. Result: 5 cleared units attributed as sweeper clears; the count of explicit clears is unchanged. +*Example B*: application code calls `ClearOutboxAsync([id1])` and it succeeds. Result: 1 cleared unit attributed as an explicit clear; the count of sweeper clears is unchanged. +*Example C*: in a process where both happen — 1 explicit clear then a sweep that clears 5 — an operator reading the metric can compute 5/6 sweeper-cleared. + +**FR-5 — Record add-to-publish latency per cleared message.** +For every cleared unit recorded under FR-3, Brighter MUST also record one latency sample equal to `dispatch time - message.Header.TimeStamp`, where *dispatch time* is the same instant passed to the `MarkDispatched` call that cleared the message. The sample MUST be recorded as a distribution (so percentiles are derivable), MUST identify the destination, and MUST carry the same sweeper-versus-explicit dimension as FR-4. `[ADR-owned: paramore.brighter.outbox.publish.latency]` +*Example*: a message whose `Header.TimeStamp` is `2026-09-18T10:00:00.000Z` is cleared at `2026-09-18T10:00:00.250Z` (per an injected `TimeProvider`). Result: exactly one latency sample of 250 ms is recorded, attributed to the message's destination. + +**FR-6 — Latency must survive process restart and cross-process dispatch.** +Because latency is computed from the persisted `message.Header.TimeStamp` and not from in-process state, a latency sample MUST be recorded even when the process that added the message no longer exists, and the sample MUST reflect the full wait, not the sweeper's own dispatch duration. +*Example*: process A adds a message at `10:00:00.000Z` and is then killed before dispatching it. Process B's sweeper picks the message up and clears it at `10:05:00.000Z`. Result: process B records exactly one latency sample of 300,000 ms (5 minutes), attributed as a sweeper clear — not a sample of the few milliseconds process B spent sending. + +**FR-7 — Deterministic handling of unusable latency inputs.** +Latency inputs can be unusable in two, and only two, ways; each has one defined outcome: +- **(a) Negative computed latency** (dispatch time is earlier than creation time, e.g. clock skew between the host that created the message and the host that dispatched it): the sample MUST be recorded as **zero**, never as a negative value. Rationale: keeps the number of latency samples aligned with the cleared count and avoids negative values that metric backends reject or render nonsensically. +- **(b) No creation time attributable to the cleared message** (for example a confirmation callback that reports an empty message id, so Brighter cannot tie the confirmation to a known message): **no latency sample** is recorded. The cleared unit of FR-3 is still recorded if and only if `MarkDispatched` succeeded. Consequently the latency sample count MAY be lower than the cleared count; the ADR MUST document this divergence. +No other input is permitted to produce a skipped, negative, or synthetic sample. +*Example (a)*: creation time `10:00:01.000Z` (fast clock on the adding host), dispatch time `10:00:00.500Z`. Result: exactly one sample of 0 ms; no negative value is recorded; no exception is thrown. +*Example (b)*: a confirmation fires reporting success with an empty message id, and no message can be resolved. Result: cleared units increase by 1 if the mark-dispatched succeeded, latency samples increase by 0, and no exception is thrown. + +**FR-8 — Failures must not be counted as successes.** +- A failed Outbox **add** MUST record 0 added units. Specifically, when `AddToOutbox` / `AddToOutboxAsync` throw `ChannelFailureException` because the resilience pipeline did not write the message, or `EndBatchAddToOutbox*` fails, nothing is counted. +- A failed **dispatch attempt** MUST record 0 cleared units and 0 latency samples. This covers: a send that returns `sent == false`, a send that throws, a bulk send that fails, a confirmation reporting failure, and a topic tripped by `IAmAnOutboxCircuitBreaker` so the sweeper never selected the message. +- A **failed mark-dispatched** MUST record 0 cleared units and 0 latency samples for that message: the metric follows the store, not the send. If the send succeeded but the `MarkDispatched` call did not succeed, nothing is counted (the message will be re-attempted by the sweeper and counted then). +*Example A*: an Outbox whose `AddAsync` always throws -> `AddToOutboxAsync` throws `ChannelFailureException` -> added units = 0. +*Example B*: a producer whose `SendAsync` throws -> `DispatchAsync` records cleared units = 0 and latency samples = 0, and existing behaviour (topic trip, log) is unchanged. +*Example C*: a sweep where `OutstandingMessagesAsync` returns 0 messages -> added units unchanged, cleared units = 0, latency samples = 0, and **no** zero-valued recordings are made at all. + +**FR-9 — Metrics must not depend on tracing or sampling.** +Added, cleared and latency recordings MUST occur when: no `IAmABrighterTracer` is registered; `requestContext.Span` is null; `Activity.Current` is null; the configured sampler drops the trace; or `InstrumentationOptions` is `None`. No recording may be conditional on a span having been created or sampled. This is the requirement that distinguishes this work from the existing span events, which are dropped outright when the ambient `Activity?` is null. +*Example*: a process configured with a `MeterProvider` (via `AddBrighterInstrumentation()`) but **no** `TracerProvider` at all adds one message and clears it. Result: 1 added unit, 1 cleared unit and 1 latency sample are recorded. + +**FR-10 — Attribute content must be safe and bounded.** +Recordings MUST NOT carry message-identifying or payload data as attributes — specifically not message id, message body, serialized headers, correlation id, partition key, or any application-supplied header-bag value. Attributes are limited to values with a bounded domain known from configuration (e.g. destination topic, the sweeper/explicit dimension, and existing service-resource attributes as already applied by `MeterProviderExtensions.GetServiceAttributes()` in `MessagingMeter`). Attribute keys MUST follow the existing conventions in `BrighterSemanticConventions` (OpenTelemetry semantic-convention keys where one exists, `paramore.brighter.*` otherwise). +*Example*: a recording for a message with id `9f8c9114-...` on topic `order.created` carries destination `order.created` and the clear-kind dimension, and carries no attribute whose value is `9f8c9114-...`. + +**FR-11 — Available through the existing registration path.** +An application that already calls `AddBrighterInstrumentation()` on its `MeterProviderBuilder` MUST receive these metrics with no additional API call, no new package reference, and no configuration flag. The instruments MUST be created on the `Paramore.Brighter` meter (`BrighterSemanticConventions.MeterName`), which is the meter that call already registers. +*Example*: an application whose only observability setup is `Sdk.CreateMeterProviderBuilder().AddBrighterInstrumentation().AddInMemoryExporter(...)` sees the added, cleared and latency instruments in the exported metric set after one add and one clear, without the source being edited. + +### Non-functional Requirements + +**NFR-1 — Zero cost when no listener is attached.** +When no meter listener is attached (the `Enabled` property of the relevant meter abstraction is `false`), the Outbox add and clear paths MUST perform no metric-related work beyond the guard check itself: no attribute/tag array construction, no string formatting, no serialization, no additional Outbox or database read, and no additional `Activity` creation. The guard MUST be observable in a test (a meter double reporting `Enabled == false` receives zero record calls). + +**NFR-2 — No behavioural change to the Outbox.** +Adding these metrics MUST NOT change: whether a message is written, sent, marked dispatched, or left for the sweeper; message ordering; producer lookup (`GetProducerLookupTopic`); circuit-breaker trip behaviour; exception types or messages; existing log entries; or existing spans and span events. The existing Outbox, sweeper and observability test suites MUST pass without modification to their assertions. + +**NFR-3 — Instrumentation faults must be isolated.** +An exception originating in metric recording MUST NOT propagate into the Outbox add path, the dispatch path, the sweeper, or a producer's confirmation callback thread, and MUST NOT prevent a message from being added, sent, or marked dispatched. Confirmation callbacks in particular run on a broker/thread-pool thread where an escaping exception can tear the thread down. + +**NFR-4 — Thread safety.** +Recording MUST be safe when the sweeper's background dispatch (`BackgroundDispatchUsingAsync`, which holds `_backgroundClearSemaphore`) runs concurrently with request-thread adds and with confirmation callbacks arriving on producer threads. No shared mutable state introduced for metrics may require callers to serialize access. + +**NFR-5 — Target-framework compatibility.** +The implementation MUST compile and behave identically across all TFMs that `Paramore.Brighter` targets, not only `net8.0+`. APIs available only on newer frameworks (e.g. `FrozenSet`, as already handled with `#if NET8_0_OR_GREATER` in `MessagingMeter`) MUST be conditionally compiled with an equivalent fallback. + +**NFR-6 — Bounded cardinality.** +The number of distinct attribute combinations per instrument MUST be bounded by `O(number of configured destinations x small constant)` — where the small constant is the product of the enumerated dimensions (clear kind, and any outcome dimension the ADR adds). No dimension may be derived from per-message data (see FR-10). + +**NFR-7 — Test determinism.** +Latency assertions MUST be deterministic and MUST NOT depend on wall-clock timing or `Thread.Sleep`. The mediator already accepts a `TimeProvider` (`_timeProvider`), and `MessageHeader.TimeStamp` is settable, so tests MUST control both creation time and dispatch time explicitly. Tests live under `tests/` (e.g. `tests/Paramore.Brighter.Core.Tests/Observability/Metrics/`) and follow the repo's `When_...` naming convention, using meter doubles in the style of the existing `SpyMessagingMeter` / `DisabledDbMeter`. + +**NFR-8 — Upstream acceptability.** +The change MUST be shaped to be acceptable as an upstream Brighter PR: MIT licence headers on new files, XML doc comments on new public members, naming and attribute style consistent with `BrighterSemanticConventions` and `MessagingMeter`/`DbMeter`, and no breaking change to public interfaces (see C-11). + +### Constraints and Assumptions + +- **C-1 (decided) — Latency basis is `message.Header.TimeStamp`.** Latency is `dispatch time - Header.TimeStamp`, not the duration of the dispatch call and not the time since the message was read from the Outbox. This is deliberate: it is the only basis that still works when the dispatching process is not the adding process — which is exactly the PI-36093 case. **Accepted imprecision:** the header timestamp is stamped during message construction, so the measured latency includes message mapping/transform time that occurs immediately before the Outbox write, and it is subject to clock skew between hosts (handled by FR-7a). These inaccuracies are accepted for this spec and MUST be documented in the ADR and in user-facing docs so operators do not misread the metric as a pure store-to-broker duration. +- **C-2 (assumption) — Creation time round-trips through the Outbox stores.** `MessageHeader.TimeStamp` is persisted and read back with the message. Assumed for all shipped Outbox implementations. Store-level precision may truncate sub-microsecond components; this is immaterial at the millisecond/second resolution operators care about. +- **C-3 — Existing metric infrastructure is trace-derived.** `MessagingMeter`, `DbMeter` and `BrighterMetricsFromTracesProcessor` (a `BaseProcessor` keyed off the `instrumentation.domain` tag) derive metrics from *ended* Brighter activities. Reusing that mechanism for Outbox metrics is permitted only if FR-9 still holds; if it cannot, the ADR must choose a direct-instrument approach. Mechanism choice is an ADR decision, FR-9 is the constraint on it. +- **C-4 — Confirmation is asynchronous and carries limited data.** For publish-confirmation producers, `MarkDispatched` happens later, in a callback, and the callback receives a `PublishConfirmationResult` rather than the `Message` — so it does not carry `Header.TimeStamp`. The ADR must therefore decide how creation time reaches the clear-recording point for these transports; FR-7b defines the fallback when it cannot. +- **C-5 — Both clear kinds share one dispatch path.** `ClearOutboxAsync` (explicit) and `BackgroundDispatchUsingAsync` (sweeper) both call `DispatchAsync` / `BulkDispatchAsync`, so FR-4 cannot be satisfied at the dispatch point without plumbing the initiating context down. How that context travels is an ADR decision, subject to NFR-2. +- **C-6 — The circuit breaker can hide backlog.** `IAmAnOutboxCircuitBreaker` supplies `TrippedTopics` to `OutstandingMessagesAsync`, so a tripped topic's messages are never selected by the sweeper. With an outstanding-count gauge out of scope (OOS-1), an operator infers a stalled topic from added volume continuing while cleared volume for that destination goes to zero. The ADR/docs should state this explicitly so RLOI-4186 can build the right alert. +- **C-7 — Names and instrument types are not settled.** Final metric names, instrument types, units and attribute keys require a proposal agreed with the Brighter maintainers via the `#interest-oss-brighter` Slack channel, recorded in the ADR. Any name appearing in this document is illustrative. Requirements here are deliberately written as capabilities so that agreeing a different name does not invalidate them. +- **C-8 — Bulk dispatch marks dispatched by id.** `BulkDispatchAsync` marks dispatched from the batch's ids, while the `Message` objects for the batch are in scope one level up. FR-3/FR-5 require per-message recording in this path, which constrains where the recording can be made. +- **C-9 — Time is injectable.** `OutboxProducerMediator` already holds a `TimeProvider` used for the `MarkDispatched` timestamp, so FR-5's "same instant" rule and NFR-7's determinism are achievable without new plumbing. +- **C-10 — Process constraint: strict TDD.** Each behaviour must have a test written and approved before its implementation, per repo convention. Acceptance criteria below are written so each maps to at least one such test. +- **C-11 — No breaking public API changes and no Outbox-implementation changes.** The work MUST NOT change the shape of `IAmAnOutbox` / `IAmAnOutboxSync` / `IAmAnOutboxAsync` or require the `Paramore.Brighter.Outbox.*` packages to be edited, and MUST NOT introduce a breaking change to `IAmAnOutboxProducerMediator`. Instrumentation sits above the Outbox abstraction, in the mediator. +- **C-12 — No new storage or schema.** No Outbox table/column additions, no extra queries against the Outbox, and no new persisted state are permitted to satisfy these requirements. + +### Out of Scope + +- **OOS-1 — Outstanding/backlog count gauge.** The RLOI-4185 *background* text mentions wanting to know "how many messages currently exist in the outbox", but the ticket's stated goal lists only the four metrics above. A backlog gauge is deferred because it requires either polling the Outbox on a timer or an observable callback that queries the store — a cost, correctness (multi-instance double counting) and API question deserving its own spec. Operators can approximate backlog growth from added minus cleared volume in the interim (C-6). +- **OOS-2 — Archiver metrics.** Counts/latency for the archive provider / Outbox archiver and for Outbox deletion/compaction. +- **OOS-3 — New publish-failure counters.** Failed sends are already observable via the existing `messaging.client.sent.messages` / `messaging.client.operation.duration` instruments with `error.type`, and via logs and circuit-breaker trips. FR-8 only requires that failures are *not* miscounted as successes. +- **OOS-4 — Inbox, message-pump, handler or mapper metrics.** Unchanged by this spec. +- **OOS-5 — Dashboards, alerts and SLOs.** Built by the consuming service under RLOI-4186. +- **OOS-6 — Changes to individual Outbox implementations** (`Paramore.Brighter.Outbox.*`) and to Outbox schemas (C-11, C-12). +- **OOS-7 — End-to-end delivery latency.** Broker-to-consumer and produce-to-process latency are not covered; "published" stops at Brighter recording the message as dispatched (Glossary). +- **OOS-8 — Exporter, collector or backend configuration.** Host application concern. +- **OOS-9 — Backfill.** No metric values are produced for messages added before the upgrade; messages already sitting in an Outbox at upgrade time will produce cleared units and latency samples when they are eventually cleared, and those samples will legitimately be large. + +## Acceptance Criteria + +Each criterion is a Given/When/Then that can be asserted directly. "Metric recorder" means a test double or in-memory exporter attached to the `Paramore.Brighter` meter; "added units", "cleared units" and "latency samples" mean the quantities defined in FR-1/FR-3/FR-5. + +**AC-1 — Single add is counted.** +*Given* a Brighter Outbox with a metric recorder attached, *when* a message with `Header.Topic == "order.created"` is added via `AddToOutboxAsync` (and again via `AddToOutbox` for the sync Outbox) and the add succeeds, *then* added units total exactly 1 per call, attributed to destination `order.created`, and no cleared unit or latency sample is recorded. + +**AC-2 — Batched add is counted per message, at flush.** +*Given* a batch started with `StartBatchAddToOutbox()` and three messages buffered with that `batchId` (topics `a`, `a`, `b`), *when* the batch has been buffered but `EndBatchAddToOutboxAsync` has not yet been called, *then* added units total 0; *and when* `EndBatchAddToOutboxAsync` succeeds, *then* added units total 3 — 2 for destination `a` and 1 for destination `b`. + +**AC-3 — Failed add is not counted.** +*Given* an Outbox whose write always fails, *when* `AddToOutboxAsync` throws `ChannelFailureException` (and likewise when `EndBatchAddToOutboxAsync` fails for a 3-message batch), *then* added units total 0 and the exception type and message are unchanged from current behaviour. + +**AC-4 — Explicit clear is counted.** +*Given* two messages in the Outbox and a non-confirmation producer that sends successfully, *when* `ClearOutboxAsync([id1, id2])` completes, *then* cleared units total exactly 2, each attributed to its destination, and each attributed as an **explicit** clear. + +**AC-5 — Sweeper clear is counted and attributed to the sweeper.** +*Given* 5 outstanding messages on topic `t` and a non-confirmation producer that sends successfully, *when* `OutboxSweeper.SweepAsync()` runs to completion, *then* cleared units total exactly 5, all attributed to destination `t` and all attributed as **sweeper** clears, and the explicit-clear total is 0. + +**AC-6 — Sweeper versus explicit ratio is derivable.** +*Given* a recorder attached, *when* one explicit clear of 1 message and one sweep clearing 5 messages have both completed in the same process, *then* the recorded values allow the ratio 5 sweeper / 6 total to be computed from the cleared metric alone, with no reference to traces or logs. + +**AC-7 — Bulk dispatch counts per message.** +*Given* a bulk-capable non-confirmation producer and 4 outstanding messages on topic `t`, *when* a sweep runs with `useBulk: true` and all 4 are marked dispatched in one send, *then* cleared units total exactly 4 (not 1) and latency samples total exactly 4. + +**AC-8 — Confirmation-based producer counts at confirmation, not at send.** +*Given* a producer implementing publish confirmation, *when* the send has returned but no confirmation has arrived, *then* cleared units total 0; *and when* the confirmation fires reporting success and the mark-dispatched succeeds, *then* cleared units total exactly 1. + +**AC-9 — Failed confirmation is not counted.** +*Given* the same confirmation-based producer, *when* the confirmation fires reporting failure, *then* cleared units total 0, latency samples total 0, existing behaviour (warning log, topic trip, message left un-dispatched) is unchanged, and no exception escapes the callback. + +**AC-10 — Failed send is not counted.** +*Given* a producer whose send returns failure (and, in a second case, throws), *when* `DispatchAsync` completes, *then* cleared units total 0 and latency samples total 0, while the existing topic trip and log behaviour are unchanged. + +**AC-11 — Failed mark-dispatched is not counted.** +*Given* a producer that sends successfully but an Outbox whose `MarkDispatchedAsync` fails, *when* `DispatchAsync` completes, *then* cleared units total 0 and latency samples total 0. + +**AC-12 — Empty sweep records nothing.** +*Given* an Outbox with no outstanding messages, *when* `SweepAsync()` runs, *then* the recorder receives no recordings at all for added units, cleared units or latency samples — including no zero-valued recordings. + +**AC-13 — Latency sample value is exact and uses the dispatch instant.** +*Given* a message whose `Header.TimeStamp` is `2026-09-18T10:00:00.000Z` and a fake `TimeProvider` set to `2026-09-18T10:00:00.250Z` at dispatch, *when* the message is cleared, *then* exactly one latency sample is recorded with value 250 ms, attributed to the message's destination and to the correct clear kind. + +**AC-14 — Latency survives cross-process dispatch.** +*Given* a message persisted in a shared Outbox with `Header.TimeStamp == 10:00:00.000Z`, written by a mediator instance that is then disposed, *when* a **second**, independently constructed mediator's sweeper clears that message with its `TimeProvider` at `10:05:00.000Z`, *then* exactly one latency sample of 300,000 ms is recorded by the second instance, attributed as a sweeper clear. + +**AC-15 — Negative latency is recorded as zero.** +*Given* a message whose `Header.TimeStamp` is `10:00:01.000Z` and a dispatch-time `TimeProvider` at `10:00:00.500Z`, *when* the message is cleared, *then* exactly one latency sample of 0 is recorded, no negative value is recorded, and no exception is thrown. + +**AC-16 — Unattributable creation time skips only the latency sample.** +*Given* a confirmation-based producer that fires a successful confirmation with an empty message id such that no creation time can be resolved, *when* the callback completes, *then* latency samples total 0, the cleared unit is recorded if and only if the mark-dispatched succeeded, and no exception escapes the callback. + +**AC-17 — Metrics are recorded with tracing absent.** +*Given* a host with a `MeterProvider` built via `AddBrighterInstrumentation()` and **no** `TracerProvider`, a mediator constructed with a null tracer, `InstrumentationOptions.None`, and a `RequestContext` whose `Span` is null (and `Activity.Current == null`), *when* a message is added and then cleared, *then* added units = 1, cleared units = 1 and latency samples = 1. + +**AC-18 — Nothing is recorded and nothing is computed when no listener is attached.** +*Given* a meter double whose `Enabled` is `false` and no `MeterListener` attached, *when* 100 messages are added and cleared, *then* the double receives zero record calls, and the instrumentation code path performs no tag construction and issues no additional Outbox or database read (asserted via the double's call counters and the Outbox double's operation counts). + +**AC-19 — No message-identifying data leaks into attributes.** +*Given* a message with a known id, a body, a partition key, a correlation id and a populated header bag, *when* it is added and cleared with a recorder attached, *then* no recorded attribute key or value contains the message id, body content, partition key, correlation id or any header-bag value, and every recorded attribute key matches an existing `BrighterSemanticConventions` constant or the `paramore.brighter.*` prefix. + +**AC-20 — Cardinality is bounded.** +*Given* 3 configured destinations and 1,000 messages spread across them, cleared by both a sweeper and explicit calls, *when* all recordings are collected, *then* the number of distinct attribute combinations per instrument is at most `3 x (number of enumerated dimension values)` and does not grow with the number of messages. + +**AC-21 — Available through the existing registration call only.** +*Given* an application whose only metrics setup is `Sdk.CreateMeterProviderBuilder().AddBrighterInstrumentation().AddInMemoryExporter(...)`, *when* one message is added and cleared, *then* the added, cleared and latency instruments appear in the exported metric set, all emitted from the meter named `Paramore.Brighter`, with no additional registration call in the application. + +**AC-22 — Instrumentation faults do not break the Outbox.** +*Given* a meter double whose record methods throw, *when* a message is added, dispatched, marked dispatched and confirmed, *then* the message is still added, still sent, still marked dispatched, no exception is observed by the caller of the add/clear API, and no exception escapes the confirmation callback thread. + +**AC-23 — Existing behaviour is unchanged.** +*Given* the pre-existing Outbox, sweeper, circuit-breaker and observability test suites, *when* the change is applied, *then* those suites pass with no assertion modified — in particular the existing Outbox span events, clear spans, confirmation spans, logs and topic-trip behaviour are unchanged. + +**AC-24 — Duplicate clears are counted per clear event.** +*Given* a message that is cleared once by an explicit clear and then, because a confirmation arrived late, cleared again by a sweeper, *when* both mark-dispatched calls succeed, *then* cleared units total 2 (1 explicit, 1 sweeper) and latency samples total 2, documenting at-least-once semantics rather than de-duplicating. + +**AC-25 — Concurrent add and sweep are safe.** +*Given* a sweep running on a background task while 50 adds run concurrently on other threads, *when* all operations complete, *then* added units total exactly 50, cleared units equal the number of messages actually marked dispatched, and no exception is thrown by any thread. + +### Traceability: FR -> AC + +| FR | Covered by | +|---|---| +| FR-1 Count single adds | AC-1, AC-17, AC-18, AC-25 | +| FR-2 Count batched adds | AC-2, AC-3 | +| FR-3 Count cleared messages | AC-4, AC-5, AC-7, AC-8, AC-17, AC-24 | +| FR-4 Sweeper vs explicit attribution | AC-4, AC-5, AC-6, AC-24 | +| FR-5 Latency per cleared message | AC-7, AC-13, AC-17 | +| FR-6 Latency across process restart | AC-14 | +| FR-7 Unusable latency inputs | AC-15 (negative -> 0), AC-16 (unattributable -> skip) | +| FR-8 Failures not counted | AC-3, AC-9, AC-10, AC-11, AC-12 | +| FR-9 Independent of tracing/sampling | AC-17 | +| FR-10 Safe, bounded attribute content | AC-19, AC-20 | +| FR-11 Existing registration path | AC-21 | +| NFR-1 Zero cost when disabled | AC-18 | +| NFR-2 No behavioural change | AC-3, AC-9, AC-10, AC-23 | +| NFR-3 Faults isolated | AC-22 | +| NFR-4 Thread safety | AC-25 | +| NFR-5 TFM compatibility | Verified by the suite building and passing on every target TFM (no dedicated AC) | +| NFR-6 Bounded cardinality | AC-20 | +| NFR-7 Test determinism | AC-13, AC-14, AC-15 (all use an injected `TimeProvider`) | +| NFR-8 Upstream acceptability | AC-19 (convention-conformant keys), AC-21 (meter name), AC-23 (no breakage) | + +## Additional Context + +**Why now / motivating incident.** Jira PI-36093 (UK): NOFR events were emitted after significant lag, causing `IgnoredOrders` before Partners / OrderPad / JetConnect could "Accept" orders. The Outbox publish lag responsible was invisible — no metric existed to dashboard or alert on, and the unsampled Outbox spans provided nothing after the fact. FR-5/FR-6 exist specifically so that this failure mode is visible next time, including when the lag is attributable to a sweeper in a different process instance. + +**Downstream consumer.** Jira RLOI-4186 ("[OrderDispatchWorker] Improve outbox observability") is blocked by this work and will build dashboards and alerts directly on these metrics. That is why FR-4 (sweeper attribution), FR-11 (no new opt-in) and NFR-6 (bounded cardinality) matter: the consuming team needs a ratio they can alert on, available from the registration call they already make, at a cardinality their metrics backend will accept. + +**Upstream collaboration.** Metric naming and instrument types need a proposal acceptable to the Brighter maintainers, coordinated in the `#interest-oss-brighter` Slack channel. This document therefore fixes *capabilities and constraints* only; names, instrument types, units and attribute keys belong in the ADR, which should present the proposal and the open questions and be updated when agreement lands (C-7). + +**Existing code the ADR will have to reconcile.** +- `src/Paramore.Brighter/OutboxProducerMediator.cs` — `AddToOutbox` / `AddToOutboxAsync`, `StartBatchAddToOutbox` / `EndBatchAddToOutbox` / `EndBatchAddToOutboxAsync`, `ClearOutbox` / `ClearOutboxAsync`, `ClearOutstandingFromOutboxAsync` -> `BackgroundDispatchUsingAsync`, `Dispatch` / `DispatchAsync` / `BulkDispatchAsync`, `HandleAsyncPublishConfirmation` / `ConfigurePublisherCallbackMaybe`, `TripTopic`, and the existing `_timeProvider` / `_tracer` / `_instrumentationOptions` fields. +- `src/Paramore.Brighter/OutboxSweeper.cs` — the sole caller of `ClearOutstandingFromOutboxAsync`, and the natural place for sweeper attribution to originate. +- `src/Paramore.Brighter/Observability/` — `MessagingMeter` / `IAmABrighterMessagingMeter`, `DbMeter` / `IAmABrighterDbMeter`, `BrighterMetricsFromTracesProcessor`, `BrighterSemanticConventions` (including `MeterName == "Paramore.Brighter"`), `BrighterTracer.WriteOutboxEvent` (the add-path event that is dropped when the span is null), and `InstrumentationOptions`. +- `src/Paramore.Brighter.Extensions.Diagnostics/BrighterMetricsBuilderExtensions.cs` and `BrighterTracerBuilderExtensions.cs` — the registration surface that FR-11 pins. +- The publish-confirmation payload (`PublishConfirmationResult`) — carries no creation time today (C-4). + +**Test landing zone.** `tests/Paramore.Brighter.Core.Tests/Observability/Metrics/` already holds metric tests (`When_ending_a_send_span_should_still_record_a_client_operation.cs`, `When_ending_a_message_pump_begin_span_should_not_record_a_client_operation.cs`) with doubles in `tests/Paramore.Brighter.Core.Tests/Observability/TestDoubles/` (`SpyMessagingMeter`, `DisabledDbMeter`) — the pattern AC-18 and AC-22 rely on. From a2a2772518c69c9216eb7014ed51c852dc8aa4ad Mon Sep 17 00:00:00 2001 From: taha-rafique Date: Wed, 7 Oct 2026 10:50:45 +0100 Subject: [PATCH 2/8] Design approved --- docs/adr/0084-outbox-metrics.md | 4 +- docs/adr/index.md | 2 +- docs/guides/outbox-metrics.md | 183 +++++++++++++++++++++ specs/0037-outbox-metrics/.design-approved | 0 4 files changed, 186 insertions(+), 3 deletions(-) create mode 100644 docs/guides/outbox-metrics.md create mode 100644 specs/0037-outbox-metrics/.design-approved diff --git a/docs/adr/0084-outbox-metrics.md b/docs/adr/0084-outbox-metrics.md index 6a2a7b45e8..92610f4327 100644 --- a/docs/adr/0084-outbox-metrics.md +++ b/docs/adr/0084-outbox-metrics.md @@ -1,7 +1,7 @@ --- id: 0084-outbox-metrics title: "Outbox Metrics" -status: Proposed +status: Accepted author: - "taha-rafique" created: 2026-09-28 @@ -19,7 +19,7 @@ Date: 2026-09-28 ## Status -Proposed +Accepted ## Context diff --git a/docs/adr/index.md b/docs/adr/index.md index 84fe619c56..a5391afed0 100644 --- a/docs/adr/index.md +++ b/docs/adr/index.md @@ -133,6 +133,6 @@ Identity is the `id` (filename stem), not the leading number — numbers may rep | `0081-report-persistent-channel-failures-in-health-checks` | Report persistent channel failures in health checks | Proposed | observability, message-pump, resilience, api-design | Track consecutive receive failures in message pumps and report Degraded once a configurable threshold is reached. Successful receives, including empty queues, reset the count; an optional capability preserves custom pump and performer compatibility. | | `0081-timed-outbox-sweeper-health-metrics` | TimedOutboxSweeper health metrics | Accepted | observability, metrics, outbox, otel | TimedOutboxSweeper records three instruments directly through a new IAmABrighterSweeperMeter role (no-op by default, registered by AddBrighterInstrumentation): tick lag, sweep duration, and sweeps by outcome (completed, failed, lock_unavailable), so a stalled or late sweeper is visible even when traces are sampled out. | | `0083-coordinate-dispatcher-startup-and-shutdown` | Coordinate Dispatcher startup and shutdown | Proposed | service-activator, concurrency | Track accepted consumer operations so shutdown cannot overtake channel creation or task registration. | -| `0084-outbox-metrics` | Outbox Metrics | Proposed | observability, metrics, otel, outbox | Outbox metrics are recorded by direct OpenTelemetry instruments (paramore.brighter.outbox.added.messages and .cleared.messages as Counter, .publish.duration as a Histogram in seconds with explicit long-tail buckets) behind a new IAmABrighterOutboxMeter/OutboxMeter role owned by OutboxProducerMediator rather than derived from traces, with the clear kind carried as an OutboxClearSource parameter on the private dispatch methods and confirmation-path creation time resolved from a bounded, TTL-evicted in-mediator pending-clear ConcurrentDictionary. | +| `0084-outbox-metrics` | Outbox Metrics | Accepted | observability, metrics, otel, outbox | Outbox metrics are recorded by direct OpenTelemetry instruments (paramore.brighter.outbox.added.messages and .cleared.messages as Counter, .publish.duration as a Histogram in seconds with explicit long-tail buckets) behind a new IAmABrighterOutboxMeter/OutboxMeter role owned by OutboxProducerMediator rather than derived from traces, with the clear kind carried as an OutboxClearSource parameter on the private dispatch methods and confirmation-path creation time resolved from a bounded, TTL-evicted in-mediator pending-clear ConcurrentDictionary. | _121 ADRs indexed._ diff --git a/docs/guides/outbox-metrics.md b/docs/guides/outbox-metrics.md new file mode 100644 index 0000000000..e22db186aa --- /dev/null +++ b/docs/guides/outbox-metrics.md @@ -0,0 +1,183 @@ +# Outbox Metrics Operator Guide + +Paramore.Brighter emits three OpenTelemetry metrics instruments from the Outbox pipeline. They are +independent of distributed tracing and sampling: you receive counts and durations even when no trace +is being captured. + +--- + +## Instruments + +| Name | Type | Unit | Description | +|---|---|---|---| +| `paramore.brighter.outbox.added.messages` | `Counter` | `{message}` | Number of messages written to the Outbox. | +| `paramore.brighter.outbox.cleared.messages` | `Counter` | `{message}` | Number of messages recorded as dispatched from the Outbox. | +| `paramore.brighter.outbox.publish.duration` | `Histogram` | `s` | Time a message waited between being created for the Outbox and being recorded as dispatched. | + +All three instruments are on the `Paramore.Brighter` meter. + +### Attributes + +Every recording carries at most two attributes. + +| Key | Values | Instruments | +|---|---|---| +| `messaging.destination.name` | The message's routing key (topic). Empty string when the routing key is absent. | All three | +| `paramore.brighter.outbox.clear_source` | `"explicit"`, `"sweeper"`, `"unknown"` | `cleared.messages`, `publish.duration` | + +**Attribute cardinality** is bounded by the number of distinct destinations × the number of clear sources (3). A deployment with 10 topics therefore produces at most 10 attribute sets on `added.messages` and at most 30 on `cleared.messages` and `publish.duration`. + +### Clear-source values + +| Value | Meaning | +|---|---| +| `"explicit"` | The clear was triggered explicitly by the application (`ClearOutbox` / `ClearOutboxAsync`). | +| `"sweeper"` | The clear was triggered by the background Outbox sweeper (`ClearOutstandingFromOutboxAsync`). | +| `"unknown"` | The clear source could not be determined. See [Unknown source](#unknown-source) below. | + +--- + +## Registration + +Outbox metrics are exported automatically through the existing `AddBrighterInstrumentation()` call. +No extra step is required: + +```csharp +services.AddOpenTelemetry() + .WithMetrics(b => b + .AddBrighterInstrumentation() // already includes the Outbox instruments + .AddOtlpExporter()); +``` + +### Providing your own OutboxMeter + +To attach service-resource attributes to every recording (e.g. `service.name`), pass a +`MeterProvider` to `OutboxMeter`: + +```csharp +// Register the meter as a singleton so the mediator uses it +services.AddSingleton(sp => +{ + var meterProvider = sp.GetRequiredService(); + return new OutboxMeter(meterProvider); +}); +``` + +The mediator receives `IAmABrighterOutboxMeter` from the container when built via `AddBrighter()`. +If no instance is registered, the mediator creates its own `OutboxMeter()` without service attributes. + +--- + +## Duration measurement + +> **Warning:** The `publish.duration` histogram does not measure store-to-broker latency. + +The measured value is `MarkDispatched` instant − `Header.TimeStamp`. `Header.TimeStamp` is set when +the message is created (before mapping and transformation). The measured duration therefore: + +- **Includes** message mapping and transform time. +- **Is subject to clock skew** when the writer and the sweeper run on different hosts. +- **Clamps negative values to zero** rather than recording them, so a small skew does not produce + impossible negative histogram values. +- **Does not measure broker round-trip time.** Use broker-provided latency metrics for that. + +On **confirmation transports** (producers implementing `ISupportPublishConfirmation`) the duration +is also measured from `Header.TimeStamp` to the `MarkDispatched` instant inside the confirmation +callback, not to the send instant. + +--- + +## Circuit breaker and the rising-added / flat-cleared pattern + +> **Warning:** A tripped circuit-breaker topic shows `added.messages` rising while +> `cleared.messages` stays flat for that destination. + +When `IAmAnOutboxCircuitBreaker` trips a destination, the sweeper skips its messages. `added.messages` +continues to increment as new messages are deposited, but `cleared.messages` stops incrementing. +The ratio `cleared / (cleared + added)` for that destination will trend toward zero. + +Alert on: `rate(paramore_brighter_outbox_added_messages[5m]) > 0` for a destination where +`rate(paramore_brighter_outbox_cleared_messages[5m]) == 0` for more than a few minutes. + +See ADR 0028 and the circuit-breaker documentation for recovery options. + +--- + +## Unknown source and fewer samples than cleared units + +On **confirmation transports**, each dispatched message is held in an in-process pending-clear map +until the broker confirmation arrives. Three conditions remove it before a confirmation can claim it: + +1. **Empty `MessageId`** in the confirmation result — the broker reported success but did not + identify which message. +2. **TTL eviction** — the entry has been in the map for more than 5 minutes without a confirmation. + This typically indicates a lost confirmation or a very slow broker. +3. **Map cap** — the map holds at most 10,000 entries. When the cap is reached a new dispatch is + not tracked; its eventual confirmation records `"unknown"`. + +When any of these conditions applies, `cleared.messages` still increments (the message _was_ +dispatched), but `publish.duration` receives **no sample** (there is no creation time to compute +the wait from). The `clear_source` is `"unknown"`. + +To compute the sweeper ratio correctly, include unknown in the denominator: + +```promql +# Sweeper fraction of all clears (Prometheus) +sum by (messaging_destination_name) ( + rate(paramore_brighter_outbox_cleared_messages_total{ + paramore_brighter_outbox_clear_source="sweeper"}[5m]) +) +/ +sum by (messaging_destination_name) ( + rate(paramore_brighter_outbox_cleared_messages_total[5m]) +) +``` + +--- + +## Counting semantics + +- `added.messages` is incremented **after** a successful Outbox write. A write that throws never + increments the counter. +- `cleared.messages` is incremented **after** a successful `MarkDispatched` call. A send that + fails (or a `MarkDispatched` that throws) never increments the counter. +- Both counters use **at-least-once semantics**. If a message is cleared twice (e.g. an explicit + clear followed by a sweeper re-dispatch with `ForceOutstanding`) both clears are counted. +- On confirmation transports, a second successful confirmation for the same message ID also + increments `cleared.messages`, but with `clear_source="unknown"` (the pending-clear entry was + already removed by the first confirmation). + +--- + +## Histogram bucket boundaries + +The `publish.duration` histogram uses the following explicit bucket boundaries (in seconds): + +``` +0.005 0.01 0.025 0.05 0.075 0.1 0.25 0.5 0.75 1 +2.5 5 7.5 10 30 60 300 600 +``` + +To override the boundaries for your own deployment, add an OpenTelemetry view: + +```csharp +builder.AddView( + instrumentName: "paramore.brighter.outbox.publish.duration", + metricStreamConfiguration: new ExplicitBucketHistogramConfiguration + { + Boundaries = [0.1, 1.0, 10.0, 60.0, 300.0] + }); +``` + +--- + +## Known gaps + +- **Sync pipeline over-count on add:** When a `RequestContext.ResilienceContext` is set, the sync + resilience pipeline may return `true` without invoking the add action. The `added.messages` + counter is incremented even though nothing was stored. This is a known limitation of the sync + resilience pipeline path and does not affect the async path. +- **Sync confirmation send cleanup:** When a sync confirming-producer send fails under + `ResilienceContext`, the pending-clear entry is not removed immediately (unlike the async path). + The entry persists until the 5-minute TTL purge. Operators may see a slightly elevated + `"unknown"` rate on sync confirmation transports. diff --git a/specs/0037-outbox-metrics/.design-approved b/specs/0037-outbox-metrics/.design-approved new file mode 100644 index 0000000000..e69de29bb2 From 5718d3aa2e8b6be6be5344c7703de905f7bb1587 Mon Sep 17 00:00:00 2001 From: taha-rafique Date: Wed, 7 Oct 2026 13:24:59 +0100 Subject: [PATCH 3/8] Adds the types of outbox clear operations --- .../Observability/OutboxClearSource.cs | 60 +++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 src/Paramore.Brighter/Observability/OutboxClearSource.cs diff --git a/src/Paramore.Brighter/Observability/OutboxClearSource.cs b/src/Paramore.Brighter/Observability/OutboxClearSource.cs new file mode 100644 index 0000000000..2e75da407e --- /dev/null +++ b/src/Paramore.Brighter/Observability/OutboxClearSource.cs @@ -0,0 +1,60 @@ +#region Licence +/* The MIT License (MIT) +Copyright © 2024 Ian Cooper + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. */ + +#endregion + +namespace Paramore.Brighter.Observability; + +/// What initiated an Outbox clear operation. +public enum OutboxClearSource +{ + /// The clear was initiated explicitly by the application (e.g. ClearOutbox / ClearOutboxAsync). + Explicit, + + /// The clear was initiated by the background Outbox sweeper (ClearOutstandingFromOutboxAsync). + Sweeper, + + /// + /// The clear source could not be determined. This occurs in three cases: + /// + /// The confirmation callback received a PublishConfirmationResult with an empty MessageId. + /// The pending-clear entry for the message was evicted by the TTL (5 minutes by default). + /// The pending-clear map was at its hard cap (10,000 entries) when the message was dispatched. + /// + /// + Unknown +} + +/// Extension methods for mapping to its attribute string value. +public static class OutboxClearSourceExtensions +{ + /// + /// Returns the bounded attribute string value for the given . + /// An unmapped value returns "unknown" without throwing, so a metrics path degrades rather than faults. + /// + public static string ToClearSourceName(this OutboxClearSource clearSource) => clearSource switch + { + OutboxClearSource.Explicit => "explicit", + OutboxClearSource.Sweeper => "sweeper", + _ => "unknown" + }; +} From 9bf045bfe5ccbe01987f7904043bbbdc106ede95 Mon Sep 17 00:00:00 2001 From: taha-rafique Date: Wed, 7 Oct 2026 13:35:53 +0100 Subject: [PATCH 4/8] Adds interface for the metrics we want to record --- .../BrighterSemanticConventions.cs | 4 ++ .../Observability/IAmABrighterOutboxMeter.cs | 68 +++++++++++++++++++ 2 files changed, 72 insertions(+) create mode 100644 src/Paramore.Brighter/Observability/IAmABrighterOutboxMeter.cs diff --git a/src/Paramore.Brighter/Observability/BrighterSemanticConventions.cs b/src/Paramore.Brighter/Observability/BrighterSemanticConventions.cs index a27fed12d0..833469cead 100644 --- a/src/Paramore.Brighter/Observability/BrighterSemanticConventions.cs +++ b/src/Paramore.Brighter/Observability/BrighterSemanticConventions.cs @@ -101,6 +101,10 @@ public static class BrighterSemanticConventions public const string NetworkPeerAddress = "network.peer.address"; public const string NetworkPeerPort = "network.peer.port"; public const string Operation = "paramore.brighter.operation"; + public const string OutboxAddedMessages = "paramore.brighter.outbox.added.messages"; + public const string OutboxClearedMessages = "paramore.brighter.outbox.cleared.messages"; + public const string OutboxPublishDuration = "paramore.brighter.outbox.publish.duration"; + public const string OutboxClearSource = "paramore.brighter.outbox.clear_source"; public const string OutboxSharedTransaction = "paramore.brighter.outbox.shared_transaction"; public const string OutboxSweeperOutcome = "paramore.brighter.outbox_sweeper.outcome"; public const string OutboxSweeperSweepDuration = "paramore.brighter.outbox_sweeper.sweep.duration"; diff --git a/src/Paramore.Brighter/Observability/IAmABrighterOutboxMeter.cs b/src/Paramore.Brighter/Observability/IAmABrighterOutboxMeter.cs new file mode 100644 index 0000000000..3ea451ead9 --- /dev/null +++ b/src/Paramore.Brighter/Observability/IAmABrighterOutboxMeter.cs @@ -0,0 +1,68 @@ +#region Licence +/* The MIT License (MIT) +Copyright © 2024 Ian Cooper + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. */ + +#endregion + +using System; + +namespace Paramore.Brighter.Observability; + +/// +/// Records Outbox metrics directly on OpenTelemetry instruments, independently of tracing and sampling. +/// +/// +/// Contract: +/// +/// may be . +/// A negative wait passed to is recorded as zero. +/// No member may throw — implementations must absorb faults internally. +/// +/// +public interface IAmABrighterOutboxMeter +{ + /// Record one message successfully written to the Outbox. + /// The message's routing key / destination topic. + void AddMessageAdded(RoutingKey destination); + + /// Record one message recorded as dispatched from the Outbox. + /// The message's routing key / destination topic. + /// What initiated the clear operation. + void AddMessageCleared(RoutingKey destination, OutboxClearSource clearSource); + + /// + /// Record how long a message waited between being created for the Outbox and being recorded as dispatched. + /// + /// + /// The duration between Header.TimeStamp (message creation) and the MarkDispatched instant. + /// This value includes message mapping/transform time and is subject to clock skew; + /// it is not a pure store-to-broker duration. Negative values are recorded as zero. + /// + /// The message's routing key / destination topic. + /// What initiated the clear operation. + void RecordPublishDuration(TimeSpan wait, RoutingKey destination, OutboxClearSource clearSource); + + /// + /// Returns false when no is attached + /// to any of the three Outbox instruments, allowing callers to skip all tag-building work at zero cost. + /// + bool Enabled { get; } +} From 0884c0abd00deab05d6c66bfc5bb37aaac5a185c Mon Sep 17 00:00:00 2001 From: taha-rafique Date: Wed, 7 Oct 2026 16:16:15 +0100 Subject: [PATCH 5/8] Registering the Outbox meter --- .../ServiceCollectionExtensions.cs | 8 +- .../Observability/OutboxMeter.cs | 161 ++++++++++++++++++ .../Paramore.Brighter.csproj | 1 + 3 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 src/Paramore.Brighter/Observability/OutboxMeter.cs diff --git a/src/Paramore.Brighter.Extensions.DependencyInjection/ServiceCollectionExtensions.cs b/src/Paramore.Brighter.Extensions.DependencyInjection/ServiceCollectionExtensions.cs index 2140b0b852..427fae03e6 100644 --- a/src/Paramore.Brighter.Extensions.DependencyInjection/ServiceCollectionExtensions.cs +++ b/src/Paramore.Brighter.Extensions.DependencyInjection/ServiceCollectionExtensions.cs @@ -851,7 +851,8 @@ private static IAmACommandProcessor BuildCommandProcessor(IServiceProvider provi TimeProvider.System, busConfiguration.InstrumentationOptions, true, - true); + true, + OutboxMeter(serviceProvider)); } /// @@ -1015,6 +1016,11 @@ private static InMemoryOutbox CreateDefaultOutbox(IAmProducersConfiguration busC return serviceProvider.GetService(); } + private static IAmABrighterOutboxMeter? OutboxMeter(IServiceProvider serviceProvider) + { + return serviceProvider.GetService(); + } + /// x /// Creates transforms. Normally you don't need to call this, it is called by the builder for Brighter or /// the Service Activator diff --git a/src/Paramore.Brighter/Observability/OutboxMeter.cs b/src/Paramore.Brighter/Observability/OutboxMeter.cs new file mode 100644 index 0000000000..38e3442afa --- /dev/null +++ b/src/Paramore.Brighter/Observability/OutboxMeter.cs @@ -0,0 +1,161 @@ +#region Licence +/* The MIT License (MIT) +Copyright © 2024 Ian Cooper + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. */ + +#endregion + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Diagnostics.Metrics; +using Microsoft.Extensions.Logging; +using OpenTelemetry.Metrics; +using Paramore.Brighter.Logging; + +namespace Paramore.Brighter.Observability; + +/// +/// Records Outbox metrics directly on OpenTelemetry instruments, independently of tracing and sampling. +/// +public sealed partial class OutboxMeter : IAmABrighterOutboxMeter, IDisposable +{ + private static readonly ILogger s_logger = ApplicationLogging.CreateLogger(); + + private static readonly double[] s_publishDurationBoundaries = + [ + 0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.25, 0.5, 0.75, 1, + 2.5, 5, 7.5, 10, 30, 60, 300, 600 + ]; + + private readonly Meter _meter; + private readonly Counter _addedMessages; + private readonly Counter _clearedMessages; + private readonly Histogram _publishDuration; + private readonly KeyValuePair[] _serviceAttributes; + + /// + /// Creates an . Pass a built through DI + /// (AddOpenTelemetry()) to include service-resource attributes on every recording. + /// When null, only the two Outbox-specific attributes are recorded. + /// + public OutboxMeter(MeterProvider? meterProvider = null) + { + _serviceAttributes = meterProvider?.GetServiceAttributes() ?? []; + _meter = new Meter(BrighterSemanticConventions.MeterName); + + _addedMessages = _meter.CreateCounter( + BrighterSemanticConventions.OutboxAddedMessages, + unit: "{message}", + description: "Number of messages written to the Outbox."); + + _clearedMessages = _meter.CreateCounter( + BrighterSemanticConventions.OutboxClearedMessages, + unit: "{message}", + description: "Number of messages recorded as dispatched from the Outbox."); + + _publishDuration = _meter.CreateHistogram( + BrighterSemanticConventions.OutboxPublishDuration, + unit: "s", + description: "Time a message waited between being created for the Outbox and being recorded as dispatched.", + advice: new InstrumentAdvice { HistogramBucketBoundaries = s_publishDurationBoundaries }); + } + + /// + public bool Enabled => + _addedMessages.Enabled || _clearedMessages.Enabled || _publishDuration.Enabled; + + /// + public void AddMessageAdded(RoutingKey destination) + { + try + { + var tags = BuildTags(destination); + _addedMessages.Add(1, in tags); + } + catch (Exception ex) + { + Log.OutboxMetricsFault(s_logger, ex); + } + } + + /// + public void AddMessageCleared(RoutingKey destination, OutboxClearSource clearSource) + { + try + { + var tags = BuildTags(destination, clearSource); + _clearedMessages.Add(1, in tags); + } + catch (Exception ex) + { + Log.OutboxMetricsFault(s_logger, ex); + } + } + + /// + public void RecordPublishDuration(TimeSpan wait, RoutingKey destination, OutboxClearSource clearSource) + { + try + { + var seconds = Math.Max(0, wait.TotalSeconds); + var tags = BuildTags(destination, clearSource); + _publishDuration.Record(seconds, in tags); + } + catch (Exception ex) + { + Log.OutboxMetricsFault(s_logger, ex); + } + } + + /// + public void Dispose() => _meter.Dispose(); + + private TagList BuildTags(RoutingKey destination) + { + var tags = new TagList + { + { BrighterSemanticConventions.MessagingDestination, destination.Value } + }; + foreach (var attr in _serviceAttributes) + tags.Add(attr.Key, attr.Value); + return tags; + } + + private TagList BuildTags(RoutingKey destination, OutboxClearSource clearSource) + { + var tags = new TagList + { + { BrighterSemanticConventions.MessagingDestination, destination.Value }, + { BrighterSemanticConventions.OutboxClearSource, clearSource.ToClearSourceName() } + }; + foreach (var attr in _serviceAttributes) + tags.Add(attr.Key, attr.Value); + return tags; + } + + private static partial class Log + { + [LoggerMessage( + LogLevel.Warning, + "Outbox metrics fault; the metric has been dropped but the Outbox operation was unaffected")] + public static partial void OutboxMetricsFault(ILogger logger, Exception ex); + } +} diff --git a/src/Paramore.Brighter/Paramore.Brighter.csproj b/src/Paramore.Brighter/Paramore.Brighter.csproj index e02893fffe..778d7c9d49 100644 --- a/src/Paramore.Brighter/Paramore.Brighter.csproj +++ b/src/Paramore.Brighter/Paramore.Brighter.csproj @@ -17,6 +17,7 @@ + From c44327078aec548877a5291df6904051a8ff4997 Mon Sep 17 00:00:00 2001 From: taha-rafique Date: Fri, 9 Oct 2026 11:30:59 +0100 Subject: [PATCH 6/8] Recording outbox metrics --- .../OutboxProducerMediator.cs | 125 +++++++++++++++--- 1 file changed, 105 insertions(+), 20 deletions(-) diff --git a/src/Paramore.Brighter/OutboxProducerMediator.cs b/src/Paramore.Brighter/OutboxProducerMediator.cs index 913683a960..a14149062a 100644 --- a/src/Paramore.Brighter/OutboxProducerMediator.cs +++ b/src/Paramore.Brighter/OutboxProducerMediator.cs @@ -94,7 +94,9 @@ public partial class OutboxProducerMediator : IAmAnOutbo private readonly Dictionary _outBoxBag; private readonly IAmABrighterTracer? _tracer; private readonly TimeProvider _timeProvider; - + private IAmABrighterOutboxMeter _outboxMeter = null!; + private bool _ownsOutboxMeter; + /// public IAmAnOutbox? Outbox => (IAmAnOutbox?)_outBox ?? _asyncOutbox; @@ -151,7 +153,8 @@ public OutboxProducerMediator( TimeProvider? timeProvider = null, InstrumentationOptions instrumentationOptions = InstrumentationOptions.All, bool ownsRegistry = false, - bool ownsTransformerFactories = false) + bool ownsTransformerFactories = false, + IAmABrighterOutboxMeter? outboxMeter = null) { _producerRegistry = producerRegistry ?? throw new ConfigurationException("Missing Producer Registry for External Bus Services"); @@ -172,6 +175,8 @@ public OutboxProducerMediator( _timeProvider = timeProvider ?? TimeProvider.System; _lastOutStandingMessageCheckAt = _timeProvider.GetUtcNow(); + _outboxMeter = outboxMeter ?? new OutboxMeter(); + _ownsOutboxMeter = outboxMeter is null; _messageMapperRegistry = mapperRegistry; _messageTransformerFactory = messageTransformerFactory; @@ -244,6 +249,9 @@ private void Dispose(bool disposing) DisposeQuietly(_messageTransformerFactory); DisposeQuietly(_messageTransformerFactoryAsync); } + + if (_ownsOutboxMeter) + DisposeQuietly(_outboxMeter); } } @@ -301,6 +309,8 @@ await _asyncOutbox if (!written) throw new ChannelFailureException($"Could not write request {message.Id} to the outbox"); + + RecordAdded(message); } /// @@ -339,6 +349,8 @@ public void AddToOutbox( if (!written) throw new ChannelFailureException($"Could not write message {message.Id} to the outbox"); + + RecordAdded(message); } /// @@ -407,7 +419,7 @@ public void ClearOutbox( try { - Dispatch([message], requestContext, args); + Dispatch([message], requestContext, OutboxClearSource.Explicit, args); } finally { @@ -476,7 +488,7 @@ public async Task ClearOutboxAsync( try { - await DispatchAsync([message], requestContext, continueOnCapturedContext, cancellationToken); + await DispatchAsync([message], requestContext, OutboxClearSource.Explicit, continueOnCapturedContext, cancellationToken); } finally { @@ -650,6 +662,9 @@ public void EndBatchAddToOutbox(string batchId, IAmABoxTransactionProvider /// The producer to add a callback for - /// The request context for the pipeline + /// The request context for the pipeline /// private void ConfigureAsyncPublisherCallbackMaybe(IAmAMessageProducerAsync producer, RequestContext requestContext) { @@ -1072,7 +1090,7 @@ private static RoutingKey GetProducerLookupTopic(Message message) return message.Header.Topic; } - private void Dispatch(IEnumerable posts, RequestContext requestContext, Dictionary? args = null) + private void Dispatch(IEnumerable posts, RequestContext requestContext, OutboxClearSource clearSource, Dictionary? args = null) { var parentSpan = requestContext.Span; var producerSpans = new ConcurrentDictionary(); @@ -1109,10 +1127,17 @@ private void Dispatch(IEnumerable posts, RequestContext requestContext, ); if (sent) { - ExecuteWithResiliencePipeline( - () => _outBox.MarkDispatched(message.Id, requestContext, _timeProvider.GetUtcNow(), args), + DateTimeOffset? dispatchedAt = null; + var marked = ExecuteWithResiliencePipeline( + () => + { + dispatchedAt = _timeProvider.GetUtcNow(); + _outBox.MarkDispatched(message.Id, requestContext, dispatchedAt.Value, args); + }, requestContext ); + if (marked && dispatchedAt.HasValue) + RecordCleared(message.Header.Topic, clearSource, message.Header.TimeStamp, dispatchedAt.Value); } } } @@ -1130,8 +1155,9 @@ private void Dispatch(IEnumerable posts, RequestContext requestContext, } private async Task BulkDispatchAsync( - IEnumerable posts, + IEnumerable posts, RequestContext requestContext, + OutboxClearSource clearSource, bool continueOnCapturedContext, CancellationToken cancellationToken) { @@ -1167,6 +1193,14 @@ private async Task BulkDispatchAsync( Log.BulkDispatchingMessages(s_logger, messages.Length, topicBatch.Key.WireTopic.Value); + Dictionary? createdAtById = null; + if (_outboxMeter.Enabled) + { + createdAtById = new Dictionary(messages.Length, StringComparer.Ordinal); + foreach (var m in messages) + createdAtById[m.Id.Value] = m.Header.TimeStamp; + } + foreach (var batch in await bulkMessageProducer.CreateBatchesAsync(messages, cancellationToken)) { var sent = await ExecuteWithResiliencePipelineAsync( @@ -1182,14 +1216,27 @@ private async Task BulkDispatchAsync( { foreach (var successfulMessage in batch.Ids()) { - await ExecuteWithResiliencePipelineAsync(async ct => + DateTimeOffset? dispatchedAt = null; + var marked = await ExecuteWithResiliencePipelineAsync(async ct => + { + dispatchedAt = _timeProvider.GetUtcNow(); await _asyncOutbox.MarkDispatchedAsync( - successfulMessage, requestContext, _timeProvider.GetUtcNow(), + successfulMessage, requestContext, dispatchedAt.Value, cancellationToken: ct - ), + ); + }, requestContext, cancellationToken: cancellationToken ); + if (marked && dispatchedAt.HasValue) + { + createdAtById?.TryGetValue(successfulMessage.Value, out var createdAt); + RecordCleared( + topicBatch.Key.WireTopic, + clearSource, + createdAtById is not null && createdAtById.TryGetValue(successfulMessage.Value, out var ct) ? ct : null, + dispatchedAt.Value); + } } } @@ -1215,6 +1262,7 @@ await _asyncOutbox.MarkDispatchedAsync( private async Task DispatchAsync( IEnumerable posts, RequestContext requestContext, + OutboxClearSource clearSource, bool continueOnCapturedContext, CancellationToken cancellationToken) { @@ -1250,14 +1298,21 @@ private async Task DispatchAsync( if (producer is not ISupportPublishConfirmation && sent) { - await ExecuteWithResiliencePipelineAsync( - async ct => await _asyncOutbox.MarkDispatchedAsync( - message.Id, requestContext, _timeProvider.GetUtcNow(), - cancellationToken: ct - ), + DateTimeOffset? dispatchedAt = null; + var marked = await ExecuteWithResiliencePipelineAsync( + async ct => + { + dispatchedAt = _timeProvider.GetUtcNow(); + await _asyncOutbox.MarkDispatchedAsync( + message.Id, requestContext, dispatchedAt.Value, + cancellationToken: ct + ); + }, requestContext, cancellationToken: cancellationToken ); + if (marked && dispatchedAt.HasValue) + RecordCleared(message.Header.Topic, clearSource, message.Header.TimeStamp, dispatchedAt.Value); } if(!sent) TripTopic(message.Header.Topic); @@ -1497,7 +1552,34 @@ private void TripTopic(RoutingKey? routingKey) if(!RoutingKey.IsNullOrEmpty(routingKey)) _outboxCircuitBreaker?.TripTopic(routingKey); } - + + private void RecordAdded(Message message) + { + if (!_outboxMeter.Enabled) return; + try { _outboxMeter.AddMessageAdded(message.Header.Topic); } + catch (Exception ex) { Log.OutboxMetricsFault(s_logger, ex); } + } + + private void RecordCleared( + RoutingKey destination, + OutboxClearSource clearSource, + DateTimeOffset? createdAt, + DateTimeOffset dispatchedAt) + { + if (!_outboxMeter.Enabled) return; + try + { + _outboxMeter.AddMessageCleared(destination, clearSource); + if (createdAt is { } created) + { + var wait = dispatchedAt - created; + _outboxMeter.RecordPublishDuration( + wait < TimeSpan.Zero ? TimeSpan.Zero : wait, destination, clearSource); + } + } + catch (Exception ex) { Log.OutboxMetricsFault(s_logger, ex); } + } + private static partial class Log { [LoggerMessage(LogLevel.Information, "Found {NumberOfMessages} to clear out of amount {AmountToClear}")] @@ -1565,6 +1647,9 @@ private static partial class Log [LoggerMessage(LogLevel.Warning, "Failed to dispose owned resource {ResourceType} while disposing the mediator; continuing with the remaining resources")] public static partial void FailedToDisposeOwnedResource(ILogger logger, string resourceType, Exception exception); + + [LoggerMessage(LogLevel.Warning, "Outbox metric recording failed; the metric has been dropped but the Outbox operation was unaffected")] + public static partial void OutboxMetricsFault(ILogger logger, Exception ex); } } } From 2f23ac200b381ec5dc80a1e8ebddf4f50d67e52f Mon Sep 17 00:00:00 2001 From: taha-rafique Date: Fri, 9 Oct 2026 16:35:52 +0100 Subject: [PATCH 7/8] Registering the outbox meter as optional --- docs/guides/outbox-metrics.md | 27 +++++++------------ .../BrighterMetricsBuilderExtensions.cs | 1 + .../OutboxProducerMediator.cs | 19 +++++++------ 3 files changed, 20 insertions(+), 27 deletions(-) diff --git a/docs/guides/outbox-metrics.md b/docs/guides/outbox-metrics.md index e22db186aa..7bd01519e3 100644 --- a/docs/guides/outbox-metrics.md +++ b/docs/guides/outbox-metrics.md @@ -39,32 +39,25 @@ Every recording carries at most two attributes. ## Registration -Outbox metrics are exported automatically through the existing `AddBrighterInstrumentation()` call. -No extra step is required: +Outbox metrics are enabled as part of your telemetry setup. The existing `AddBrighterInstrumentation()` +call registers an `OutboxMeter` (with the `MeterProvider`'s service-resource attributes, e.g. `service.name`) +alongside Brighter's other meters. No extra step is required: ```csharp services.AddOpenTelemetry() .WithMetrics(b => b - .AddBrighterInstrumentation() // already includes the Outbox instruments + .AddBrighterInstrumentation() // registers IAmABrighterOutboxMeter .AddOtlpExporter()); ``` -### Providing your own OutboxMeter - -To attach service-resource attributes to every recording (e.g. `service.name`), pass a -`MeterProvider` to `OutboxMeter`: +The mediator receives `IAmABrighterOutboxMeter` from the container when built via `AddBrighter()`. +The meter is optional: if none is registered, or you construct `OutboxProducerMediator` yourself without +passing one, no Outbox metrics are recorded and the recording calls return immediately. -```csharp -// Register the meter as a singleton so the mediator uses it -services.AddSingleton(sp => -{ - var meterProvider = sp.GetRequiredService(); - return new OutboxMeter(meterProvider); -}); -``` +### Providing your own OutboxMeter -The mediator receives `IAmABrighterOutboxMeter` from the container when built via `AddBrighter()`. -If no instance is registered, the mediator creates its own `OutboxMeter()` without service attributes. +Register your own `IAmABrighterOutboxMeter` before calling `AddBrighterInstrumentation()`; the built-in +registration does not replace an existing one. --- diff --git a/src/Paramore.Brighter.Extensions.Diagnostics/BrighterMetricsBuilderExtensions.cs b/src/Paramore.Brighter.Extensions.Diagnostics/BrighterMetricsBuilderExtensions.cs index 7172875e8c..1e0e1adda3 100644 --- a/src/Paramore.Brighter.Extensions.Diagnostics/BrighterMetricsBuilderExtensions.cs +++ b/src/Paramore.Brighter.Extensions.Diagnostics/BrighterMetricsBuilderExtensions.cs @@ -40,6 +40,7 @@ public static MeterProviderBuilder AddBrighterInstrumentation(this MeterProvider services.TryAddSingleton(); services.TryAddSingleton(); services.TryAddSingleton(); + services.TryAddSingleton(); builder.AddMeter(BrighterSemanticConventions.MeterName); }); diff --git a/src/Paramore.Brighter/OutboxProducerMediator.cs b/src/Paramore.Brighter/OutboxProducerMediator.cs index a14149062a..60cf1d96d0 100644 --- a/src/Paramore.Brighter/OutboxProducerMediator.cs +++ b/src/Paramore.Brighter/OutboxProducerMediator.cs @@ -94,8 +94,7 @@ public partial class OutboxProducerMediator : IAmAnOutbo private readonly Dictionary _outBoxBag; private readonly IAmABrighterTracer? _tracer; private readonly TimeProvider _timeProvider; - private IAmABrighterOutboxMeter _outboxMeter = null!; - private bool _ownsOutboxMeter; + private readonly IAmABrighterOutboxMeter? _outboxMeter; /// public IAmAnOutbox? Outbox => (IAmAnOutbox?)_outBox ?? _asyncOutbox; @@ -135,6 +134,10 @@ public partial class OutboxProducerMediator : IAmAnOutbo /// Does this mediator own the transform factories, so that should dispose them? /// Defaults to false for the manual-wiring path; the DI path passes true. /// + /// + /// Records Outbox metrics. Optional; when null, no Outbox metrics are recorded. Registered by + /// AddBrighterInstrumentation() when Brighter metrics are configured. + /// public OutboxProducerMediator( IAmAProducerRegistry producerRegistry, ResiliencePipelineRegistry resiliencePipelineRegistry, @@ -175,8 +178,7 @@ public OutboxProducerMediator( _timeProvider = timeProvider ?? TimeProvider.System; _lastOutStandingMessageCheckAt = _timeProvider.GetUtcNow(); - _outboxMeter = outboxMeter ?? new OutboxMeter(); - _ownsOutboxMeter = outboxMeter is null; + _outboxMeter = outboxMeter; _messageMapperRegistry = mapperRegistry; _messageTransformerFactory = messageTransformerFactory; @@ -249,9 +251,6 @@ private void Dispose(bool disposing) DisposeQuietly(_messageTransformerFactory); DisposeQuietly(_messageTransformerFactoryAsync); } - - if (_ownsOutboxMeter) - DisposeQuietly(_outboxMeter); } } @@ -1194,7 +1193,7 @@ private async Task BulkDispatchAsync( Log.BulkDispatchingMessages(s_logger, messages.Length, topicBatch.Key.WireTopic.Value); Dictionary? createdAtById = null; - if (_outboxMeter.Enabled) + if (_outboxMeter is { Enabled: true }) { createdAtById = new Dictionary(messages.Length, StringComparer.Ordinal); foreach (var m in messages) @@ -1555,7 +1554,7 @@ private void TripTopic(RoutingKey? routingKey) private void RecordAdded(Message message) { - if (!_outboxMeter.Enabled) return; + if (_outboxMeter is not { Enabled: true }) return; try { _outboxMeter.AddMessageAdded(message.Header.Topic); } catch (Exception ex) { Log.OutboxMetricsFault(s_logger, ex); } } @@ -1566,7 +1565,7 @@ private void RecordCleared( DateTimeOffset? createdAt, DateTimeOffset dispatchedAt) { - if (!_outboxMeter.Enabled) return; + if (_outboxMeter is not { Enabled: true }) return; try { _outboxMeter.AddMessageCleared(destination, clearSource); From 2c5e520a2b848348bdd9471f92cdc1b57de21a4a Mon Sep 17 00:00:00 2001 From: taha-rafique Date: Fri, 9 Oct 2026 16:56:08 +0100 Subject: [PATCH 8/8] Adds tests --- ...n_a_send_fails_should_not_count_cleared.cs | 131 ++++++++++ ...the_outbox_async_should_count_one_added.cs | 88 +++++++ ...outbox_add_fails_should_not_count_added.cs | 151 ++++++++++++ ..._outbox_should_count_cleared_as_sweeper.cs | 89 +++++++ .../TestDoubles/CountingOutbox.cs | 223 ++++++++++++++++++ .../TestDoubles/NonConfirmingProducerAsync.cs | 58 +++++ .../TestDoubles/NonConfirmingProducerSync.cs | 45 ++++ .../TestDoubles/SpyOutboxMeter.cs | 31 +++ 8 files changed, 816 insertions(+) create mode 100644 tests/Paramore.Brighter.Core.Tests/Observability/Metrics/Outbox/When_a_send_fails_should_not_count_cleared.cs create mode 100644 tests/Paramore.Brighter.Core.Tests/Observability/Metrics/Outbox/When_adding_a_message_to_the_outbox_async_should_count_one_added.cs create mode 100644 tests/Paramore.Brighter.Core.Tests/Observability/Metrics/Outbox/When_an_outbox_add_fails_should_not_count_added.cs create mode 100644 tests/Paramore.Brighter.Core.Tests/Observability/Metrics/Outbox/When_the_sweeper_clears_the_outbox_should_count_cleared_as_sweeper.cs create mode 100644 tests/Paramore.Brighter.Core.Tests/Observability/TestDoubles/CountingOutbox.cs create mode 100644 tests/Paramore.Brighter.Core.Tests/Observability/TestDoubles/NonConfirmingProducerAsync.cs create mode 100644 tests/Paramore.Brighter.Core.Tests/Observability/TestDoubles/NonConfirmingProducerSync.cs create mode 100644 tests/Paramore.Brighter.Core.Tests/Observability/TestDoubles/SpyOutboxMeter.cs diff --git a/tests/Paramore.Brighter.Core.Tests/Observability/Metrics/Outbox/When_a_send_fails_should_not_count_cleared.cs b/tests/Paramore.Brighter.Core.Tests/Observability/Metrics/Outbox/When_a_send_fails_should_not_count_cleared.cs new file mode 100644 index 0000000000..43e0d439e7 --- /dev/null +++ b/tests/Paramore.Brighter.Core.Tests/Observability/Metrics/Outbox/When_a_send_fails_should_not_count_cleared.cs @@ -0,0 +1,131 @@ +#region Licence +/* The MIT License (MIT) +Copyright © 2024 Ian Cooper + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. */ +#endregion + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.Threading.Tasks; +using System.Transactions; +using Microsoft.Extensions.Time.Testing; +using Paramore.Brighter.Core.Tests.Observability.TestDoubles; +using Paramore.Brighter.Extensions; +using Paramore.Brighter.Observability; +using Polly.Registry; +using Xunit; + +namespace Paramore.Brighter.Core.Tests.Observability.Metrics.Outbox; + +public class When_a_send_fails_should_not_count_cleared +{ + private readonly FakeTimeProvider _timeProvider = new(); + + [Fact] + public async Task Async_send_throws_cleared_stays_empty() + { + var topic = new RoutingKey("order.async-fail"); + var meter = new SpyOutboxMeter(); + var producer = new NonConfirmingProducerAsync(topic) { ThrowOnSend = true }; + var outbox = new CountingOutbox(_timeProvider); + + using var mediator = new OutboxProducerMediator( + new ProducerRegistry(new Dictionary { { topic, producer } }), + new ResiliencePipelineRegistry().AddBrighterDefault(), + new MessageMapperRegistry(new SimpleMessageMapperFactory(_ => null!), null), + new EmptyMessageTransformerFactory(), + new EmptyMessageTransformerFactoryAsync(), + tracer: null, + new FindPublicationByPublicationTopicOrRequestType(), + outbox: outbox, + timeProvider: _timeProvider, + outboxMeter: meter); + + var ctx = new RequestContext(); + var msg = new Message( + new MessageHeader(Id.Random(), topic, MessageType.MT_COMMAND, + timeStamp: _timeProvider.GetUtcNow()), + new MessageBody("{}")); + + await mediator.AddToOutboxAsync(msg, ctx); + Assert.Single(meter.Added); + + await mediator.ClearOutboxAsync([msg.Id], ctx); + + Assert.Empty(meter.Cleared); + Assert.Empty(meter.Durations); + } + + [Fact] + public void Sync_send_throws_cleared_stays_empty() + { + var topic = new RoutingKey("order.sync-fail"); + var meter = new SpyOutboxMeter(); + var producer = new ThrowingSyncProducer(topic); + var outbox = new CountingOutbox(_timeProvider); + + using var mediator = new OutboxProducerMediator( + new ProducerRegistry(new Dictionary { { topic, producer } }), + new ResiliencePipelineRegistry().AddBrighterDefault(), + new MessageMapperRegistry(new SimpleMessageMapperFactory(_ => null!), null), + new EmptyMessageTransformerFactory(), + new EmptyMessageTransformerFactoryAsync(), + tracer: null, + new FindPublicationByPublicationTopicOrRequestType(), + outbox: outbox, + timeProvider: _timeProvider, + outboxMeter: meter); + + var ctx = new RequestContext(); + var msg = new Message( + new MessageHeader(Id.Random(), topic, MessageType.MT_COMMAND, + timeStamp: _timeProvider.GetUtcNow()), + new MessageBody("{}")); + + mediator.AddToOutbox(msg, ctx); + Assert.Single(meter.Added); + + mediator.ClearOutbox([msg.Id], ctx); + + Assert.Empty(meter.Cleared); + Assert.Empty(meter.Durations); + } + + private sealed class ThrowingSyncProducer : IAmAMessageProducerSync + { + public ThrowingSyncProducer(RoutingKey topic) + { + Publication = new Publication { Topic = topic }; + } + + public Publication Publication { get; } + public Activity? Span { get; set; } + public IAmAMessageScheduler? Scheduler { get; set; } + + public void Send(Message message) => + throw new InvalidOperationException("ThrowingSyncProducer forced throw"); + + public void SendWithDelay(Message message, TimeSpan? delay) => + throw new InvalidOperationException("ThrowingSyncProducer forced throw"); + + public void Dispose() { } + } +} diff --git a/tests/Paramore.Brighter.Core.Tests/Observability/Metrics/Outbox/When_adding_a_message_to_the_outbox_async_should_count_one_added.cs b/tests/Paramore.Brighter.Core.Tests/Observability/Metrics/Outbox/When_adding_a_message_to_the_outbox_async_should_count_one_added.cs new file mode 100644 index 0000000000..0c6f37c846 --- /dev/null +++ b/tests/Paramore.Brighter.Core.Tests/Observability/Metrics/Outbox/When_adding_a_message_to_the_outbox_async_should_count_one_added.cs @@ -0,0 +1,88 @@ +#region Licence +/* The MIT License (MIT) +Copyright © 2024 Ian Cooper + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. */ +#endregion + +using System.Collections.Generic; +using System.Threading.Tasks; +using System.Transactions; +using Microsoft.Extensions.Time.Testing; +using Paramore.Brighter.Core.Tests.Observability.TestDoubles; +using Paramore.Brighter.Extensions; +using Paramore.Brighter.Observability; +using Polly.Registry; +using Xunit; + +namespace Paramore.Brighter.Core.Tests.Observability.Metrics.Outbox; + +public class When_adding_a_message_to_the_outbox_async_should_count_one_added +{ + private readonly RoutingKey _topic = new("order.created"); + private readonly SpyOutboxMeter _meter = new(); + private readonly OutboxProducerMediator _mediator; + private readonly CountingOutbox _outbox; + private readonly FakeTimeProvider _timeProvider = new(); + + public When_adding_a_message_to_the_outbox_async_should_count_one_added() + { + var producer = new NonConfirmingProducerAsync(_topic); + _outbox = new CountingOutbox(_timeProvider); + + _mediator = new OutboxProducerMediator( + new ProducerRegistry(new Dictionary { { _topic, producer } }), + new ResiliencePipelineRegistry().AddBrighterDefault(), + new MessageMapperRegistry(new SimpleMessageMapperFactory(_ => null!), null), + new EmptyMessageTransformerFactory(), + new EmptyMessageTransformerFactoryAsync(), + tracer: null, + new FindPublicationByPublicationTopicOrRequestType(), + outbox: _outbox, + timeProvider: _timeProvider, + outboxMeter: _meter); + } + + [Fact] + public async Task When_adding_a_message_to_the_outbox_async_should_count_one_added_() + { + var ctx = new RequestContext(); + var msg1 = new Message( + new MessageHeader(Id.Random(), _topic, MessageType.MT_COMMAND, + timeStamp: _timeProvider.GetUtcNow()), + new MessageBody("{}")); + var msg2 = new Message( + new MessageHeader(Id.Random(), _topic, MessageType.MT_COMMAND, + timeStamp: _timeProvider.GetUtcNow()), + new MessageBody("{}")); + + await _mediator.AddToOutboxAsync(msg1, ctx); + + Assert.Single(_meter.Added); + Assert.Equal(_topic, _meter.Added[0]); + Assert.Empty(_meter.Cleared); + Assert.Empty(_meter.Durations); + + await _mediator.AddToOutboxAsync(msg2, ctx); + + Assert.Equal(2, _meter.Added.Count); + Assert.Empty(_meter.Cleared); + Assert.Empty(_meter.Durations); + } +} diff --git a/tests/Paramore.Brighter.Core.Tests/Observability/Metrics/Outbox/When_an_outbox_add_fails_should_not_count_added.cs b/tests/Paramore.Brighter.Core.Tests/Observability/Metrics/Outbox/When_an_outbox_add_fails_should_not_count_added.cs new file mode 100644 index 0000000000..59f4289160 --- /dev/null +++ b/tests/Paramore.Brighter.Core.Tests/Observability/Metrics/Outbox/When_an_outbox_add_fails_should_not_count_added.cs @@ -0,0 +1,151 @@ +#region Licence +/* The MIT License (MIT) +Copyright © 2024 Ian Cooper + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. */ +#endregion + +// ACCEPTANCE — RED-first exemption: RecordAdded is placed after the !written throw. + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using System.Transactions; +using Microsoft.Extensions.Time.Testing; +using Paramore.Brighter.Core.Tests.Observability.TestDoubles; +using Paramore.Brighter.Extensions; +using Paramore.Brighter.Observability; +using Polly.Registry; +using Xunit; + +namespace Paramore.Brighter.Core.Tests.Observability.Metrics.Outbox; + +public class When_an_outbox_add_fails_should_not_count_added +{ + private readonly RoutingKey _topic = new("order.created"); + private readonly SpyOutboxMeter _meter = new(); + private readonly FakeTimeProvider _timeProvider = new(); + + private OutboxProducerMediator BuildMediator(CountingOutbox outbox) + { + var producerAsync = new NonConfirmingProducerAsync(_topic); + var producerSync = new NonConfirmingProducerSync(_topic); + return new OutboxProducerMediator( + new ProducerRegistry(new Dictionary + { + { _topic, producerAsync } + }), + new ResiliencePipelineRegistry().AddBrighterDefault(), + new MessageMapperRegistry(new SimpleMessageMapperFactory(_ => null!), null), + new EmptyMessageTransformerFactory(), + new EmptyMessageTransformerFactoryAsync(), + tracer: null, + new FindPublicationByPublicationTopicOrRequestType(), + outbox: outbox, + timeProvider: _timeProvider, + outboxMeter: _meter); + } + + private OutboxProducerMediator BuildSyncMediator(CountingOutbox outbox) + { + var producer = new NonConfirmingProducerSync(_topic); + return new OutboxProducerMediator( + new ProducerRegistry(new Dictionary + { + { _topic, producer } + }), + new ResiliencePipelineRegistry().AddBrighterDefault(), + new MessageMapperRegistry(new SimpleMessageMapperFactory(_ => null!), null), + new EmptyMessageTransformerFactory(), + new EmptyMessageTransformerFactoryAsync(), + tracer: null, + new FindPublicationByPublicationTopicOrRequestType(), + outbox: outbox, + timeProvider: _timeProvider, + outboxMeter: _meter); + } + + [Fact] + public async Task When_async_single_add_fails_should_not_count() + { + var outbox = new CountingOutbox(_timeProvider) { FailAdd = true }; + using var mediator = BuildMediator(outbox); + var ctx = new RequestContext(); + var msg = new Message( + new MessageHeader(Id.Random(), _topic, MessageType.MT_COMMAND, + timeStamp: _timeProvider.GetUtcNow()), new MessageBody("{}")); + + await Assert.ThrowsAsync( + () => mediator.AddToOutboxAsync(msg, ctx)); + Assert.Empty(_meter.Added); + } + + [Fact] + public async Task When_async_batch_add_fails_should_not_count() + { + var outbox = new CountingOutbox(_timeProvider) { FailAdd = true }; + using var mediator = BuildMediator(outbox); + var ctx = new RequestContext(); + var batchId = mediator.StartBatchAddToOutbox(); + for (var i = 0; i < 3; i++) + { + var msg = new Message( + new MessageHeader(Id.Random(), _topic, MessageType.MT_COMMAND, + timeStamp: _timeProvider.GetUtcNow()), new MessageBody("{}")); + await mediator.AddToOutboxAsync(msg, ctx, batchId: batchId); + } + + await Assert.ThrowsAsync( + () => mediator.EndBatchAddToOutboxAsync(batchId, null, ctx, CancellationToken.None)); + Assert.Empty(_meter.Added); + } + + [Fact] + public void When_sync_single_add_fails_should_not_count() + { + var outbox = new CountingOutbox(_timeProvider) { FailAdd = true }; + using var mediator = BuildSyncMediator(outbox); + var ctx = new RequestContext(); + var msg = new Message( + new MessageHeader(Id.Random(), _topic, MessageType.MT_COMMAND, + timeStamp: _timeProvider.GetUtcNow()), new MessageBody("{}")); + + Assert.Throws(() => mediator.AddToOutbox(msg, ctx)); + Assert.Empty(_meter.Added); + } + + [Fact] + public void When_sync_batch_add_fails_should_not_count() + { + var outbox = new CountingOutbox(_timeProvider) { FailAdd = true }; + using var mediator = BuildSyncMediator(outbox); + var ctx = new RequestContext(); + var batchId = mediator.StartBatchAddToOutbox(); + for (var i = 0; i < 3; i++) + { + var msg = new Message( + new MessageHeader(Id.Random(), _topic, MessageType.MT_COMMAND, + timeStamp: _timeProvider.GetUtcNow()), new MessageBody("{}")); + mediator.AddToOutbox(msg, ctx, batchId: batchId); + } + + Assert.Throws(() => mediator.EndBatchAddToOutbox(batchId, null, ctx)); + Assert.Empty(_meter.Added); + } +} diff --git a/tests/Paramore.Brighter.Core.Tests/Observability/Metrics/Outbox/When_the_sweeper_clears_the_outbox_should_count_cleared_as_sweeper.cs b/tests/Paramore.Brighter.Core.Tests/Observability/Metrics/Outbox/When_the_sweeper_clears_the_outbox_should_count_cleared_as_sweeper.cs new file mode 100644 index 0000000000..13a3acc93c --- /dev/null +++ b/tests/Paramore.Brighter.Core.Tests/Observability/Metrics/Outbox/When_the_sweeper_clears_the_outbox_should_count_cleared_as_sweeper.cs @@ -0,0 +1,89 @@ +#region Licence +/* The MIT License (MIT) +Copyright © 2024 Ian Cooper + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. */ +#endregion + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using System.Transactions; +using Microsoft.Extensions.Time.Testing; +using Paramore.Brighter.Core.Tests.Observability.TestDoubles; +using Paramore.Brighter.Extensions; +using Paramore.Brighter.Observability; +using Polly.Registry; +using Xunit; + +namespace Paramore.Brighter.Core.Tests.Observability.Metrics.Outbox; + +public class When_the_sweeper_clears_the_outbox_should_count_cleared_as_sweeper +{ + private readonly RoutingKey _topic = new("t"); + private readonly SpyOutboxMeter _meter = new(); + private readonly OutboxProducerMediator _mediator; + private readonly CountingOutbox _outbox; + private readonly FakeTimeProvider _timeProvider = new(); + + public When_the_sweeper_clears_the_outbox_should_count_cleared_as_sweeper() + { + _outbox = new CountingOutbox(_timeProvider); + + _mediator = new OutboxProducerMediator( + new ProducerRegistry(new Dictionary + { { _topic, new NonConfirmingProducerAsync(_topic) } }), + new ResiliencePipelineRegistry().AddBrighterDefault(), + new MessageMapperRegistry(new SimpleMessageMapperFactory(_ => null!), null), + new EmptyMessageTransformerFactory(), + new EmptyMessageTransformerFactoryAsync(), + tracer: null, + new FindPublicationByPublicationTopicOrRequestType(), + outbox: _outbox, + timeProvider: _timeProvider, + outboxMeter: _meter); + } + + [Fact] + public async Task When_the_sweeper_clears_the_outbox_should_count_cleared_as_sweeper_() + { + var ctx = new RequestContext(); + for (var i = 0; i < 5; i++) + { + var msg = new Message( + new MessageHeader(Id.Random(), _topic, MessageType.MT_COMMAND, + timeStamp: _timeProvider.GetUtcNow()), new MessageBody("{}")); + await _outbox.AddAsync(msg, ctx); + } + + var sweeper = new OutboxSweeper( + System.TimeSpan.Zero, _mediator, new InMemoryRequestContextFactory()); + await sweeper.SweepAsync(); + + Assert.Equal(5, _meter.Cleared.Count); + Assert.All(_meter.Cleared, c => + { + Assert.Equal(_topic, c.Destination); + Assert.Equal(OutboxClearSource.Sweeper, c.Source); + }); + Assert.False(_meter.Cleared.Any(c => c.Source == OutboxClearSource.Explicit)); + Assert.Equal(5, _meter.Durations.Count); + Assert.All(_meter.Durations, d => Assert.Equal(OutboxClearSource.Sweeper, d.Source)); + } +} diff --git a/tests/Paramore.Brighter.Core.Tests/Observability/TestDoubles/CountingOutbox.cs b/tests/Paramore.Brighter.Core.Tests/Observability/TestDoubles/CountingOutbox.cs new file mode 100644 index 0000000000..65e2d82265 --- /dev/null +++ b/tests/Paramore.Brighter.Core.Tests/Observability/TestDoubles/CountingOutbox.cs @@ -0,0 +1,223 @@ +#region Licence +/* The MIT License (MIT) +Copyright © 2024 Ian Cooper + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. */ +#endregion + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using System.Transactions; +using Paramore.Brighter.Observability; + +namespace Paramore.Brighter.Core.Tests.Observability.TestDoubles; + +public class CountingOutbox : IAmAnOutboxSync, IAmAnOutboxAsync +{ + private readonly InMemoryOutbox _inner; + private readonly List _allAddedIds = new(); + private readonly object _lock = new(); + + public int AddCount { get; private set; } + public int AddAsyncCount { get; private set; } + public int GetCount { get; private set; } + public int GetAsyncCount { get; private set; } + public int OutstandingMessagesCount { get; private set; } + public int OutstandingMessagesAsyncCount { get; private set; } + public int MarkDispatchedCount { get; private set; } + public int MarkDispatchedAsyncCount { get; private set; } + + public bool FailAdd { get; set; } + public bool FailMarkDispatched { get; set; } + public bool ForceOutstanding { get; set; } + + public Dictionary MarkDispatchedTimestamps { get; } = new(); + + public bool ContinueOnCapturedContext + { + get => _inner.ContinueOnCapturedContext; + set => _inner.ContinueOnCapturedContext = value; + } + + public IAmABrighterTracer? Tracer + { + set => _inner.Tracer = value; + } + + public CountingOutbox(TimeProvider? timeProvider = null) + { + _inner = new InMemoryOutbox(timeProvider ?? TimeProvider.System); + } + + public void Add(Message message, RequestContext requestContext, int outBoxTimeout = -1, + IAmABoxTransactionProvider? transactionProvider = null) + { + if (FailAdd) throw new ChannelFailureException("forced add failure"); + AddCount++; + lock (_lock) _allAddedIds.Add(message.Id); + _inner.Add(message, requestContext, outBoxTimeout, transactionProvider); + } + + public void Add(IEnumerable messages, RequestContext? requestContext, int outBoxTimeout = -1, + IAmABoxTransactionProvider? transactionProvider = null) + { + if (FailAdd) throw new ChannelFailureException("forced add failure"); + var list = messages.ToList(); + AddCount += list.Count; + lock (_lock) _allAddedIds.AddRange(list.Select(m => m.Id)); + _inner.Add(list, requestContext, outBoxTimeout, transactionProvider); + } + + public Task AddAsync(Message message, RequestContext requestContext, int outBoxTimeout = -1, + IAmABoxTransactionProvider? transactionProvider = null, + CancellationToken cancellationToken = default) + { + if (FailAdd) throw new ChannelFailureException("forced add failure"); + AddAsyncCount++; + lock (_lock) _allAddedIds.Add(message.Id); + return _inner.AddAsync(message, requestContext, outBoxTimeout, transactionProvider, cancellationToken); + } + + public Task AddAsync(IEnumerable messages, RequestContext? requestContext, int outBoxTimeout = -1, + IAmABoxTransactionProvider? transactionProvider = null, + CancellationToken cancellationToken = default) + { + if (FailAdd) throw new ChannelFailureException("forced add failure"); + var list = messages.ToList(); + AddAsyncCount += list.Count; + lock (_lock) _allAddedIds.AddRange(list.Select(m => m.Id)); + return _inner.AddAsync(list, requestContext, outBoxTimeout, transactionProvider, cancellationToken); + } + + public void Delete(Id[] messageIds, RequestContext? requestContext, Dictionary? args = null) + => _inner.Delete(messageIds, requestContext, args); + + public Task DeleteAsync(Id[] messageIds, RequestContext requestContext, + Dictionary? args = null, CancellationToken cancellationToken = default) + => _inner.DeleteAsync(messageIds, requestContext, args, cancellationToken); + + public IEnumerable DispatchedMessages(TimeSpan dispatchedSince, RequestContext requestContext, + int pageSize = 100, int pageNumber = 1, int outBoxTimeout = -1, Dictionary? args = null) + => _inner.DispatchedMessages(dispatchedSince, requestContext, pageSize, pageNumber, outBoxTimeout, args); + + public Task> DispatchedMessagesAsync(TimeSpan dispatchedSince, RequestContext requestContext, + int pageSize = 100, int pageNumber = 1, int outboxTimeout = -1, Dictionary? args = null, + CancellationToken cancellationToken = default) + => _inner.DispatchedMessagesAsync(dispatchedSince, requestContext, pageSize, pageNumber, outboxTimeout, args, cancellationToken); + + public Message Get(Id messageId, RequestContext requestContext, int outBoxTimeout = -1, + Dictionary? args = null) + { + GetCount++; + return _inner.Get(messageId, requestContext, outBoxTimeout, args); + } + + public IEnumerable Get(IEnumerable messageIds, RequestContext requestContext, + int outBoxTimeout = -1, Dictionary? args = null) + { + GetCount++; + return _inner.Get(messageIds, requestContext, outBoxTimeout, args); + } + + public Task GetAsync(Id messageId, RequestContext requestContext, int outBoxTimeout = -1, + Dictionary? args = null, CancellationToken cancellationToken = default) + { + GetAsyncCount++; + return _inner.GetAsync(messageId, requestContext, outBoxTimeout, args, cancellationToken); + } + + public Task> GetAsync(IEnumerable messageId, RequestContext requestContext, + int outBoxTimeout = -1, Dictionary? args = null, + CancellationToken cancellationToken = default) + { + GetAsyncCount++; + return _inner.GetAsync(messageId, requestContext, outBoxTimeout, args, cancellationToken); + } + + public void MarkDispatched(Id id, RequestContext requestContext, DateTimeOffset? dispatchedAt = null, + Dictionary? args = null) + { + if (FailMarkDispatched) throw new InvalidOperationException("forced mark failure"); + MarkDispatchedCount++; + MarkDispatchedTimestamps.TryAdd(id.Value, dispatchedAt ?? DateTimeOffset.UtcNow); + _inner.MarkDispatched(id, requestContext, dispatchedAt, args); + } + + public Task MarkDispatchedAsync(Id id, RequestContext requestContext, DateTimeOffset? dispatchedAt = null, + Dictionary? args = null, CancellationToken cancellationToken = default) + { + if (FailMarkDispatched) throw new InvalidOperationException("forced mark failure"); + MarkDispatchedAsyncCount++; + MarkDispatchedTimestamps.TryAdd(id.Value, dispatchedAt ?? DateTimeOffset.UtcNow); + return _inner.MarkDispatchedAsync(id, requestContext, dispatchedAt, args, cancellationToken); + } + + public Task MarkDispatchedAsync(IEnumerable ids, RequestContext requestContext, + DateTimeOffset? dispatchedAt = null, Dictionary? args = null, + CancellationToken cancellationToken = default) + { + if (FailMarkDispatched) throw new InvalidOperationException("forced mark failure"); + var list = ids.ToList(); + MarkDispatchedAsyncCount += list.Count; + var ts = dispatchedAt ?? DateTimeOffset.UtcNow; + foreach (var id in list) MarkDispatchedTimestamps.TryAdd(id.Value, ts); + return _inner.MarkDispatchedAsync(list, requestContext, dispatchedAt, args, cancellationToken); + } + + public IEnumerable OutstandingMessages(TimeSpan dispatchedSince, RequestContext? requestContext, + int pageSize = 100, int pageNumber = 1, IEnumerable? trippedTopics = null, + Dictionary? args = null) + { + OutstandingMessagesCount++; + if (ForceOutstanding) + { + List ids; + lock (_lock) ids = _allAddedIds.ToList(); + return _inner.Get(ids, requestContext ?? new RequestContext()); + } + return _inner.OutstandingMessages(dispatchedSince, requestContext, pageSize, pageNumber, trippedTopics, args); + } + + public Task> OutstandingMessagesAsync(TimeSpan dispatchedSince, + RequestContext requestContext, int pageSize = 100, int pageNumber = 1, + IEnumerable? trippedTopics = null, Dictionary? args = null, + CancellationToken cancellationToken = default) + { + OutstandingMessagesAsyncCount++; + if (ForceOutstanding) + { + List ids; + lock (_lock) ids = _allAddedIds.ToList(); + return _inner.GetAsync(ids, requestContext, cancellationToken: cancellationToken); + } + return _inner.OutstandingMessagesAsync(dispatchedSince, requestContext, pageSize, pageNumber, trippedTopics, args, cancellationToken); + } + + public int GetOutstandingMessageCount(TimeSpan dispatchedSince, RequestContext? requestContext, + int maxCount = 100, Dictionary? args = null) + => _inner.GetOutstandingMessageCount(dispatchedSince, requestContext, maxCount, args); + + public Task GetOutstandingMessageCountAsync(TimeSpan dispatchedSince, RequestContext? requestContext, + int maxCount = 100, Dictionary? args = null, + CancellationToken cancellationToken = default) + => _inner.GetOutstandingMessageCountAsync(dispatchedSince, requestContext, maxCount, args, cancellationToken); +} diff --git a/tests/Paramore.Brighter.Core.Tests/Observability/TestDoubles/NonConfirmingProducerAsync.cs b/tests/Paramore.Brighter.Core.Tests/Observability/TestDoubles/NonConfirmingProducerAsync.cs new file mode 100644 index 0000000000..d8e0186d49 --- /dev/null +++ b/tests/Paramore.Brighter.Core.Tests/Observability/TestDoubles/NonConfirmingProducerAsync.cs @@ -0,0 +1,58 @@ +#region Licence +/* The MIT License (MIT) +Copyright © 2024 Ian Cooper + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. */ +#endregion + +using System; +using System.Diagnostics; +using System.Threading; +using System.Threading.Tasks; + +namespace Paramore.Brighter.Core.Tests.Observability.TestDoubles; + +public class NonConfirmingProducerAsync : IAmAMessageProducerAsync +{ + public NonConfirmingProducerAsync(RoutingKey topic) + { + Publication = new Publication { Topic = topic }; + } + + public Publication Publication { get; } + public Activity? Span { get; set; } + public IAmAMessageScheduler? Scheduler { get; set; } + public bool ThrowOnSend { get; set; } + + public Task SendAsync(Message message, CancellationToken cancellationToken = default) + { + if (ThrowOnSend) + throw new InvalidOperationException("NonConfirmingProducerAsync.SendAsync forced throw"); + return Task.CompletedTask; + } + + public Task SendWithDelayAsync(Message message, TimeSpan? delay, CancellationToken cancellationToken = default) + { + if (ThrowOnSend) + throw new InvalidOperationException("NonConfirmingProducerAsync.SendAsync forced throw"); + return Task.CompletedTask; + } + + public ValueTask DisposeAsync() => default; +} diff --git a/tests/Paramore.Brighter.Core.Tests/Observability/TestDoubles/NonConfirmingProducerSync.cs b/tests/Paramore.Brighter.Core.Tests/Observability/TestDoubles/NonConfirmingProducerSync.cs new file mode 100644 index 0000000000..6a56b4635b --- /dev/null +++ b/tests/Paramore.Brighter.Core.Tests/Observability/TestDoubles/NonConfirmingProducerSync.cs @@ -0,0 +1,45 @@ +#region Licence +/* The MIT License (MIT) +Copyright © 2024 Ian Cooper + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. */ +#endregion + +using System; +using System.Diagnostics; + +namespace Paramore.Brighter.Core.Tests.Observability.TestDoubles; + +public class NonConfirmingProducerSync : IAmAMessageProducerSync +{ + public NonConfirmingProducerSync(RoutingKey topic) + { + Publication = new Publication { Topic = topic }; + } + + public Publication Publication { get; } + public Activity? Span { get; set; } + public IAmAMessageScheduler? Scheduler { get; set; } + + public void Send(Message message) { } + + public void SendWithDelay(Message message, TimeSpan? delay) { } + + public void Dispose() { } +} diff --git a/tests/Paramore.Brighter.Core.Tests/Observability/TestDoubles/SpyOutboxMeter.cs b/tests/Paramore.Brighter.Core.Tests/Observability/TestDoubles/SpyOutboxMeter.cs new file mode 100644 index 0000000000..785eef7ccb --- /dev/null +++ b/tests/Paramore.Brighter.Core.Tests/Observability/TestDoubles/SpyOutboxMeter.cs @@ -0,0 +1,31 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using Paramore.Brighter.Observability; + +namespace Paramore.Brighter.Core.Tests.Observability.TestDoubles; + +/// +/// Spy that records every Outbox metric call for assertion in tests. +/// Thread-safe: all list access is behind a lock (for AC-25 concurrent add tests). +/// +public class SpyOutboxMeter : IAmABrighterOutboxMeter, IDisposable +{ + private readonly object _lock = new(); + private readonly List _added = new(); + private readonly List<(RoutingKey Destination, OutboxClearSource Source)> _cleared = new(); + private readonly List<(TimeSpan Wait, RoutingKey Destination, OutboxClearSource Source)> _durations = new(); + + public bool Enabled { get; set; } = true; + public bool Disposed { get; private set; } + + public IReadOnlyList Added { get { lock (_lock) return _added.ToList(); } } + public IReadOnlyList<(RoutingKey Destination, OutboxClearSource Source)> Cleared { get { lock (_lock) return _cleared.ToList(); } } + public IReadOnlyList<(TimeSpan Wait, RoutingKey Destination, OutboxClearSource Source)> Durations { get { lock (_lock) return _durations.ToList(); } } + + public void AddMessageAdded(RoutingKey destination) { lock (_lock) _added.Add(destination); } + public void AddMessageCleared(RoutingKey destination, OutboxClearSource clearSource) { lock (_lock) _cleared.Add((destination, clearSource)); } + public void RecordPublishDuration(TimeSpan wait, RoutingKey destination, OutboxClearSource clearSource) { lock (_lock) _durations.Add((wait, destination, clearSource)); } + + public void Dispose() => Disposed = true; +}