From 845a59538a1aba1c96ebc8165b0f25c37574a194 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Fri, 18 Sep 2026 18:32:19 -0700 Subject: [PATCH 001/158] feat(events): persist recorded events to an append-only log (tier 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Tiers 1 and 2 both need the process to survive. When it does not, everything recorded since the last delivery goes with it -- including the crash report an application was recording when it died. Tier 3 is the answer to that: events are written to a log under Application Support, and a later launch delivers whatever the previous run left behind. The write is a plain append into the page cache, never an fsync, so a commit point costs tens of microseconds rather than milliseconds. track and identify are commit points: at .immediate the write happens on the calling thread, so a process that dies the instant after track returns still reports that event. Evaluations are not -- they are held as objects and encoded in runs, so the hottest path in the SDK pays a counter bump and nothing else. This is off by default. Writing events changes what a track costs and leaves a directory of the application's data on the device, and a minor version cannot decide either on an application's behalf. LDConfig.eventPersistence has three levels: disabled, which is what Tiers 1 and 2 describe and is the default; deferred, which writes off the calling thread; and immediate, which is the whole of the above. Delivery is held for the round trip rather than just its start, because a batch now stays on disk until its response is handled and a second delivery beginning inside that window would send it again under the same payload ID. Spec: Event Durability §13, §14; Page-Cache Event Persistence. Co-authored-by: Cursor --- LaunchDarkly.xcodeproj/project.pbxproj | 14 + .../GeneratedCode/mocks.generated.swift | 7 + LaunchDarkly/LaunchDarkly/LDClient.swift | 25 +- .../LaunchDarkly/Models/LDConfig.swift | 37 +- .../ObjectiveC/ObjcLDConfig.swift | 13 +- .../ServiceObjects/EventReporter.swift | 612 +++++++++++--- .../ServiceObjects/EventStore.swift | 763 ++++++++++++++++++ .../EventJSONWriterTests.swift | 35 +- .../Mocks/DarklyServiceMock.swift | 20 + .../Models/DiagnosticEventSpec.swift | 6 +- .../Models/LDConfigSpec.swift | 3 + .../ServiceObjects/EventReporterSpec.swift | 465 +++++++++-- .../ServiceObjects/EventStoreSpec.swift | 442 ++++++++++ LaunchDarkly/LaunchDarklyTests/TestUtil.swift | 24 + 14 files changed, 2264 insertions(+), 202 deletions(-) create mode 100644 LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift create mode 100644 LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift diff --git a/LaunchDarkly.xcodeproj/project.pbxproj b/LaunchDarkly.xcodeproj/project.pbxproj index 94c6b3b21..331de333c 100644 --- a/LaunchDarkly.xcodeproj/project.pbxproj +++ b/LaunchDarkly.xcodeproj/project.pbxproj @@ -7,6 +7,7 @@ objects = { /* Begin PBXBuildFile section */ + 1E65A889B0528E5AB03C517A /* EventStoreSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = EE6BAECBB739A0F24DB5A944 /* EventStoreSpec.swift */; }; 29F9D19E2812E005008D12C0 /* ObjcLDValue.swift in Sources */ = {isa = PBXBuildFile; fileRef = 29F9D19D2812E005008D12C0 /* ObjcLDValue.swift */; }; 29F9D19F2812E005008D12C0 /* ObjcLDValue.swift in Sources */ = {isa = PBXBuildFile; fileRef = 29F9D19D2812E005008D12C0 /* ObjcLDValue.swift */; }; 29F9D1A02812E005008D12C0 /* ObjcLDValue.swift in Sources */ = {isa = PBXBuildFile; fileRef = 29F9D19D2812E005008D12C0 /* ObjcLDValue.swift */; }; @@ -15,6 +16,7 @@ 29FE1299280413D4008CC918 /* Util.swift in Sources */ = {isa = PBXBuildFile; fileRef = 29FE1297280413D4008CC918 /* Util.swift */; }; 29FE129A280413D4008CC918 /* Util.swift in Sources */ = {isa = PBXBuildFile; fileRef = 29FE1297280413D4008CC918 /* Util.swift */; }; 29FE129B280413D4008CC918 /* Util.swift in Sources */ = {isa = PBXBuildFile; fileRef = 29FE1297280413D4008CC918 /* Util.swift */; }; + 2CA9D36B3566D78850D6F0AF /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; 3D2406142E0D90E000F91253 /* Plugin.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3D2406102E0D90E000F91253 /* Plugin.swift */; }; 3D2406152E0D90E000F91253 /* PluginMetadata.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3D2406112E0D90E000F91253 /* PluginMetadata.swift */; }; 3D2406162E0D90E000F91253 /* EnvironmentMetadata.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3D24060F2E0D90E000F91253 /* EnvironmentMetadata.swift */; }; @@ -46,6 +48,7 @@ 50EE85C42EA0487F007CC662 /* TimeoutExecutor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50EE85C12EA0487F007CC662 /* TimeoutExecutor.swift */; }; 50EE85C52EA0487F007CC662 /* TimeoutExecutor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50EE85C12EA0487F007CC662 /* TimeoutExecutor.swift */; }; 50EE85C72EA0749C007CC662 /* TimeoutExecutorSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50EE85C62EA0749C007CC662 /* TimeoutExecutorSpec.swift */; }; + 5AEA1F832DCA974314E07D21 /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; 7185A3C30E7FFC4F725769F1 /* BackgroundActivity.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9A771B316E19ED598CD4FB0B /* BackgroundActivity.swift */; }; 830BF933202D188E006DF9B1 /* HTTPURLRequest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 830BF932202D188E006DF9B1 /* HTTPURLRequest.swift */; }; 830DB3AC22380A3E00D65D25 /* HTTPHeadersSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 830DB3AB22380A3E00D65D25 /* HTTPHeadersSpec.swift */; }; @@ -360,6 +363,7 @@ A3FFE1132B7D4BA2009EF93F /* LDValueDecoderSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = A3FFE1122B7D4BA2009EF93F /* LDValueDecoderSpec.swift */; }; A8449C0800DF492B8F0619E4 /* LDContextJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88DAB8D1864C14D9C7E06F11 /* LDContextJSONWriter.swift */; }; AFCE752CB1A75DEBABC23034 /* LDContextJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88DAB8D1864C14D9C7E06F11 /* LDContextJSONWriter.swift */; }; + B0B1AD7CADE8DBB596C2DE9B /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; B40B419C249ADA6B00CD0726 /* DiagnosticCacheSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = B40B419B249ADA6B00CD0726 /* DiagnosticCacheSpec.swift */; }; B4265EB124E7390C001CFD2C /* TestUtil.swift in Sources */ = {isa = PBXBuildFile; fileRef = B4265EB024E7390C001CFD2C /* TestUtil.swift */; }; B468E71024B3C3AC00E0C883 /* ObjcLDEvaluationDetail.swift in Sources */ = {isa = PBXBuildFile; fileRef = B468E70F24B3C3AC00E0C883 /* ObjcLDEvaluationDetail.swift */; }; @@ -409,6 +413,7 @@ C443A41123186A4F00145710 /* ConnectionModeChangeObserver.swift in Sources */ = {isa = PBXBuildFile; fileRef = C443A40E23186A4F00145710 /* ConnectionModeChangeObserver.swift */; }; C443A41223186A4F00145710 /* ConnectionModeChangeObserver.swift in Sources */ = {isa = PBXBuildFile; fileRef = C443A40E23186A4F00145710 /* ConnectionModeChangeObserver.swift */; }; C4AF311E4DA8C3AB7040637D /* EventJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = B430884DBF2E541B919C08FA /* EventJSONWriter.swift */; }; + CF9A1EB116D8AA3B7B5535B2 /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; D7A500FCA04E3412F931003F /* ContextEncodingCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = D93B7F5F6CF5C3DE79FB4286 /* ContextEncodingCache.swift */; }; DAF5695D58E28B8CF9221213 /* LDContextJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88DAB8D1864C14D9C7E06F11 /* LDContextJSONWriter.swift */; }; EBFA09F4DFC59FCB6C0AE209 /* LDContextJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88DAB8D1864C14D9C7E06F11 /* LDContextJSONWriter.swift */; }; @@ -627,12 +632,14 @@ B4C9D4322489C8FD004A9B03 /* DiagnosticCache.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DiagnosticCache.swift; sourceTree = ""; }; B4C9D4372489E20A004A9B03 /* DiagnosticReporter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DiagnosticReporter.swift; sourceTree = ""; }; B4F689132497B2FC004D3CE0 /* DiagnosticEventSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DiagnosticEventSpec.swift; sourceTree = ""; }; + C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventStore.swift; sourceTree = ""; }; C408884623033B3600420721 /* ConnectionInformationStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConnectionInformationStore.swift; sourceTree = ""; }; C408884823033B7500420721 /* ConnectionInformation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConnectionInformation.swift; sourceTree = ""; }; C43C37E0236BA050003C1624 /* LDEvaluationDetail.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LDEvaluationDetail.swift; sourceTree = ""; }; C443A4092315AA4D00145710 /* NetworkReporter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NetworkReporter.swift; sourceTree = ""; }; C443A40E23186A4F00145710 /* ConnectionModeChangeObserver.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConnectionModeChangeObserver.swift; sourceTree = ""; }; D93B7F5F6CF5C3DE79FB4286 /* ContextEncodingCache.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = ContextEncodingCache.swift; sourceTree = ""; }; + EE6BAECBB739A0F24DB5A944 /* EventStoreSpec.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventStoreSpec.swift; sourceTree = ""; }; FED0000000000000000000C0 /* TrackSeriesContext.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TrackSeriesContext.swift; sourceTree = ""; }; FED0000000000000000000D0 /* LDClientTrackHookSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LDClientTrackHookSpec.swift; sourceTree = ""; }; /* End PBXFileReference section */ @@ -729,6 +736,7 @@ 831AAE2F20A9E75D00B46DBA /* ThrottlerSpec.swift */, 8354AC75224316C700CDE602 /* Cache */, A3C6F7612B7FA803005B3B61 /* SheddingQueueSpec.swift */, + EE6BAECBB739A0F24DB5A944 /* EventStoreSpec.swift */, ); path = ServiceObjects; sourceTree = ""; @@ -1003,6 +1011,7 @@ B430884DBF2E541B919C08FA /* EventJSONWriter.swift */, D93B7F5F6CF5C3DE79FB4286 /* ContextEncodingCache.swift */, 9A771B316E19ED598CD4FB0B /* BackgroundActivity.swift */, + C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */, ); path = ServiceObjects; sourceTree = ""; @@ -1526,6 +1535,7 @@ 94C1C136BA8989F1B8AFE0A8 /* EventJSONWriter.swift in Sources */, 40F4D07AD934AF976687011D /* ContextEncodingCache.swift in Sources */, A32FCDB22D53263851DDB727 /* BackgroundActivity.swift in Sources */, + CF9A1EB116D8AA3B7B5535B2 /* EventStore.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -1617,6 +1627,7 @@ C4AF311E4DA8C3AB7040637D /* EventJSONWriter.swift in Sources */, 4882B223C23B63F30BC60941 /* ContextEncodingCache.swift in Sources */, 7185A3C30E7FFC4F725769F1 /* BackgroundActivity.swift in Sources */, + 2CA9D36B3566D78850D6F0AF /* EventStore.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -1707,6 +1718,7 @@ 9BE9637664F4E97681CFCD45 /* EventJSONWriter.swift in Sources */, D7A500FCA04E3412F931003F /* ContextEncodingCache.swift in Sources */, A20146E7AF098560627C614E /* BackgroundActivity.swift in Sources */, + 5AEA1F832DCA974314E07D21 /* EventStore.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -1770,6 +1782,7 @@ 838F967A1FBA551A009CFC45 /* ClientServiceMockFactory.swift in Sources */, A31088292837DCA900184942 /* KindSpec.swift in Sources */, A0BAD9EE25943C43A5154E6F /* EventJSONWriterTests.swift in Sources */, + 1E65A889B0528E5AB03C517A /* EventStoreSpec.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -1860,6 +1873,7 @@ A032A73E446055D14A546CA0 /* EventJSONWriter.swift in Sources */, BAE0B4F45562EBF4AC786B5D /* ContextEncodingCache.swift in Sources */, 3D34EEEF37158657D49335BB /* BackgroundActivity.swift in Sources */, + B0B1AD7CADE8DBB596C2DE9B /* EventStore.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/LaunchDarkly/GeneratedCode/mocks.generated.swift b/LaunchDarkly/GeneratedCode/mocks.generated.swift index d8cc0b4e5..b52960c84 100644 --- a/LaunchDarkly/GeneratedCode/mocks.generated.swift +++ b/LaunchDarkly/GeneratedCode/mocks.generated.swift @@ -230,6 +230,13 @@ final class EventReportingMock: EventReporting { flushReportingOutcomeReceivedCompletion = completion try! flushReportingOutcomeCallback?() } + + var commitRecordedEventsCallCount = 0 + var commitRecordedEventsCallback: (() throws -> Void)? + func commitRecordedEvents() { + commitRecordedEventsCallCount += 1 + try! commitRecordedEventsCallback?() + } } // MARK: - FeatureFlagCachingMock diff --git a/LaunchDarkly/LaunchDarkly/LDClient.swift b/LaunchDarkly/LaunchDarkly/LDClient.swift index 1ec13a4cb..3e01d3fa6 100644 --- a/LaunchDarkly/LaunchDarkly/LDClient.swift +++ b/LaunchDarkly/LaunchDarkly/LDClient.swift @@ -263,9 +263,12 @@ public class LDClient { @objc private func didEnterBackground() { os_log("%s", log: config.logger, type: .debug, typeName(and: #function)) - // A backgrounded process is suspended as soon as it goes idle, so a delivery started here reaches the network - // only inside an activity assertion. Without one, whatever is queued waits in memory for a foreground the - // process may not live to see. + // A backgrounded application is one the OS may kill without warning, so whatever it has recorded is made + // durable now rather than waiting for the next report interval to come around. + eventReporter.commitRecordedEvents() + // The commit above makes the events survivable; this tries to make surviving unnecessary. A backgrounded + // process is suspended as soon as it goes idle, so a delivery started here reaches the network only inside an + // activity assertion. If it does not get out, the bytes are on disk and the next launch sends them. BackgroundActivity.run(reason: "LaunchDarkly event delivery") { [weak self] finished in guard let self = self else { @@ -274,7 +277,7 @@ public class LDClient { } self.eventReporter.flushReportingOutcome { delivered in if !delivered { - os_log("%s events did not reach LaunchDarkly before suspension", log: self.config.logger, type: .debug, self.typeName(and: #function)) + os_log("%s events did not reach LaunchDarkly before suspension; they remain on disk", log: self.config.logger, type: .debug, self.typeName(and: #function)) } finished() } @@ -1097,11 +1100,21 @@ extension LDClient { } /** - Tells the SDK to immediately send any currently queued events to LaunchDarkly. + Writes down all pending events and sends them to LaunchDarkly. There should not normally be a need to call this function. While online, the LDClient automatically reports events on an interval defined by `LDConfig.eventFlushInterval`. Note that this function does not block until events are - sent, it only triggers a background task to send events immediately. + sent, it only triggers a background task to send them immediately. + + Where `LDConfig.eventPersistence` is on, it also writes before it returns. Recording an event does not on its own + make it outlive the process: events are written in runs, so one recorded shortly before the process ends may + never have been written at all. This call writes everything recorded so far, and those events then survive + whether or not the delivery does. Where it is off, events live in memory only and nothing survives the process, + whether or not this was called. + + That is what makes it worth calling where the process is about to end deliberately. It is not a crash-time + mechanism — Apple gives the SDK no crash hook, and a trap or a `SIGKILL` takes whatever was recorded since the + last write. Those events are not lost so much as late: they reach LaunchDarkly on the next launch. */ public func flush() { LDClient.instancesQueue.sync(flags: .barrier) { diff --git a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift index f4edfa3bf..e802f95a1 100644 --- a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift @@ -19,6 +19,27 @@ public enum LDStreamingMode { case polling } +/// How far the SDK goes to make a recorded event outlive the process that recorded it. +/// +/// An event lives in memory until it is delivered, so a process that dies before the next flush takes everything +/// recorded since the last one — including the crash an application was reporting when it died. Writing events to +/// disk is what closes that gap: they are delivered on a later launch instead of being lost. +@objc(LDEventPersistence) public enum EventPersistence: Int { + /// Events are kept in memory only, and nothing survives the process ending. + case disabled + /// Events are written to disk, off the thread that recorded them. + /// + /// Recording stays as cheap as it is without persistence, and an event becomes durable a moment after it was + /// recorded rather than immediately. A process that dies inside that window still loses it. + case deferred + /// Events are written to disk before `track` and `identify` return. + /// + /// The write happens on the calling thread, costing tens of microseconds depending on how many evaluations are + /// waiting to be encoded alongside it. In exchange there is no window: a process that dies the instant after + /// `track` returns still reports that event on the next launch. Evaluating a flag stays in memory regardless. + case immediate +} + /** Enable / disable options for Auto Environment Attributes functionality. When enabled, the SDK will automatically provide data about the mobile environment where the application is running. This data makes it simpler to target @@ -192,7 +213,10 @@ public struct LDConfig { static let sendEvents = true /// The default maximum number of events the LDClient can store - static let eventCapacity = 100 + static let eventCapacity = 1000 + + /// The default behavior for the SDK is to keep events in memory only. + static let eventPersistence = EventPersistence.disabled /// The default timeout interval for flag requests and event reports. (10 seconds) static let connectionTimeout: TimeInterval = 10.0 @@ -314,9 +338,17 @@ public struct LDConfig { /// only the sending of client-side events, not streaming or polling for events from the server. public var sendEvents: Bool = Defaults.sendEvents - /// The maximum number of analytics events the LDClient can store. When the LDClient event store reaches the eventCapacity, the SDK discards events until it successfully reports them to LaunchDarkly. (Default: 100) + /// The maximum number of analytics events the LDClient can store. When the LDClient event store reaches the eventCapacity, the SDK discards events until it successfully reports them to LaunchDarkly. (Default: 1000) public var eventCapacity: Int = Defaults.eventCapacity + /// How far the SDK goes to make a recorded event outlive the process that recorded it. + /// + /// Anything other than `.disabled` appends events to a log under Application Support and delivers them on a later + /// run, which is what lets the crash an application was reporting when it died reach LaunchDarkly at all. + /// `.immediate` additionally puts the write on the thread that called `track` or `identify`, so there is no window + /// in which the event exists only in memory. (Default: `.disabled`) + public var eventPersistence: EventPersistence = Defaults.eventPersistence + /// The timeout interval for flag requests and event reports. (Default: 10 seconds) public var connectionTimeout: TimeInterval = Defaults.connectionTimeout /// The time interval between event reports (Default: 30 seconds) @@ -546,6 +578,7 @@ extension LDConfig: Equatable { && lhs.eventsUrl == rhs.eventsUrl && lhs.streamUrl == rhs.streamUrl && lhs.eventCapacity == rhs.eventCapacity + && lhs.eventPersistence == rhs.eventPersistence && lhs.sendEvents == rhs.sendEvents && lhs.connectionTimeout == rhs.connectionTimeout && lhs.eventFlushInterval == rhs.eventFlushInterval diff --git a/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift b/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift index 2616c1391..673364af4 100644 --- a/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift @@ -35,12 +35,23 @@ public final class ObjcLDConfig: NSObject { set { config.streamUrl = newValue } } - /// The maximum number of analytics events the LDClient can store. When the LDClient event store reaches the eventCapacity, the SDK discards events until it successfully reports them to LaunchDarkly. (Default: 100) + /// The maximum number of analytics events the LDClient can store. When the LDClient event store reaches the eventCapacity, the SDK discards events until it successfully reports them to LaunchDarkly. (Default: 1000) @objc public var eventCapacity: Int { get { config.eventCapacity } set { config.eventCapacity = newValue } } + /// How far the SDK goes to make a recorded event outlive the process that recorded it. + /// + /// Without this, an event lives in memory until it is delivered, and a process that dies before the next flush + /// takes everything recorded since the last one, including the crash an application was reporting when it died. + /// `.immediate` additionally puts the write on the thread that called `track` or `identify`, so there is no window + /// in which the event exists only in memory. (Default: `.disabled`) + @objc public var eventPersistence: EventPersistence { + get { config.eventPersistence } + set { config.eventPersistence = newValue } + } + /// The timeout interval for flag requests and event reports. (Default: 10 seconds) @objc public var connectionTimeout: TimeInterval { get { config.connectionTimeout } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift index b10f4d0e0..92674f640 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift @@ -16,11 +16,17 @@ protocol EventReporting { func recordFlagEvaluationEvents(flagKey: LDFlagKey, value: LDValue, defaultValue: LDValue, featureFlag: FeatureFlag?, context: LDContext, includeReason: Bool) func flush(completion: CompletionClosure?) - /// Same as `flush`, and reports whether the pending events left the SDK's hands. + /// Same as `flush`, and reports whether the pending batches left the SDK's hands. /// /// `true` if they were delivered, refused for good, or there were none. `false` if the SDK is offline or a - /// retryable failure means they are still waiting to be sent. + /// retryable failure left batches on disk. func flushReportingOutcome(completion: @escaping FlushOutcomeClosure) + + /// Makes everything recorded so far outlive the process, without waiting for a delivery. + /// + /// The SDK does this itself at the points where an application is most likely to be about to die. It is worth + /// calling directly before deliberately terminating the process. + func commitRecordedEvents() } class NullEventReporter: EventReporting { @@ -40,65 +46,188 @@ class NullEventReporter: EventReporting { func flushReportingOutcome(completion: @escaping FlushOutcomeClosure) { completion(true) } + + func commitRecordedEvents() { + } } +/// Records analytics events and delivers them to LaunchDarkly. +/// +/// Recorded events end up in an `EventStore`, an on-disk log, rather than held in memory until a delivery succeeds. +/// That is what lets an application that dies moments after recording an event still report it: the events are on disk +/// before the process is gone, and the next run of the application delivers them. +/// +/// Recording an event summarizes it and, if it has to be delivered in full, holds it as an `Event`. Nothing is encoded +/// on the thread that recorded it; a commit turns the whole held run into bytes at once. So an evaluation of an +/// untracked flag costs a counter update, and an evaluation of a tracked one costs that plus an array append. +/// +/// Where the line falls is between the events an application asked for by name and the ones it did not. Recording a +/// custom or identify event is a *commit point*, because an application reporting something it chose to report is +/// saying this matters more than the microseconds it costs, and the crash it describes may be moments away. A commit +/// takes the whole held run with it, so the exposures leading up to the error go down alongside it. Evaluations get no +/// such promise, and are committed once `pendingCommitThreshold` of them have accumulated, when a delivery starts, or +/// on `flush`. +/// +/// Whether a commit point runs on the caller's thread is the application's choice, through +/// `LDConfig.eventPersistence`. Only at `.immediate` can `track` promise the event is on disk by the time it returns. class EventReporter: EventReporting { + /// How many full events may be held unencoded before one of them pays for a commit. + /// + /// It sets two things at once. The first is how many evaluations a termination can take -- not everything a crash + /// could take, since an application that flushes on its way out commits the whole run, but the window for the + /// terminations that run nothing on the way out, such as the system reclaiming a backgrounded process. Recording a + /// custom or identify event closes it too, because that commits. + /// + /// The second is the worst a commit point can cost, since it encodes whatever is held before returning. The common + /// case is far below the bound, because the commit queue keeps the run drained; raising this trades that tail + /// against the number of writes. + private static let pendingCommitThreshold = 32 + var isOnline: Bool { get { timerQueue.sync { eventReportTimer != nil } } set { timerQueue.sync { newValue ? startReporting() : stopReporting() } } } - private(set) var lastEventResponseDate: Date + var lastEventResponseDate: Date { + stateLock.lock() + defer { stateLock.unlock() } + return responseDate + } let service: DarklyServiceProvider + let store: EventStoring - private let eventQueue = DispatchQueue(label: "com.launchdarkly.eventSyncQueue", qos: .userInitiated) - // These fields should only be used synchronized on the eventQueue - private(set) var eventStore: [Event] = [] + /// Guards the summarizer and the last response date. + /// + /// This is a lock rather than a queue because it is taken once per evaluation, and a `DispatchQueue.sync` costs + /// enough at that rate to be worth avoiding: the same measurement that made `EvaluationExposureDeduper` a lock + /// applies here. It is never held across encoding an event or across a syscall, so an evaluation contends only with + /// another evaluation's bookkeeping. + private let stateLock = UnfairLock() + /// Only to be used while holding `stateLock`. private(set) var contextSummarizer: ContextSummarizer + /// Only to be used while holding `stateLock`. + private var responseDate: Date + + /// Held for the whole of a commit, so that only one runs at a time. + /// + /// This is what a commit point's guarantee rests on. Without it a commit already in flight could take the caller's + /// event out of `pending` before the caller got there, leaving the caller nothing to write and returning while + /// those bytes were still being produced somewhere else. Waiting here instead means that when the call returns the + /// event is on disk, whichever commit put it there. It also keeps two encoders from staging their runs in + /// whichever order they happened to finish. + /// + /// Taken before `pendingLock` and before anything the store locks, never after. + private let commitLock = UnfairLock() + + /// Guards the held events. Held only long enough to append one or to hand the run over, never across the encoder. + private let pendingLock = UnfairLock() + + /// Full events recorded but not yet encoded. Only to be used while holding `pendingLock`. + /// + /// Held rather than encoded because encoding early would not make them durable: the store stages bytes into memory + /// too, and only a commit reaches the file. Both forms are equally lost to a crash, so the encode may as well + /// happen where it is cheapest. + private var pending: [Event] = [] + + /// Whether a commit is already queued, so a run of recordings past the threshold asks for one write rather than + /// one each. Only to be used while holding `pendingLock`. + private var isCommitScheduled = false + + /// How many events the SDK will hold in total, across `pending` and the store. + private let capacity: Int + + /// Where a commit runs when no caller is waiting on it. + private let commitQueue: DispatchQueue private var timerQueue = DispatchQueue(label: "com.launchdarkly.EventReporter.timerQueue") private var eventReportTimer: TimeResponding? var isReportingActive: Bool { eventReportTimer != nil } + /// Deliveries run here, off whatever thread recorded an event. + private let deliveryQueue = DispatchQueue(label: "com.launchdarkly.eventSyncQueue", qos: .userInitiated) + + /// Whether the store held events from a previous run of the application when this reporter started. + /// + /// Only to be used on `deliveryQueue`, which is what makes it safe without a lock: the recovery that answers the + /// question and the delivery that acts on it are both enqueued there, in that order. + private var hasEventsFromPreviousRun = false + + /// Whether a delivery is running, from the request being sent until its response has been handled. + /// + /// `deliveryQueue` serializes the *start* of a delivery but not the round trip it waits on, and a batch stays on + /// disk until its response arrives. Without this, a delivery beginning inside that window would list the same batch + /// again and send it a second time -- under the same payload ID, leaving it to LaunchDarkly to notice that two + /// requests arriving at once are the same one. + /// + /// Only to be used on `deliveryQueue`, like the two below. + private var isDelivering = false + + /// Whether a delivery was asked for while one was already running. + private var hasWaitingRequest = false + + /// Callers waiting on the pass that `hasWaitingRequest` will start. + private var waitingCompletions: [FlushOutcomeClosure] = [] + private let onSyncComplete: EventSyncCompleteClosure? - /// The reflective encoder, built once and then only read. + /// The encoder every recorded event goes through. /// - /// `JSONEncoder` is `@unchecked Sendable` and constructs a fresh internal encoder for each `encode` call, so one - /// instance can be shared — but only while nothing mutates it, which is why `userInfo` is set here rather than per - /// call. What it holds cannot go stale: `config` is a `let` on the service, so the privacy settings the encoding - /// depends on are fixed for as long as this reporter exists. + /// Built once and then only read. `JSONEncoder` is `@unchecked Sendable` and constructs a fresh internal encoder + /// for each `encode` call, so the threads recording events can share this one — but only while nothing mutates it, + /// which is why `userInfo` is set here rather than per event. What it holds cannot go stale: `config` is a `let` on + /// the service, so the privacy settings the encoding depends on are fixed for as long as this reporter exists. private let encoder: JSONEncoder let encoding: Encoding - private let handWrittenEncoder: EventJSONWriter + fileprivate let handWrittenEncoder: EventJSONWriter init(service: DarklyServiceProvider, onSyncComplete: EventSyncCompleteClosure?, - encoding: Encoding = .handWrittenCachingContext) { + store: EventStoring? = nil, + encoding: Encoding = .handWrittenCachingContext, + commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.EventReporter.commitQueue", qos: .userInitiated)) { self.service = service self.onSyncComplete = onSyncComplete - self.lastEventResponseDate = Date() + self.responseDate = Date() self.encoding = encoding + self.capacity = service.config.eventCapacity + self.commitQueue = commitQueue self.encoder = EventReporter.makeEncoder(config: service.config) self.handWrittenEncoder = EventJSONWriter(config: service.config, cachingContexts: encoding == .handWrittenCachingContext) self.contextSummarizer = ContextSummarizer(logger: service.config.logger) + self.store = store ?? EventReporter.makeStore(config: service.config) + + // A log left open by a previous run has to be closed before it can be delivered, but nothing waits on that: + // doing it on the caller's thread would put file I/O in the way of the client starting up. + let store = self.store + deliveryQueue.async { [weak self] in + store.recoverInterruptedLog() + self?.hasEventsFromPreviousRun = !store.pendingBatches().isEmpty + } } - func record(_ event: Event) { - // The eventReporter is created when the LDClient singleton is created, and kept for the app's lifetime. So while the use of self in the async block does setup a retain cycle, it's not going to cause a memory leak - eventQueue.sync { recordNoSync(event) } + private static func makeStore(config: LDConfig) -> EventStoring { + guard let directory = EventStore.defaultDirectory(mobileKey: config.mobileKey) + else { + os_log("Events cannot be persisted: no writable directory was available", log: config.logger, type: .debug) + return NullEventStore() + } + return EventStore(directory: directory, + capacity: config.eventCapacity, + persistEvents: config.eventPersistence != .disabled, + logger: config.logger) } - func recordNoSync(_ event: Event) { - if self.eventStore.count >= self.service.config.eventCapacity { - os_log("%s aborted. Event store is full", log: service.config.logger, type: .debug, typeName(and: #function)) - self.service.diagnosticCache?.incrementDroppedEventCount() - return + // MARK: Recording + + func record(_ event: Event) { + hold(event) + + if EventReporter.isCommitPoint(event.kind) { + commitAtCommitPoint() } - self.eventStore.append(event) } // swiftlint:disable:next function_parameter_count @@ -106,23 +235,189 @@ class EventReporter: EventReporting { let recordingFeatureEvent = featureFlag?.trackEvents == true let recordingDebugEvent = featureFlag?.shouldCreateDebugEvents(lastEventReportResponseTime: lastEventResponseDate) ?? false - eventQueue.sync { - contextSummarizer.trackRequest(flagKey: flagKey, reportedValue: value, featureFlag: featureFlag, defaultValue: defaultValue, context: context) - if recordingFeatureEvent { - let featureEvent = FeatureEvent(key: flagKey, context: context, value: value, defaultValue: defaultValue, featureFlag: featureFlag, includeReason: includeReason, isDebug: false) - recordNoSync(featureEvent) - } - if recordingDebugEvent { - let debugEvent = FeatureEvent(key: flagKey, context: context, value: value, defaultValue: defaultValue, featureFlag: featureFlag, includeReason: includeReason, isDebug: true) - recordNoSync(debugEvent) - } + stateLock.lock() + contextSummarizer.trackRequest(flagKey: flagKey, reportedValue: value, featureFlag: featureFlag, defaultValue: defaultValue, context: context) + stateLock.unlock() + + if recordingFeatureEvent { + hold(FeatureEvent(key: flagKey, context: context, value: value, defaultValue: defaultValue, featureFlag: featureFlag, includeReason: includeReason, isDebug: false)) + } + if recordingDebugEvent { + hold(FeatureEvent(key: flagKey, context: context, value: value, defaultValue: defaultValue, featureFlag: featureFlag, includeReason: includeReason, isDebug: true)) + } + // Deliberately no commit. An evaluation is expected to cost what bookkeeping costs, and it is usually the main + // thread doing it; the run is encoded and written on the commit queue once enough of them have piled up, and + // the next event recorded at a commit point makes them durable along with itself. + } + + /// Holds an event for the next commit to encode, counting it as dropped if the SDK is already full. + /// + /// Capacity is consulted before anything else, so an event that will not be kept is never encoded. That ordering is + /// what bounds an application re-evaluating a tracked flag in a render loop: once the limit is reached an + /// evaluation costs no more than its summary counter, however fast the loop runs. + private func hold(_ event: Event) { + pendingLock.lock() + + guard pending.count + store.pendingEventCount < capacity + else { + pendingLock.unlock() + os_log("%s aborted. Event store is full", log: service.config.logger, type: .debug, typeName(and: #function)) + service.diagnosticCache?.incrementDroppedEventCount() + return + } + + pending.append(event) + let needsCommit = pending.count >= EventReporter.pendingCommitThreshold + pendingLock.unlock() + + if needsCommit { + scheduleCommit() + } + } + + /// Commits at a commit point, on the caller's thread or off it as the application asked. + /// + /// Committing on the caller's thread is what lets `track` promise its event is on disk by the time it returns. + /// Scheduling it instead keeps the encode and the write off that thread, and the event is durable a moment later + /// rather than immediately — which is nothing at all when persistence is off, since there is no disk for an early + /// commit to reach. + private func commitAtCommitPoint() { + if service.config.eventPersistence == .immediate { + commitRecordedEvents() + } else { + scheduleCommit() + } + } + + /// Queues a commit unless one is already queued, so a run of recordings asks for one write rather than one each. + private func scheduleCommit() { + pendingLock.lock() + guard !isCommitScheduled + else { + pendingLock.unlock() + return + } + isCommitScheduled = true + pendingLock.unlock() + + commitQueue.async { [weak self] in + guard let self + else { return } + // Cleared before the commit rather than after it, so that an event held while this one is in flight can ask + // for another commit instead of finding one apparently already on its way. + self.pendingLock.lock() + self.isCommitScheduled = false + self.pendingLock.unlock() + + self.commitRecordedEvents() + } + } + + func commitRecordedEvents() { + commitLock.lock() + defer { commitLock.unlock() } + + stagePendingEvents() + stageSummaries() + store.commit() + } + + /// Encodes the held events as one run and stages the bytes. + /// + /// The run is taken under `pendingLock` and encoded outside it, so recording does not wait on the encoder. Staging + /// bypasses capacity because the decision to keep these events was made in `hold`, and refusing them here would + /// drop events the SDK has already counted as accepted. + /// + /// Requires `commitLock`: two threads draining separate runs would stage them in whichever order they finished + /// encoding, which is not the order they were recorded in. + private func stagePendingEvents() { + pendingLock.lock() + if pending.isEmpty { + pendingLock.unlock() + return + } + let run = pending + pending = [] + pendingLock.unlock() + + for event in run { + guard let encoded = encode(event) + else { continue } + stage(encoded, bypassingCapacity: true) + } + } + + /// Whether recording this kind of event should encode and write the held run before it returns. + /// + /// Custom and identify events are recorded because the application did something it chose to report, which is both + /// rare enough to afford a write and the moment it is least acceptable to lose. Evaluations are neither. + private static func isCommitPoint(_ kind: Event.Kind) -> Bool { + switch kind { + case .custom, .identify: return true + case .feature, .debug, .summary: return false + } + } + + private func stage(_ encodedEvent: Data, bypassingCapacity: Bool = false) { + guard store.stage(encodedEvent, bypassingCapacity: bypassingCapacity) + else { + os_log("%s aborted. Event store is full", log: service.config.logger, type: .debug, typeName(and: #function)) + service.diagnosticCache?.incrementDroppedEventCount() + return } } + /// Turns the evaluations counted so far into summary events and stages them, leaving the counters empty. + /// + /// Counters live only in memory, so a crash takes whatever has not been summarized with it. Summarizing at each + /// commit point rather than only at a delivery is what bounds that loss, and it costs nothing in accuracy: + /// LaunchDarkly sums the counters of every summary it receives, so a session that produced several summaries is + /// counted the same as one that produced a single summary covering the same evaluations. + private func stageSummaries() { + stateLock.lock() + let summaries = contextSummarizer.hasLoggedRequests ? contextSummarizer.getSummaries() : [] + if !summaries.isEmpty { + contextSummarizer.clear() + } + stateLock.unlock() + + for summary in summaries { + let summaryEvent = SummaryEvent(flagRequestTracker: summary.tracker, context: summary.context) + guard let encoded = encode(summaryEvent) + else { continue } + // Summaries are an aggregate of evaluations that were already counted, so refusing one for capacity would + // lose evaluations the SDK promised to report rather than shed new load. + stage(encoded, bypassingCapacity: true) + } + } + + private func encode(_ event: Event) -> Data? { + let encoded: Data? = encoding == .codable ? try? encoder.encode(event) : handWrittenEncoder.encode(event) + guard let encoded = encoded + else { + os_log("%s Failed to serialize event for publication: %s", log: service.config.logger, type: .debug, typeName(and: #function), String(describing: event)) + return nil + } + return encoded + } + + // MARK: Reporting + private func startReporting() { guard eventReportTimer == nil else { return } - eventReportTimer = LDTimer(withTimeInterval: service.config.eventFlushInterval, fireQueue: eventQueue, execute: reportEvents) + eventReportTimer = LDTimer(withTimeInterval: service.config.eventFlushInterval, fireQueue: deliveryQueue, execute: reportEvents) + + // Events a previous run left behind are already late by however long the application was gone, so they do not + // wait out a report interval on top of that: an application that crashed reports it as soon as it is next able + // to. Only the first time online brings a delivery forward, so going online again later, as a network comes and + // goes, keeps to the ordinary cadence. + deliveryQueue.async { [weak self] in + guard let self, self.hasEventsFromPreviousRun + else { return } + self.hasEventsFromPreviousRun = false + self.reportEvents() + } } private func stopReporting() { @@ -135,7 +430,10 @@ class EventReporter: EventReporting { } func flushReportingOutcome(completion: @escaping FlushOutcomeClosure) { - eventQueue.async { + // Flush is a commit point: everything accepted before this call must be on disk before control returns, + // even when delivery cannot run because the client is offline. + commitRecordedEvents() + deliveryQueue.async { self.reportEvents(completion: completion) } } @@ -145,6 +443,17 @@ class EventReporter: EventReporting { } private func reportEvents(completion: FlushOutcomeClosure?) { + guard !isDelivering + else { + // Everything this caller recorded is already committed, so one pass after the current delivery finishes + // covers it. Starting one now would only re-send what is still in flight. + hasWaitingRequest = true + if let completion { + waitingCompletions.append(completion) + } + return + } + guard isOnline else { os_log("%s aborted. EventReporter is offline", log: service.config.logger, type: .debug, typeName(and: #function)) @@ -153,16 +462,11 @@ class EventReporter: EventReporting { return } - if contextSummarizer.hasLoggedRequests { - let summaries = contextSummarizer.getSummaries() - for summary in summaries { - let summaryEvent = SummaryEvent(flagRequestTracker: summary.tracker, context: summary.context) - self.eventStore.append(summaryEvent) - } - contextSummarizer.clear() - } + commitRecordedEvents() + _ = store.closeBatch() - guard !eventStore.isEmpty + let batches = store.pendingBatches() + guard !batches.isEmpty else { os_log("%s aborted. Event store is empty", log: service.config.logger, type: .debug, typeName(and: #function)) reportSyncComplete(nil) @@ -171,104 +475,134 @@ class EventReporter: EventReporting { } os_log("%s starting", log: service.config.logger, type: .debug, typeName(and: #function)) + isDelivering = true + deliver(batches) { [weak self] delivered in + guard let self + else { + completion?(false) + return + } + // `deliver` reports from whichever queue the response arrived on. + self.deliveryQueue.async { + self.finishDelivery(delivered, completion) + } + } + } - let toPublish = self.eventStore - self.eventStore = [] + /// Releases the in-flight claim and, if a delivery was asked for while it was held, makes the one pass that covers + /// every caller that waited. + private func finishDelivery(_ delivered: Bool, _ completion: FlushOutcomeClosure?) { + isDelivering = false + completion?(delivered) - service.diagnosticCache?.recordEventsInLastBatch(eventsInLastBatch: toPublish.count) + guard hasWaitingRequest + else { return } - DispatchQueue.global().async { - self.publish(toPublish, UUID().uuidString, completion) + let waiting = waitingCompletions + hasWaitingRequest = false + waitingCompletions = [] + reportEvents { result in + waiting.forEach { $0(result) } } } - private func publish(_ events: [Event], _ payloadId: String, _ completion: FlushOutcomeClosure?) { - guard let eventData = encode(events) + /// Delivers batches oldest first, stopping at the first one that failed in a way worth retrying. + /// + /// Stopping matters: the batches that are left keep their place in the log, and a later delivery attempts them + /// again rather than the SDK spending the rest of the session's requests on a service that is refusing them. + private func deliver(_ batches: [EventBatch], _ completion: FlushOutcomeClosure?) { + var remaining = batches + guard !remaining.isEmpty else { - os_log("%s Failed to serialize event(s) for publication: %s", log: service.config.logger, type: .debug, typeName(and: #function), String(describing: events)) - // Nothing is left waiting to be sent: no later attempt would produce bytes this one could not. completion?(true) return } - self.service.publishEventData(eventData, payloadId) { response in - switch self.processEventResponse(sentEvents: events.count, response: response.urlResponse as? HTTPURLResponse, error: response.error, isRetry: false) { - case .settled: - completion?(true) - case .dropped: + + let batch = remaining.removeFirst() + guard let body = store.body(of: batch) + else { + // Nothing deliverable in it, so it will never become deliverable. + store.remove(batch) + deliver(remaining, completion) + return + } + + service.diagnosticCache?.recordEventsInLastBatch(eventsInLastBatch: batch.eventCount) + publish(batch, body) { shouldContinue in + if shouldContinue { + self.deliver(remaining, completion) + } else { completion?(false) + } + } + } + + private func publish(_ batch: EventBatch, _ body: Data, _ completion: @escaping FlushOutcomeClosure) { + service.publishEventData(body, batch.payloadId) { response in + let outcome = self.outcome(sentEvents: batch.eventCount, response: response.urlResponse as? HTTPURLResponse, error: response.error, isRetry: false) + switch outcome { + case .accepted, .refused: + self.store.remove(batch) + completion(true) case .retryable: os_log("%s Retrying event post after delay.", log: self.service.config.logger, type: .debug, self.typeName(and: #function)) DispatchQueue.global().asyncAfter(deadline: DispatchTime.now() + 1.0) { - self.service.publishEventData(eventData, payloadId) { response in - let outcome = self.processEventResponse(sentEvents: events.count, response: response.urlResponse as? HTTPURLResponse, error: response.error, isRetry: true) - completion?(outcome == .settled) + self.service.publishEventData(body, batch.payloadId) { retryResponse in + let retried = self.outcome(sentEvents: batch.eventCount, response: retryResponse.urlResponse as? HTTPURLResponse, error: retryResponse.error, isRetry: true) + switch retried { + case .accepted, .refused: + self.store.remove(batch) + completion(true) + case .retryable: + // The batch stays on disk. A later delivery, in this run of the application or the next + // one, sends it under the same payload ID, so LaunchDarkly can tell it is not new. + os_log("%s Keeping %d event(s) on disk to retry later", log: self.service.config.logger, type: .debug, self.typeName(and: #function), batch.eventCount) + completion(false) + } } } } } } - /// What a delivery attempt's response means for the events it carried. private enum DeliveryOutcome { - /// Accepted, or refused in a way no further attempt would get past. Either way they are off our hands. - case settled - /// A failure another attempt might get past. + /// LaunchDarkly took the events. + case accepted + /// LaunchDarkly will never take these events, so keeping them only wastes space. + case refused + /// The events may still be deliverable. case retryable - /// A failure with no attempt left to make. - case dropped } - /// Encodes a run of events as the array the events endpoint takes. - /// - /// One `JSONWriter` covers the whole run rather than one per event, so its byte buffer and the capacity it has - /// grown into are reused. That, and the context cache, is most of what the hand-written path saves over the - /// reflective one — a run of evaluations is nearly always the same context encoded again and again. - private func encode(_ events: [Event]) -> Data? { - guard encoding != .codable - else { return try? encoder.encode(events) } - - let writer = JSONWriter() - var payload = Data([UInt8(ascii: "[")]) - for (index, event) in events.enumerated() { - guard let encoded = handWrittenEncoder.encode(event, into: writer) - else { return nil } - if index > 0 { - payload.append(UInt8(ascii: ",")) - } - payload.append(encoded) - } - payload.append(UInt8(ascii: "]")) - return payload - } - - private func processEventResponse(sentEvents: Int, response: HTTPURLResponse?, error: Error?, isRetry: Bool) -> DeliveryOutcome { + private func outcome(sentEvents: Int, response: HTTPURLResponse?, error: Error?, isRetry: Bool) -> DeliveryOutcome { if error == nil && (200..<300).contains(response?.statusCode ?? 0) { - let serverTime = response?.headerDate ?? self.lastEventResponseDate - if serverTime > self.lastEventResponseDate { - self.lastEventResponseDate = serverTime + let serverTime = response?.headerDate + stateLock.lock() + if let serverTime = serverTime, serverTime > responseDate { + responseDate = serverTime } + stateLock.unlock() os_log("%s Completed sending %d event(s)", log: service.config.logger, type: .debug, typeName(and: #function), sentEvents) self.reportSyncComplete(nil) - return .settled + return .accepted } if let statusCode = response?.statusCode, (400..<500).contains(statusCode) && ![400, 408, 429].contains(statusCode) { os_log("%s dropping events due to non-retriable response: %s", log: service.config.logger, type: .debug, typeName(and: #function), String(describing: response)) self.reportSyncComplete(.response(response)) - return .settled + return .refused } os_log("%s Sending events failed with error: %s response: %s", log: service.config.logger, type: .debug, typeName(and: #function), String(describing: error), String(describing: response)) if isRetry { - os_log("%s dropping events due to failed retry", log: service.config.logger, type: .debug, typeName(and: #function)) if let error = error { reportSyncComplete(.request(error)) } else { reportSyncComplete(.response(response)) } - return .dropped + return .retryable } return .retryable @@ -316,10 +650,90 @@ extension EventReporter { extension EventReporter: TypeIdentifying { } +/// Stands in where the platform gave the SDK nowhere to write. Events are recorded and delivered from memory for the +/// life of the process, which is the behavior the SDK had before it kept a log. +private final class NullEventStore: EventStoring { + private let lock = UnfairLock() + private var staged: [Data] = [] + private var closed: [String: [Data]] = [:] + + var pendingEventCount: Int { + lock.lock() + defer { lock.unlock() } + return staged.count + closed.values.reduce(0) { $0 + $1.count } + } + + func stage(_ encodedEvent: Data, bypassingCapacity: Bool) -> Bool { + lock.lock() + defer { lock.unlock() } + staged.append(encodedEvent) + return true + } + + func commit() { + } + + func closeBatch() -> EventBatch? { + lock.lock() + defer { lock.unlock() } + + guard !staged.isEmpty + else { return nil } + + let payloadId = UUID().uuidString + let events = staged + staged = [] + closed[payloadId] = events + return EventBatch(payloadId: payloadId, eventCount: events.count) + } + + func pendingBatches() -> [EventBatch] { + lock.lock() + defer { lock.unlock() } + return closed.map { EventBatch(payloadId: $0.key, eventCount: $0.value.count) } + } + + func body(of batch: EventBatch) -> Data? { + lock.lock() + defer { lock.unlock() } + + guard let events = closed[batch.payloadId] + else { return nil } + + var body = Data("[".utf8) + for (index, event) in events.enumerated() { + if index > 0 { + body.append(UInt8(ascii: ",")) + } + body.append(event) + } + body.append(UInt8(ascii: "]")) + return body + } + + func remove(_ batch: EventBatch) { + lock.lock() + defer { lock.unlock() } + closed[batch.payloadId] = nil + } + + func recoverInterruptedLog() { + } +} + #if DEBUG extension EventReporter { + /// Full events accepted but not yet handed to a commit. + var pendingEventsForTesting: [Event] { + pendingLock.lock() + defer { pendingLock.unlock() } + return pending + } + func setLastEventResponseDate(_ date: Date) { - lastEventResponseDate = date + stateLock.lock() + responseDate = date + stateLock.unlock() } } #endif diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift new file mode 100644 index 000000000..ea2a296f2 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -0,0 +1,763 @@ +import Foundation +import OSLog + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#elseif canImport(Musl) +import Musl +#endif + +/// A group of encoded events that has been closed off for delivery. +/// +/// The identifier doubles as the payload ID sent to LaunchDarkly, so a batch that is retried after the process died +/// mid-delivery is recognized upstream as the same delivery rather than counted twice. +struct EventBatch: Equatable { + let payloadId: String + let eventCount: Int +} + +protocol EventStoring { + /// Encoded events that have been recorded but not yet accepted by LaunchDarkly, whether they are still staged in + /// memory, written to the open log, or sitting in a closed batch. + var pendingEventCount: Int { get } + + /// Stages an already encoded event, returning false if it was dropped because `capacity` is reached. + /// + /// Staging is a copy into a buffer; `commit()` is what makes the event outlive the process. + func stage(_ encodedEvent: Data, bypassingCapacity: Bool) -> Bool + + /// Hands every staged byte to the kernel, so that the events recorded so far survive the process dying. + func commit() + + /// Commits, then closes the open log into a batch to deliver. Returns nil when there is nothing to send. + func closeBatch() -> EventBatch? + + /// Batches awaiting delivery, oldest first, including any recovered from a previous run of the application. + func pendingBatches() -> [EventBatch] + + /// The `[...]` request body for a batch, or nil if the file has since become unreadable. + func body(of batch: EventBatch) -> Data? + + /// Forgets a batch, which is only correct once LaunchDarkly has accepted it or permanently refused it. + func remove(_ batch: EventBatch) + + /// Closes any log left open by a previous run of the application, so its events join the batches to deliver. + func recoverInterruptedLog() +} + +/// A crash-durable append log of encoded analytics events. +/// +/// Events are appended to an open log, which is closed into a batch when the reporter is ready to deliver one. A batch +/// is deleted only once LaunchDarkly has accepted it, so a delivery interrupted by the process dying is retried on the +/// next run rather than lost. +/// +/// Bytes reach the kernel through `write(2)` and are never `fsync`ed. What this defends against is the process dying -- +/// which is what `fatalError`, an uncaught exception, and the OS killing a backgrounded application all are -- and a +/// process cannot take back bytes the kernel already holds; the kernel flushes them on its own schedule. Only losing +/// the kernel itself, to a panic or a power cut, can discard them, and covering that would mean `F_FULLFSYNC` on every +/// event, which costs milliseconds where this costs microseconds. The trade that buys is stated where the SDK records +/// events: a commit point is a point where staged bytes are committed, and the events an application cares most +/// about are recorded at one. +final class EventStore: EventStoring { + /// Names a log that has been closed off for delivery; the rest of the name is the batch's payload ID. + private static let batchPrefix = "ready-" + /// How many staged bytes are allowed to accumulate before one of them pays for a write. + /// + /// This is what keeps a syscall off most recordings while bounding what a crash can take with it, for an + /// application that only evaluates flags and so never reaches a commit point of its own. + private static let stagingThreshold = 16 * 1024 + + let directory: URL + private let capacity: Int + private let logger: OSLog + /// Where a commit runs when it was not asked for by a caller who needs it to have happened. + /// + /// Evaluating a flag must not put a write on whichever thread evaluated it, and that thread is usually the main one. + /// A caller at a commit point still commits synchronously, because for them having returned is the guarantee. + private let commitQueue: DispatchQueue + + /// Guards the buffer and the counters. It is held for a copy and never across a syscall, so an evaluation + /// recording an event waits on another thread's memcpy at worst, never on the disk. + private let bufferLock = UnfairLock() + /// Serializes commits so that two threads committing at once cannot interleave a frame or write frames out of + /// order. Taken before `bufferLock`, never after it. + private let ioLock = UnfairLock() + + /// Only to be used while holding `bufferLock`. + private var bufferData = Data() + private var bufferedEventCount = 0 + private var committedEvents = 0 + private var closedEvents = 0 + /// Whether events are being written to disk, which is what the application asked for until a write fails and the + /// store gives up on persistence for the rest of the session. + private var persistEvents = true + /// Whether a commit is already on its way, so that a burst of recordings queues one write rather than one each. + private var isCommitScheduled = false + + /// Only to be used while holding `ioLock`. + private var descriptor: Int32 = -1 + + /// Batches held in memory because the filesystem would not take them. + /// + /// Persistence failing should cost durability and nothing else, so the store falls back to what the SDK did before + /// it kept a log: hold the events, deliver them, lose them only if the process dies. Kept in the order they were + /// closed, so delivery still goes oldest first. + /// + /// Only to be used while holding `ioLock`. + private var inMemoryBatches: [HeldBatch] = [] + + /// How many events are in each batch this process closed or recovered, so that listing them does not have to read + /// them back. + /// + /// A batch is never appended to once it is closed, so a count taken at the close holds until the batch is + /// delivered. Without this, listing reads every batch file in full while holding `ioLock`, and a commit at a + /// commit point can end up waiting behind those reads. + /// + /// Only to be used while holding `ioLock`. + private var eventCounts: [String: Int] = [:] + + private var currentLogUrl: URL { directory.appendingPathComponent("current") } + + /// A batch is identified by its payload ID rather than by a path, so that a batch listed from the directory and the + /// same batch as it was closed are one thing. + private func url(of payloadId: String) -> URL { + directory.appendingPathComponent("\(EventStore.batchPrefix)\(payloadId)") + } + + init(directory: URL, + capacity: Int, + persistEvents: Bool = true, + logger: OSLog, + commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.EventStore.commitQueue", qos: .userInitiated)) { + self.directory = directory + self.capacity = capacity + self.logger = logger + self.commitQueue = commitQueue + // An application that has not asked for persistence gets the same store running the same way it runs once a + // write has failed: events are held, delivered, and lost only if the process dies. Batches a run with + // persistence turned on left behind are still recovered and delivered, which is why this sets the runtime + // flag rather than skipping the store's reads. + self.persistEvents = persistEvents + } + + /// The directory the store for a mobile key belongs in, or nil where the platform gave us nowhere to write. + /// + /// tvOS is the reason this is not simply Application Support: a tvOS application is only allowed to persist to + /// purgeable caches, so there the events are kept somewhere the system may reclaim, which is the most durability + /// the platform offers. + static func defaultDirectory(mobileKey: String) -> URL? { + #if os(tvOS) + let searchPath = FileManager.SearchPathDirectory.cachesDirectory + #else + let searchPath = FileManager.SearchPathDirectory.applicationSupportDirectory + #endif + + let root = FileManager.default.urls(for: searchPath, in: .userDomainMask).first + ?? FileManager.default.temporaryDirectory + + var keyComponent = Util.sha256(mobileKey).base64UrlEncodedString + if let bundleId = Bundle.main.bundleIdentifier { + keyComponent = "\(Util.sha256(bundleId).base64UrlEncodedString).\(keyComponent)" + } + + return root + .appendingPathComponent("com.launchdarkly.events", isDirectory: true) + .appendingPathComponent(keyComponent, isDirectory: true) + } + + // MARK: Recording + + var pendingEventCount: Int { + bufferLock.lock() + defer { bufferLock.unlock() } + return bufferedEventCount + committedEvents + closedEvents + } + + func stage(_ encodedEvent: Data, bypassingCapacity: Bool = false) -> Bool { + guard encodedEvent.count <= EventLogFormat.maxFrameSize + else { + os_log("%s dropping an event larger than a log frame allows", log: logger, type: .debug, typeName(and: #function)) + return false + } + + bufferLock.lock() + + guard bypassingCapacity || bufferedEventCount + committedEvents + closedEvents < capacity + else { + bufferLock.unlock() + return false + } + + bufferData.append(EventLogFormat.frame(for: encodedEvent)) + bufferedEventCount += 1 + // Nothing to schedule once persistence has been given up on: there is nowhere for a commit to put these bytes, + // so they stay in memory until a delivery closes them into a batch. + let needsCommit = persistEvents && bufferData.count >= EventStore.stagingThreshold && !isCommitScheduled + if needsCommit { + isCommitScheduled = true + } + bufferLock.unlock() + + // Handed to another thread rather than done here. Recording an event is something a flag evaluation does, and an + // evaluation is expected to be a memory operation: the caller is usually the main thread, where a write that + // happens to find a busy filesystem is a stall an application cannot do anything about. + if needsCommit { + commitQueue.async { [weak self] in + self?.commitScheduled() + } + } + return true + } + + private func commitScheduled() { + // Cleared before the commit rather than after it, so that an event staged while this write is in flight can ask + // for another one instead of finding a commit apparently already on its way and waiting for a commit point. + bufferLock.lock() + isCommitScheduled = false + bufferLock.unlock() + + commit() + } + + func commit() { + ioLock.lock() + defer { ioLock.unlock() } + commitHoldingIoLock() + } + + func closeBatch() -> EventBatch? { + ioLock.lock() + defer { ioLock.unlock() } + + commitHoldingIoLock() + + bufferLock.lock() + let events = committedEvents + let stillBuffered = bufferedEventCount + bufferLock.unlock() + + if events > 0 { + return closeOpenLogHoldingIoLock(events) + } + if stillBuffered > 0 { + // Only reachable once persistence has been given up on; a healthy commit leaves nothing staged behind. + return closeInMemoryBatchHoldingIoLock() + } + return nil + } + + /// Requires `ioLock`. + private func closeOpenLogHoldingIoLock(_ events: Int) -> EventBatch? { + closeDescriptor() + + let payloadId = UUID().uuidString + do { + try FileManager.default.moveItem(at: currentLogUrl, to: url(of: payloadId)) + } catch { + os_log("%s could not close the event log: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) + // The log is still there to try again with, unless it is the log that went missing -- a purged cache + // directory on tvOS, say. Then its events are gone, and they have to stop counting against capacity or the + // store would refuse events for the rest of the session. + if !FileManager.default.fileExists(atPath: currentLogUrl.path) { + bufferLock.lock() + committedEvents = 0 + bufferLock.unlock() + } + return nil + } + + eventCounts[payloadId] = events + + bufferLock.lock() + committedEvents = 0 + closedEvents += events + bufferLock.unlock() + + return EventBatch(payloadId: payloadId, eventCount: events) + } + + // MARK: Delivery + + func pendingBatches() -> [EventBatch] { + ioLock.lock() + defer { ioLock.unlock() } + + let contents = (try? FileManager.default.contentsOfDirectory(at: directory, + includingPropertiesForKeys: [.contentModificationDateKey], + options: [.skipsHiddenFiles])) ?? [] + + var batches = contents + .filter { $0.lastPathComponent.hasPrefix(EventStore.batchPrefix) } + .compactMap { file -> (EventBatch, Date)? in + let payloadId = String(file.lastPathComponent.dropFirst(EventStore.batchPrefix.count)) + // Reading is only for a batch this process has not counted: one a previous run left behind, or one + // belonging to another process sharing the environment. + guard let events = eventCounts[payloadId] ?? EventLogFormat.eventCount(in: file) + else { + // Written by a version of the SDK whose format this one does not read, or damaged beyond what the + // torn tail recovery tolerates. Either way it can never be delivered, so it is not kept. + try? FileManager.default.removeItem(at: file) + return nil + } + eventCounts[payloadId] = events + let modified = (try? file.resourceValues(forKeys: [.contentModificationDateKey]))?.contentModificationDate + return (EventBatch(payloadId: payloadId, eventCount: events), modified ?? Date.distantPast) + } + .sorted { $0.1 < $1.1 } + .map { $0.0 } + + // A batch that left the directory without going through `remove` -- a purged cache directory on tvOS, say -- + // would otherwise keep its count for the rest of the session. + let listed = Set(batches.map { $0.payloadId }) + eventCounts = eventCounts.filter { listed.contains($0.key) } + + // After the files, since anything held in memory was closed by this run and so is newer than whatever a + // previous run left on the disk. + batches.append(contentsOf: inMemoryBatches.map { + EventBatch(payloadId: $0.payloadId, eventCount: $0.eventCount) + }) + + bufferLock.lock() + closedEvents = batches.reduce(0) { $0 + $1.eventCount } + bufferLock.unlock() + + return batches + } + + func body(of batch: EventBatch) -> Data? { + ioLock.lock() + defer { ioLock.unlock() } + + if let held = inMemoryBatches.first(where: { $0.payloadId == batch.payloadId }) { + return held.body + } + + guard let file = try? Data(contentsOf: url(of: batch.payloadId)) + else { return nil } + return EventLogFormat.assembleBody(from: file) + } + + func remove(_ batch: EventBatch) { + ioLock.lock() + defer { ioLock.unlock() } + + if let index = inMemoryBatches.firstIndex(where: { $0.payloadId == batch.payloadId }) { + inMemoryBatches.remove(at: index) + } else { + try? FileManager.default.removeItem(at: url(of: batch.payloadId)) + eventCounts[batch.payloadId] = nil + } + + bufferLock.lock() + closedEvents = max(0, closedEvents - batch.eventCount) + bufferLock.unlock() + } + + /// Closes any log left open by a previous run of the application, so its events join the batches to deliver. + /// + /// The events in it were recorded by a process that is gone, so there is no one left to add to it. + func recoverInterruptedLog() { + ioLock.lock() + defer { ioLock.unlock() } + + guard FileManager.default.fileExists(atPath: currentLogUrl.path), + let events = EventLogFormat.eventCount(in: currentLogUrl) + else { + // Nothing recoverable. An unreadable log cannot be appended to either, so it goes. + if FileManager.default.fileExists(atPath: currentLogUrl.path) { + try? FileManager.default.removeItem(at: currentLogUrl) + } + return + } + + guard events > 0 + else { + try? FileManager.default.removeItem(at: currentLogUrl) + return + } + + let payloadId = UUID().uuidString + guard (try? FileManager.default.moveItem(at: currentLogUrl, to: url(of: payloadId))) != nil + else { return } + + eventCounts[payloadId] = events + os_log("%s recovered %d event(s) from a previous run", log: logger, type: .debug, typeName(and: #function), events) + } + + // MARK: Writing + + /// Requires `ioLock`. + private func commitHoldingIoLock() { + bufferLock.lock() + guard persistEvents, bufferedEventCount > 0 + else { + // Staged bytes are left where they are. With nowhere durable to put them, memory is better than dropping + // them: a delivery can still close them into a batch and send them. + bufferLock.unlock() + return + } + let bytes = bufferData + let events = bufferedEventCount + bufferData = Data() + bufferedEventCount = 0 + bufferLock.unlock() + + if append(bytes) { + bufferLock.lock() + committedEvents += events + bufferLock.unlock() + } else { + // Put back, to be delivered from memory rather than lost. Ahead of anything staged while the write was in + // flight, so the events keep the order they were recorded in. + bufferLock.lock() + let laterFrames = bufferData + bufferData = bytes + bufferData.append(laterFrames) + bufferedEventCount += events + bufferLock.unlock() + } + } + + /// Requires `ioLock`. + private func append(_ bytes: Data) -> Bool { + guard let descriptor = descriptorForAppending() + else { + disablePersistenceHoldingIoLock() + return false + } + return append(bytes, to: descriptor) + } + + /// Requires `ioLock`. + private func append(_ bytes: Data, to descriptor: Int32) -> Bool { + var offset = 0 + let failure: Int32? = bytes.withUnsafeBytes { raw -> Int32? in + guard let base = raw.baseAddress + else { return nil } + while offset < raw.count { + let result = writeBytes(descriptor, base.advanced(by: offset), raw.count - offset) + if result > 0 { + offset += result + continue + } + // A signal interrupting the write is not a failure; anything else is. + if errno == EINTR { + continue + } + return errno + } + return nil + } + + if let failure = failure { + // Never trap on a full disk. The SDK gives up on persistence for the rest of the session rather than + // taking the application down with it, which is how other SDKs have crashed their hosts. + os_log("%s giving up on persisting events: errno %d", log: logger, type: .debug, typeName(and: #function), failure) + disablePersistenceHoldingIoLock() + return false + } + + return true + } + + /// Requires `ioLock`. + private func descriptorForAppending() -> Int32? { + if descriptor >= 0 { + return descriptor + } + + do { + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + } catch { + os_log("%s could not create the event directory: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) + return nil + } + excludeFromBackup() + + // O_APPEND is what makes each write land at the end of the file as one step, so that clients for several + // environments writing their own logs, or a commit racing a reader, cannot produce a spliced frame. + let opened = currentLogUrl.withUnsafeFileSystemRepresentation { path -> Int32 in + guard let path = path + else { return -1 } + return openFile(path, O_WRONLY | O_APPEND | O_CREAT, 0o600) + } + + guard opened >= 0 + else { + os_log("%s could not open the event log: errno %d", log: logger, type: .debug, typeName(and: #function), errno) + return nil + } + + descriptor = opened + + if lseek(opened, 0, SEEK_END) == 0 { + guard append(EventLogFormat.fileHeader, to: opened) + else { return nil } + } + + return descriptor + } + + /// Requires `ioLock`. + private func closeDescriptor() { + guard descriptor >= 0 + else { return } + closeFile(descriptor) + descriptor = -1 + } + + private func excludeFromBackup() { + #if canImport(Darwin) + var url = directory + var values = URLResourceValues() + values.isExcludedFromBackup = true + try? url.setResourceValues(values) + #endif + } + + deinit { + // One acquisition: the lock is not recursive, and `commit()` would take it again. + ioLock.lock() + commitHoldingIoLock() + closeDescriptor() + ioLock.unlock() + } +} + +/// A batch the filesystem would not take, kept where the SDK used to keep all of them. +private struct HeldBatch { + let payloadId: String + let body: Data + let eventCount: Int +} + +/// What the store does once the filesystem has refused it. +/// +/// Losing the disk should cost durability and nothing else, so the store falls back to what the SDK did before it kept +/// a log: hold the events in memory, deliver them from there, and lose them only if the process dies. That is a worse +/// guarantee than Tier 3 and a better one than dropping the events, which is what a full device used to mean here. +private extension EventStore { + /// Gives up on the disk for the rest of the session, leaving the events themselves alone. + /// + /// Requires `ioLock`. + func disablePersistenceHoldingIoLock() { + closeDescriptor() + bufferLock.lock() + persistEvents = false + bufferLock.unlock() + } + + /// Closes whatever is staged into a batch that never reaches a file. + /// + /// Requires `ioLock`. + func closeInMemoryBatchHoldingIoLock() -> EventBatch? { + bufferLock.lock() + let frames = bufferData + let events = bufferedEventCount + bufferData = Data() + bufferedEventCount = 0 + bufferLock.unlock() + + // Assembled through the same reader a batch on disk goes through, so a held batch and a written one produce + // byte-identical request bodies. + var log = EventLogFormat.fileHeader + log.append(frames) + guard let body = EventLogFormat.assembleBody(from: log) + else { return nil } + + let payloadId = UUID().uuidString + inMemoryBatches.append(HeldBatch(payloadId: payloadId, body: body, eventCount: events)) + + bufferLock.lock() + closedEvents += events + bufferLock.unlock() + + return EventBatch(payloadId: payloadId, eventCount: events) + } +} + +/// The on-disk shape of an event log. +/// +/// A log opens with a magic and a format version, and holds a sequence of frames: +/// +/// +- 4 bytes -+- 2 bytes -+ +- 2 bytes -+- 4 bytes -+- n bytes -+ +/// | LDEV | version | ... | type | length (n) | payload | +/// +-----------+-----------+ +-----------+-------------+-----------+ +/// +/// The length prefix is what makes a log recoverable: a reader can tell a frame the writer finished from one it did not, +/// without having to parse the event to find where it ends. The version is what makes an upgrade safe -- a log this +/// version does not understand is discarded rather than misread -- and the frame type leaves room for a later version to +/// write something new into a log this one still reads. +enum EventLogFormat { + /// Bump this whenever the framing or the meaning of a frame changes. + static let version: UInt16 = 1 + static let magic = Data("LDEV".utf8) + static let fileHeaderSize = 6 + static let frameHeaderSize = 6 + /// The only frame type written today; a reader skips a frame whose type it does not know. + static let eventFrame: UInt16 = 1 + /// A ceiling on a single frame, so a corrupt length cannot make recovery allocate wildly. + static let maxFrameSize = 8 * 1024 * 1024 + + static var fileHeader: Data { + var header = magic + header.append(bigEndian(version)) + return header + } + + static func frame(for encodedEvent: Data) -> Data { + var frame = Data(capacity: frameHeaderSize + encodedEvent.count) + frame.append(bigEndian(eventFrame)) + frame.append(bigEndian(UInt32(encodedEvent.count))) + frame.append(encodedEvent) + return frame + } + + /// Walks the frames of a log, stopping at the first one that is not wholly there. + /// + /// A process that died partway through a write leaves a torn frame at the end. Everything before it is intact, so + /// recovery keeps that and drops only the tail. Returns false when the file is not a log this version reads. + static func forEachFrame(in file: Data, _ visit: (UInt16, Data) -> Void) -> Bool { + guard file.count >= fileHeaderSize, + file.prefix(magic.count) == magic, + readUInt16(file, at: magic.count) == version + else { return false } + + var cursor = fileHeaderSize + while cursor + frameHeaderSize <= file.count { + let type = readUInt16(file, at: cursor) + let length = Int(readUInt32(file, at: cursor + 2)) + let start = cursor + frameHeaderSize + guard length > 0, length <= maxFrameSize, start + length <= file.count + else { break } + + visit(type, file.subdata(in: start..<(start + length))) + cursor = start + length + } + return true + } + + static func eventCount(in url: URL) -> Int? { + guard let file = try? Data(contentsOf: url) + else { return nil } + + var events = 0 + guard forEachFrame(in: file, { type, _ in + if type == eventFrame { + events += 1 + } + }) + else { return nil } + return events + } + + /// Assembles the JSON array LaunchDarkly expects out of the frames, without parsing the events themselves: they were + /// serialized on the way in and are shipped exactly as they were recorded. + static func assembleBody(from file: Data) -> Data? { + var body = Data("[".utf8) + var isFirst = true + let readable = forEachFrame(in: file) { type, payload in + guard type == eventFrame + else { return } + if !isFirst { + body.append(UInt8(ascii: ",")) + } + body.append(payload) + isFirst = false + } + + guard readable, !isFirst + else { return nil } + + body.append(UInt8(ascii: "]")) + return body + } + + static func bigEndian(_ value: UInt16) -> Data { + Data([UInt8(truncatingIfNeeded: value >> 8), UInt8(truncatingIfNeeded: value)]) + } + + static func bigEndian(_ value: UInt32) -> Data { + Data([UInt8(truncatingIfNeeded: value >> 24), + UInt8(truncatingIfNeeded: value >> 16), + UInt8(truncatingIfNeeded: value >> 8), + UInt8(truncatingIfNeeded: value)]) + } + + static func readUInt16(_ data: Data, at offset: Int) -> UInt16 { + let start = data.startIndex + offset + return UInt16(data[start]) << 8 | UInt16(data[start + 1]) + } + + static func readUInt32(_ data: Data, at offset: Int) -> UInt32 { + let start = data.startIndex + offset + return UInt32(data[start]) << 24 | UInt32(data[start + 1]) << 16 | UInt32(data[start + 2]) << 8 | UInt32(data[start + 3]) + } +} + +extension EventStore { + /// Every event the store is holding, encoded exactly as it will be sent. + /// + /// Reading the log means committing what is staged, so this is not free and is meant for tests and for diagnosing a + /// store rather than for the recording path. + func pendingEventPayloads() -> [Data] { + commit() + + ioLock.lock() + defer { ioLock.unlock() } + + let contents = (try? FileManager.default.contentsOfDirectory(at: directory, + includingPropertiesForKeys: nil, + options: [.skipsHiddenFiles])) ?? [] + var logs = contents.filter { $0.lastPathComponent.hasPrefix(EventStore.batchPrefix) }.sorted { $0.path < $1.path } + if FileManager.default.fileExists(atPath: currentLogUrl.path) { + logs.append(currentLogUrl) + } + + var payloads: [Data] = [] + for log in logs { + guard let file = try? Data(contentsOf: log) + else { continue } + _ = EventLogFormat.forEachFrame(in: file) { type, payload in + if type == EventLogFormat.eventFrame { + payloads.append(payload) + } + } + } + return payloads + } +} + +extension EventStore: TypeIdentifying { } + +// The POSIX calls are wrapped so that the names cannot be shadowed by a member of the type using them, and so the +// platform differences stay in one place. Foundation's FileHandle is deliberately not used: it raises an Objective-C +// exception when a write fails, which is not something a Swift caller can catch, and has taken host applications down +// with it when a device ran out of space. +#if canImport(Darwin) +private func openFile(_ path: UnsafePointer, _ flags: Int32, _ mode: mode_t) -> Int32 { + Darwin.open(path, flags, mode) +} + +private func writeBytes(_ descriptor: Int32, _ bytes: UnsafeRawPointer, _ count: Int) -> Int { + Darwin.write(descriptor, bytes, count) +} + +private func closeFile(_ descriptor: Int32) { + _ = Darwin.close(descriptor) +} +#else +private func openFile(_ path: UnsafePointer, _ flags: Int32, _ mode: mode_t) -> Int32 { + open(path, flags, mode) +} + +private func writeBytes(_ descriptor: Int32, _ bytes: UnsafeRawPointer, _ count: Int) -> Int { + write(descriptor, bytes, count) +} + +private func closeFile(_ descriptor: Int32) { + _ = close(descriptor) +} +#endif diff --git a/LaunchDarkly/LaunchDarklyTests/EventJSONWriterTests.swift b/LaunchDarkly/LaunchDarklyTests/EventJSONWriterTests.swift index 23052c790..852d5efd1 100644 --- a/LaunchDarkly/LaunchDarklyTests/EventJSONWriterTests.swift +++ b/LaunchDarkly/LaunchDarklyTests/EventJSONWriterTests.swift @@ -384,30 +384,27 @@ final class EventJSONWriterTests: XCTestCase { config.privateContextAttributes = [Reference("email")] for (name, event) in events(context: context) { - let expected = try XCTUnwrap(published(event, config: config, encoding: .codable), - "nothing published for \(name)") - let actual = try XCTUnwrap(published(event, config: config, encoding: .handWritten), - "nothing published for \(name)") + let codableStore = EventStore.temporary(capacity: .max) + defer { codableStore.deleteEverything() } + let handWrittenStore = EventStore.temporary(capacity: .max) + defer { handWrittenStore.deleteEverything() } - XCTAssertEqual(try canonical(expected), try canonical(actual), "mismatch through the reporter for \(name)") - } - } + let service = DarklyServiceMock() + service.config = config - /// Records one event and returns the payload the reporter handed to the service. - private func published(_ event: Event, config: LDConfig, encoding: EventReporter.Encoding) -> Data? { - let service = DarklyServiceMock() - service.config = config - service.stubEventResponse(success: true) + let codableReporter = EventReporter(service: service, onSyncComplete: nil, store: codableStore, encoding: .codable) + let handWrittenReporter = EventReporter(service: service, onSyncComplete: nil, store: handWrittenStore, encoding: .handWritten) - let reporter = EventReporter(service: service, onSyncComplete: nil, encoding: encoding) - reporter.isOnline = true - reporter.record(event) + codableReporter.record(event) + handWrittenReporter.record(event) + codableReporter.commitRecordedEvents() + handWrittenReporter.commitRecordedEvents() - let published = expectation(description: "published") - reporter.flush { published.fulfill() } - wait(for: [published], timeout: 5) + let expected = try XCTUnwrap(codableStore.pendingEventPayloads().first, "nothing recorded for \(name)") + let actual = try XCTUnwrap(handWrittenStore.pendingEventPayloads().first, "nothing recorded for \(name)") - return service.publishedEventData + XCTAssertEqual(try canonical(expected), try canonical(actual), "mismatch through the reporter for \(name)") + } } // MARK: Helpers diff --git a/LaunchDarkly/LaunchDarklyTests/Mocks/DarklyServiceMock.swift b/LaunchDarkly/LaunchDarklyTests/Mocks/DarklyServiceMock.swift index e1e36ee4e..0182eaf33 100644 --- a/LaunchDarkly/LaunchDarklyTests/Mocks/DarklyServiceMock.swift +++ b/LaunchDarkly/LaunchDarklyTests/Mocks/DarklyServiceMock.swift @@ -136,12 +136,32 @@ final class DarklyServiceMock: DarklyServiceProvider { var stubbedEventResponse: ServiceResponse? var publishEventDataCallCount = 0 var publishedEventData: Data? + /// Every payload ID used, in order, so a test can tell a retry of a delivery from a new one. + var publishedPayloadIds: [String] = [] + /// Holds responses back instead of answering inline, so a test can act while a delivery is in flight. + var holdsEventCompletions = false + private var heldEventCompletions: [ServiceCompletionHandler] = [] func publishEventData(_ eventData: Data, _ payloadId: String, completion: ServiceCompletionHandler?) { publishEventDataCallCount += 1 publishedEventData = eventData + publishedPayloadIds.append(payloadId) + guard !holdsEventCompletions + else { + if let completion { + heldEventCompletions.append(completion) + } + return + } completion?(stubbedEventResponse ?? (nil, nil, nil, nil)) } + /// Answers every held request, as the network coming back would. + func releaseHeldEventCompletions() { + let held = heldEventCompletions + heldEventCompletions = [] + held.forEach { $0(stubbedEventResponse ?? (nil, nil, nil, nil)) } + } + var stubbedDiagnosticResponse: ServiceResponse? var publishDiagnosticCallCount = 0 var publishedDiagnostic: DiagnosticEvent? diff --git a/LaunchDarkly/LaunchDarklyTests/Models/DiagnosticEventSpec.swift b/LaunchDarkly/LaunchDarklyTests/Models/DiagnosticEventSpec.swift index b76e4186e..d693b8dde 100644 --- a/LaunchDarkly/LaunchDarklyTests/Models/DiagnosticEventSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/Models/DiagnosticEventSpec.swift @@ -174,7 +174,7 @@ final class DiagnosticEventSpec: QuickSpec { customConfig.baseUrl = URL(string: "https://clientstream.launchdarkly.com")! customConfig.eventsUrl = URL(string: "https://app.launchdarkly.com")! customConfig.streamUrl = URL(string: "https://mobile.launchdarkly.com")! - customConfig.eventCapacity = 1_000 + customConfig.eventCapacity = 2_000 customConfig.connectionTimeout = 30.0 customConfig.eventFlushInterval = 60.0 customConfig.streamingMode = .polling @@ -202,7 +202,7 @@ final class DiagnosticEventSpec: QuickSpec { expect(diagnosticConfig.customBaseURI) == false expect(diagnosticConfig.customEventsURI) == false expect(diagnosticConfig.customStreamURI) == false - expect(diagnosticConfig.eventsCapacity) == 100 + expect(diagnosticConfig.eventsCapacity) == 1_000 expect(diagnosticConfig.connectTimeoutMillis) == 10_000 expect(diagnosticConfig.eventsFlushIntervalMillis) == 30_000 expect(diagnosticConfig.streamingDisabled) == false @@ -224,7 +224,7 @@ final class DiagnosticEventSpec: QuickSpec { expect(diagnosticConfig.customBaseURI) == true expect(diagnosticConfig.customEventsURI) == true expect(diagnosticConfig.customStreamURI) == true - expect(diagnosticConfig.eventsCapacity) == 1_000 + expect(diagnosticConfig.eventsCapacity) == 2_000 expect(diagnosticConfig.connectTimeoutMillis) == 30_000 expect(diagnosticConfig.eventsFlushIntervalMillis) == 60_000 expect(diagnosticConfig.streamingDisabled) == true diff --git a/LaunchDarkly/LaunchDarklyTests/Models/LDConfigSpec.swift b/LaunchDarkly/LaunchDarklyTests/Models/LDConfigSpec.swift index f5457044d..3a0eed8d7 100644 --- a/LaunchDarkly/LaunchDarklyTests/Models/LDConfigSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/Models/LDConfigSpec.swift @@ -7,6 +7,7 @@ final class LDConfigSpec: XCTestCase { struct Constants { fileprivate static let alternateMockUrl = URL(string: "https://dummy.alternate.com")! fileprivate static let eventCapacity = 10 + fileprivate static let eventPersistence = EventPersistence.immediate fileprivate static let connectionTimeout: TimeInterval = 0.01 fileprivate static let eventFlushInterval: TimeInterval = 0.01 fileprivate static let flagPollingInterval: TimeInterval = 0.01 @@ -37,6 +38,7 @@ final class LDConfigSpec: XCTestCase { ("event URL", Constants.alternateMockUrl, { c, v in c.eventsUrl = v as! URL }), ("stream URL", Constants.alternateMockUrl, { c, v in c.streamUrl = v as! URL }), ("event capacity", Constants.eventCapacity, { c, v in c.eventCapacity = v as! Int }), + ("event persistence", Constants.eventPersistence, { c, v in c.eventPersistence = v as! EventPersistence }), ("connection timeout", Constants.connectionTimeout, { c, v in c.connectionTimeout = v as! TimeInterval }), ("event flush interval", Constants.eventFlushInterval, { c, v in c.eventFlushInterval = v as! TimeInterval }), ("poll interval", Constants.flagPollingInterval, { c, v in c.flagPollingInterval = v as! TimeInterval }), @@ -62,6 +64,7 @@ final class LDConfigSpec: XCTestCase { XCTAssertEqual(config.eventsUrl, LDConfig.Defaults.eventsUrl) XCTAssertEqual(config.streamUrl, LDConfig.Defaults.streamUrl) XCTAssertEqual(config.eventCapacity, LDConfig.Defaults.eventCapacity) + XCTAssertEqual(config.eventPersistence, LDConfig.Defaults.eventPersistence) XCTAssertEqual(config.connectionTimeout, LDConfig.Defaults.connectionTimeout) XCTAssertEqual(config.eventFlushInterval, LDConfig.Defaults.eventFlushInterval) XCTAssertEqual(config.flagPollingInterval, LDConfig.Defaults.flagPollingInterval) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift index 499d4e215..2d05a6313 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift @@ -14,6 +14,7 @@ final class EventReporterSpec: QuickSpec { var config: LDConfig! var context: LDContext! var serviceMock: DarklyServiceMock! + var store: EventStore! var events: [Event] = [] var lastEventResponseDate: Date var eventStubResponseDate: Date? @@ -27,10 +28,17 @@ final class EventReporterSpec: QuickSpec { stubResponseOnly: Bool = false, stubResponseErrorOnly: Bool = false, eventStubResponseDate: Date? = nil, - onSyncComplete: EventSyncCompleteClosure? = nil) { + onSyncComplete: EventSyncCompleteClosure? = nil, + store: EventStore? = nil, + eventCapacity: Int = Event.Kind.allKinds.count, + eventPersistence: EventPersistence = .immediate, + commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.tests.EventReporter.commitQueue")) { config = LDConfig.stub - config.eventCapacity = Event.Kind.allKinds.count + config.eventCapacity = eventCapacity + // The durable behaviour is what most of these are about, so it is the default here even though an + // application has to ask for it. + config.eventPersistence = eventPersistence config.eventFlushInterval = eventFlushInterval ?? Constants.eventFlushInterval context = LDContext.stub() @@ -44,7 +52,9 @@ final class EventReporterSpec: QuickSpec { serviceMock.diagnosticCache = diagnosticCache self.lastEventResponseDate = lastEventResponseDate.adjustedForHttpUrlHeaderUse - eventReporter = EventReporter(service: serviceMock, onSyncComplete: onSyncComplete) + self.store = store ?? EventStore.temporary(capacity: config.eventCapacity) + eventReporter = EventReporter(service: serviceMock, onSyncComplete: onSyncComplete, + store: self.store, commitQueue: commitQueue) (0.. [LDValue] { + eventReporter.commitRecordedEvents() + return store.pendingEventPayloads().compactMap { try? JSONDecoder().decode(LDValue.self, from: $0) } + } + + /// The events recorded so far, in the JSON they encode to, for comparison with `committedEvents()`. + func recordedEventsAsJSON() -> [LDValue] { + events.compactMap { encodeToLDValue($0) } + } + + func cleanUp() { + eventReporter.isOnline = false + store.deleteEverything() + } } override func spec() { @@ -69,6 +95,8 @@ final class EventReporterSpec: QuickSpec { testRecordFlagEvaluationEvents() reportEventsSpec() reportTimerSpec() + durabilitySpec() + flushReportingOutcomeSpec() } private func initSpec() { @@ -76,7 +104,7 @@ final class EventReporterSpec: QuickSpec { var testContext: TestContext! beforeEach { testContext = TestContext() - testContext.eventReporter = EventReporter(service: testContext.serviceMock) { _ in } + testContext.eventReporter = EventReporter(service: testContext.serviceMock, onSyncComplete: { _ in }, store: testContext.store) } it("starts offline without reporting events") { expect(testContext.eventReporter.service) === testContext.serviceMock @@ -94,7 +122,7 @@ final class EventReporterSpec: QuickSpec { testContext = TestContext() } afterEach { - testContext.eventReporter.isOnline = false + testContext.cleanUp() } context("online to offline") { beforeEach { @@ -157,7 +185,7 @@ final class EventReporterSpec: QuickSpec { expect(testContext.eventReporter.isOnline) == false expect(testContext.eventReporter.isReportingActive) == false expect(testContext.serviceMock.publishEventDataCallCount) == 0 - expect(testContext.eventReporter.eventStore) == testContext.events + expect(testContext.committedEvents()) == testContext.recordedEventsAsJSON() } } } @@ -175,7 +203,7 @@ final class EventReporterSpec: QuickSpec { expect(testContext.eventReporter.isOnline) == false expect(testContext.eventReporter.isReportingActive) == false expect(testContext.serviceMock.publishEventDataCallCount) == 0 - expect(testContext.eventReporter.eventStore) == testContext.events + expect(testContext.committedEvents()) == testContext.recordedEventsAsJSON() } it("does not record a dropped event to diagnosticCache") { expect(testContext.diagnosticCache.incrementDroppedEventCountCallCount) == 0 @@ -193,7 +221,7 @@ final class EventReporterSpec: QuickSpec { expect(testContext.eventReporter.isOnline) == false expect(testContext.eventReporter.isReportingActive) == false expect(testContext.serviceMock.publishEventDataCallCount) == 0 - expect(testContext.eventReporter.eventStore) == testContext.events + expect(testContext.committedEvents()) == testContext.recordedEventsAsJSON() } it("records a dropped event to diagnosticCache") { expect(testContext.diagnosticCache.incrementDroppedEventCountCallCount) == 1 @@ -210,7 +238,7 @@ final class EventReporterSpec: QuickSpec { eventStubResponseDate = Date().addingTimeInterval(-TimeInterval(3)) } afterEach { - testContext.eventReporter.isOnline = false + testContext.cleanUp() } let erOnline = { expect(testContext.eventReporter.isOnline) == true @@ -252,7 +280,7 @@ final class EventReporterSpec: QuickSpec { } expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 - expect(testContext.eventReporter.eventStore.isEmpty) == true + expect(testContext.store.pendingEventCount) == 0 expect(testContext.eventReporter.lastEventResponseDate) == testContext.eventStubResponseDate expect(testContext.eventReporter.contextSummarizer.hasLoggedRequests) == false expect(testContext.syncResult).to(beNil()) @@ -277,7 +305,7 @@ final class EventReporterSpec: QuickSpec { expect(published) == encodeToLDValue(testContext.events) expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == testContext.events.count - expect(testContext.eventReporter.eventStore.isEmpty) == true + expect(testContext.store.pendingEventCount) == 0 expect(testContext.eventReporter.lastEventResponseDate) == testContext.eventStubResponseDate expect(testContext.eventReporter.contextSummarizer.hasLoggedRequests) == false expect(testContext.syncResult).to(beNil()) @@ -315,7 +343,7 @@ final class EventReporterSpec: QuickSpec { } expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == 1 - expect(testContext.eventReporter.eventStore.isEmpty) == true + expect(testContext.store.pendingEventCount) == 0 expect(testContext.eventReporter.lastEventResponseDate) == testContext.eventStubResponseDate expect(testContext.eventReporter.contextSummarizer.hasLoggedRequests) == false expect(testContext.syncResult).to(beNil()) @@ -336,7 +364,7 @@ final class EventReporterSpec: QuickSpec { erOnline() expect(testContext.serviceMock.publishEventDataCallCount) == 0 expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 0 - expect(testContext.eventReporter.eventStore.isEmpty) == true + expect(testContext.store.pendingEventCount) == 0 expect(testContext.eventReporter.lastEventResponseDate) == Date.distantPast expect(testContext.eventReporter.contextSummarizer.hasLoggedRequests) == false expect(testContext.syncResult).to(beNil()) @@ -365,10 +393,11 @@ final class EventReporterSpec: QuickSpec { testContext.eventReporter.flush(completion: nil) } } - it("drops events after the failure") { + it("keeps the events to retry after the failure") { erOnline() expect(testContext.serviceMock.publishEventDataCallCount) == 2 // 1 retry attempt - expect(testContext.eventReporter.eventStore.isEmpty) == true + // The events are not lost by a failed delivery; they stay in the log, summary included. + expect(testContext.store.pendingEventCount) == testContext.events.count + 1 let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) valueIsArray(published) { valueArray in expect(valueArray.count) == testContext.events.count + 1 @@ -410,10 +439,11 @@ final class EventReporterSpec: QuickSpec { testContext.eventReporter.flush(completion: nil) } } - it("drops events after the failure") { + it("keeps the events to retry after the failure") { erOnline() expect(testContext.serviceMock.publishEventDataCallCount) == 2 // 1 retry attempt - expect(testContext.eventReporter.eventStore.isEmpty) == true + // The events are not lost by a failed delivery; they stay in the log, summary included. + expect(testContext.store.pendingEventCount) == testContext.events.count + 1 let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) valueIsArray(published) { valueArray in expect(valueArray.count) == testContext.events.count + 1 @@ -458,10 +488,11 @@ final class EventReporterSpec: QuickSpec { testContext.eventReporter.flush(completion: nil) } } - it("drops events events after the failure") { + it("keeps the events to retry after the failure") { erOnline() expect(testContext.serviceMock.publishEventDataCallCount) == 2 // 1 retry attempt - expect(testContext.eventReporter.eventStore.isEmpty) == true + // The events are not lost by a failed delivery; they stay in the log, summary included. + expect(testContext.store.pendingEventCount) == testContext.events.count + 1 let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) valueIsArray(published) { valueArray in expect(valueArray.count) == testContext.events.count + 1 @@ -504,14 +535,19 @@ final class EventReporterSpec: QuickSpec { testContext.eventReporter.flush(completion: nil) } } - it("doesn't report events") { + it("doesn't report events, but persists them") { expect(testContext.eventReporter.isOnline) == false expect(testContext.eventReporter.isReportingActive) == false expect(testContext.serviceMock.publishEventDataCallCount) == 0 expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 0 - expect(testContext.eventReporter.eventStore) == testContext.events expect(testContext.eventReporter.lastEventResponseDate) == Date.distantPast - expect(testContext.eventReporter.contextSummarizer.hasLoggedRequests) == true + + // Being offline delays delivery, not durability: a flush is a commit point, so the recorded events and + // the evaluations counted so far are on disk even though nothing can be sent. + let pending = testContext.committedEvents() + expect(pending.dropLast().map { $0 }) == testContext.recordedEventsAsJSON() + expect(pending.last?.kindField) == "summary" + expect(testContext.eventReporter.contextSummarizer.hasLoggedRequests) == false guard case .isOffline = testContext.syncResult else { fail("Expected error .isOffline result for event send") @@ -525,11 +561,19 @@ final class EventReporterSpec: QuickSpec { func testRecordFlagEvaluationEvents() { let context = LDContext.stub() let serviceMock = DarklyServiceMock() + + /// A reporter writing to a store of its own, and that store, so a test can read back what was recorded. + func makeReporter() -> (EventReporter, EventStore) { + let store = EventStore.temporary() + return (EventReporter(service: serviceMock, onSyncComplete: nil, store: store), store) + } + describe("recordFlagEvaluationEvents") { it("unknown flag") { - let reporter = EventReporter(service: serviceMock, onSyncComplete: nil) + let (reporter, store) = makeReporter() + defer { store.deleteEverything() } reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: nil, context: context, includeReason: true) - expect(reporter.eventStore.count) == 0 + expect(recordedEvents(store)).to(beEmpty()) expect(reporter.contextSummarizer.hasLoggedRequests) == true let summaries = reporter.contextSummarizer.getSummaries() expect(summaries.count) == 1 @@ -540,10 +584,11 @@ final class EventReporterSpec: QuickSpec { expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: nil, version: nil)]?.value) == "a" } it("untracked flag") { - let reporter = EventReporter(service: serviceMock, onSyncComplete: nil) + let (reporter, store) = makeReporter() + defer { store.deleteEverything() } let flag = FeatureFlag(flagKey: "unused", value: nil, variation: 1, flagVersion: 2, trackEvents: false) reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: true) - expect(reporter.eventStore.count) == 0 + expect(recordedEvents(store)).to(beEmpty()) expect(reporter.contextSummarizer.hasLoggedRequests) == true let summaries = reporter.contextSummarizer.getSummaries() expect(summaries.count) == 1 @@ -554,17 +599,13 @@ final class EventReporterSpec: QuickSpec { expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.value) == "a" } it("tracked flag") { - let reporter = EventReporter(service: serviceMock, onSyncComplete: nil) + let (reporter, store) = makeReporter() + defer { store.deleteEverything() } let flag = FeatureFlag(flagKey: "unused", value: nil, variation: 1, flagVersion: 2, trackEvents: true) reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: true) - expect(reporter.eventStore.count) == 1 - expect((reporter.eventStore[0] as? FeatureEvent)?.kind) == .feature - expect((reporter.eventStore[0] as? FeatureEvent)?.key) == "flag-key" - expect((reporter.eventStore[0] as? FeatureEvent)?.context.contextHash()) == context.contextHash() - expect((reporter.eventStore[0] as? FeatureEvent)?.value) == "a" - expect((reporter.eventStore[0] as? FeatureEvent)?.defaultValue) == "b" - expect((reporter.eventStore[0] as? FeatureEvent)?.featureFlag) == flag - expect((reporter.eventStore[0] as? FeatureEvent)?.includeReason) == true + let expected = FeatureEvent(key: "flag-key", context: context, value: "a", defaultValue: "b", featureFlag: flag, includeReason: true, isDebug: false) + expect(expectedEvents(reporter.pendingEventsForTesting)) == expectedEvents([expected]) + expect(recordedEvents(store)).to(beEmpty()) expect(reporter.contextSummarizer.hasLoggedRequests) == true let summaries = reporter.contextSummarizer.getSummaries() expect(summaries.count) == 1 @@ -575,10 +616,11 @@ final class EventReporterSpec: QuickSpec { expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.value) == "a" } it("debug until past date") { - let reporter = EventReporter(service: serviceMock, onSyncComplete: nil) + let (reporter, store) = makeReporter() + defer { store.deleteEverything() } let flag = FeatureFlag(flagKey: "unused", value: nil, variation: 1, flagVersion: 2, trackEvents: false, debugEventsUntilDate: Date().addingTimeInterval(-1.0)) reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: true) - expect(reporter.eventStore.count) == 0 + expect(recordedEvents(store)).to(beEmpty()) expect(reporter.contextSummarizer.hasLoggedRequests) == true let summaries = reporter.contextSummarizer.getSummaries() expect(summaries.count) == 1 @@ -589,17 +631,13 @@ final class EventReporterSpec: QuickSpec { expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.value) == "a" } it("debug until future date") { - let reporter = EventReporter(service: serviceMock, onSyncComplete: nil) + let (reporter, store) = makeReporter() + defer { store.deleteEverything() } let flag = FeatureFlag(flagKey: "unused", value: nil, variation: 1, flagVersion: 2, trackEvents: false, debugEventsUntilDate: Date().addingTimeInterval(3.0)) reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: false) - expect(reporter.eventStore.count) == 1 - expect((reporter.eventStore[0] as? FeatureEvent)?.kind) == .debug - expect((reporter.eventStore[0] as? FeatureEvent)?.key) == "flag-key" - expect((reporter.eventStore[0] as? FeatureEvent)?.context) == context - expect((reporter.eventStore[0] as? FeatureEvent)?.value) == "a" - expect((reporter.eventStore[0] as? FeatureEvent)?.defaultValue) == "b" - expect((reporter.eventStore[0] as? FeatureEvent)?.featureFlag) == flag - expect((reporter.eventStore[0] as? FeatureEvent)?.includeReason) == false + let expected = FeatureEvent(key: "flag-key", context: context, value: "a", defaultValue: "b", featureFlag: flag, includeReason: false, isDebug: true) + expect(expectedEvents(reporter.pendingEventsForTesting)) == expectedEvents([expected]) + expect(recordedEvents(store)).to(beEmpty()) expect(reporter.contextSummarizer.hasLoggedRequests) == true let summaries = reporter.contextSummarizer.getSummaries() expect(summaries.count) == 1 @@ -610,11 +648,12 @@ final class EventReporterSpec: QuickSpec { expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.value) == "a" } it("debug until future date earlier than service date") { - let reporter = EventReporter(service: serviceMock, onSyncComplete: nil) + let (reporter, store) = makeReporter() + defer { store.deleteEverything() } reporter.setLastEventResponseDate(Date().addingTimeInterval(10.0)) let flag = FeatureFlag(flagKey: "unused", value: nil, variation: 1, flagVersion: 2, trackEvents: false, debugEventsUntilDate: Date().addingTimeInterval(3.0)) reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: true) - expect(reporter.eventStore.count) == 0 + expect(recordedEvents(store)).to(beEmpty()) expect(reporter.contextSummarizer.hasLoggedRequests) == true let summaries = reporter.contextSummarizer.getSummaries() expect(summaries.count) == 1 @@ -625,27 +664,15 @@ final class EventReporterSpec: QuickSpec { expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.value) == "a" } it("tracked flag and debug date in future") { - let reporter = EventReporter(service: serviceMock, onSyncComplete: nil) + let (reporter, store) = makeReporter() + defer { store.deleteEverything() } reporter.setLastEventResponseDate(Date().addingTimeInterval(-3.0)) let flag = FeatureFlag(flagKey: "unused", value: nil, variation: 1, flagVersion: 2, trackEvents: true, debugEventsUntilDate: Date()) reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: false) - expect(reporter.eventStore.count) == 2 - let featureEvent = reporter.eventStore.first { $0.kind == .feature } as? FeatureEvent - let debugEvent = reporter.eventStore.first { $0.kind == .debug } as? FeatureEvent - expect(featureEvent?.kind) == .feature - expect(featureEvent?.key) == "flag-key" - expect(featureEvent?.context.contextHash()) == context.contextHash() - expect(featureEvent?.value) == "a" - expect(featureEvent?.defaultValue) == "b" - expect(featureEvent?.featureFlag) == flag - expect(featureEvent?.includeReason) == false - expect(debugEvent?.kind) == .debug - expect(debugEvent?.key) == "flag-key" - expect(debugEvent?.context) == context - expect(debugEvent?.value) == "a" - expect(debugEvent?.defaultValue) == "b" - expect(debugEvent?.featureFlag) == flag - expect(debugEvent?.includeReason) == false + let expectedFeature = FeatureEvent(key: "flag-key", context: context, value: "a", defaultValue: "b", featureFlag: flag, includeReason: false, isDebug: false) + let expectedDebug = FeatureEvent(key: "flag-key", context: context, value: "a", defaultValue: "b", featureFlag: flag, includeReason: false, isDebug: true) + expect(expectedEvents(reporter.pendingEventsForTesting)) == expectedEvents([expectedFeature, expectedDebug]) + expect(recordedEvents(store)).to(beEmpty()) expect(reporter.contextSummarizer.hasLoggedRequests) == true let summaries = reporter.contextSummarizer.getSummaries() expect(summaries.count) == 1 @@ -656,7 +683,8 @@ final class EventReporterSpec: QuickSpec { expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.value) == "a" } it("records events concurrently") { - let reporter = EventReporter(service: serviceMock, onSyncComplete: nil) + let (reporter, store) = makeReporter() + defer { store.deleteEverything() } reporter.setLastEventResponseDate(Date()) let flag = FeatureFlag(flagKey: "unused", trackEvents: true, debugEventsUntilDate: Date().addingTimeInterval(3.0)) @@ -667,9 +695,11 @@ final class EventReporterSpec: QuickSpec { } (0..<10).forEach { _ in counter.wait() } - expect(reporter.eventStore.count) == 20 - expect(reporter.eventStore.filter { $0.kind == .feature }.count) == 10 - expect(reporter.eventStore.filter { $0.kind == .debug }.count) == 10 + let held = expectedEvents(reporter.pendingEventsForTesting) + expect(held.count) == 20 + expect(held.filter { $0.kindField == "feature" }.count) == 10 + expect(held.filter { $0.kindField == "debug" }.count) == 10 + expect(recordedEvents(store)).to(beEmpty()) expect(reporter.contextSummarizer.hasLoggedRequests) == true let summaries = reporter.contextSummarizer.getSummaries() expect(summaries.count) == 1 @@ -686,7 +716,7 @@ final class EventReporterSpec: QuickSpec { describe("report timer fires") { var testContext: TestContext! afterEach { - testContext.eventReporter.isOnline = false + testContext.cleanUp() } context("with events") { beforeEach { @@ -696,7 +726,7 @@ final class EventReporterSpec: QuickSpec { } it("reports events") { expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) - expect(testContext.eventReporter.eventStore.isEmpty).toEventually(beTrue()) + expect(testContext.store.pendingEventCount).toEventually(equal(0)) expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount).toEventually(equal(1)) expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch).toEventually(equal(testContext.events.count)) let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) @@ -721,6 +751,297 @@ final class EventReporterSpec: QuickSpec { } } +extension EventReporterSpec { + /// What the SDK is able to report after the process it was recording in is gone. + /// + /// These are written against a second store reading the same directory, which is what the next run of the + /// application amounts to: it sees the bytes that reached the disk and nothing that was still in memory. + private func durabilitySpec() { + describe("surviving the process") { + var testContext: TestContext! + afterEach { + testContext.cleanUp() + } + + /// The events the next run of the application would find. + func eventsLeftOnDisk() -> [LDValue] { + let reader = EventStore(directory: testContext.store.directory, capacity: 100, logger: .disabled) + return reader.pendingEventPayloads().compactMap { try? JSONDecoder().decode(LDValue.self, from: $0) } + } + + it("has written a tracked event by the time recording it returns") { + testContext = TestContext() + + // The sequence that loses events today: something is tracked and the process ends immediately after. + testContext.eventReporter.record(CustomEvent(key: "fatal-error", context: testContext.context, data: ["message": "boom"])) + + let onDisk = eventsLeftOnDisk() + expect(onDisk.count) == 1 + expect(onDisk.first?.kindField) == "custom" + } + + it("still writes a tracked event when the write is not on the caller's thread") { + testContext = TestContext(eventPersistence: .deferred) + + testContext.eventReporter.record(CustomEvent(key: "fatal-error", context: testContext.context, data: ["message": "boom"])) + + // The promise is weaker by exactly one scheduling hop: the event reaches the disk, just not before + // recording it returned. + expect(eventsLeftOnDisk().compactMap { $0.kindField }).toEventually(equal(["custom"])) + } + + it("has written the evaluations that came before a tracked event") { + // A response date of its own, because the distant past that a fresh context defaults to also satisfies + // the debug window comparison and would add a debug event to what is asserted below. + testContext = TestContext(lastEventResponseDate: Date()) + let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: false) + + // An untracked flag is only ever reported as a summary, so this is the case where the exposure exists + // nowhere but in memory until something forces it out. + testContext.eventReporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: testContext.context, includeReason: false) + testContext.eventReporter.record(CustomEvent(key: "fatal-error", context: testContext.context, data: nil)) + + expect(eventsLeftOnDisk().compactMap { $0.kindField }) == ["custom", "summary"] + } + + it("persists feature and summary events synchronously on flush") { + testContext = TestContext(lastEventResponseDate: Date()) + let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: true) + + // The deliberate trade: an evaluation does not pay for a write, so it is only staged until something + // else commits it. + testContext.eventReporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: testContext.context, includeReason: false) + expect(eventsLeftOnDisk()).to(beEmpty()) + + testContext.eventReporter.flush(completion: nil) + expect(eventsLeftOnDisk().compactMap { $0.kindField }) == ["feature", "summary"] + } + + it("does not write on the main thread when a flag is evaluated there") { + // Suspended for the length of the evaluations, so that a write this thread was going to cause is a + // write that cannot happen: an evaluation is expected to be a memory operation, and the thread + // evaluating is usually this one. + let commits = DispatchQueue(label: "com.launchdarkly.tests.suspended") + commits.suspend() + testContext = TestContext(lastEventResponseDate: Date(), + store: EventStore.temporary(capacity: .max), + eventCapacity: .max, + commitQueue: commits) + + let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: true) + expect(Thread.isMainThread) == true + // Far past the pending threshold, so one coalesced commit is waiting. + for _ in 0..<400 { + testContext.eventReporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: testContext.context, includeReason: false) + } + + expect(eventsLeftOnDisk()).to(beEmpty()) + + commits.resume() + + expect(eventsLeftOnDisk().compactMap { $0.kindField }.filter { $0 == "feature" }.count) + .toEventually(equal(400)) + } + + it("reports the events a previous run left behind") { + testContext = TestContext() + testContext.recordEvents(2) + testContext.eventReporter.commitRecordedEvents() + + // Nothing delivered them and nothing closed the log, as would be the case had the process died here. + let recovered = EventStore(directory: testContext.store.directory, capacity: 100, logger: .disabled) + let nextRun = EventReporter(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) + nextRun.isOnline = true + + waitUntil { done in + nextRun.flush(completion: done) + } + + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) + valueIsArray(published) { events in + expect(events.count) == 2 + } + expect(recovered.pendingEventCount) == 0 + nextRun.isOnline = false + } + + it("delivers what a previous run left behind as soon as it is online, without waiting for a report interval") { + testContext = TestContext() + testContext.recordEvents(2) + testContext.eventReporter.commitRecordedEvents() + + let recovered = EventStore(directory: testContext.store.directory, capacity: 100, logger: .disabled) + let nextRun = EventReporter(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) + + // Nothing asks for a flush, and the report interval is several times longer than this waits, so the + // only thing that can deliver these is the reporter coming online. + nextRun.isOnline = true + + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) + expect(recovered.pendingEventCount).toEventually(equal(0)) + nextRun.isOnline = false + } + + it("does not bring a delivery forward when a previous run left nothing behind") { + testContext = TestContext() + + testContext.eventReporter.isOnline = true + + // Coming online is not itself a reason to send: an ordinary start has nothing waiting, and the events + // this run records keep to the report interval. + testContext.recordEvents(1) + Thread.sleep(forTimeInterval: 0.2) + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + } + + it("delivers a batch a failed run left behind under the same payload id") { + testContext = TestContext(stubResponseSuccess: false) + testContext.recordEvents(1) + + waitUntil(timeout: .seconds(10)) { done in + testContext.eventReporter.isOnline = true + testContext.eventReporter.flush(completion: done) + } + + let kept = testContext.store.pendingBatches() + expect(kept.count) == 1 + + // A payload LaunchDarkly may have already seen is retried under the identifier it was first sent with, + // which is how the events are not counted twice. + let payloadIds = Set(testContext.serviceMock.publishedPayloadIds) + expect(payloadIds.count) == 1 + expect(payloadIds.first) == kept.first?.payloadId + } + + it("does not send a batch again when a delivery is asked for while one is in flight") { + testContext = TestContext() + testContext.recordEvents(1) + testContext.eventReporter.isOnline = true + // The response is held back, so the first delivery stays in flight and its batch stays on disk -- + // the window in which a second delivery would find that batch and send it again. + testContext.serviceMock.holdsEventCompletions = true + + var firstFinished = false + testContext.eventReporter.flush { firstFinished = true } + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) + + var secondFinished = false + testContext.eventReporter.flush { secondFinished = true } + // Nothing more goes out while the first delivery is unanswered. + Thread.sleep(forTimeInterval: 0.3) + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + + testContext.serviceMock.holdsEventCompletions = false + testContext.serviceMock.releaseHeldEventCompletions() + + // The second caller is answered by the pass that runs once the first delivery is done. It finds an + // empty store, so the batch is sent once in total. + expect(firstFinished).toEventually(beTrue()) + expect(secondFinished).toEventually(beTrue()) + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + expect(testContext.store.pendingEventCount) == 0 + } + } + } + + private func flushReportingOutcomeSpec() { + describe("flush reporting outcome") { + var testContext: TestContext! + afterEach { + testContext.cleanUp() + } + + it("reports true when there is nothing to deliver") { + testContext = TestContext() + testContext.eventReporter.isOnline = true + var delivered: Bool? + waitUntil { done in + testContext.eventReporter.flushReportingOutcome { result in + delivered = result + done() + } + } + expect(delivered) == true + } + + it("reports false while offline") { + testContext = TestContext() + testContext.recordEvents(1) + var delivered: Bool? + waitUntil { done in + testContext.eventReporter.flushReportingOutcome { result in + delivered = result + done() + } + } + expect(delivered) == false + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + } + + it("reports true when LaunchDarkly accepts the batch") { + testContext = TestContext() + testContext.recordEvents(1) + testContext.eventReporter.isOnline = true + var delivered: Bool? + waitUntil { done in + testContext.eventReporter.flushReportingOutcome { result in + delivered = result + done() + } + } + expect(delivered) == true + expect(testContext.store.pendingEventCount) == 0 + } + + it("reports false when a retryable failure leaves the batch on disk") { + testContext = TestContext(stubResponseSuccess: false) + testContext.recordEvents(1) + testContext.eventReporter.isOnline = true + var delivered: Bool? + waitUntil(timeout: .seconds(10)) { done in + testContext.eventReporter.flushReportingOutcome { result in + delivered = result + done() + } + } + expect(delivered) == false + expect(testContext.store.pendingBatches().count) == 1 + } + } + } +} + +/// The events a store is holding, as JSON with `creationDate` dropped. +/// +/// Dropping the timestamp is what lets an expectation be written as the event the SDK should have produced, rather than +/// as a list of whichever fields the test remembered to check: the recorded event and the expected one are constructed +/// moments apart and differ only in when they were made. +private func recordedEvents(_ store: EventStore) -> [LDValue] { + store.pendingEventPayloads() + .compactMap { try? JSONDecoder().decode(LDValue.self, from: $0) } + .map(withoutCreationDate) +} + +private func expectedEvents(_ events: [Event]) -> [LDValue] { + events.compactMap { encodeToLDValue($0) }.map(withoutCreationDate) +} + +private func withoutCreationDate(_ value: LDValue) -> LDValue { + guard case .object(var fields) = value + else { return value } + fields["creationDate"] = nil + return .object(fields) +} + +private extension LDValue { + var kindField: String? { + guard case .object(let fields) = self, case .string(let kind) = fields["kind"] + else { return nil } + return kind + } +} + private extension Date { var adjustedForHttpUrlHeaderUse: Date { let headerDateFormatter = DateFormatter.httpUrlHeaderFormatter diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift new file mode 100644 index 000000000..3bacac5b7 --- /dev/null +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -0,0 +1,442 @@ +import Foundation +import Quick +import Nimble +@testable import LaunchDarkly + +final class EventStoreSpec: QuickSpec { + override func spec() { + stagingSpec() + recoverySpec() + corruptionSpec() + batchSpec() + capacitySpec() + fallbackSpec() + optOutSpec() + } + + private func optOutSpec() { + describe("a store the application did not ask to persist") { + var store: EventStore! + beforeEach { + store = EventStore.temporary(persistEvents: false) + } + afterEach { + store.deleteEverything() + } + + it("writes nothing to the disk, so the directory is never even created") { + _ = store.stage(EventStoreSpec.payload("held")) + store.commit() + + expect(FileManager.default.fileExists(atPath: store.directory.path)) == false + expect(EventStoreSpec.keys(of: EventStoreSpec.reader(sharing: store))).to(beEmpty()) + } + + it("delivers the events from memory, so opting out costs durability and nothing else") { + _ = store.stage(EventStoreSpec.payload("held")) + let batch = store.closeBatch()! + + expect(batch.eventCount) == 1 + expect(store.pendingEventCount) == 1 + expect(store.body(of: batch)) == Data("[{\"kind\":\"custom\",\"key\":\"held\"}]".utf8) + } + } + } + + /// A store that can never write, because the directory it was given sits underneath a regular file and so cannot + /// be created. Standing in for the full disk and the revoked permission, neither of which a test can arrange. + private static func unwritableStore(capacity: Int = 100) -> EventStore { + let blocker = FileManager.default.temporaryDirectory + .appendingPathComponent("com.launchdarkly.tests.events", isDirectory: true) + .appendingPathComponent(UUID().uuidString) + try? FileManager.default.createDirectory(at: blocker.deletingLastPathComponent(), + withIntermediateDirectories: true) + FileManager.default.createFile(atPath: blocker.path, contents: Data("not a directory".utf8)) + + return EventStore(directory: blocker.appendingPathComponent("store", isDirectory: true), + capacity: capacity, + logger: .disabled) + } + + /// A store reading the same directory as another, standing in for the next run of the application: it sees only + /// what actually reached the disk, which is the whole question a crash asks. + private static func reader(sharing store: EventStore) -> EventStore { + EventStore(directory: store.directory, capacity: 100, logger: .disabled) + } + + private static func payload(_ key: String) -> Data { + Data("{\"kind\":\"custom\",\"key\":\"\(key)\"}".utf8) + } + + private static func keys(of store: EventStore) -> [String] { + store.pendingEventPayloads().compactMap { + guard case .object(let fields) = try? JSONDecoder().decode(LDValue.self, from: $0), + case .string(let key) = fields["key"] + else { return nil } + return key + } + } + + private func stagingSpec() { + describe("staging") { + var store: EventStore! + beforeEach { + store = EventStore.temporary() + } + afterEach { + store.deleteEverything() + } + + it("keeps a staged event out of the log until it is committed") { + _ = store.stage(EventStoreSpec.payload("staged-only")) + + expect(EventStoreSpec.keys(of: EventStoreSpec.reader(sharing: store))).to(beEmpty()) + expect(store.pendingEventCount) == 1 + } + + it("puts a committed event where another process would find it") { + _ = store.stage(EventStoreSpec.payload("committed")) + store.commit() + + expect(EventStoreSpec.keys(of: EventStoreSpec.reader(sharing: store))) == ["committed"] + } + + it("commits without being asked once enough has been staged") { + // No commit point is reached and nothing is flushed, so only the staging threshold can have written these. + let event = EventStoreSpec.payload(String(repeating: "p", count: 1_024)) + for _ in 0..<32 { + _ = store.stage(event) + } + + expect(EventStoreSpec.reader(sharing: store).pendingEventPayloads()).toEventuallyNot(beEmpty()) + } + + it("does not write on the thread that staged the event") { + // Suspended, so any write this thread was going to cause is a write that cannot happen. + let commits = DispatchQueue(label: "com.launchdarkly.tests.suspended") + commits.suspend() + let deferring = EventStore.temporary(commitQueue: commits) + // Resumed in the body rather than here: a queue resumed twice traps. + defer { deferring.deleteEverything() } + + let event = EventStoreSpec.payload(String(repeating: "p", count: 1_024)) + for _ in 0..<64 { + _ = deferring.stage(event) + } + + expect(EventStoreSpec.reader(sharing: deferring).pendingEventPayloads()).to(beEmpty()) + expect(deferring.pendingEventCount) == 64 + + commits.resume() + + expect(EventStoreSpec.reader(sharing: deferring).pendingEventPayloads().count).toEventually(equal(64)) + } + + it("still writes on the caller's thread when the caller is at a commit point") { + // The same suspended queue, to show a commit point does not depend on it: `commit()` is the caller's + // guarantee that the event outlived them, so it cannot be handed to someone else. + let commits = DispatchQueue(label: "com.launchdarkly.tests.suspended") + commits.suspend() + let committing = EventStore.temporary(commitQueue: commits) + defer { + commits.resume() + committing.deleteEverything() + } + + _ = committing.stage(EventStoreSpec.payload("tracked")) + committing.commit() + + expect(EventStoreSpec.keys(of: EventStoreSpec.reader(sharing: committing))) == ["tracked"] + } + + it("keeps events in the order they were recorded") { + for index in 0..<5 { + _ = store.stage(EventStoreSpec.payload("event-\(index)")) + } + store.commit() + + expect(EventStoreSpec.keys(of: EventStoreSpec.reader(sharing: store))) == ["event-0", "event-1", "event-2", "event-3", "event-4"] + } + } + } + + private func recoverySpec() { + describe("recovering a log a previous run left open") { + var store: EventStore! + afterEach { + store.deleteEverything() + } + + it("delivers the events as a batch") { + store = EventStore.temporary() + _ = store.stage(EventStoreSpec.payload("survived")) + store.commit() + + // Nothing closed the log, as nothing would if the process had died here. + let next = EventStoreSpec.reader(sharing: store) + next.recoverInterruptedLog() + + let batches = next.pendingBatches() + expect(batches.count) == 1 + expect(batches.first?.eventCount) == 1 + expect(next.body(of: batches[0])) == Data("[{\"kind\":\"custom\",\"key\":\"survived\"}]".utf8) + } + + it("leaves nothing behind when the log held no events") { + store = EventStore.temporary() + _ = store.stage(EventStoreSpec.payload("only")) + store.commit() + let batch = store.closeBatch() + store.remove(batch!) + + let next = EventStoreSpec.reader(sharing: store) + next.recoverInterruptedLog() + + expect(next.pendingBatches()).to(beEmpty()) + expect(next.pendingEventCount) == 0 + } + } + } + + private func corruptionSpec() { + describe("a damaged log") { + var store: EventStore! + afterEach { + store.deleteEverything() + } + + it("keeps every event before a frame the writer did not finish") { + store = EventStore.temporary() + for index in 0..<3 { + _ = store.stage(EventStoreSpec.payload("event-\(index)")) + } + store.commit() + + // A process killed partway through a write leaves the last frame short of its declared length. + let log = store.directory.appendingPathComponent("current") + let whole = try Data(contentsOf: log) + try whole.dropLast(8).write(to: log) + + expect(EventStoreSpec.keys(of: EventStoreSpec.reader(sharing: store))) == ["event-0", "event-1"] + } + + it("is discarded when it was written in a format this version does not read") { + store = EventStore.temporary() + _ = store.stage(EventStoreSpec.payload("unreadable")) + store.commit() + _ = store.closeBatch() + + let batchUrl = try FileManager.default + .contentsOfDirectory(at: store.directory, includingPropertiesForKeys: nil) + .first { $0.lastPathComponent.hasPrefix("ready-") } + var corrupted = try Data(contentsOf: batchUrl!) + corrupted[4] = 0xFF + corrupted[5] = 0xFF + try corrupted.write(to: batchUrl!) + + let next = EventStoreSpec.reader(sharing: store) + expect(next.pendingBatches()).to(beEmpty()) + // A batch that can never be delivered is not left to occupy the device either. + expect(FileManager.default.fileExists(atPath: batchUrl!.path)) == false + } + } + } + + private func batchSpec() { + describe("batches") { + var store: EventStore! + beforeEach { + store = EventStore.temporary() + } + afterEach { + store.deleteEverything() + } + + it("assembles the events into one request body") { + _ = store.stage(EventStoreSpec.payload("first")) + _ = store.stage(EventStoreSpec.payload("second")) + + let batch = store.closeBatch() + expect(batch?.eventCount) == 2 + + let body = store.body(of: batch!) + expect(body) == Data("[{\"kind\":\"custom\",\"key\":\"first\"},{\"kind\":\"custom\",\"key\":\"second\"}]".utf8) + } + + it("has nothing to close when nothing was recorded") { + expect(store.closeBatch()).to(beNil()) + } + + it("holds a batch until it is removed, so a failed delivery can be retried") { + _ = store.stage(EventStoreSpec.payload("retry-me")) + let batch = store.closeBatch()! + + expect(store.pendingBatches()) == [batch] + expect(store.pendingEventCount) == 1 + expect(store.body(of: batch)).toNot(beNil()) + + store.remove(batch) + + expect(store.pendingBatches()).to(beEmpty()) + expect(store.pendingEventCount) == 0 + } + + it("counts a batch it closed without reading the file back") { + _ = store.stage(EventStoreSpec.payload("first")) + _ = store.stage(EventStoreSpec.payload("second")) + let batch = store.closeBatch()! + + // Nothing appends to a batch once it is closed, so the count taken at the close still holds and + // listing it has no reason to go to the disk. Damaging the file is how the test tells the two apart: + // a listing that read would refuse this batch, which is exactly what the store below does. + let batchUrl = store.directory.appendingPathComponent("ready-\(batch.payloadId)") + try Data("not an event log".utf8).write(to: batchUrl) + + expect(store.pendingBatches()) == [batch] + + // A store that did not close it has no count to go on, so it reads, and finds the damage. + expect(EventStoreSpec.reader(sharing: store).pendingBatches()).to(beEmpty()) + } + + it("gives a batch its own payload id, so a retry is not read as a new delivery") { + _ = store.stage(EventStoreSpec.payload("first")) + let first = store.closeBatch()! + _ = store.stage(EventStoreSpec.payload("second")) + let second = store.closeBatch()! + + expect(first.payloadId).toNot(equal(second.payloadId)) + } + + it("keeps recording into a fresh log after one is closed") { + _ = store.stage(EventStoreSpec.payload("before")) + _ = store.closeBatch() + _ = store.stage(EventStoreSpec.payload("after")) + store.commit() + + expect(EventStoreSpec.keys(of: store)) == ["before", "after"] + expect(store.pendingEventCount) == 2 + } + } + } + + private func capacitySpec() { + describe("capacity") { + var store: EventStore! + afterEach { + store.deleteEverything() + } + + it("refuses events once it is reached") { + store = EventStore.temporary(capacity: 2) + + expect(store.stage(EventStoreSpec.payload("first"))) == true + expect(store.stage(EventStoreSpec.payload("second"))) == true + expect(store.stage(EventStoreSpec.payload("third"))) == false + expect(store.pendingEventCount) == 2 + } + + it("counts the events in a batch that has not been delivered yet") { + store = EventStore.temporary(capacity: 1) + + expect(store.stage(EventStoreSpec.payload("first"))) == true + _ = store.closeBatch() + + expect(store.stage(EventStoreSpec.payload("second"))) == false + } + + it("takes a summary even when it is full, since those evaluations were already counted") { + store = EventStore.temporary(capacity: 1) + + expect(store.stage(EventStoreSpec.payload("first"))) == true + expect(store.stage(EventStoreSpec.payload("summary"), bypassingCapacity: true)) == true + expect(store.pendingEventCount) == 2 + } + } + } + + /// Losing the disk should cost durability and nothing else. These are the events an application would otherwise + /// have reported before any of this existed, so the store keeps delivering them from memory. + private func fallbackSpec() { + describe("a store the filesystem will not take") { + var store: EventStore! + beforeEach { + store = EventStoreSpec.unwritableStore() + } + afterEach { + // The blocking file, which is the parent of the directory the store never got to create. + try? FileManager.default.removeItem(at: store.directory.deletingLastPathComponent()) + } + + it("delivers the events from memory rather than dropping them") { + _ = store.stage(EventStoreSpec.payload("first")) + _ = store.stage(EventStoreSpec.payload("second")) + store.commit() + + // The commit had nowhere to go, so the events are still the store's to deliver. + expect(store.pendingEventCount) == 2 + + let batch = store.closeBatch() + expect(batch?.eventCount) == 2 + expect(store.body(of: batch!)) + == Data("[{\"kind\":\"custom\",\"key\":\"first\"},{\"kind\":\"custom\",\"key\":\"second\"}]".utf8) + } + + it("produces the same body a batch on disk would have") { + let written = EventStore.temporary() + defer { written.deleteEverything() } + _ = written.stage(EventStoreSpec.payload("same")) + let onDisk = written.closeBatch()! + + _ = store.stage(EventStoreSpec.payload("same")) + let held = store.closeBatch()! + + expect(store.body(of: held)) == written.body(of: onDisk) + } + + it("keeps accepting events after persistence has been given up on") { + _ = store.stage(EventStoreSpec.payload("before")) + store.commit() + + expect(store.stage(EventStoreSpec.payload("after"))) == true + } + + it("keeps the events in the order they were recorded across a failed commit") { + _ = store.stage(EventStoreSpec.payload("first")) + store.commit() + _ = store.stage(EventStoreSpec.payload("second")) + + let batch = store.closeBatch()! + expect(store.body(of: batch)) + == Data("[{\"kind\":\"custom\",\"key\":\"first\"},{\"kind\":\"custom\",\"key\":\"second\"}]".utf8) + } + + it("holds a batch until it is removed, so a failed delivery can be retried") { + _ = store.stage(EventStoreSpec.payload("retry-me")) + let batch = store.closeBatch()! + + expect(store.pendingBatches()) == [batch] + expect(store.pendingEventCount) == 1 + + store.remove(batch) + + expect(store.pendingBatches()).to(beEmpty()) + expect(store.pendingEventCount) == 0 + } + + it("still refuses events once capacity is reached, so memory stays bounded") { + let bounded = EventStoreSpec.unwritableStore(capacity: 2) + defer { try? FileManager.default.removeItem(at: bounded.directory.deletingLastPathComponent()) } + + expect(bounded.stage(EventStoreSpec.payload("first"))) == true + bounded.commit() + expect(bounded.stage(EventStoreSpec.payload("second"))) == true + expect(bounded.stage(EventStoreSpec.payload("third"))) == false + expect(bounded.pendingEventCount) == 2 + } + + it("has nothing to close when nothing was recorded") { + expect(store.closeBatch()).to(beNil()) + } + } + } +} diff --git a/LaunchDarkly/LaunchDarklyTests/TestUtil.swift b/LaunchDarkly/LaunchDarklyTests/TestUtil.swift index 523fa82a7..2e9d1892d 100644 --- a/LaunchDarkly/LaunchDarklyTests/TestUtil.swift +++ b/LaunchDarkly/LaunchDarklyTests/TestUtil.swift @@ -48,3 +48,27 @@ func valueIsArray(_ value: LDValue?, asserts: ([LDValue]) -> Void) { } asserts(arr) } + +extension EventStore { + /// A store in a directory of its own under the temporary directory, so that tests neither see each other's events + /// nor leave any in the directory a real client would use. + static func temporary( + capacity: Int = 100, + persistEvents: Bool = true, + commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.tests.commitQueue") + ) -> EventStore { + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("com.launchdarkly.tests.events", isDirectory: true) + .appendingPathComponent(UUID().uuidString, isDirectory: true) + return EventStore(directory: directory, + capacity: capacity, + persistEvents: persistEvents, + logger: .disabled, + commitQueue: commitQueue) + } + + /// Removes the store's directory, including any batch still waiting to be delivered. + func deleteEverything() { + try? FileManager.default.removeItem(at: directory) + } +} From 1307ed403bc02fcb9c7f815f4cc1237ead0a2b61 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 24 Sep 2026 16:19:53 -0700 Subject: [PATCH 002/158] test: point the unserializable-event specs at the event store Tier 3 stages encoded bytes into the store instead of holding events in memory, so the assertion that nothing was buffered has to ask the store. --- .../ServiceObjects/EventReporterSpec.swift | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift index c7b2dabeb..220048d23 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift @@ -240,6 +240,7 @@ final class EventReporterSpec: QuickSpec { var serviceMock: DarklyServiceMock! var ldContext: LDContext! var reporter: EventReporter! + var store: EventStore! func makeReporter(encoding: EventReporter.Encoding) -> EventReporter { var config = LDConfig.stub @@ -248,7 +249,8 @@ final class EventReporterSpec: QuickSpec { serviceMock = DarklyServiceMock() serviceMock.config = config serviceMock.stubEventResponse(success: true) - return EventReporter(service: serviceMock, onSyncComplete: nil, encoding: encoding) + store = EventStore.temporary(capacity: config.eventCapacity) + return EventReporter(service: serviceMock, onSyncComplete: nil, store: store, encoding: encoding) } afterEach { @@ -266,7 +268,7 @@ final class EventReporterSpec: QuickSpec { reporter.flush(completion: done) } expect(serviceMock.publishEventDataCallCount) == 0 - expect(reporter.eventStore.isEmpty) == true + expect(store.pendingEventCount) == 0 waitUntil { done in reporter.record(CustomEvent(key: "after-poison", context: ldContext, metricValue: 1.0)) @@ -348,7 +350,7 @@ final class EventReporterSpec: QuickSpec { reporter.flush(completion: done) } expect(serviceMock.publishEventDataCallCount) == 0 - expect(reporter.eventStore.isEmpty) == true + expect(store.pendingEventCount) == 0 expect(reporter.contextSummarizer.hasLoggedRequests) == false waitUntil { done in From 2c46b87502d238461db21d570cbfdda6bbeb6664 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 29 Sep 2026 14:39:35 -0700 Subject: [PATCH 003/158] Encode held events only where they are persisted; record evaluations atomically; reserve capacity during encode - Without persistence, nothing but a delivery drains the held run: no threshold commit, and track, identify, flush and backgrounding no longer encode on the caller's thread. - An evaluation's summary counter and its full/debug events are recorded under pendingLock, and a commit takes both under it. - A commit reserves capacity for the run it takes, so events being encoded still count against eventCapacity. Co-authored-by: Cursor --- .../GeneratedCode/mocks.generated.swift | 7 + LaunchDarkly/LaunchDarkly/LDClient.swift | 2 +- .../ServiceObjects/EventReporter.swift | 203 +++++++++++------- .../ServiceObjects/EventStore.swift | 57 ++++- .../ServiceObjects/EventReporterSpec.swift | 145 +++++++++++++ 5 files changed, 332 insertions(+), 82 deletions(-) diff --git a/LaunchDarkly/GeneratedCode/mocks.generated.swift b/LaunchDarkly/GeneratedCode/mocks.generated.swift index b52960c84..cbdada187 100644 --- a/LaunchDarkly/GeneratedCode/mocks.generated.swift +++ b/LaunchDarkly/GeneratedCode/mocks.generated.swift @@ -237,6 +237,13 @@ final class EventReportingMock: EventReporting { commitRecordedEventsCallCount += 1 try! commitRecordedEventsCallback?() } + + var commitAtCommitPointCallCount = 0 + var commitAtCommitPointCallback: (() throws -> Void)? + func commitAtCommitPoint() { + commitAtCommitPointCallCount += 1 + try! commitAtCommitPointCallback?() + } } // MARK: - FeatureFlagCachingMock diff --git a/LaunchDarkly/LaunchDarkly/LDClient.swift b/LaunchDarkly/LaunchDarkly/LDClient.swift index 5c2919615..3b99a0734 100644 --- a/LaunchDarkly/LaunchDarkly/LDClient.swift +++ b/LaunchDarkly/LaunchDarkly/LDClient.swift @@ -265,7 +265,7 @@ public class LDClient { os_log("%s", log: config.logger, type: .debug, typeName(and: #function)) // A backgrounded application is one the OS may kill without warning, so whatever it has recorded is made // durable now rather than waiting for the next report interval to come around. - eventReporter.commitRecordedEvents() + eventReporter.commitAtCommitPoint() // The commit above makes the events survivable; this tries to make surviving unnecessary. A backgrounded // process is suspended as soon as it goes idle, so a delivery started here reaches the network only inside an // activity assertion. If it does not get out, the bytes are on disk and the next launch sends them. diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift index c12eb75af..795921da0 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift @@ -27,6 +27,10 @@ protocol EventReporting { /// The SDK does this itself at the points where an application is most likely to be about to die. It is worth /// calling directly before deliberately terminating the process. func commitRecordedEvents() + + /// Commits as a commit point does: before returning at `.immediate`, on a background queue at `.deferred`, and not + /// at all where nothing is persisted, where a commit would only move the encode earlier. + func commitAtCommitPoint() } class NullEventReporter: EventReporting { @@ -49,6 +53,9 @@ class NullEventReporter: EventReporting { func commitRecordedEvents() { } + + func commitAtCommitPoint() { + } } /// Records analytics events and delivers them to LaunchDarkly. @@ -70,6 +77,7 @@ class NullEventReporter: EventReporting { /// /// Whether a commit point runs on the caller's thread is the application's choice, through /// `LDConfig.eventPersistence`. Only at `.immediate` can `track` promise the event is on disk by the time it returns. +/// At `.disabled` nothing commits early: the held run waits for the delivery, which encodes it. class EventReporter: EventReporting { /// How many full events may be held unencoded before one of them pays for a commit. /// @@ -97,14 +105,9 @@ class EventReporter: EventReporting { let service: DarklyServiceProvider let store: EventStoring - /// Guards the summarizer and the last response date. - /// - /// This is a lock rather than a queue because it is taken once per evaluation, and a `DispatchQueue.sync` costs - /// enough at that rate to be worth avoiding: the same measurement that made `EvaluationExposureDeduper` a lock - /// applies here. It is never held across encoding an event or across a syscall, so an evaluation contends only with - /// another evaluation's bookkeeping. + /// Guards the last response date. private let stateLock = UnfairLock() - /// Only to be used while holding `stateLock`. + /// Only to be used while holding `pendingLock`. private(set) var contextSummarizer: ContextSummarizer /// Only to be used while holding `stateLock`. private var responseDate: Date @@ -120,7 +123,14 @@ class EventReporter: EventReporting { /// Taken before `pendingLock` and before anything the store locks, never after. private let commitLock = UnfairLock() - /// Guards the held events. Held only long enough to append one or to hand the run over, never across the encoder. + /// Guards everything one recording writes: the held events and the summarizer. + /// + /// One evaluation can produce a counter, a full event and a debug event, and the three have to land together. Taken + /// separately, a commit landing between them splits one evaluation across two payloads. + /// + /// Held for the appends and the handover of the run, never across the encoder or a syscall. It is a lock rather + /// than a queue because it is taken once per evaluation, and a `DispatchQueue.sync` costs enough at that rate to be + /// worth avoiding: the same measurement that made `EvaluationExposureDeduper` a lock applies here. private let pendingLock = UnfairLock() /// Full events recorded but not yet encoded. Only to be used while holding `pendingLock`. @@ -221,10 +231,18 @@ class EventReporter: EventReporting { // MARK: Recording func record(_ event: Event) { - hold(event) + pendingLock.lock() + let dropped = !holdHoldingPendingLock(event) + let needsCommit = pending.count >= EventReporter.pendingCommitThreshold + pendingLock.unlock() + if dropped { + reportDropped() + } if EventReporter.isCommitPoint(event.kind) { commitAtCommitPoint() + } else if needsCommit { + scheduleCommitWherePersisting() } } @@ -232,57 +250,69 @@ class EventReporter: EventReporting { func recordFlagEvaluationEvents(flagKey: LDFlagKey, value: LDValue, defaultValue: LDValue, featureFlag: FeatureFlag?, context: LDContext, includeReason: Bool) { let recordingFeatureEvent = featureFlag?.trackEvents == true let recordingDebugEvent = featureFlag?.shouldCreateDebugEvents(lastEventReportResponseTime: lastEventResponseDate) ?? false + // Built before the lock is taken, so that the critical section is only the writes. + let featureEvent = recordingFeatureEvent ? FeatureEvent(key: flagKey, context: context, value: value, defaultValue: defaultValue, featureFlag: featureFlag, includeReason: includeReason, isDebug: false) : nil + let debugEvent = recordingDebugEvent ? FeatureEvent(key: flagKey, context: context, value: value, defaultValue: defaultValue, featureFlag: featureFlag, includeReason: includeReason, isDebug: true) : nil - stateLock.lock() + var dropped = 0 + pendingLock.lock() contextSummarizer.trackRequest(flagKey: flagKey, reportedValue: value, featureFlag: featureFlag, defaultValue: defaultValue, context: context) - stateLock.unlock() + for event in [featureEvent, debugEvent].compactMap({ $0 }) where !holdHoldingPendingLock(event) { + dropped += 1 + } + let needsCommit = pending.count >= EventReporter.pendingCommitThreshold + pendingLock.unlock() - if recordingFeatureEvent { - hold(FeatureEvent(key: flagKey, context: context, value: value, defaultValue: defaultValue, featureFlag: featureFlag, includeReason: includeReason, isDebug: false)) + for _ in 0.. Bool { guard pending.count + store.pendingEventCount < capacity - else { - pendingLock.unlock() - os_log("%s aborted. Event store is full", log: service.config.logger, type: .debug, typeName(and: #function)) - service.diagnosticCache?.incrementDroppedEventCount() - return - } + else { return false } pending.append(event) - let needsCommit = pending.count >= EventReporter.pendingCommitThreshold - pendingLock.unlock() + return true + } - if needsCommit { - scheduleCommit() - } + private func reportDropped() { + os_log("%s aborted. Event store is full", log: service.config.logger, type: .debug, typeName(and: #function)) + service.diagnosticCache?.incrementDroppedEventCount() } /// Commits at a commit point, on the caller's thread or off it as the application asked. /// /// Committing on the caller's thread is what lets `track` promise its event is on disk by the time it returns. /// Scheduling it instead keeps the encode and the write off that thread, and the event is durable a moment later - /// rather than immediately — which is nothing at all when persistence is off, since there is no disk for an early - /// commit to reach. - private func commitAtCommitPoint() { + /// rather than immediately. + func commitAtCommitPoint() { if service.config.eventPersistence == .immediate { commitRecordedEvents() } else { + scheduleCommitWherePersisting() + } + } + + /// Queues a commit where there is a disk for it to reach. + /// + /// Without persistence a commit makes nothing durable. All it would do is move the encode into the middle of the + /// application's evaluations, where it competes with them for the CPU and splits their counters across summaries; + /// left alone, the run waits for the delivery, which encodes it anyway, and capacity still bounds how much is held. + private func scheduleCommitWherePersisting() { + if store.isPersisting { scheduleCommit() } } @@ -311,37 +341,47 @@ class EventReporter: EventReporting { } } + /// Encodes everything recorded since the last commit, stages it, and commits it. + /// + /// The run and the counters are taken in one critical section, so that an evaluation's counter and its full event + /// are staged by the same commit, and encoded outside it, so recording does not wait on the encoder. func commitRecordedEvents() { commitLock.lock() defer { commitLock.unlock() } - stagePendingEvents() - stageSummaries() + pendingLock.lock() + let run = pending + pending = [] + let summaries = contextSummarizer.hasLoggedRequests ? contextSummarizer.getSummaries() : [] + if !summaries.isEmpty { + contextSummarizer.clear() + } + // Reserved in the same critical section the run leaves `pending` in, so capacity counts it in one place or the + // other throughout the encode. + store.reserve(run.count) + pendingLock.unlock() + + stage(run) + store.releaseReservations() + stageSummaries(summaries) store.commit() } /// Encodes the held events as one run and stages the bytes. /// - /// The run is taken under `pendingLock` and encoded outside it, so recording does not wait on the encoder. Staging - /// bypasses capacity because the decision to keep these events was made in `hold`, and refusing them here would - /// drop events the SDK has already counted as accepted. + /// Staging uses the reservation taken with the run rather than checking capacity, because the decision to keep + /// these events was made when they were held, and refusing them here would drop events the SDK has already counted + /// as accepted. /// /// Requires `commitLock`: two threads draining separate runs would stage them in whichever order they finished /// encoding, which is not the order they were recorded in. - private func stagePendingEvents() { - pendingLock.lock() - if pending.isEmpty { - pendingLock.unlock() - return - } - let run = pending - pending = [] - pendingLock.unlock() - + private func stage(_ run: [Event]) { for event in run { guard let encoded = encode(event) else { continue } - stage(encoded, bypassingCapacity: true) + if !store.stageReserved(encoded) { + reportDropped() + } } } @@ -356,36 +396,22 @@ class EventReporter: EventReporting { } } - private func stage(_ encodedEvent: Data, bypassingCapacity: Bool = false) { - guard store.stage(encodedEvent, bypassingCapacity: bypassingCapacity) - else { - os_log("%s aborted. Event store is full", log: service.config.logger, type: .debug, typeName(and: #function)) - service.diagnosticCache?.incrementDroppedEventCount() - return - } - } - - /// Turns the evaluations counted so far into summary events and stages them, leaving the counters empty. + /// Turns the counters taken with the run into summary events and stages them. /// /// Counters live only in memory, so a crash takes whatever has not been summarized with it. Summarizing at each /// commit point rather than only at a delivery is what bounds that loss, and it costs nothing in accuracy: /// LaunchDarkly sums the counters of every summary it receives, so a session that produced several summaries is /// counted the same as one that produced a single summary covering the same evaluations. - private func stageSummaries() { - stateLock.lock() - let summaries = contextSummarizer.hasLoggedRequests ? contextSummarizer.getSummaries() : [] - if !summaries.isEmpty { - contextSummarizer.clear() - } - stateLock.unlock() - + private func stageSummaries(_ summaries: [(tracker: FlagRequestTracker, context: LDContext)]) { for summary in summaries { let summaryEvent = SummaryEvent(flagRequestTracker: summary.tracker, context: summary.context) guard let encoded = encode(summaryEvent) else { continue } // Summaries are an aggregate of evaluations that were already counted, so refusing one for capacity would // lose evaluations the SDK promised to report rather than shed new load. - stage(encoded, bypassingCapacity: true) + if !store.stage(encoded, bypassingCapacity: true) { + reportDropped() + } } } @@ -428,9 +454,9 @@ class EventReporter: EventReporting { } func flushReportingOutcome(completion: @escaping FlushOutcomeClosure) { - // Flush is a commit point: everything accepted before this call must be on disk before control returns, - // even when delivery cannot run because the client is offline. - commitRecordedEvents() + // Flush is a commit point, so at `.immediate` everything accepted before this call is on disk before control + // returns, even when delivery cannot run because the client is offline. + commitAtCommitPoint() deliveryQueue.async { self.reportEvents(completion: completion) } @@ -443,8 +469,8 @@ class EventReporter: EventReporting { private func reportEvents(completion: FlushOutcomeClosure?) { guard !isDelivering else { - // Everything this caller recorded is already committed, so one pass after the current delivery finishes - // covers it. Starting one now would only re-send what is still in flight. + // The pass that runs once the current delivery finishes commits whatever this caller recorded, so it + // covers them. Starting one now would only re-send what is still in flight. hasWaitingRequest = true if let completion { waitingCompletions.append(completion) @@ -648,14 +674,17 @@ extension EventReporter: TypeIdentifying { } private final class NullEventStore: EventStoring { private let lock = UnfairLock() private var staged: [Data] = [] + private var reserved = 0 private var closed: [String: [Data]] = [:] var pendingEventCount: Int { lock.lock() defer { lock.unlock() } - return staged.count + closed.values.reduce(0) { $0 + $1.count } + return reserved + staged.count + closed.values.reduce(0) { $0 + $1.count } } + var isPersisting: Bool { false } + func stage(_ encodedEvent: Data, bypassingCapacity: Bool) -> Bool { lock.lock() defer { lock.unlock() } @@ -663,6 +692,26 @@ private final class NullEventStore: EventStoring { return true } + func reserve(_ events: Int) { + lock.lock() + defer { lock.unlock() } + reserved += events + } + + func stageReserved(_ encodedEvent: Data) -> Bool { + lock.lock() + defer { lock.unlock() } + reserved = max(0, reserved - 1) + staged.append(encodedEvent) + return true + } + + func releaseReservations() { + lock.lock() + defer { lock.unlock() } + reserved = 0 + } + func commit() { } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index ea2a296f2..c17c0501c 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -19,15 +19,28 @@ struct EventBatch: Equatable { } protocol EventStoring { - /// Encoded events that have been recorded but not yet accepted by LaunchDarkly, whether they are still staged in - /// memory, written to the open log, or sitting in a closed batch. + /// Events that have been recorded but not yet accepted by LaunchDarkly, whether they are reserved while being + /// encoded, staged in memory, written to the open log, or sitting in a closed batch. var pendingEventCount: Int { get } + /// Whether a commit writes anything, which is false where persistence is off and once a write has failed and the + /// store has given up on it. + var isPersisting: Bool { get } + /// Stages an already encoded event, returning false if it was dropped because `capacity` is reached. /// /// Staging is a copy into a buffer; `commit()` is what makes the event outlive the process. func stage(_ encodedEvent: Data, bypassingCapacity: Bool) -> Bool + /// Counts events that have been accepted but are still being encoded, so capacity sees them until they are staged. + func reserve(_ events: Int) + + /// Stages an event using up one reservation, rather than checking capacity again. + func stageReserved(_ encodedEvent: Data) -> Bool + + /// Gives back every reservation not used up by `stageReserved(_:)`, such as one for an event that failed to encode. + func releaseReservations() + /// Hands every staged byte to the kernel, so that the events recorded so far survive the process dying. func commit() @@ -88,6 +101,7 @@ final class EventStore: EventStoring { /// Only to be used while holding `bufferLock`. private var bufferData = Data() private var bufferedEventCount = 0 + private var reservedEvents = 0 private var committedEvents = 0 private var closedEvents = 0 /// Whether events are being written to disk, which is what the application asked for until a write fails and the @@ -172,24 +186,59 @@ final class EventStore: EventStoring { var pendingEventCount: Int { bufferLock.lock() defer { bufferLock.unlock() } - return bufferedEventCount + committedEvents + closedEvents + return reservedEvents + bufferedEventCount + committedEvents + closedEvents + } + + var isPersisting: Bool { + bufferLock.lock() + defer { bufferLock.unlock() } + return persistEvents + } + + func reserve(_ events: Int) { + bufferLock.lock() + reservedEvents += events + bufferLock.unlock() + } + + func releaseReservations() { + bufferLock.lock() + reservedEvents = 0 + bufferLock.unlock() } func stage(_ encodedEvent: Data, bypassingCapacity: Bool = false) -> Bool { + stage(encodedEvent, bypassingCapacity: bypassingCapacity, usingReservation: false) + } + + func stageReserved(_ encodedEvent: Data) -> Bool { + stage(encodedEvent, bypassingCapacity: true, usingReservation: true) + } + + private func stage(_ encodedEvent: Data, bypassingCapacity: Bool, usingReservation: Bool) -> Bool { guard encodedEvent.count <= EventLogFormat.maxFrameSize else { os_log("%s dropping an event larger than a log frame allows", log: logger, type: .debug, typeName(and: #function)) + if usingReservation { + bufferLock.lock() + reservedEvents = max(0, reservedEvents - 1) + bufferLock.unlock() + } return false } bufferLock.lock() - guard bypassingCapacity || bufferedEventCount + committedEvents + closedEvents < capacity + guard bypassingCapacity || reservedEvents + bufferedEventCount + committedEvents + closedEvents < capacity else { bufferLock.unlock() return false } + // In the same critical section as the append, so the event is never counted twice or not at all. + if usingReservation { + reservedEvents = max(0, reservedEvents - 1) + } bufferData.append(EventLogFormat.frame(for: encodedEvent)) bufferedEventCount += 1 // Nothing to schedule once persistence has been given up on: there is nowhere for a commit to put these bytes, diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift index d8fe542e3..ded3d4f52 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift @@ -97,6 +97,7 @@ final class EventReporterSpec: QuickSpec { unserializableEventSpec() reportTimerSpec() durabilitySpec() + commitSpec() flushReportingOutcomeSpec() } @@ -1140,6 +1141,122 @@ extension EventReporterSpec { } } + private func commitSpec() { + describe("committing held events") { + var testContext: TestContext! + afterEach { + testContext?.cleanUp() + testContext = nil + } + + it("leaves held events for the delivery to encode when they are not persisted") { + let commits = DispatchQueue(label: "com.launchdarkly.tests.commits") + testContext = TestContext(lastEventResponseDate: Date(), + store: EventStore.temporary(capacity: .max, persistEvents: false), + eventCapacity: .max, + eventPersistence: .disabled, + commitQueue: commits) + let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: true) + + // Past the pending threshold and then a commit point: either would commit where events are persisted. + for _ in 0..<100 { + testContext.eventReporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: testContext.context, includeReason: false) + } + testContext.eventReporter.record(CustomEvent(key: "custom", context: testContext.context)) + testContext.eventReporter.flush(completion: nil) + commits.sync { } + + // Encoding them early would buy nothing, since a commit here writes nothing to disk. + expect(testContext.store.pendingEventCount) == 0 + + testContext.eventReporter.isOnline = true + waitUntil { done in + testContext.eventReporter.flush(completion: done) + } + let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) + valueIsArray(published) { events in + expect(events.compactMap { $0.kindField }.filter { $0 == "feature" }.count) == 100 + expect(events.compactMap { $0.kindField }.filter { $0 != "feature" }) == ["custom", "summary"] + } + } + + it("commits an evaluation's counter together with its full event") { + testContext = TestContext(lastEventResponseDate: Date(), + store: EventStore.temporary(capacity: .max), + eventCapacity: .max) + let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: true) + let reporter = testContext.eventReporter! + let context = testContext.context! + + let recorded = DispatchSemaphore(value: 0) + DispatchQueue.global().async { + for _ in 0..<2_000 { + reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: false) + } + recorded.signal() + } + while recorded.wait(timeout: .now()) == .timedOut { + reporter.commitRecordedEvents() + } + reporter.commitRecordedEvents() + + // A commit stages its run and then its summary, so each summary has to count exactly the feature + // events since the one before it; a commit that caught an evaluation half recorded would not. + var featuresSinceSummary = 0 + var summaries = 0 + for event in recordedEvents(testContext.store) { + switch event.kindField { + case "feature": + featuresSinceSummary += 1 + case "summary": + summaries += 1 + expect(summaryCount(event)) == featuresSinceSummary + featuresSinceSummary = 0 + default: + break + } + } + expect(featuresSinceSummary) == 0 + expect(summaries) > 1 + } + + it("counts a run toward capacity while it is being encoded") { + var config = LDConfig.stub + config.eventCapacity = 1 + config.eventPersistence = .deferred + let serviceMock = DarklyServiceMock() + serviceMock.config = config + let diagnosticCache = DiagnosticCachingMock() + serviceMock.diagnosticCache = diagnosticCache + let store = EventStore.temporary(capacity: config.eventCapacity) + defer { store.deleteEverything() } + let commits = DispatchQueue(label: "com.launchdarkly.tests.suspended") + commits.suspend() + defer { commits.resume() } + let reporter = EventReporter(service: serviceMock, onSyncComplete: nil, store: store, encoding: .codable, commitQueue: commits) + + let slow = EncodingGate(key: "slow", context: LDContext.stub()) + reporter.record(slow) + let committed = DispatchSemaphore(value: 0) + DispatchQueue.global().async { + reporter.commitRecordedEvents() + committed.signal() + } + expect(slow.started.wait(timeout: .now() + 5)) == .success + + // Neither held nor staged at this moment, the slow event is only in the commit encoding it. + reporter.record(CustomEvent(key: "late", context: LDContext.stub())) + + slow.proceed.signal() + expect(committed.wait(timeout: .now() + 5)) == .success + reporter.commitRecordedEvents() + + expect(store.pendingEventCount) == 1 + expect(diagnosticCache.incrementDroppedEventCountCallCount) == 1 + } + } + } + private func flushReportingOutcomeSpec() { describe("flush reporting outcome") { var testContext: TestContext! @@ -1218,6 +1335,34 @@ private func recordedEvents(_ store: EventStore) -> [LDValue] { .map(withoutCreationDate) } +/// How many evaluations a summary event counts, across all of its flags and variations. +private func summaryCount(_ summary: LDValue) -> Int { + guard case .object(let fields) = summary, case .object(let features)? = fields["features"] + else { return 0 } + var total = 0 + for case .object(let flag) in features.values { + guard case .array(let counters)? = flag["counters"] else { continue } + for case .object(let counter) in counters { + if case .number(let count)? = counter["count"] { + total += Int(count) + } + } + } + return total +} + +/// A custom event whose encoding waits to be let through, so a test can act while a commit is encoding it. +private final class EncodingGate: CustomEvent { + let started = DispatchSemaphore(value: 0) + let proceed = DispatchSemaphore(value: 0) + + override func encode(to encoder: Encoder) throws { + started.signal() + _ = proceed.wait(timeout: .now() + 5) + try super.encode(to: encoder) + } +} + private func expectedEvents(_ events: [Event]) -> [LDValue] { events.compactMap { encodeToLDValue($0) }.map(withoutCreationDate) } From 4bac99396ed907f74be9c039648b0693bb87ce09 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Wed, 7 Oct 2026 14:44:57 -0700 Subject: [PATCH 004/158] tier 3 fixes --- LaunchDarkly/LaunchDarkly/LDClient.swift | 23 +++-- .../ServiceObjects/EventReporter.swift | 17 +++- .../LaunchDarklyTests/LDClientSpec.swift | 4 + .../ServiceObjects/EventReporterSpec.swift | 89 +++++++++++++++++++ 4 files changed, 122 insertions(+), 11 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/LDClient.swift b/LaunchDarkly/LaunchDarkly/LDClient.swift index 421b461c3..1a20e4af3 100644 --- a/LaunchDarkly/LaunchDarkly/LDClient.swift +++ b/LaunchDarkly/LaunchDarkly/LDClient.swift @@ -1040,7 +1040,7 @@ public class LDClient { service = self.serviceFactory.makeDarklyServiceProvider(config: config, context: context, envReporter: environmentReporter) diagnosticReporter = self.serviceFactory.makeDiagnosticReporter(config: config, service: service, environmentReporter: environmentReporter) - eventReporter = self.serviceFactory.makeEventReporter(config: config, service: service) + eventReporter = self.serviceFactory.makeEventReporter(config: config, service: service, onSyncComplete: LDClient.eventSyncLogger(configuration.logger)) connectionInformation = self.serviceFactory.makeConnectionInformation() let cachedData = flagCache.getCachedData(cacheKey: context.fullyQualifiedHashedKey(), contextHash: context.contextHash()) @@ -1059,7 +1059,6 @@ public class LDClient { NotificationCenter.default.addObserver(self, selector: #selector(didCloseEventSource), name: Notification.Name(FlagSynchronizer.Constants.didCloseEventSourceName), object: nil) - eventReporter = self.serviceFactory.makeEventReporter(config: configuration, service: service, onSyncComplete: onEventSyncComplete) service.resetFlagResponseCache(etag: cachedData.etag) flagSynchronizer = self.serviceFactory.makeFlagSynchronizer(streamingMode: config.allowStreamingMode ? config.streamingMode : .polling, pollingInterval: config.flagPollingInterval(runMode: runMode), @@ -1087,16 +1086,22 @@ public class LDClient { extension LDClient: TypeIdentifying { } -// MARK: - Event delivery -extension LDClient { - private func onEventSyncComplete(result: SynchronizingError?) { - if let synchronizingError = result { - os_log("%s result: %s", log: config.logger, type: .debug, typeName(and: #function), String(describing: synchronizingError)) - } else { - os_log("%s result: success", log: config.logger, type: .debug, typeName(and: #function)) +private extension LDClient { + /// Logs each event delivery's outcome. Static so it can be given to the event reporter before `self` is available, + /// which lets the client create a single reporter, and with it a single store for the event directory. + static func eventSyncLogger(_ logger: OSLog) -> EventSyncCompleteClosure { + return { result in + if let synchronizingError = result { + os_log("%s result: %s", log: logger, type: .debug, typeName(and: "onEventSyncComplete"), String(describing: synchronizingError)) + } else { + os_log("%s result: success", log: logger, type: .debug, typeName(and: "onEventSyncComplete")) + } } } +} +// MARK: - Event delivery +extension LDClient { /** Writes down all pending events and sends them to LaunchDarkly. diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift index 0b69892a0..685354699 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift @@ -353,7 +353,21 @@ class EventReporter: EventReporting { func commitRecordedEvents() { commitLock.lock() defer { commitLock.unlock() } + commitRecordedEventsHoldingCommitLock() + } + + /// Commits what is held and closes it into a batch, as one step with respect to other commits. + /// + /// Closing takes everything staged so far, and a commit stages its run one event at a time, so a commit running + /// between the two would be closed partway: an evaluation's event in this batch and its summary in the next. + private func commitAndCloseBatch() { + commitLock.lock() + defer { commitLock.unlock() } + commitRecordedEventsHoldingCommitLock() + _ = store.closeBatch() + } + private func commitRecordedEventsHoldingCommitLock() { pendingLock.lock() let run = pending pending = [] @@ -491,8 +505,7 @@ class EventReporter: EventReporting { return } - commitRecordedEvents() - _ = store.closeBatch() + commitAndCloseBatch() let batches = store.pendingBatches() guard !batches.isEmpty diff --git a/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift b/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift index e226f27cb..254eea2ff 100644 --- a/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift @@ -76,6 +76,10 @@ final class LDClientSpec: QuickSpec { expect(testContext.subject.flagSynchronizer.isOnline) == testContext.subject.isOnline expect(testContext.subject.eventReporter.isOnline) == testContext.subject.isOnline } + it("creates one event reporter, so one store owns the event directory") { + expect(testContext.serviceFactoryMock.makeEventReporterCallCount) == 1 + expect(testContext.serviceFactoryMock.onEventSyncComplete).toNot(beNil()) + } it("saves the config") { expect(testContext.subject.config) == testContext.config expect(testContext.subject.service.config) == testContext.config diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift index 6fb9939ee..1a4e48a6a 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift @@ -1280,6 +1280,47 @@ extension EventReporterSpec { expect(store.pendingEventCount) == 1 expect(diagnosticCache.incrementDroppedEventCountCallCount) == 1 } + + it("does not close a batch partway through another commit's run") { + var config = LDConfig.stub + config.eventPersistence = .deferred + let serviceMock = DarklyServiceMock() + serviceMock.config = config + serviceMock.stubEventResponse(success: true) + let inner = EventStore.temporary(capacity: config.eventCapacity) + defer { inner.deleteEverything() } + let store = ClosingHookStore(inner) + let commits = DispatchQueue(label: "com.launchdarkly.tests.suspended") + commits.suspend() + defer { commits.resume() } + let reporter = EventReporter(service: serviceMock, onSyncComplete: nil, store: store, encoding: .codable, commitQueue: commits) + + let slow = EncodingGate(key: "slow", context: LDContext.stub()) + let competingCommit = DispatchSemaphore(value: 0) + store.beforeClosing = { + reporter.record(CustomEvent(key: "first", context: LDContext.stub())) + reporter.record(slow) + DispatchQueue.global().async { + reporter.commitRecordedEvents() + competingCommit.signal() + } + // Long enough for an unguarded commit to stage "first" and block encoding "slow". + _ = slow.started.wait(timeout: .now() + 0.5) + } + store.afterClosing = { slow.proceed.signal() } + + reporter.isOnline = true + waitUntil(timeout: .seconds(10)) { done in + reporter.flushReportingOutcome { _ in done() } + } + expect(competingCommit.wait(timeout: .now() + 5)) == .success + + for keys in store.closedKeys { + expect(keys.contains("first")) == keys.contains("slow") + } + let rest = inner.closeBatch().flatMap { inner.body(of: $0) }.map(customKeys) ?? [] + expect(rest) == ["first", "slow"] + } } } @@ -1416,6 +1457,54 @@ private final class EncodingGate: CustomEvent { } } +/// A store that runs a hook on either side of closing a batch, and remembers the custom event keys of each batch closed. +private final class ClosingHookStore: EventStoring { + let inner: EventStoring + var beforeClosing: (() -> Void)? + var afterClosing: (() -> Void)? + private(set) var closedKeys: [[String]] = [] + + init(_ inner: EventStoring) { + self.inner = inner + } + + var pendingEventCount: Int { inner.pendingEventCount } + var isPersisting: Bool { inner.isPersisting } + func stage(_ encodedEvent: Data, bypassingCapacity: Bool) -> Bool { inner.stage(encodedEvent, bypassingCapacity: bypassingCapacity) } + func reserve(_ events: Int) { inner.reserve(events) } + func stageReserved(_ encodedEvent: Data) -> Bool { inner.stageReserved(encodedEvent) } + func releaseReservations() { inner.releaseReservations() } + func commit() { inner.commit() } + func pendingBatches() -> [EventBatch] { inner.pendingBatches() } + func body(of batch: EventBatch) -> Data? { inner.body(of: batch) } + func remove(_ batch: EventBatch) { inner.remove(batch) } + func recoverInterruptedLog() { inner.recoverInterruptedLog() } + + func closeBatch() -> EventBatch? { + let before = beforeClosing + beforeClosing = nil + before?() + let batch = inner.closeBatch() + if let batch, let body = inner.body(of: batch) { + closedKeys.append(customKeys(body)) + } + let after = afterClosing + afterClosing = nil + after?() + return batch + } +} + +private func customKeys(_ body: Data) -> [String] { + guard case .array(let events)? = try? JSONDecoder().decode(LDValue.self, from: body) + else { return [] } + return events.compactMap { event in + guard event.kindField == "custom", case .object(let fields) = event, case .string(let key) = fields["key"] + else { return nil } + return key + } +} + private func expectedEvents(_ events: [Event]) -> [LDValue] { events.compactMap { encodeToLDValue($0) }.map(withoutCreationDate) } From e995b26553dfed74a77e5eaa51f4d3e9fc92d6c9 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Wed, 7 Oct 2026 15:01:33 -0700 Subject: [PATCH 005/158] fixes --- .../ServiceObjects/EventStore.swift | 14 ++++++++ .../ServiceObjects/EventStoreSpec.swift | 33 +++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index c17c0501c..56b1424f0 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -112,6 +112,8 @@ final class EventStore: EventStoring { /// Only to be used while holding `ioLock`. private var descriptor: Int32 = -1 + /// Whether a log left by a previous run has been dealt with. Only to be used while holding `ioLock`. + private var hasRecoveredInterruptedLog = false /// Batches held in memory because the filesystem would not take them. /// @@ -410,6 +412,14 @@ final class EventStore: EventStoring { func recoverInterruptedLog() { ioLock.lock() defer { ioLock.unlock() } + recoverInterruptedLogHoldingIoLock() + } + + /// Requires `ioLock`. Does its work once per store: after that, `current` is a log this run opened. + private func recoverInterruptedLogHoldingIoLock() { + guard !hasRecoveredInterruptedLog + else { return } + hasRecoveredInterruptedLog = true guard FileManager.default.fileExists(atPath: currentLogUrl.path), let events = EventLogFormat.eventCount(in: currentLogUrl) @@ -517,6 +527,10 @@ final class EventStore: EventStoring { return descriptor } + // Before opening, because `current` may still be the log a previous run died writing. Appended to, this run's + // events would sit behind that run's torn last frame, where a reader can no longer find where they start. + recoverInterruptedLogHoldingIoLock() + do { try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) } catch { diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 3bacac5b7..1b2f5009b 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -195,6 +195,39 @@ final class EventStoreSpec: QuickSpec { expect(next.pendingBatches()).to(beEmpty()) expect(next.pendingEventCount) == 0 } + + it("keeps this run's events out of it, even when this run commits before recovery is asked for") { + store = EventStore.temporary() + for index in 0..<2 { + _ = store.stage(EventStoreSpec.payload("previous-\(index)")) + } + store.commit() + // The previous run died partway through writing its last frame. + let log = store.directory.appendingPathComponent("current") + try Data(contentsOf: log).dropLast(3).write(to: log) + + let next = EventStoreSpec.reader(sharing: store) + _ = next.stage(EventStoreSpec.payload("this-run")) + next.commit() + next.recoverInterruptedLog() + _ = next.stage(EventStoreSpec.payload("later")) + next.commit() + _ = next.closeBatch() + + let bodies = next.pendingBatches().map { EventStoreSpec.keys(ofBody: next.body(of: $0)) } + expect(bodies) == [["previous-0"], ["this-run", "later"]] + expect(next.pendingEventCount) == 3 + } + } + } + + private static func keys(ofBody body: Data?) -> [String] { + guard let body, case .array(let events)? = try? JSONDecoder().decode(LDValue.self, from: body) + else { return [] } + return events.compactMap { + guard case .object(let fields) = $0, case .string(let key) = fields["key"] + else { return nil } + return key } } From 6406d5d4d2d3e70bd4026f8876248ccb303c507f Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 8 Oct 2026 07:58:24 -0700 Subject: [PATCH 006/158] fix(events): Keep stored events whose read failed, and deliver what a failed write left in memory A read can fail on a file that is intact, as it does in a process out of file descriptors, and every failure was treated as a batch that can never be delivered and deleted. Listing now skips such a batch until a later listing can read it, delivery retries it, and startup recovery closes the previous run's log off uncounted rather than deleting it. Only a file that is gone, or one written in a format this version does not read, is nothing to keep. closeBatch also closed only the log where a failed write had left events both there and in memory, so a flush could report success and close() discard the rest. Co-authored-by: Cursor --- .../ServiceObjects/EventReporter.swift | 16 +- .../ServiceObjects/EventStore.swift | 157 +++++++++++++----- .../ServiceObjects/EventReporterSpec.swift | 42 ++++- .../ServiceObjects/EventStoreSpec.swift | 103 ++++++++++-- LaunchDarkly/LaunchDarklyTests/TestUtil.swift | 8 +- 5 files changed, 262 insertions(+), 64 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift index c9e25ed16..7efbd42c4 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift @@ -570,11 +570,21 @@ class EventReporter: EventReporting { } let batch = remaining.removeFirst() - guard let body = store.body(of: batch) + let read: Data? + do { + read = try store.body(of: batch) + } catch { + // Kept: the batch is still there, and a read that failed may not fail next time. + os_log("%s could not read stored events: %s", log: service.config.logger, type: .debug, typeName(and: #function), String(describing: error)) + completion?(false) + return + } + + guard let body = read else { - // Nothing deliverable in it, so it will never become deliverable: those events are lost. + // Gone, or holding nothing this version can send. Either way there is nothing to send and nothing to keep. store.remove(batch) - deliver(remaining, lostAnything: true, completion) + deliver(remaining, lostAnything: lostAnything, completion) return } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index 56b1424f0..6b6e49cfe 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -44,14 +44,20 @@ protocol EventStoring { /// Hands every staged byte to the kernel, so that the events recorded so far survive the process dying. func commit() - /// Commits, then closes the open log into a batch to deliver. Returns nil when there is nothing to send. + /// Commits, then closes the open log into a batch to deliver, and whatever is still staged in memory into another. + /// + /// Returns the batch closed from the log, or the one closed from memory where the log held nothing, and nil when + /// there is nothing to send. func closeBatch() -> EventBatch? /// Batches awaiting delivery, oldest first, including any recovered from a previous run of the application. func pendingBatches() -> [EventBatch] - /// The `[...]` request body for a batch, or nil if the file has since become unreadable. - func body(of batch: EventBatch) -> Data? + /// The `[...]` request body for a batch, or nil if it is gone or holds nothing this version can send, neither of + /// which a later attempt changes. + /// + /// - throws: if the batch is there but could not be read this time, which a later attempt may manage. + func body(of batch: EventBatch) throws -> Data? /// Forgets a batch, which is only correct once LaunchDarkly has accepted it or permanently refused it. func remove(_ batch: EventBatch) @@ -85,6 +91,11 @@ final class EventStore: EventStoring { let directory: URL private let capacity: Int private let logger: OSLog + /// How a stored log is read, injectable so a test can make a read fail while the file is intact. + private let readFile: (URL) throws -> Data + /// How bytes reach the open log, injectable so a test can make a write fail partway through a session, as a full + /// disk does. Answers with the errno it stopped on, or nil where every byte landed. + private let writeLog: (Int32, Data) -> Int32? /// Where a commit runs when it was not asked for by a caller who needs it to have happened. /// /// Evaluating a flag must not put a write on whichever thread evaluated it, and that thread is usually the main one. @@ -146,11 +157,15 @@ final class EventStore: EventStoring { capacity: Int, persistEvents: Bool = true, logger: OSLog, - commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.EventStore.commitQueue", qos: .userInitiated)) { + commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.EventStore.commitQueue", qos: .userInitiated), + readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }, + writeLog: @escaping (Int32, Data) -> Int32? = EventStore.writeAll) { self.directory = directory self.capacity = capacity self.logger = logger self.commitQueue = commitQueue + self.readFile = readFile + self.writeLog = writeLog // An application that has not asked for persistence gets the same store running the same way it runs once a // write has failed: events are held, delivered, and lost only if the process dies. Batches a run with // persistence turned on left behind are still recovered and delivered, which is why this sets the runtime @@ -289,14 +304,13 @@ final class EventStore: EventStoring { let stillBuffered = bufferedEventCount bufferLock.unlock() - if events > 0 { - return closeOpenLogHoldingIoLock(events) - } - if stillBuffered > 0 { - // Only reachable once persistence has been given up on; a healthy commit leaves nothing staged behind. - return closeInMemoryBatchHoldingIoLock() - } - return nil + let fromLog = events > 0 ? closeOpenLogHoldingIoLock(events) : nil + // Closed after the log, so the older events are delivered first. Both can hold events at once: a write that + // fails after earlier ones landed gives up on persistence with those earlier events in the log and the failed + // ones back in memory, and closing only the log would leave the rest to a delivery that reports success + // without them. + let fromMemory = stillBuffered > 0 ? closeInMemoryBatchHoldingIoLock() : nil + return fromLog ?? fromMemory } /// Requires `ioLock`. @@ -331,6 +345,25 @@ final class EventStore: EventStoring { // MARK: Delivery + /// What reading a stored log produced. + /// + /// A read that failed is kept apart from a file that is not there, because the two arrive the same way: a process + /// out of file descriptors fails to open a file that is perfectly intact with the error a missing one gets. Only + /// what is missing, or what this version cannot make sense of, is gone for good. + private enum LogRead { + case bytes(Data) + case missing + case failed(Error) + } + + private func read(_ url: URL) -> LogRead { + do { + return .bytes(try readFile(url)) + } catch { + return FileManager.default.fileExists(atPath: url.path) ? .failed(error) : .missing + } + } + func pendingBatches() -> [EventBatch] { ioLock.lock() defer { ioLock.unlock() } @@ -343,15 +376,9 @@ final class EventStore: EventStoring { .filter { $0.lastPathComponent.hasPrefix(EventStore.batchPrefix) } .compactMap { file -> (EventBatch, Date)? in let payloadId = String(file.lastPathComponent.dropFirst(EventStore.batchPrefix.count)) - // Reading is only for a batch this process has not counted: one a previous run left behind, or one - // belonging to another process sharing the environment. - guard let events = eventCounts[payloadId] ?? EventLogFormat.eventCount(in: file) - else { - // Written by a version of the SDK whose format this one does not read, or damaged beyond what the - // torn tail recovery tolerates. Either way it can never be delivered, so it is not kept. - try? FileManager.default.removeItem(at: file) - return nil - } + // Reading is only for a batch this process has not counted: one a previous run left behind. + guard let events = eventCounts[payloadId] ?? countOfEventsHoldingIoLock(in: file) + else { return nil } eventCounts[payloadId] = events let modified = (try? file.resourceValues(forKeys: [.contentModificationDateKey]))?.contentModificationDate return (EventBatch(payloadId: payloadId, eventCount: events), modified ?? Date.distantPast) @@ -377,7 +404,32 @@ final class EventStore: EventStoring { return batches } - func body(of batch: EventBatch) -> Data? { + /// How many events a batch file holds, or nil when it is not one to list this time. + /// + /// Requires `ioLock`. + private func countOfEventsHoldingIoLock(in file: URL) -> Int? { + switch read(file) { + case .missing: + // Gone since the directory was listed. + return nil + case .failed(let error): + // Left for a later listing rather than deleted: the read may pass then, and the events in it may have been + // intact all along. Without a count it cannot be delivered now. + os_log("%s could not read stored events, will try again later: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) + return nil + case .bytes(let log): + guard let events = EventLogFormat.eventCount(in: log) + else { + // Written by a version of the SDK whose format this one does not read, or damaged beyond what the + // torn tail recovery tolerates. Either way it can never be delivered, so it is not kept. + try? FileManager.default.removeItem(at: file) + return nil + } + return events + } + } + + func body(of batch: EventBatch) throws -> Data? { ioLock.lock() defer { ioLock.unlock() } @@ -385,9 +437,11 @@ final class EventStore: EventStoring { return held.body } - guard let file = try? Data(contentsOf: url(of: batch.payloadId)) - else { return nil } - return EventLogFormat.assembleBody(from: file) + switch read(url(of: batch.payloadId)) { + case .missing: return nil + case .failed(let error): throw error + case .bytes(let file): return EventLogFormat.assembleBody(from: file) + } } func remove(_ batch: EventBatch) { @@ -421,14 +475,25 @@ final class EventStore: EventStoring { else { return } hasRecoveredInterruptedLog = true - guard FileManager.default.fileExists(atPath: currentLogUrl.path), - let events = EventLogFormat.eventCount(in: currentLogUrl) - else { - // Nothing recoverable. An unreadable log cannot be appended to either, so it goes. - if FileManager.default.fileExists(atPath: currentLogUrl.path) { + let events: Int + switch read(currentLogUrl) { + case .missing: + return + case .failed(let error): + // It cannot stay under this name, which this run is about to append its own events to, and deleting it + // would lose events that may be intact. Closed off uncounted instead, it is read and counted by whichever + // listing first manages to. + os_log("%s could not read events from a previous run, will try again later: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) + try? FileManager.default.moveItem(at: currentLogUrl, to: url(of: UUID().uuidString)) + return + case .bytes(let log): + guard let counted = EventLogFormat.eventCount(in: log) + else { + // Unreadable, and a log that cannot be read cannot be appended to either. try? FileManager.default.removeItem(at: currentLogUrl) + return } - return + events = counted } guard events > 0 @@ -491,8 +556,20 @@ final class EventStore: EventStoring { /// Requires `ioLock`. private func append(_ bytes: Data, to descriptor: Int32) -> Bool { + guard let failure = writeLog(descriptor, bytes) + else { return true } + + // Never trap on a full disk. The SDK gives up on persistence for the rest of the session rather than taking + // the application down with it, which is how other SDKs have crashed their hosts. + os_log("%s giving up on persisting events: errno %d", log: logger, type: .debug, typeName(and: #function), failure) + disablePersistenceHoldingIoLock() + return false + } + + /// Writes every byte to the log, returning the errno it stopped on or nil where they all landed. + static func writeAll(_ descriptor: Int32, _ bytes: Data) -> Int32? { var offset = 0 - let failure: Int32? = bytes.withUnsafeBytes { raw -> Int32? in + return bytes.withUnsafeBytes { raw -> Int32? in guard let base = raw.baseAddress else { return nil } while offset < raw.count { @@ -509,16 +586,6 @@ final class EventStore: EventStoring { } return nil } - - if let failure = failure { - // Never trap on a full disk. The SDK gives up on persistence for the rest of the session rather than - // taking the application down with it, which is how other SDKs have crashed their hosts. - os_log("%s giving up on persisting events: errno %d", log: logger, type: .debug, typeName(and: #function), failure) - disablePersistenceHoldingIoLock() - return false - } - - return true } /// Requires `ioLock`. @@ -702,10 +769,8 @@ enum EventLogFormat { return true } - static func eventCount(in url: URL) -> Int? { - guard let file = try? Data(contentsOf: url) - else { return nil } - + /// How many events a log holds, or nil where it is not one this version reads. + static func eventCount(in file: Data) -> Int? { var events = 0 guard forEachFrame(in: file, { type, _ in if type == eventFrame { diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift index f3f11983b..2459dd4aa 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift @@ -1318,7 +1318,7 @@ extension EventReporterSpec { for keys in store.closedKeys { expect(keys.contains("first")) == keys.contains("slow") } - let rest = inner.closeBatch().flatMap { inner.body(of: $0) }.map(customKeys) ?? [] + let rest = try inner.closeBatch().flatMap { try inner.body(of: $0) }.map(customKeys) ?? [] expect(rest) == ["first", "slow"] } } @@ -1373,6 +1373,42 @@ extension EventReporterSpec { expect(testContext.store.pendingEventCount) == 0 } + it("reports false and keeps the batch when it could not be read, and delivers it once it can be") { + var failing = false + let store = EventStore.temporary(readFile: { url in + // What a process out of file descriptors gets for a file that is perfectly intact. + guard !failing + else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(EMFILE)) } + return try Data(contentsOf: url) + }) + testContext = TestContext(store: store) + testContext.recordEvents(1) + testContext.eventReporter.isOnline = true + + failing = true + var delivered: Bool? + waitUntil { done in + testContext.eventReporter.flushReportingOutcome { result in + delivered = result + done() + } + } + expect(delivered) == false + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + expect(store.pendingBatches().count) == 1 + + failing = false + waitUntil { done in + testContext.eventReporter.flushReportingOutcome { result in + delivered = result + done() + } + } + expect(delivered) == true + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + expect(store.pendingEventCount) == 0 + } + it("reports false when a retryable failure leaves the batch on disk") { testContext = TestContext(stubResponseSuccess: false) testContext.recordEvents(1) @@ -1498,7 +1534,7 @@ private final class ClosingHookStore: EventStoring { func releaseReservations() { inner.releaseReservations() } func commit() { inner.commit() } func pendingBatches() -> [EventBatch] { inner.pendingBatches() } - func body(of batch: EventBatch) -> Data? { inner.body(of: batch) } + func body(of batch: EventBatch) throws -> Data? { try inner.body(of: batch) } func remove(_ batch: EventBatch) { inner.remove(batch) } func recoverInterruptedLog() { inner.recoverInterruptedLog() } @@ -1507,7 +1543,7 @@ private final class ClosingHookStore: EventStoring { beforeClosing = nil before?() let batch = inner.closeBatch() - if let batch, let body = inner.body(of: batch) { + if let batch, let body = try? inner.body(of: batch) ?? nil { closedKeys.append(customKeys(body)) } let after = afterClosing diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 1b2f5009b..4b59fd50f 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -38,7 +38,7 @@ final class EventStoreSpec: QuickSpec { expect(batch.eventCount) == 1 expect(store.pendingEventCount) == 1 - expect(store.body(of: batch)) == Data("[{\"kind\":\"custom\",\"key\":\"held\"}]".utf8) + expect(try store.body(of: batch)) == Data("[{\"kind\":\"custom\",\"key\":\"held\"}]".utf8) } } } @@ -60,8 +60,18 @@ final class EventStoreSpec: QuickSpec { /// A store reading the same directory as another, standing in for the next run of the application: it sees only /// what actually reached the disk, which is the whole question a crash asks. - private static func reader(sharing store: EventStore) -> EventStore { - EventStore(directory: store.directory, capacity: 100, logger: .disabled) + private static func reader(sharing store: EventStore, + readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }) -> EventStore { + EventStore(directory: store.directory, capacity: 100, logger: .disabled, readFile: readFile) + } + + /// What a process out of file descriptors gets for a file that is perfectly intact. + private static func readFailing(while failing: @escaping () -> Bool) -> (URL) throws -> Data { + { url in + guard !failing() + else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(EMFILE)) } + return try Data(contentsOf: url) + } } private static func payload(_ key: String) -> Data { @@ -179,7 +189,7 @@ final class EventStoreSpec: QuickSpec { let batches = next.pendingBatches() expect(batches.count) == 1 expect(batches.first?.eventCount) == 1 - expect(next.body(of: batches[0])) == Data("[{\"kind\":\"custom\",\"key\":\"survived\"}]".utf8) + expect(try next.body(of: batches[0])) == Data("[{\"kind\":\"custom\",\"key\":\"survived\"}]".utf8) } it("leaves nothing behind when the log held no events") { @@ -214,7 +224,7 @@ final class EventStoreSpec: QuickSpec { next.commit() _ = next.closeBatch() - let bodies = next.pendingBatches().map { EventStoreSpec.keys(ofBody: next.body(of: $0)) } + let bodies = next.pendingBatches().map { EventStoreSpec.keys(ofBody: try? next.body(of: $0) ?? nil) } expect(bodies) == [["previous-0"], ["this-run", "later"]] expect(next.pendingEventCount) == 3 } @@ -253,6 +263,55 @@ final class EventStoreSpec: QuickSpec { expect(EventStoreSpec.keys(of: EventStoreSpec.reader(sharing: store))) == ["event-0", "event-1"] } + it("is kept for a later listing when the read failed rather than the log") { + store = EventStore.temporary() + _ = store.stage(EventStoreSpec.payload("kept")) + store.commit() + _ = store.closeBatch() + + var failing = true + let next = EventStoreSpec.reader(sharing: store, + readFile: EventStoreSpec.readFailing(while: { failing })) + expect(next.pendingBatches()).to(beEmpty()) + + // A read can fail on an intact file, as it does in a process out of file descriptors, so the batch has + // to still be there once reads work again. + failing = false + let batches = next.pendingBatches() + expect(batches.count) == 1 + expect(EventStoreSpec.keys(ofBody: try next.body(of: batches[0]))) == ["kept"] + } + + it("is a failure to retry, not nothing to send, when a body cannot be read") { + var failing = false + store = EventStore.temporary(readFile: EventStoreSpec.readFailing(while: { failing })) + _ = store.stage(EventStoreSpec.payload("kept")) + let batch = store.closeBatch()! + + failing = true + expect { try store.body(of: batch) }.to(throwError()) + // Still the store's to deliver: reporting nothing to send would have it dropped. + expect(store.pendingBatches()) == [batch] + } + + it("is closed off uncounted when a log a previous run left open cannot be read") { + store = EventStore.temporary() + _ = store.stage(EventStoreSpec.payload("from-the-previous-run")) + store.commit() + + var failing = true + let next = EventStoreSpec.reader(sharing: store, + readFile: EventStoreSpec.readFailing(while: { failing })) + next.recoverInterruptedLog() + + // It cannot stay under the name this run appends to, and deleting it would lose events that are + // intact, so it is closed off for whichever listing first manages to read it. + failing = false + let batches = next.pendingBatches() + expect(batches.count) == 1 + expect(EventStoreSpec.keys(ofBody: try next.body(of: batches[0]))) == ["from-the-previous-run"] + } + it("is discarded when it was written in a format this version does not read") { store = EventStore.temporary() _ = store.stage(EventStoreSpec.payload("unreadable")) @@ -292,7 +351,7 @@ final class EventStoreSpec: QuickSpec { let batch = store.closeBatch() expect(batch?.eventCount) == 2 - let body = store.body(of: batch!) + let body = try store.body(of: batch!) expect(body) == Data("[{\"kind\":\"custom\",\"key\":\"first\"},{\"kind\":\"custom\",\"key\":\"second\"}]".utf8) } @@ -306,7 +365,7 @@ final class EventStoreSpec: QuickSpec { expect(store.pendingBatches()) == [batch] expect(store.pendingEventCount) == 1 - expect(store.body(of: batch)).toNot(beNil()) + expect(try store.body(of: batch)).toNot(beNil()) store.remove(batch) @@ -410,7 +469,7 @@ final class EventStoreSpec: QuickSpec { let batch = store.closeBatch() expect(batch?.eventCount) == 2 - expect(store.body(of: batch!)) + expect(try store.body(of: batch!)) == Data("[{\"kind\":\"custom\",\"key\":\"first\"},{\"kind\":\"custom\",\"key\":\"second\"}]".utf8) } @@ -423,7 +482,7 @@ final class EventStoreSpec: QuickSpec { _ = store.stage(EventStoreSpec.payload("same")) let held = store.closeBatch()! - expect(store.body(of: held)) == written.body(of: onDisk) + expect(try store.body(of: held)) == (try written.body(of: onDisk)) } it("keeps accepting events after persistence has been given up on") { @@ -439,10 +498,34 @@ final class EventStoreSpec: QuickSpec { _ = store.stage(EventStoreSpec.payload("second")) let batch = store.closeBatch()! - expect(store.body(of: batch)) + expect(try store.body(of: batch)) == Data("[{\"kind\":\"custom\",\"key\":\"first\"},{\"kind\":\"custom\",\"key\":\"second\"}]".utf8) } + it("delivers what memory was left holding beside what the disk already took") { + var writes = 0 + let interrupted = EventStore.temporary(writeLog: { descriptor, bytes in + writes += 1 + // The file header and the first event land; the device is full from then on. + return writes > 2 ? ENOSPC : EventStore.writeAll(descriptor, bytes) + }) + defer { interrupted.deleteEverything() } + + _ = interrupted.stage(EventStoreSpec.payload("written")) + interrupted.commit() + _ = interrupted.stage(EventStoreSpec.payload("held")) + interrupted.commit() + expect(interrupted.isPersisting) == false + + _ = interrupted.closeBatch() + + // A failed write leaves events on both sides, and closing only the log would leave the rest to a + // delivery that reported success without them. + let bodies = interrupted.pendingBatches().map { EventStoreSpec.keys(ofBody: try? interrupted.body(of: $0) ?? nil) } + expect(bodies) == [["written"], ["held"]] + expect(interrupted.pendingEventCount) == 2 + } + it("holds a batch until it is removed, so a failed delivery can be retried") { _ = store.stage(EventStoreSpec.payload("retry-me")) let batch = store.closeBatch()! diff --git a/LaunchDarkly/LaunchDarklyTests/TestUtil.swift b/LaunchDarkly/LaunchDarklyTests/TestUtil.swift index 2e9d1892d..2addbc113 100644 --- a/LaunchDarkly/LaunchDarklyTests/TestUtil.swift +++ b/LaunchDarkly/LaunchDarklyTests/TestUtil.swift @@ -55,7 +55,9 @@ extension EventStore { static func temporary( capacity: Int = 100, persistEvents: Bool = true, - commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.tests.commitQueue") + commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.tests.commitQueue"), + readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }, + writeLog: @escaping (Int32, Data) -> Int32? = EventStore.writeAll ) -> EventStore { let directory = FileManager.default.temporaryDirectory .appendingPathComponent("com.launchdarkly.tests.events", isDirectory: true) @@ -64,7 +66,9 @@ extension EventStore { capacity: capacity, persistEvents: persistEvents, logger: .disabled, - commitQueue: commitQueue) + commitQueue: commitQueue, + readFile: readFile, + writeLog: writeLog) } /// Removes the store's directory, including any batch still waiting to be delivered. From 0c8ea6695b0b25387abe3d58b761f676e53e5f98 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 8 Oct 2026 08:02:57 -0700 Subject: [PATCH 007/158] fix(events): Report a flush false when an event the SDK accepted could not be kept A batch LaunchDarkly refuses in a way that may pass is kept and retried, so it is not a loss. What is lost is what the store never received: an event or a summary that could not be serialized when it was committed, or that serialized too large to store. A delivery that finds nothing to send cannot tell those apart from events that arrived, so flushAndWait reported success for them. Co-authored-by: Cursor --- .../ServiceObjects/EventReporter.swift | 54 ++++++++++++++++--- .../ServiceObjects/EventReporterSpec.swift | 44 +++++++++++++++ 2 files changed, 91 insertions(+), 7 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift index 7efbd42c4..5fc3a85c8 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift @@ -17,7 +17,8 @@ protocol EventReporting { func flush(completion: CompletionClosure?) /// Like `flush`. Reports `true` if LaunchDarkly accepted the pending batches, or there were none to send. Reports - /// `false` if the reporter is offline, a batch was refused for good, or a retryable failure left batches on disk. + /// `false` if the reporter is offline, a batch was refused for good, a retryable failure left batches on disk, or + /// an event the SDK had accepted could not be kept to send at all. /// /// The completion runs on the reporter's delivery queue. Keep it short: a long one holds up the deliveries behind /// it, though it cannot block an evaluation, which records under a lock rather than on that queue. @@ -106,13 +107,22 @@ class EventReporter: EventReporting { let service: DarklyServiceProvider let store: EventStoring - /// Guards the last response date. + /// Guards the last response date and the record of what has been lost. private let stateLock = UnfairLock() /// Only to be used while holding `pendingLock`. private(set) var contextSummarizer: ContextSummarizer /// Only to be used while holding `stateLock`. private var responseDate: Date + /// Whether events the SDK accepted were dropped for good, cleared once a caller who hears an outcome has been + /// told so. Only to be used while holding `stateLock`. + /// + /// A batch LaunchDarkly refuses in a way that may pass is kept and retried, so it is not a loss. What is lost is + /// what the store never received: an event or a summary that could not be serialized when it was committed, or + /// that serialized too large to store. A delivery that finds nothing to send cannot tell those apart from events + /// that arrived, so without this it would report success for them. + private var eventsLostSinceLastAnswer = false + /// Held for the whole of a commit, so that only one runs at a time. /// /// This is what a commit point's guarantee rests on. Without it a commit already in flight could take the caller's @@ -300,6 +310,24 @@ class EventReporter: EventReporting { service.diagnosticCache?.incrementDroppedEventCount() } + /// Records the loss of an event the SDK accepted and then could not keep, which no later attempt recovers. + private func reportLost() { + os_log("%s dropping an event the store would not take", log: service.config.logger, type: .error, typeName(and: #function)) + service.diagnosticCache?.incrementDroppedEventCount() + stateLock.lock() + eventsLostSinceLastAnswer = true + stateLock.unlock() + } + + /// Whether anything has been lost since the last caller was told, which that caller is now the one to hear about. + private func takeEventsLost() -> Bool { + stateLock.lock() + defer { stateLock.unlock() } + let lost = eventsLostSinceLastAnswer + eventsLostSinceLastAnswer = false + return lost + } + /// Commits at a commit point, on the caller's thread or off it as the application asked. /// /// Committing on the caller's thread is what lets `track` promise its event is on disk by the time it returns. @@ -399,9 +427,13 @@ class EventReporter: EventReporting { private func stage(_ run: [Event]) { for event in run { guard let encoded = encode(event) - else { continue } + else { + reportLost() + continue + } + // Capacity is bypassed here, so what is left is a frame too large to store, which no later attempt fixes. if !store.stageReserved(encoded) { - reportDropped() + reportLost() } } } @@ -427,11 +459,14 @@ class EventReporter: EventReporting { for summary in summaries { let summaryEvent = SummaryEvent(flagRequestTracker: summary.tracker, context: summary.context) guard let encoded = encode(summaryEvent) - else { continue } + else { + reportLost() + continue + } // Summaries are an aggregate of evaluations that were already counted, so refusing one for capacity would // lose evaluations the SDK promised to report rather than shed new load. if !store.stage(encoded, bypassingCapacity: true) { - reportDropped() + reportLost() } } } @@ -479,7 +514,12 @@ class EventReporter: EventReporting { // returns, even when delivery cannot run because the client is offline. commitAtCommitPoint() deliveryQueue.async { - self.reportEvents(completion: completion) + self.reportEvents { delivered in + // Committing first is what makes this the answer for everything recorded before the call: whatever + // could not be kept has been counted by now. Cleared as it is reported, so that the next caller is not + // told again about a loss this answer has already accounted for. + completion(delivered && !self.takeEventsLost()) + } } } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift index 2459dd4aa..ab0ee5d56 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift @@ -1409,6 +1409,50 @@ extension EventReporterSpec { expect(store.pendingEventCount) == 0 } + it("reports false when an event could not be serialized, though the rest were accepted") { + testContext = TestContext() + testContext.eventReporter.record(CustomEvent(key: "poison", context: LDContext.stub(), metricValue: .nan)) + testContext.recordEvents(1) + testContext.eventReporter.isOnline = true + var delivered: Bool? + waitUntil { done in + testContext.eventReporter.flushReportingOutcome { result in + delivered = result + done() + } + } + expect(delivered) == false + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + + // Told once, to the caller who was there to hear it. + waitUntil { done in + testContext.eventReporter.flushReportingOutcome { result in + delivered = result + done() + } + } + expect(delivered) == true + } + + it("reports false when an event serialized too large to store") { + testContext = TestContext() + // Larger than a log frame holds, so the store refuses it however much room is left. + let oversized = String(repeating: "x", count: EventLogFormat.maxFrameSize) + testContext.eventReporter.record(CustomEvent(key: oversized, context: LDContext.stub())) + testContext.recordEvents(1) + testContext.eventReporter.isOnline = true + var delivered: Bool? + waitUntil { done in + testContext.eventReporter.flushReportingOutcome { result in + delivered = result + done() + } + } + expect(delivered) == false + // The event beside it still went. + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + } + it("reports false when a retryable failure leaves the batch on disk") { testContext = TestContext(stubResponseSuccess: false) testContext.recordEvents(1) From 6a310faecf9b5b214cdb33a1263aa390bc2c30e9 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 8 Oct 2026 08:21:41 -0700 Subject: [PATCH 008/158] test(events): Match Android's coverage of the recoverable failures Android pins the failed write at the processor level, where the bug was visible: the flush reports success and both the written and the held events arrive. It also pins that a body reads correctly once reads work again, and that a previous run's log moved aside leaves this run the name to append to. Co-authored-by: Cursor --- .../Mocks/DarklyServiceMock.swift | 3 ++ .../ServiceObjects/EventReporterSpec.swift | 31 +++++++++++++++++-- .../ServiceObjects/EventStoreSpec.swift | 12 +++++-- 3 files changed, 42 insertions(+), 4 deletions(-) diff --git a/LaunchDarkly/LaunchDarklyTests/Mocks/DarklyServiceMock.swift b/LaunchDarkly/LaunchDarklyTests/Mocks/DarklyServiceMock.swift index a302defa0..c4ca606ae 100644 --- a/LaunchDarkly/LaunchDarklyTests/Mocks/DarklyServiceMock.swift +++ b/LaunchDarkly/LaunchDarklyTests/Mocks/DarklyServiceMock.swift @@ -136,6 +136,8 @@ final class DarklyServiceMock: DarklyServiceProvider { var stubbedEventResponse: ServiceResponse? var publishEventDataCallCount = 0 var publishedEventData: Data? + /// Every payload published, in order, so a test can see what a delivery of several batches sent. + var publishedEventPayloads: [Data] = [] /// Every payload ID used, in order, so a test can tell a retry of a delivery from a new one. var publishedPayloadIds: [String] = [] /// Holds responses so a test can act while a delivery is in flight. @@ -144,6 +146,7 @@ final class DarklyServiceMock: DarklyServiceProvider { func publishEventData(_ eventData: Data, _ payloadId: String, completion: ServiceCompletionHandler?) { publishEventDataCallCount += 1 publishedEventData = eventData + publishedEventPayloads.append(eventData) publishedPayloadIds.append(payloadId) guard !holdsEventCompletions else { diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift index ab0ee5d56..ab6c8cd71 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift @@ -1409,6 +1409,33 @@ extension EventReporterSpec { expect(store.pendingEventCount) == 0 } + it("delivers what memory was left holding beside what the disk took, and reports true") { + var writes = 0 + let store = EventStore.temporary(writeLog: { descriptor, bytes in + writes += 1 + // The file header and the first event land; the device is full from then on. + return writes > 2 ? ENOSPC : EventStore.writeAll(descriptor, bytes) + }) + testContext = TestContext(store: store) + testContext.eventReporter.record(CustomEvent(key: "written", context: LDContext.stub())) + // Its write fails, so persistence is given up with "written" in the log and this in memory. + testContext.eventReporter.record(CustomEvent(key: "held", context: LDContext.stub())) + expect(store.isPersisting) == false + + testContext.eventReporter.isOnline = true + var delivered: Bool? + waitUntil { done in + testContext.eventReporter.flushReportingOutcome { result in + delivered = result + done() + } + } + expect(delivered) == true + // Neither left behind, and oldest first. + expect(testContext.serviceMock.publishedEventPayloads.flatMap(customKeys)) == ["written", "held"] + expect(store.pendingEventCount) == 0 + } + it("reports false when an event could not be serialized, though the rest were accepted") { testContext = TestContext() testContext.eventReporter.record(CustomEvent(key: "poison", context: LDContext.stub(), metricValue: .nan)) @@ -1439,7 +1466,7 @@ extension EventReporterSpec { // Larger than a log frame holds, so the store refuses it however much room is left. let oversized = String(repeating: "x", count: EventLogFormat.maxFrameSize) testContext.eventReporter.record(CustomEvent(key: oversized, context: LDContext.stub())) - testContext.recordEvents(1) + testContext.eventReporter.record(CustomEvent(key: "fine", context: LDContext.stub())) testContext.eventReporter.isOnline = true var delivered: Bool? waitUntil { done in @@ -1450,7 +1477,7 @@ extension EventReporterSpec { } expect(delivered) == false // The event beside it still went. - expect(testContext.serviceMock.publishEventDataCallCount) == 1 + expect(testContext.serviceMock.publishedEventPayloads.flatMap(customKeys)) == ["fine"] } it("reports false when a retryable failure leaves the batch on disk") { diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 4b59fd50f..24df6b3a6 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -292,6 +292,9 @@ final class EventStoreSpec: QuickSpec { expect { try store.body(of: batch) }.to(throwError()) // Still the store's to deliver: reporting nothing to send would have it dropped. expect(store.pendingBatches()) == [batch] + + failing = false + expect(EventStoreSpec.keys(ofBody: try store.body(of: batch))) == ["kept"] } it("is closed off uncounted when a log a previous run left open cannot be read") { @@ -303,13 +306,18 @@ final class EventStoreSpec: QuickSpec { let next = EventStoreSpec.reader(sharing: store, readFile: EventStoreSpec.readFailing(while: { failing })) next.recoverInterruptedLog() + // Moved out from under the name this run appends to, and not deleted. + expect(FileManager.default.fileExists(atPath: store.directory.appendingPathComponent("current").path)) == false - // It cannot stay under the name this run appends to, and deleting it would lose events that are - // intact, so it is closed off for whichever listing first manages to read it. + // So this run has the name to itself, and the events that are intact are only waiting for a read that + // works to be counted and delivered. + _ = next.stage(EventStoreSpec.payload("this-run")) + next.commit() failing = false let batches = next.pendingBatches() expect(batches.count) == 1 expect(EventStoreSpec.keys(ofBody: try next.body(of: batches[0]))) == ["from-the-previous-run"] + expect(EventStoreSpec.keys(of: EventStoreSpec.reader(sharing: store))) == ["from-the-previous-run", "this-run"] } it("is discarded when it was written in a format this version does not read") { From fe918c68220812ca51f29521e7d946b9ed70f704 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Fri, 9 Oct 2026 05:37:50 -0700 Subject: [PATCH 009/158] format --- .../ServiceObjects/EventReporter.swift | 21 ++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift index 0fe02e768..6d26fd9fc 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift @@ -13,7 +13,12 @@ protocol EventReporting { func record(_ event: Event) // swiftlint:disable:next function_parameter_count - func recordFlagEvaluationEvents(flagKey: LDFlagKey, value: LDValue, defaultValue: LDValue, featureFlag: FeatureFlag?, context: LDContext, includeReason: Bool) + func recordFlagEvaluationEvents(flagKey: LDFlagKey, + value: LDValue, + defaultValue: LDValue, + featureFlag: FeatureFlag?, + context: LDContext, + includeReason: Bool) func flush(completion: CompletionClosure?) /// Like `flush`. Reports `true` if LaunchDarkly accepted the pending batches, or there were none to send. Reports @@ -47,7 +52,12 @@ class NullEventReporter: EventReporting { func record(_ event: Event) { } - func recordFlagEvaluationEvents(flagKey: LDFlagKey, value: LDValue, defaultValue: LDValue, featureFlag: FeatureFlag?, context: LDContext, includeReason: Bool) { + func recordFlagEvaluationEvents(flagKey: LDFlagKey, + value: LDValue, + defaultValue: LDValue, + featureFlag: FeatureFlag?, + context: LDContext, + includeReason: Bool) { } func flush(completion: CompletionClosure?) { @@ -268,7 +278,12 @@ class EventReporter: EventReporting { } // swiftlint:disable:next function_parameter_count - func recordFlagEvaluationEvents(flagKey: LDFlagKey, value: LDValue, defaultValue: LDValue, featureFlag: FeatureFlag?, context: LDContext, includeReason: Bool) { + func recordFlagEvaluationEvents(flagKey: LDFlagKey, + value: LDValue, + defaultValue: LDValue, + featureFlag: FeatureFlag?, + context: LDContext, + includeReason: Bool) { let recordingFeatureEvent = featureFlag?.trackEvents == true let recordingDebugEvent = featureFlag?.shouldCreateDebugEvents(lastEventReportResponseTime: lastEventResponseDate) ?? false // Built before the lock is taken, so that the critical section is only the writes. From 2b892dfc5d816de0db1f5929769a8c05de779250 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Fri, 9 Oct 2026 06:24:25 -0700 Subject: [PATCH 010/158] Assert lock ownership instead of encoding it in method names Functions that must run under a lock now have plain names, keep "Requires `xLock`" in their docs, and check it with UnfairLock.assertOwned(), which traps in debug builds when the calling thread does not hold the lock. Co-authored-by: Cursor --- .../ServiceObjects/EventReporter.swift | 19 +++++---- .../ServiceObjects/EventStore.swift | 42 ++++++++++++------- LaunchDarkly/LaunchDarkly/UnfairLock.swift | 13 ++++++ 3 files changed, 51 insertions(+), 23 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift index 6d26fd9fc..9498d65bc 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift @@ -263,7 +263,7 @@ class EventReporter: EventReporting { func record(_ event: Event) { pendingLock.lock() - let dropped = !holdHoldingPendingLock(event) + let dropped = !admitToPending(event) let needsCommit = pending.count >= EventReporter.pendingCommitThreshold pendingLock.unlock() @@ -293,7 +293,7 @@ class EventReporter: EventReporting { var dropped = 0 pendingLock.lock() let counted = contextSummarizer.trackRequest(flagKey: flagKey, reportedValue: value, featureFlag: featureFlag, defaultValue: defaultValue, context: context) - for event in [featureEvent, debugEvent].compactMap({ $0 }) where !holdHoldingPendingLock(event) { + for event in [featureEvent, debugEvent].compactMap({ $0 }) where !admitToPending(event) { dropped += 1 } let needsCommit = pending.count >= EventReporter.pendingCommitThreshold @@ -316,12 +316,14 @@ class EventReporter: EventReporting { } } - /// Holds an event for the next commit to encode, returning false if the SDK is already full. Requires `pendingLock`. + /// Adds an event to `pending` for the next commit to encode, returning false if the SDK is already full. Requires + /// `pendingLock`. /// /// Capacity is consulted before anything else, so an event that will not be kept is never encoded. That ordering is /// what bounds an application re-evaluating a tracked flag in a render loop: once the limit is reached an /// evaluation costs no more than its summary counter, however fast the loop runs. - private func holdHoldingPendingLock(_ event: Event) -> Bool { + private func admitToPending(_ event: Event) -> Bool { + pendingLock.assertOwned() guard pending.count + store.pendingEventCount < capacity else { return false } @@ -407,7 +409,7 @@ class EventReporter: EventReporting { func commitRecordedEvents() { commitLock.lock() defer { commitLock.unlock() } - commitRecordedEventsHoldingCommitLock() + encodeAndCommitPending() } /// Commits what is held and closes it into a batch, as one step with respect to other commits. @@ -417,11 +419,13 @@ class EventReporter: EventReporting { private func commitAndCloseBatch() { commitLock.lock() defer { commitLock.unlock() } - commitRecordedEventsHoldingCommitLock() + encodeAndCommitPending() _ = store.closeBatch() } - private func commitRecordedEventsHoldingCommitLock() { + /// Requires `commitLock`. + private func encodeAndCommitPending() { + commitLock.assertOwned() pendingLock.lock() let run = pending pending = [] @@ -449,6 +453,7 @@ class EventReporter: EventReporting { /// Requires `commitLock`: two threads draining separate runs would stage them in whichever order they finished /// encoding, which is not the order they were recorded in. private func stage(_ run: [Event]) { + commitLock.assertOwned() for event in run { guard let encoded = encode(event) else { diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index 6b6e49cfe..de9fafc68 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -290,31 +290,32 @@ final class EventStore: EventStoring { func commit() { ioLock.lock() defer { ioLock.unlock() } - commitHoldingIoLock() + writeStagedBytes() } func closeBatch() -> EventBatch? { ioLock.lock() defer { ioLock.unlock() } - commitHoldingIoLock() + writeStagedBytes() bufferLock.lock() let events = committedEvents let stillBuffered = bufferedEventCount bufferLock.unlock() - let fromLog = events > 0 ? closeOpenLogHoldingIoLock(events) : nil + let fromLog = events > 0 ? closeOpenLog(events) : nil // Closed after the log, so the older events are delivered first. Both can hold events at once: a write that // fails after earlier ones landed gives up on persistence with those earlier events in the log and the failed // ones back in memory, and closing only the log would leave the rest to a delivery that reports success // without them. - let fromMemory = stillBuffered > 0 ? closeInMemoryBatchHoldingIoLock() : nil + let fromMemory = stillBuffered > 0 ? closeInMemoryBatch() : nil return fromLog ?? fromMemory } /// Requires `ioLock`. - private func closeOpenLogHoldingIoLock(_ events: Int) -> EventBatch? { + private func closeOpenLog(_ events: Int) -> EventBatch? { + ioLock.assertOwned() closeDescriptor() let payloadId = UUID().uuidString @@ -377,7 +378,7 @@ final class EventStore: EventStoring { .compactMap { file -> (EventBatch, Date)? in let payloadId = String(file.lastPathComponent.dropFirst(EventStore.batchPrefix.count)) // Reading is only for a batch this process has not counted: one a previous run left behind. - guard let events = eventCounts[payloadId] ?? countOfEventsHoldingIoLock(in: file) + guard let events = eventCounts[payloadId] ?? countOfEvents(in: file) else { return nil } eventCounts[payloadId] = events let modified = (try? file.resourceValues(forKeys: [.contentModificationDateKey]))?.contentModificationDate @@ -407,7 +408,8 @@ final class EventStore: EventStoring { /// How many events a batch file holds, or nil when it is not one to list this time. /// /// Requires `ioLock`. - private func countOfEventsHoldingIoLock(in file: URL) -> Int? { + private func countOfEvents(in file: URL) -> Int? { + ioLock.assertOwned() switch read(file) { case .missing: // Gone since the directory was listed. @@ -466,11 +468,12 @@ final class EventStore: EventStoring { func recoverInterruptedLog() { ioLock.lock() defer { ioLock.unlock() } - recoverInterruptedLogHoldingIoLock() + recoverInterruptedLogOnce() } /// Requires `ioLock`. Does its work once per store: after that, `current` is a log this run opened. - private func recoverInterruptedLogHoldingIoLock() { + private func recoverInterruptedLogOnce() { + ioLock.assertOwned() guard !hasRecoveredInterruptedLog else { return } hasRecoveredInterruptedLog = true @@ -513,7 +516,8 @@ final class EventStore: EventStoring { // MARK: Writing /// Requires `ioLock`. - private func commitHoldingIoLock() { + private func writeStagedBytes() { + ioLock.assertOwned() bufferLock.lock() guard persistEvents, bufferedEventCount > 0 else { @@ -546,9 +550,10 @@ final class EventStore: EventStoring { /// Requires `ioLock`. private func append(_ bytes: Data) -> Bool { + ioLock.assertOwned() guard let descriptor = descriptorForAppending() else { - disablePersistenceHoldingIoLock() + disablePersistence() return false } return append(bytes, to: descriptor) @@ -556,13 +561,14 @@ final class EventStore: EventStoring { /// Requires `ioLock`. private func append(_ bytes: Data, to descriptor: Int32) -> Bool { + ioLock.assertOwned() guard let failure = writeLog(descriptor, bytes) else { return true } // Never trap on a full disk. The SDK gives up on persistence for the rest of the session rather than taking // the application down with it, which is how other SDKs have crashed their hosts. os_log("%s giving up on persisting events: errno %d", log: logger, type: .debug, typeName(and: #function), failure) - disablePersistenceHoldingIoLock() + disablePersistence() return false } @@ -590,13 +596,14 @@ final class EventStore: EventStoring { /// Requires `ioLock`. private func descriptorForAppending() -> Int32? { + ioLock.assertOwned() if descriptor >= 0 { return descriptor } // Before opening, because `current` may still be the log a previous run died writing. Appended to, this run's // events would sit behind that run's torn last frame, where a reader can no longer find where they start. - recoverInterruptedLogHoldingIoLock() + recoverInterruptedLogOnce() do { try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) @@ -632,6 +639,7 @@ final class EventStore: EventStoring { /// Requires `ioLock`. private func closeDescriptor() { + ioLock.assertOwned() guard descriptor >= 0 else { return } closeFile(descriptor) @@ -650,7 +658,7 @@ final class EventStore: EventStoring { deinit { // One acquisition: the lock is not recursive, and `commit()` would take it again. ioLock.lock() - commitHoldingIoLock() + writeStagedBytes() closeDescriptor() ioLock.unlock() } @@ -672,7 +680,8 @@ private extension EventStore { /// Gives up on the disk for the rest of the session, leaving the events themselves alone. /// /// Requires `ioLock`. - func disablePersistenceHoldingIoLock() { + func disablePersistence() { + ioLock.assertOwned() closeDescriptor() bufferLock.lock() persistEvents = false @@ -682,7 +691,8 @@ private extension EventStore { /// Closes whatever is staged into a batch that never reaches a file. /// /// Requires `ioLock`. - func closeInMemoryBatchHoldingIoLock() -> EventBatch? { + func closeInMemoryBatch() -> EventBatch? { + ioLock.assertOwned() bufferLock.lock() let frames = bufferData let events = bufferedEventCount diff --git a/LaunchDarkly/LaunchDarkly/UnfairLock.swift b/LaunchDarkly/LaunchDarkly/UnfairLock.swift index e35bfa325..2cb96d992 100644 --- a/LaunchDarkly/LaunchDarkly/UnfairLock.swift +++ b/LaunchDarkly/LaunchDarkly/UnfairLock.swift @@ -23,8 +23,21 @@ final class UnfairLock { func unlock() { os_unfair_lock_unlock(unfairLock) } + + /// Traps unless the calling thread holds the lock. Debug builds only, since it is called on recording paths. + @inline(__always) + func assertOwned() { + #if DEBUG + os_unfair_lock_assert_owner(unfairLock) + #endif + } } #else /// `os_unfair_lock` is Darwin only, so elsewhere this is Foundation's mutex. typealias UnfairLock = NSLock + +extension NSLock { + /// `NSLock` cannot tell which thread holds it, so there is nothing to check. + func assertOwned() {} +} #endif From fd73b60da1e8d449bc3f424734810aabc12451b3 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Fri, 9 Oct 2026 07:45:33 -0700 Subject: [PATCH 011/158] tests --- .../LaunchDarkly/Models/LDConfig.swift | 3 + .../ServiceObjects/EventStore.swift | 49 ++- .../Models/LDConfigSpec.swift | 4 + .../ServiceObjects/EventReporterSpec.swift | 360 ++++++++++++++++++ .../ServiceObjects/EventStoreSpec.swift | 41 ++ 5 files changed, 442 insertions(+), 15 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift index e802f95a1..65c52f595 100644 --- a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift @@ -26,6 +26,9 @@ public enum LDStreamingMode { /// disk is what closes that gap: they are delivered on a later launch instead of being lost. @objc(LDEventPersistence) public enum EventPersistence: Int { /// Events are kept in memory only, and nothing survives the process ending. + /// + /// Events an earlier launch wrote to disk with persistence turned on are left there, and delivered by the next + /// launch that has it turned on. case disabled /// Events are written to disk, off the thread that recorded them. /// diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index de9fafc68..b30640fe8 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -118,6 +118,13 @@ final class EventStore: EventStoring { /// Whether events are being written to disk, which is what the application asked for until a write fails and the /// store gives up on persistence for the rest of the session. private var persistEvents = true + /// Whether what is on the disk is this store's to recover, list, and deliver. + /// + /// False only where the application did not ask for persistence. What a run with persistence turned on left behind + /// waits for the next run that has it turned on, rather than being delivered by a run that was told to keep + /// events off the disk. A store that gave up on the disk partway through a session still reads it, since the + /// batches there are its own. + private let readsDisk: Bool /// Whether a commit is already on its way, so that a burst of recordings queues one write rather than one each. private var isCommitScheduled = false @@ -167,10 +174,9 @@ final class EventStore: EventStoring { self.readFile = readFile self.writeLog = writeLog // An application that has not asked for persistence gets the same store running the same way it runs once a - // write has failed: events are held, delivered, and lost only if the process dies. Batches a run with - // persistence turned on left behind are still recovered and delivered, which is why this sets the runtime - // flag rather than skipping the store's reads. + // write has failed: events are held, delivered, and lost only if the process dies. self.persistEvents = persistEvents + self.readsDisk = persistEvents } /// The directory the store for a mobile key belongs in, or nil where the platform gave us nowhere to write. @@ -369,11 +375,31 @@ final class EventStore: EventStoring { ioLock.lock() defer { ioLock.unlock() } + var batches = readsDisk ? batchesOnDisk() : [] + + // After the files, since anything held in memory was closed by this run and so is newer than whatever a + // previous run left on the disk. + batches.append(contentsOf: inMemoryBatches.map { + EventBatch(payloadId: $0.payloadId, eventCount: $0.eventCount) + }) + + bufferLock.lock() + closedEvents = batches.reduce(0) { $0 + $1.eventCount } + bufferLock.unlock() + + return batches + } + + /// The batch files in the directory, oldest first. + /// + /// Requires `ioLock`. + private func batchesOnDisk() -> [EventBatch] { + ioLock.assertOwned() let contents = (try? FileManager.default.contentsOfDirectory(at: directory, includingPropertiesForKeys: [.contentModificationDateKey], options: [.skipsHiddenFiles])) ?? [] - var batches = contents + let batches = contents .filter { $0.lastPathComponent.hasPrefix(EventStore.batchPrefix) } .compactMap { file -> (EventBatch, Date)? in let payloadId = String(file.lastPathComponent.dropFirst(EventStore.batchPrefix.count)) @@ -392,16 +418,6 @@ final class EventStore: EventStoring { let listed = Set(batches.map { $0.payloadId }) eventCounts = eventCounts.filter { listed.contains($0.key) } - // After the files, since anything held in memory was closed by this run and so is newer than whatever a - // previous run left on the disk. - batches.append(contentsOf: inMemoryBatches.map { - EventBatch(payloadId: $0.payloadId, eventCount: $0.eventCount) - }) - - bufferLock.lock() - closedEvents = batches.reduce(0) { $0 + $1.eventCount } - bufferLock.unlock() - return batches } @@ -474,7 +490,7 @@ final class EventStore: EventStoring { /// Requires `ioLock`. Does its work once per store: after that, `current` is a log this run opened. private func recoverInterruptedLogOnce() { ioLock.assertOwned() - guard !hasRecoveredInterruptedLog + guard readsDisk, !hasRecoveredInterruptedLog else { return } hasRecoveredInterruptedLog = true @@ -846,6 +862,9 @@ extension EventStore { ioLock.lock() defer { ioLock.unlock() } + guard readsDisk + else { return [] } + let contents = (try? FileManager.default.contentsOfDirectory(at: directory, includingPropertiesForKeys: nil, options: [.skipsHiddenFiles])) ?? [] diff --git a/LaunchDarkly/LaunchDarklyTests/Models/LDConfigSpec.swift b/LaunchDarkly/LaunchDarklyTests/Models/LDConfigSpec.swift index 3a0eed8d7..7f86d0cc7 100644 --- a/LaunchDarkly/LaunchDarklyTests/Models/LDConfigSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/Models/LDConfigSpec.swift @@ -57,6 +57,10 @@ final class LDConfigSpec: XCTestCase { ("wrapper version", Constants.wrapperVersion, { c, v in c.wrapperVersion = v as! String? }), ("additional headers", Constants.additionalHeaders, { c, v in c.additionalHeaders = v as! [String: String]})] + func testEventPersistenceIsOffUnlessTheApplicationAsksForIt() { + XCTAssertEqual(LDConfig(mobileKey: LDConfig.Constants.mockMobileKey, autoEnvAttributes: .disabled).eventPersistence, .disabled) + } + func testInitDefault() { let config = LDConfig(mobileKey: LDConfig.Constants.mockMobileKey, autoEnvAttributes: .disabled) XCTAssertEqual(config.mobileKey, LDConfig.Constants.mockMobileKey) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift index d048b11af..df6b5c1b1 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift @@ -103,6 +103,19 @@ final class EventReporterSpec: QuickSpec { return recorded } + /// Flushes and returns what the flush answered. + func flushOutcome() -> Bool? { + var delivered: Bool? + // Long enough for a refused post and its retry a second later. + waitUntil(timeout: .seconds(10)) { done in + eventReporter.flushReportingOutcome { result in + delivered = result + done() + } + } + return delivered + } + func cleanUp() { eventReporter.isOnline = false store.deleteEverything() @@ -120,6 +133,7 @@ final class EventReporterSpec: QuickSpec { durabilitySpec() commitSpec() flushReportingOutcomeSpec() + outageSpec() } private func initSpec() { @@ -1185,6 +1199,120 @@ extension EventReporterSpec { expect(testContext.serviceMock.publishEventDataCallCount) == 1 expect(testContext.store.pendingEventCount) == 0 } + + it("writes a deferred track while a delivery holds the delivery queue") { + testContext = TestContext(eventPersistence: .deferred) + // Stands in for a post to a network that does not answer, which holds the queue for as long as its + // timeouts allow. + let released = DispatchSemaphore(value: 0) + defer { released.signal() } + testContext.eventReporter.occupyQueue(until: released) + + testContext.eventReporter.record(CustomEvent(key: "tracked", context: testContext.context)) + + expect(eventsLeftOnDisk().compactMap { $0.kindField }).toEventually(equal(["custom"])) + } + + it("has written everything by the time a flush returns, even while a delivery holds the delivery queue") { + testContext = TestContext(lastEventResponseDate: Date()) + let released = DispatchSemaphore(value: 0) + defer { released.signal() } + testContext.eventReporter.occupyQueue(until: released) + let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: true) + testContext.eventReporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: testContext.context, includeReason: false) + + // What `LDClient.close()` relies on: the write is the caller's, so a delivery that never finishes + // cannot keep it from happening. + testContext.eventReporter.flush(completion: nil) + + expect(eventsLeftOnDisk().compactMap { $0.kindField }) == ["feature", "summary"] + } + + it("shares writes between tracks recorded at once, and has each on disk when recording it returns") { + let trackers = 8 + let writesLock = NSLock() + var writes = 0 + // A slow disk, so that the other tracks arrive while a write is under way. + let store = EventStore.temporary(writeLog: { descriptor, bytes in + writesLock.lock() + writes += 1 + writesLock.unlock() + Thread.sleep(forTimeInterval: 0.05) + return EventStore.writeAll(descriptor, bytes) + }) + testContext = TestContext(store: store, eventCapacity: 100) + let reporter = testContext.eventReporter! + let context = testContext.context! + + let start = DispatchSemaphore(value: 0) + let finished = DispatchGroup() + let missingLock = NSLock() + var missingOnReturn: [String] = [] + for index in 0..= 3 + expect(posts) <= Int(window / interval) + 1 + Int(window / 1.0) + 1 + + // Offline stops it entirely, kept batch or not. Waits out a retry already scheduled before counting. + testContext.eventReporter.isOnline = false + waitLettingTimersFire(1.0 + interval) + let whenOffline = testContext.serviceMock.publishEventDataCallCount + waitLettingTimersFire(interval * 3) + expect(testContext.serviceMock.publishEventDataCallCount) == whenOffline + } + } + } } /// The events a store is holding, as JSON with `creationDate` dropped. @@ -1717,6 +2056,11 @@ private final class ClosingHookStore: EventStoring { } } +/// Waits while letting the main run loop turn, since that is where the report timer fires. +private func waitLettingTimersFire(_ seconds: TimeInterval) { + RunLoop.current.run(until: Date().addingTimeInterval(seconds)) +} + private func customKeys(_ body: Data) -> [String] { guard case .array(let events)? = try? JSONDecoder().decode(LDValue.self, from: body) else { return [] } @@ -1744,6 +2088,22 @@ private extension LDValue { else { return nil } return kind } + + var keyField: String? { + guard case .object(let fields) = self, case .string(let key) = fields["key"] + else { return nil } + return key + } +} + +private extension DarklyServiceMock { + /// Answers every event post with this status and no error, the way a service that is reachable answers. + func respondToEvents(with status: Int) { + stubbedEventResponse = (nil, + HTTPURLResponse(url: config.eventsUrl, statusCode: status, httpVersion: Constants.httpVersion, headerFields: nil), + nil, + nil) + } } private extension Date { diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 24df6b3a6..3b87f22ab 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -40,6 +40,30 @@ final class EventStoreSpec: QuickSpec { expect(store.pendingEventCount) == 1 expect(try store.body(of: batch)) == Data("[{\"kind\":\"custom\",\"key\":\"held\"}]".utf8) } + + it("leaves alone what a run that did persist left behind, for the next run that persists") { + let persisting = EventStore.temporary() + defer { persisting.deleteEverything() } + _ = persisting.stage(EventStoreSpec.payload("closed")) + persisting.commit() + expect(persisting.closeBatch()).toNot(beNil()) + _ = persisting.stage(EventStoreSpec.payload("open")) + persisting.commit() + + let withoutPersistence = EventStore(directory: persisting.directory, capacity: 100, persistEvents: false, logger: .disabled) + withoutPersistence.recoverInterruptedLog() + expect(withoutPersistence.pendingBatches()).to(beEmpty()) + expect(withoutPersistence.pendingEventCount) == 0 + + _ = withoutPersistence.stage(EventStoreSpec.payload("in-memory")) + let own = withoutPersistence.closeBatch() + expect(withoutPersistence.pendingBatches().map(\.payloadId)) == [own?.payloadId] + + let persistingAgain = EventStoreSpec.reader(sharing: persisting) + persistingAgain.recoverInterruptedLog() + let bodies = persistingAgain.pendingBatches().map { EventStoreSpec.keys(ofBody: try? persistingAgain.body(of: $0) ?? nil) } + expect(bodies) == [["closed"], ["open"]] + } } } @@ -416,6 +440,23 @@ final class EventStoreSpec: QuickSpec { expect(EventStoreSpec.keys(of: store)) == ["before", "after"] expect(store.pendingEventCount) == 2 } + + it("treats a batch something else already removed as nothing to send, and removing it again as harmless") { + _ = store.stage(EventStoreSpec.payload("delivered")) + let batch = store.closeBatch()! + _ = store.stage(EventStoreSpec.payload("still-pending")) + let other = store.closeBatch()! + + // Delivered through a store reading the same directory, as the next run would. + EventStoreSpec.reader(sharing: store).remove(batch) + + // Nothing, which a delivery takes as already delivered rather than as a failure to retry. + expect(try store.body(of: batch)).to(beNil()) + store.remove(batch) + + expect(store.pendingBatches()) == [other] + expect(store.pendingEventCount) == 1 + } } } From 2b25a35d018d95f60335d4b3c471bde4dce014b2 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Fri, 9 Oct 2026 11:40:27 -0700 Subject: [PATCH 012/158] fix(events): Answer every flush off the caller's thread The completion contract promises a background queue, but NullEventReporter answered inline, as did flush(completion:) on a closed client. The threading a caller saw then depended on whether events were turned on, and one taking a lock its completion also takes would deadlock only with them off. flushAndWait(timeout: 0) is documented not to wait, and so to report false, but it waited against an expired deadline, which succeeds where an answer already landed. It now gives up explicitly, after starting the deliveries: at .immediate those put the events on disk before returning, which is what makes a zero timeout worth calling. Co-authored-by: Cursor --- LaunchDarkly/LaunchDarkly/LDClientFlush.swift | 10 +++++- .../ServiceObjects/EventReporter.swift | 9 ++++-- .../LaunchDarklyTests/LDClientSpec.swift | 31 ++++++++++++++++++- .../ServiceObjects/EventReporterSpec.swift | 26 ++++++++++++++++ 4 files changed, 72 insertions(+), 4 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/LDClientFlush.swift b/LaunchDarkly/LaunchDarkly/LDClientFlush.swift index fefcdda68..8ab4abce6 100644 --- a/LaunchDarkly/LaunchDarkly/LDClientFlush.swift +++ b/LaunchDarkly/LaunchDarkly/LDClientFlush.swift @@ -46,7 +46,9 @@ extension LDClient { guard !clients.isEmpty else { os_log("%s called on a closed client", log: config.logger, type: .debug, self.typeName(and: #function)) - completion(false) + // Answered off the caller's thread even with nothing to wait for, so that the contract holds whether or + // not the client turned out to be closed. + DispatchQueue.global().async { completion(false) } return } @@ -144,6 +146,12 @@ extension LDClient { } } + // Started, then given up on. The deliveries are what makes a zero timeout worth calling at all: at + // `.immediate` the events are on disk by the time each `flushReportingOutcome` returns. Waiting is what is + // skipped, and a reporter that answered during the loop above must not turn that into a `true`. + guard timeout > 0 + else { return false } + // Every wait is against the one deadline rather than a fresh copy of it, so the timeout the caller asked for // is the time this call can take however many environments there are. A delivery still running when the wait // gives up is left to finish rather than canceled: its events are already in flight, so abandoning it would diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift index 9498d65bc..474bc4fdb 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift @@ -60,12 +60,17 @@ class NullEventReporter: EventReporting { includeReason: Bool) { } + // There is never anything to send, but a completion still answers on a background queue, as `EventReporter`'s + // does. Were it answered inline, the threading a caller sees would depend on whether events are turned on, and a + // caller doing what the contract allows -- taking a lock it already holds, say -- would deadlock only there. func flush(completion: CompletionClosure?) { - completion?() + guard let completion + else { return } + DispatchQueue.global().async { completion() } } func flushReportingOutcome(completion: @escaping FlushOutcomeClosure) { - completion(true) + DispatchQueue.global().async { completion(true) } } func commitRecordedEvents() { diff --git a/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift b/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift index 2492f10a2..2c896aae4 100644 --- a/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift @@ -1697,10 +1697,28 @@ final class LDClientSpec: QuickSpec { var delivered: Bool? stale.subject.flush { delivered = $0 } - expect(delivered) == false + expect(delivered).toEventually(beFalse()) expect(current.eventReporterMock.flushReportingOutcomeCallCount) == 0 } + it("answers a closed client off the caller's thread, as it does one with a delivery to wait for") { + let testContext = TestContext() + testContext.start() + testContext.subject.close() + + // The contract holds whether or not the client turned out to be closed: a caller told its completion + // is elsewhere may take a lock there that it is holding now. + let caller = Thread.current + var answeredOnCaller: Bool? + waitUntil { done in + testContext.subject.flush { _ in + answeredOnCaller = Thread.current === caller + done() + } + } + expect(answeredOnCaller) == false + } + it("reports once every environment has answered, and only once") { let testContext = TestContext() try testContext.config.setSecondaryMobileKeys(["alternate": Constants.alternateMockMobileKey]) @@ -1752,6 +1770,17 @@ final class LDClientSpec: QuickSpec { expect(testContext.subject.flushAndWait(timeout: 0.05)) == false } + it("does not wait without a budget, though it still starts the delivery") { + let testContext = TestContext() + testContext.start() + // Answers inside the call, so a zero budget that waited at all would report true. + answer(testContext.eventReporterMock, with: true) + + expect(testContext.subject.flushAndWait(timeout: 0)) == false + // Still worth calling: at `.immediate` the events are on disk by the time the delivery is queued. + expect(testContext.eventReporterMock.flushReportingOutcomeCallCount) == 1 + } + it("reports false once closed") { let testContext = TestContext() testContext.start() diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift index df6b5c1b1..fb48328b6 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift @@ -134,6 +134,32 @@ final class EventReporterSpec: QuickSpec { commitSpec() flushReportingOutcomeSpec() outageSpec() + nullEventReporterSpec() + } + + private func nullEventReporterSpec() { + describe("NullEventReporter") { + it("answers every flush off the caller's thread") { + let reporter = NullEventReporter() + let caller = Thread.current + let lock = UnfairLock() + var answers: [(delivered: Bool, onCaller: Bool)] = [] + let record: (Bool) -> Void = { delivered in + lock.withLock { answers.append((delivered, Thread.current === caller)) } + } + + // Events are off, so there is never anything to wait for. The threading a caller sees still must not + // depend on that: one holding a lock its completion takes would deadlock only with events off. + reporter.flush { record(true) } + reporter.flushReportingOutcome(completion: record) + + expect(lock.withLock { answers.count }).toEventually(equal(2)) + lock.withLock { + expect(answers.map { $0.delivered }) == [true, true] + expect(answers.contains { $0.onCaller }) == false + } + } + } } private func initSpec() { From e5a4e4353837919c4ef41e04d1b6c8a02ecd9af5 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Fri, 9 Oct 2026 19:19:56 -0700 Subject: [PATCH 013/158] fix(events): Give each process a log of its own Every process shared one open log, named current. A second process starting up recovered it as if a previous run had left it, renaming it into a batch while the first was still appending through its open descriptor, so the first's later events landed in a batch already sent or deleted. The open log is now open--, as on Android, and recovery only touches this process's own. Closed batches stay shared, so a process that never runs again still has its events delivered by another. Co-authored-by: Cursor --- .../ServiceObjects/EventStore.swift | 64 ++++++++++---- .../ServiceObjects/EventStoreSpec.swift | 85 ++++++++++++++++++- 2 files changed, 130 insertions(+), 19 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index b30640fe8..7d6f6a793 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -62,7 +62,7 @@ protocol EventStoring { /// Forgets a batch, which is only correct once LaunchDarkly has accepted it or permanently refused it. func remove(_ batch: EventBatch) - /// Closes any log left open by a previous run of the application, so its events join the batches to deliver. + /// Closes any log left open by a previous run of this process, so its events join the batches to deliver. func recoverInterruptedLog() } @@ -82,6 +82,8 @@ protocol EventStoring { final class EventStore: EventStoring { /// Names a log that has been closed off for delivery; the rest of the name is the batch's payload ID. private static let batchPrefix = "ready-" + /// Names the log a single process appends to; the rest of the name is that process. + private static let openLogPrefix = "open-" /// How many staged bytes are allowed to accumulate before one of them pays for a write. /// /// This is what keeps a syscall off most recordings while bounding what a crash can take with it, for an @@ -152,7 +154,14 @@ final class EventStore: EventStoring { /// Only to be used while holding `ioLock`. private var eventCounts: [String: Int] = [:] - private var currentLogUrl: URL { directory.appendingPathComponent("current") } + /// The log this process appends to. + /// + /// One per process rather than one per directory, so that only one process ever writes a given log and recovery + /// only ever touches its own. With one shared log, a second process starting up would close off, as if a previous + /// run had left it, a log the first is still appending to through its open descriptor, and the first's later + /// events would land in a batch already sent or deleted. Another process's log is left for that process to recover + /// the next time it runs. Two instances of one macOS application run under one name, and so still share a log. + let openLogUrl: URL /// A batch is identified by its payload ID rather than by a path, so that a batch listed from the directory and the /// same batch as it was closed are one thing. @@ -163,11 +172,13 @@ final class EventStore: EventStoring { init(directory: URL, capacity: Int, persistEvents: Bool = true, + processName: String = ProcessInfo.processInfo.processName, logger: OSLog, commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.EventStore.commitQueue", qos: .userInitiated), readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }, writeLog: @escaping (Int32, Data) -> Int32? = EventStore.writeAll) { self.directory = directory + self.openLogUrl = directory.appendingPathComponent(EventStore.openLogPrefix + EventStore.logName(for: processName)) self.capacity = capacity self.logger = logger self.commitQueue = commitQueue @@ -179,6 +190,26 @@ final class EventStore: EventStoring { self.readsDisk = persistEvents } + /// Names the log belonging to one process. + /// + /// The readable part is the process name with anything awkward in a filename replaced, which keeps a directory of + /// these diagnosable. That reduction is not one-to-one, and two processes sharing a log is the one thing this name + /// exists to prevent, so a digest of the original name is appended. + static func logName(for processName: String) -> String { + let trimmed = processName.trimmingCharacters(in: .whitespacesAndNewlines) + let name = trimmed.isEmpty ? "unknown" : trimmed + let readable = String(name.unicodeScalars.map { scalar -> Character in + switch scalar { + case "a"..."z", "A"..."Z", "0"..."9", "-", "_": + return Character(scalar) + default: + return "_" + } + }) + let digest = Util.sha256(name).prefix(4).map { String(format: "%02x", $0) }.joined() + return "\(readable)-\(digest)" + } + /// The directory the store for a mobile key belongs in, or nil where the platform gave us nowhere to write. /// /// tvOS is the reason this is not simply Application Support: a tvOS application is only allowed to persist to @@ -326,13 +357,13 @@ final class EventStore: EventStoring { let payloadId = UUID().uuidString do { - try FileManager.default.moveItem(at: currentLogUrl, to: url(of: payloadId)) + try FileManager.default.moveItem(at: openLogUrl, to: url(of: payloadId)) } catch { os_log("%s could not close the event log: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) // The log is still there to try again with, unless it is the log that went missing -- a purged cache // directory on tvOS, say. Then its events are gone, and they have to stop counting against capacity or the // store would refuse events for the rest of the session. - if !FileManager.default.fileExists(atPath: currentLogUrl.path) { + if !FileManager.default.fileExists(atPath: openLogUrl.path) { bufferLock.lock() committedEvents = 0 bufferLock.unlock() @@ -478,16 +509,17 @@ final class EventStore: EventStoring { bufferLock.unlock() } - /// Closes any log left open by a previous run of the application, so its events join the batches to deliver. + /// Closes any log left open by a previous run of this process, so its events join the batches to deliver. /// - /// The events in it were recorded by a process that is gone, so there is no one left to add to it. + /// The events in it were recorded by a process that is gone, so there is no one left to add to it. Only this + /// process's own log is touched: another process's may still be open in a process that is alive. func recoverInterruptedLog() { ioLock.lock() defer { ioLock.unlock() } recoverInterruptedLogOnce() } - /// Requires `ioLock`. Does its work once per store: after that, `current` is a log this run opened. + /// Requires `ioLock`. Does its work once per store: after that, the open log is one this run opened. private func recoverInterruptedLogOnce() { ioLock.assertOwned() guard readsDisk, !hasRecoveredInterruptedLog @@ -495,7 +527,7 @@ final class EventStore: EventStoring { hasRecoveredInterruptedLog = true let events: Int - switch read(currentLogUrl) { + switch read(openLogUrl) { case .missing: return case .failed(let error): @@ -503,13 +535,13 @@ final class EventStore: EventStoring { // would lose events that may be intact. Closed off uncounted instead, it is read and counted by whichever // listing first manages to. os_log("%s could not read events from a previous run, will try again later: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) - try? FileManager.default.moveItem(at: currentLogUrl, to: url(of: UUID().uuidString)) + try? FileManager.default.moveItem(at: openLogUrl, to: url(of: UUID().uuidString)) return case .bytes(let log): guard let counted = EventLogFormat.eventCount(in: log) else { // Unreadable, and a log that cannot be read cannot be appended to either. - try? FileManager.default.removeItem(at: currentLogUrl) + try? FileManager.default.removeItem(at: openLogUrl) return } events = counted @@ -517,12 +549,12 @@ final class EventStore: EventStoring { guard events > 0 else { - try? FileManager.default.removeItem(at: currentLogUrl) + try? FileManager.default.removeItem(at: openLogUrl) return } let payloadId = UUID().uuidString - guard (try? FileManager.default.moveItem(at: currentLogUrl, to: url(of: payloadId))) != nil + guard (try? FileManager.default.moveItem(at: openLogUrl, to: url(of: payloadId))) != nil else { return } eventCounts[payloadId] = events @@ -617,7 +649,7 @@ final class EventStore: EventStoring { return descriptor } - // Before opening, because `current` may still be the log a previous run died writing. Appended to, this run's + // Before opening, because the open log may still be the one a previous run died writing. Appended to, this run's // events would sit behind that run's torn last frame, where a reader can no longer find where they start. recoverInterruptedLogOnce() @@ -631,7 +663,7 @@ final class EventStore: EventStoring { // O_APPEND is what makes each write land at the end of the file as one step, so that clients for several // environments writing their own logs, or a commit racing a reader, cannot produce a spliced frame. - let opened = currentLogUrl.withUnsafeFileSystemRepresentation { path -> Int32 in + let opened = openLogUrl.withUnsafeFileSystemRepresentation { path -> Int32 in guard let path = path else { return -1 } return openFile(path, O_WRONLY | O_APPEND | O_CREAT, 0o600) @@ -869,8 +901,8 @@ extension EventStore { includingPropertiesForKeys: nil, options: [.skipsHiddenFiles])) ?? [] var logs = contents.filter { $0.lastPathComponent.hasPrefix(EventStore.batchPrefix) }.sorted { $0.path < $1.path } - if FileManager.default.fileExists(atPath: currentLogUrl.path) { - logs.append(currentLogUrl) + if FileManager.default.fileExists(atPath: openLogUrl.path) { + logs.append(openLogUrl) } var payloads: [Data] = [] diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 3b87f22ab..f7a10976b 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -12,6 +12,85 @@ final class EventStoreSpec: QuickSpec { capacitySpec() fallbackSpec() optOutSpec() + multiProcessSpec() + } + + private func multiProcessSpec() { + describe("several processes sharing a directory") { + var app: EventStore! + afterEach { + app.deleteEverything() + } + + /// Another process of the same application, using the same directory under its own name. + func otherProcess(_ name: String = "widget") -> EventStore { + EventStore(directory: app.directory, capacity: 100, processName: name, logger: .disabled) + } + + beforeEach { + app = EventStore(directory: EventStore.temporary().directory, capacity: 100, processName: "app", logger: .disabled) + } + + it("leaves a log another process still has open where it is, so that process's later events reach it") { + _ = app.stage(EventStoreSpec.payload("before")) + app.commit() + + // Starting up, the other process recovers what its own previous run left open, and nothing else. + let widget = otherProcess() + widget.recoverInterruptedLog() + expect(widget.pendingBatches()).to(beEmpty()) + + _ = app.stage(EventStoreSpec.payload("after")) + app.commit() + let batch = app.closeBatch() + expect(batch?.eventCount) == 2 + expect(EventStoreSpec.keys(ofBody: batch.flatMap { try? app.body(of: $0) })) == ["before", "after"] + } + + it("keeps each process's events in a log of its own") { + let widget = otherProcess() + _ = app.stage(EventStoreSpec.payload("from-app")) + app.commit() + _ = widget.stage(EventStoreSpec.payload("from-widget")) + widget.commit() + + expect(app.openLogUrl) != widget.openLogUrl + let fromApp = app.closeBatch() + let fromWidget = widget.closeBatch() + expect(EventStoreSpec.keys(ofBody: fromApp.flatMap { try? app.body(of: $0) })) == ["from-app"] + expect(EventStoreSpec.keys(ofBody: fromWidget.flatMap { try? widget.body(of: $0) })) == ["from-widget"] + } + + it("delivers a batch another process closed, since that process may never run again") { + _ = app.stage(EventStoreSpec.payload("closed-by-app")) + let closed = app.closeBatch() + + let widget = otherProcess() + widget.recoverInterruptedLog() + expect(closed).toNot(beNil()) + expect(widget.pendingBatches().map(\.payloadId)) == [closed?.payloadId].compactMap { $0 } + } + + it("recovers a process's own log on its next run") { + _ = app.stage(EventStoreSpec.payload("left-open")) + app.commit() + + let nextRun = otherProcess("app") + nextRun.recoverInterruptedLog() + let batches = nextRun.pendingBatches() + expect(batches.count) == 1 + expect(EventStoreSpec.keys(ofBody: batches.first.flatMap { try? nextRun.body(of: $0) })) == ["left-open"] + } + + it("names logs so that process names reducing to the same characters stay apart") { + let colon = EventStore.logName(for: "com.app:remote") + let dot = EventStore.logName(for: "com.app.remote") + expect(colon) != dot + expect(colon.hasPrefix("com_app_remote-")) == true + expect(dot.hasPrefix("com_app_remote-")) == true + expect(EventStore.logName(for: " ").hasPrefix("unknown-")) == true + } + } } private func optOutSpec() { @@ -237,7 +316,7 @@ final class EventStoreSpec: QuickSpec { } store.commit() // The previous run died partway through writing its last frame. - let log = store.directory.appendingPathComponent("current") + let log = store.openLogUrl try Data(contentsOf: log).dropLast(3).write(to: log) let next = EventStoreSpec.reader(sharing: store) @@ -280,7 +359,7 @@ final class EventStoreSpec: QuickSpec { store.commit() // A process killed partway through a write leaves the last frame short of its declared length. - let log = store.directory.appendingPathComponent("current") + let log = store.openLogUrl let whole = try Data(contentsOf: log) try whole.dropLast(8).write(to: log) @@ -331,7 +410,7 @@ final class EventStoreSpec: QuickSpec { readFile: EventStoreSpec.readFailing(while: { failing })) next.recoverInterruptedLog() // Moved out from under the name this run appends to, and not deleted. - expect(FileManager.default.fileExists(atPath: store.directory.appendingPathComponent("current").path)) == false + expect(FileManager.default.fileExists(atPath: store.openLogUrl.path)) == false // So this run has the name to itself, and the events that are intact are only waiting for a read that // works to be counted and delivered. From 61ba495e561fcd9a44540a3cf08fbc0d20da29a9 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Fri, 9 Oct 2026 21:08:32 -0700 Subject: [PATCH 014/158] Lock the open event log on a Mac, where instances share a process name A Mac runs a second instance of an application under the same process name, so a log named for the process alone would still be shared. Where that can happen (macOS, and iPhone or iPad applications running on a Mac), the open log is held under a non-blocking flock. An instance that finds the name's log locked opens one named for its process ID instead, and recovery closes off any open log no live process holds, whatever it is named. The kernel drops the lock when the process dies, so a crash never leaves a stale one. iOS, tvOS and watchOS do not lock: a suspended application holding a lock on a file in a shared container is killed (0xdead10cc), and there a process name is one live process. Co-authored-by: Cursor --- .../ServiceObjects/EventStore.swift | 164 ++++++++++++++++-- .../ServiceObjects/EventReporterSpec.swift | 10 +- .../ServiceObjects/EventStoreSpec.swift | 85 ++++++++- LaunchDarkly/LaunchDarklyTests/TestUtil.swift | 5 + 4 files changed, 235 insertions(+), 29 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index 7d6f6a793..8db83ed2c 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -160,8 +160,32 @@ final class EventStore: EventStoring { /// only ever touches its own. With one shared log, a second process starting up would close off, as if a previous /// run had left it, a log the first is still appending to through its open descriptor, and the first's later /// events would land in a batch already sent or deleted. Another process's log is left for that process to recover - /// the next time it runs. Two instances of one macOS application run under one name, and so still share a log. - let openLogUrl: URL + /// the next time it runs. + /// + /// Named for the process, unless `locksOpenLog` found another live instance holding that name, in which case it is + /// named for this instance's process ID as well. Only to be used while holding `ioLock`. + private(set) var openLogUrl: URL + /// The readable, collision-proof form of the process name every log this process opens is named after. + private let processLogName: String + /// Whether the open log is locked while this store has it open, so that another live process can tell it is taken. + /// + /// Only where two live processes can run under one name, which a process name alone cannot keep apart: a Mac runs + /// a second instance of an application on request. Elsewhere a name is one live process, and a lock is avoided + /// rather than added for nothing, because iOS kills a suspended application that holds a lock on a file in a + /// shared container. + private let locksOpenLog: Bool + + /// Whether this platform can run two live instances of an application under one process name. True on a Mac, + /// which includes an iPhone or iPad application running on one, whether built with Mac Catalyst or not. + static var instancesCanShareProcessName: Bool { + #if os(macOS) + return true + #elseif canImport(Darwin) + return ProcessInfo.processInfo.isMacCatalystApp + #else + return false + #endif + } /// A batch is identified by its payload ID rather than by a path, so that a batch listed from the directory and the /// same batch as it was closed are one thing. @@ -173,12 +197,15 @@ final class EventStore: EventStoring { capacity: Int, persistEvents: Bool = true, processName: String = ProcessInfo.processInfo.processName, + locksOpenLog: Bool = EventStore.instancesCanShareProcessName, logger: OSLog, commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.EventStore.commitQueue", qos: .userInitiated), readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }, writeLog: @escaping (Int32, Data) -> Int32? = EventStore.writeAll) { self.directory = directory - self.openLogUrl = directory.appendingPathComponent(EventStore.openLogPrefix + EventStore.logName(for: processName)) + self.processLogName = EventStore.logName(for: processName) + self.openLogUrl = directory.appendingPathComponent(EventStore.openLogPrefix + processLogName) + self.locksOpenLog = locksOpenLog self.capacity = capacity self.logger = logger self.commitQueue = commitQueue @@ -353,7 +380,9 @@ final class EventStore: EventStoring { /// Requires `ioLock`. private func closeOpenLog(_ events: Int) -> EventBatch? { ioLock.assertOwned() - closeDescriptor() + // Closed after the rename rather than before it, so that a locked log is still locked while it moves. Another + // instance recovering unlocked logs could otherwise take it in between. + defer { closeDescriptor() } let payloadId = UUID().uuidString do { @@ -512,7 +541,8 @@ final class EventStore: EventStoring { /// Closes any log left open by a previous run of this process, so its events join the batches to deliver. /// /// The events in it were recorded by a process that is gone, so there is no one left to add to it. Only this - /// process's own log is touched: another process's may still be open in a process that is alive. + /// process's own log is touched: another process's may still be open in a process that is alive. Where logs are + /// locked, any log no live process holds is recovered, whatever it is named. func recoverInterruptedLog() { ioLock.lock() defer { ioLock.unlock() } @@ -526,8 +556,30 @@ final class EventStore: EventStoring { else { return } hasRecoveredInterruptedLog = true + guard locksOpenLog + else { + recover(logAt: openLogUrl) + return + } + + // A log that can be locked is one no live process has open, so whoever wrote it is gone. That includes a log a + // second instance opened under its process ID, which no later run would otherwise come back for by name. + let contents = (try? FileManager.default.contentsOfDirectory(at: directory, + includingPropertiesForKeys: nil, + options: [.skipsHiddenFiles])) ?? [] + for log in contents where log.lastPathComponent.hasPrefix(EventStore.openLogPrefix) { + guard let held = EventStore.openLocked(log, flags: O_RDONLY) + else { continue } + recover(logAt: log) + closeFile(held) + } + } + + /// Closes off a log nobody is writing to any more. Requires `ioLock`. + private func recover(logAt leftOpen: URL) { + ioLock.assertOwned() let events: Int - switch read(openLogUrl) { + switch read(leftOpen) { case .missing: return case .failed(let error): @@ -535,13 +587,13 @@ final class EventStore: EventStoring { // would lose events that may be intact. Closed off uncounted instead, it is read and counted by whichever // listing first manages to. os_log("%s could not read events from a previous run, will try again later: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) - try? FileManager.default.moveItem(at: openLogUrl, to: url(of: UUID().uuidString)) + try? FileManager.default.moveItem(at: leftOpen, to: url(of: UUID().uuidString)) return case .bytes(let log): guard let counted = EventLogFormat.eventCount(in: log) else { // Unreadable, and a log that cannot be read cannot be appended to either. - try? FileManager.default.removeItem(at: openLogUrl) + try? FileManager.default.removeItem(at: leftOpen) return } events = counted @@ -549,12 +601,12 @@ final class EventStore: EventStoring { guard events > 0 else { - try? FileManager.default.removeItem(at: openLogUrl) + try? FileManager.default.removeItem(at: leftOpen) return } let payloadId = UUID().uuidString - guard (try? FileManager.default.moveItem(at: openLogUrl, to: url(of: payloadId))) != nil + guard (try? FileManager.default.moveItem(at: leftOpen, to: url(of: payloadId))) != nil else { return } eventCounts[payloadId] = events @@ -661,15 +713,7 @@ final class EventStore: EventStoring { } excludeFromBackup() - // O_APPEND is what makes each write land at the end of the file as one step, so that clients for several - // environments writing their own logs, or a commit racing a reader, cannot produce a spliced frame. - let opened = openLogUrl.withUnsafeFileSystemRepresentation { path -> Int32 in - guard let path = path - else { return -1 } - return openFile(path, O_WRONLY | O_APPEND | O_CREAT, 0o600) - } - - guard opened >= 0 + guard let opened = openLogForAppending() else { os_log("%s could not open the event log: errno %d", log: logger, type: .debug, typeName(and: #function), errno) return nil @@ -685,6 +729,64 @@ final class EventStore: EventStoring { return descriptor } + /// Opens the log this process appends to, locked where logs are. Requires `ioLock`. + private func openLogForAppending() -> Int32? { + ioLock.assertOwned() + // O_APPEND is what makes each write land at the end of the file as one step, so that clients for several + // environments writing their own logs, or a commit racing a reader, cannot produce a spliced frame. + let flags = O_WRONLY | O_APPEND | O_CREAT + guard locksOpenLog + else { + let opened = EventStore.openDescriptor(at: openLogUrl, flags: flags) + return opened >= 0 ? opened : nil + } + + if let opened = EventStore.openLocked(openLogUrl, flags: flags) { + return opened + } + + // Another live instance of the application holds the log for this process name, so this one takes a log of its + // own. The process ID keeps it apart from every other live process, and the suffix from another store in this + // one, as a closed client's can be while it is still going away. Recovery finds the log by its lock rather than + // its name, so it is not orphaned when this instance ends. + let ownLog = directory.appendingPathComponent( + "\(EventStore.openLogPrefix)\(processLogName)-\(getpid())-\(UUID().uuidString.prefix(8))") + guard let opened = EventStore.openLocked(ownLog, flags: flags) + else { return nil } + openLogUrl = ownLog + return opened + } + + private static func openDescriptor(at url: URL, flags: Int32) -> Int32 { + url.withUnsafeFileSystemRepresentation { path -> Int32 in + guard let path = path + else { return -1 } + return openFile(path, flags, 0o600) + } + } + + /// Opens a file and takes its lock without waiting, or nil where another live process holds it. + /// + /// The lock has to be on the file that is still at `url`: one renamed or removed between the open and the lock, + /// by another process closing off or recovering it, is no longer a log anyone appends to. + private static func openLocked(_ url: URL, flags: Int32) -> Int32? { + for _ in 0..<3 { + let opened = openDescriptor(at: url, flags: flags) + guard opened >= 0 + else { return nil } + guard lockFile(opened) + else { + closeFile(opened) + return nil + } + if isFile(opened, at: url) { + return opened + } + closeFile(opened) + } + return nil + } + /// Requires `ioLock`. private func closeDescriptor() { ioLock.assertOwned() @@ -937,6 +1039,12 @@ private func writeBytes(_ descriptor: Int32, _ bytes: UnsafeRawPointer, _ count: private func closeFile(_ descriptor: Int32) { _ = Darwin.close(descriptor) } + +/// Takes an exclusive lock without waiting. Released by the kernel when the descriptor closes, including when the +/// process dies, so a crashed process never leaves one behind. +private func lockFile(_ descriptor: Int32) -> Bool { + flock(descriptor, LOCK_EX | LOCK_NB) == 0 +} #else private func openFile(_ path: UnsafePointer, _ flags: Int32, _ mode: mode_t) -> Int32 { open(path, flags, mode) @@ -949,4 +1057,22 @@ private func writeBytes(_ descriptor: Int32, _ bytes: UnsafeRawPointer, _ count: private func closeFile(_ descriptor: Int32) { _ = close(descriptor) } + +private func lockFile(_ descriptor: Int32) -> Bool { + flock(descriptor, LOCK_EX | LOCK_NB) == 0 +} #endif + +/// Whether a descriptor is open on the file currently at `url`, rather than one since renamed or removed. +private func isFile(_ descriptor: Int32, at url: URL) -> Bool { + var opened = stat() + var named = stat() + guard fstat(descriptor, &opened) == 0 + else { return false } + let found = url.withUnsafeFileSystemRepresentation { path -> Bool in + guard let path = path + else { return false } + return stat(path, &named) == 0 + } + return found && opened.st_ino == named.st_ino && opened.st_dev == named.st_dev +} diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift index fb48328b6..2d83d2e83 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift @@ -1048,7 +1048,7 @@ extension EventReporterSpec { /// The events the next run of the application would find. func eventsLeftOnDisk() -> [LDValue] { - let reader = EventStore(directory: testContext.store.directory, capacity: 100, logger: .disabled) + let reader = EventStore(directory: testContext.store.directory, capacity: 100, locksOpenLog: false, logger: .disabled) return reader.pendingEventPayloads().compactMap { try? JSONDecoder().decode(LDValue.self, from: $0) } } @@ -1132,7 +1132,7 @@ extension EventReporterSpec { testContext.eventReporter.commitRecordedEvents() // Nothing delivered them and nothing closed the log, as would be the case had the process died here. - let recovered = EventStore(directory: testContext.store.directory, capacity: 100, logger: .disabled) + let recovered = EventStore(directory: testContext.store.directory, capacity: 100, locksOpenLog: false, logger: .disabled) let nextRun = EventReporter(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) nextRun.isOnline = true @@ -1154,7 +1154,7 @@ extension EventReporterSpec { testContext.recordEvents(2) testContext.eventReporter.commitRecordedEvents() - let recovered = EventStore(directory: testContext.store.directory, capacity: 100, logger: .disabled) + let recovered = EventStore(directory: testContext.store.directory, capacity: 100, locksOpenLog: false, logger: .disabled) let nextRun = EventReporter(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) // Nothing asks for a flush, and the report interval is several times longer than this waits, so the @@ -1306,7 +1306,7 @@ extension EventReporterSpec { testContext.serviceMock.respondToEvents(with: 503) testContext.eventReporter.record(CustomEvent(key: "kept", context: testContext.context)) - let recovered = EventStore(directory: testContext.store.directory, capacity: 100, logger: .disabled) + let recovered = EventStore(directory: testContext.store.directory, capacity: 100, locksOpenLog: false, logger: .disabled) let nextRun = EventReporter(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) nextRun.isOnline = true defer { nextRun.isOnline = false } @@ -1329,7 +1329,7 @@ extension EventReporterSpec { // The same directory, as the application's next launch would find it. testContext.serviceMock.stubEventResponse(success: true) - let recovered = EventStore(directory: testContext.store.directory, capacity: 100, logger: .disabled) + let recovered = EventStore(directory: testContext.store.directory, capacity: 100, locksOpenLog: false, logger: .disabled) let nextRun = EventReporter(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) nextRun.isOnline = true defer { nextRun.isOnline = false } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index f7a10976b..23d1c3038 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -13,6 +13,80 @@ final class EventStoreSpec: QuickSpec { fallbackSpec() optOutSpec() multiProcessSpec() + sameNameInstancesSpec() + } + + private func sameNameInstancesSpec() { + // As on a Mac, where a second instance of an application runs under the first one's process name. + describe("two live instances under one process name, where logs are locked") { + var first: EventStore! + afterEach { + first.deleteEverything() + } + + func instance() -> EventStore { + EventStore(directory: first.directory, capacity: 100, processName: "app", locksOpenLog: true, logger: .disabled) + } + + beforeEach { + first = EventStore(directory: EventStore.temporary().directory, + capacity: 100, + processName: "app", + locksOpenLog: true, + logger: .disabled) + } + + it("gives the second a log of its own, named for its process, and leaves the first's alone") { + _ = first.stage(EventStoreSpec.payload("first")) + first.commit() + + let second = instance() + second.recoverInterruptedLog() + expect(second.pendingBatches()).to(beEmpty()) + + _ = second.stage(EventStoreSpec.payload("second")) + second.commit() + expect(second.openLogUrl) != first.openLogUrl + expect(second.openLogUrl.lastPathComponent).to(contain("-\(getpid())-")) + + _ = first.stage(EventStoreSpec.payload("first-again")) + first.commit() + let fromFirst = first.closeBatch() + let fromSecond = second.closeBatch() + expect(EventStoreSpec.keys(ofBody: fromFirst.flatMap { try? first.body(of: $0) })) == ["first", "first-again"] + expect(EventStoreSpec.keys(ofBody: fromSecond.flatMap { try? second.body(of: $0) })) == ["second"] + } + + it("recovers the log an instance that has ended left open, though no run is named for it") { + _ = first.stage(EventStoreSpec.payload("first")) + first.commit() + + var ended: EventStore? = instance() + _ = ended?.stage(EventStoreSpec.payload("from-the-ended-instance")) + ended?.commit() + // Going away closes its descriptor, which releases the lock, as the process dying would. + ended = nil + + let next = instance() + next.recoverInterruptedLog() + let batches = next.pendingBatches() + expect(batches.count) == 1 + expect(EventStoreSpec.keys(ofBody: batches.first.flatMap { try? next.body(of: $0) })) == ["from-the-ended-instance"] + } + + it("recovers the log a previous run under the name left open, once nothing holds it") { + _ = first.stage(EventStoreSpec.payload("left-open")) + first.commit() + let directory = first.directory + first = nil + first = EventStore(directory: directory, capacity: 100, processName: "app", locksOpenLog: true, logger: .disabled) + + first.recoverInterruptedLog() + let batches = first.pendingBatches() + expect(batches.count) == 1 + expect(EventStoreSpec.keys(ofBody: batches.first.flatMap { try? first.body(of: $0) })) == ["left-open"] + } + } } private func multiProcessSpec() { @@ -22,13 +96,14 @@ final class EventStoreSpec: QuickSpec { app.deleteEverything() } - /// Another process of the same application, using the same directory under its own name. + /// Another process of the same application, using the same directory under its own name. Unlocked, as on an + /// iPhone or iPad, where a process name is one live process and the name alone keeps the logs apart. func otherProcess(_ name: String = "widget") -> EventStore { - EventStore(directory: app.directory, capacity: 100, processName: name, logger: .disabled) + EventStore(directory: app.directory, capacity: 100, processName: name, locksOpenLog: false, logger: .disabled) } beforeEach { - app = EventStore(directory: EventStore.temporary().directory, capacity: 100, processName: "app", logger: .disabled) + app = EventStore(directory: EventStore.temporary().directory, capacity: 100, processName: "app", locksOpenLog: false, logger: .disabled) } it("leaves a log another process still has open where it is, so that process's later events reach it") { @@ -129,7 +204,7 @@ final class EventStoreSpec: QuickSpec { _ = persisting.stage(EventStoreSpec.payload("open")) persisting.commit() - let withoutPersistence = EventStore(directory: persisting.directory, capacity: 100, persistEvents: false, logger: .disabled) + let withoutPersistence = EventStore(directory: persisting.directory, capacity: 100, persistEvents: false, locksOpenLog: false, logger: .disabled) withoutPersistence.recoverInterruptedLog() expect(withoutPersistence.pendingBatches()).to(beEmpty()) expect(withoutPersistence.pendingEventCount) == 0 @@ -165,7 +240,7 @@ final class EventStoreSpec: QuickSpec { /// what actually reached the disk, which is the whole question a crash asks. private static func reader(sharing store: EventStore, readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }) -> EventStore { - EventStore(directory: store.directory, capacity: 100, logger: .disabled, readFile: readFile) + EventStore(directory: store.directory, capacity: 100, locksOpenLog: false, logger: .disabled, readFile: readFile) } /// What a process out of file descriptors gets for a file that is perfectly intact. diff --git a/LaunchDarkly/LaunchDarklyTests/TestUtil.swift b/LaunchDarkly/LaunchDarklyTests/TestUtil.swift index 2addbc113..e67b2135c 100644 --- a/LaunchDarkly/LaunchDarklyTests/TestUtil.swift +++ b/LaunchDarkly/LaunchDarklyTests/TestUtil.swift @@ -52,9 +52,13 @@ func valueIsArray(_ value: LDValue?, asserts: ([LDValue]) -> Void) { extension EventStore { /// A store in a directory of its own under the temporary directory, so that tests neither see each other's events /// nor leave any in the directory a real client would use. + /// + /// Unlocked unless asked, whatever the platform. Tests stand in for the next run of the application with a second + /// store while the first is still alive, and on a Mac a locked log would rightly be left to the live one. static func temporary( capacity: Int = 100, persistEvents: Bool = true, + locksOpenLog: Bool = false, commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.tests.commitQueue"), readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }, writeLog: @escaping (Int32, Data) -> Int32? = EventStore.writeAll @@ -65,6 +69,7 @@ extension EventStore { return EventStore(directory: directory, capacity: capacity, persistEvents: persistEvents, + locksOpenLog: locksOpenLog, logger: .disabled, commitQueue: commitQueue, readFile: readFile, From 12cb3e2887dc142cfa297a85ea9f54a1b09e339c Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Fri, 9 Oct 2026 22:37:16 -0700 Subject: [PATCH 015/158] split --- LaunchDarkly.xcodeproj/project.pbxproj | 30 ++ .../ServiceObjects/EventLogFile.swift | 138 +++++ .../ServiceObjects/EventLogFormat.swift | 118 +++++ .../ServiceObjects/EventStore.swift | 488 ++---------------- .../ServiceObjects/OpenEventLog.swift | 233 +++++++++ .../ServiceObjects/EventReporterSpec.swift | 14 +- .../ServiceObjects/EventStoreSpec.swift | 29 +- LaunchDarkly/LaunchDarklyTests/TestUtil.swift | 44 +- 8 files changed, 630 insertions(+), 464 deletions(-) create mode 100644 LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFile.swift create mode 100644 LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift create mode 100644 LaunchDarkly/LaunchDarkly/ServiceObjects/OpenEventLog.swift diff --git a/LaunchDarkly.xcodeproj/project.pbxproj b/LaunchDarkly.xcodeproj/project.pbxproj index cc72b5d3e..78f70e577 100644 --- a/LaunchDarkly.xcodeproj/project.pbxproj +++ b/LaunchDarkly.xcodeproj/project.pbxproj @@ -17,6 +17,9 @@ 29FE129A280413D4008CC918 /* Util.swift in Sources */ = {isa = PBXBuildFile; fileRef = 29FE1297280413D4008CC918 /* Util.swift */; }; 29FE129B280413D4008CC918 /* Util.swift in Sources */ = {isa = PBXBuildFile; fileRef = 29FE1297280413D4008CC918 /* Util.swift */; }; 2CA9D36B3566D78850D6F0AF /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; + B8F228923BE11E444D1ECB8E /* OpenEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* OpenEventLog.swift */; }; + 3C55384C9E3485B493E1F86A /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; + AB4827D11084D58661E24B05 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; 3D2406142E0D90E000F91253 /* Plugin.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3D2406102E0D90E000F91253 /* Plugin.swift */; }; 3D2406152E0D90E000F91253 /* PluginMetadata.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3D2406112E0D90E000F91253 /* PluginMetadata.swift */; }; 3D2406162E0D90E000F91253 /* EnvironmentMetadata.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3D24060F2E0D90E000F91253 /* EnvironmentMetadata.swift */; }; @@ -49,6 +52,9 @@ 50EE85C52EA0487F007CC662 /* TimeoutExecutor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50EE85C12EA0487F007CC662 /* TimeoutExecutor.swift */; }; 50EE85C72EA0749C007CC662 /* TimeoutExecutorSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50EE85C62EA0749C007CC662 /* TimeoutExecutorSpec.swift */; }; 5AEA1F832DCA974314E07D21 /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; + D4225F1F05F2C55E9518941B /* OpenEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* OpenEventLog.swift */; }; + DDA788E3056F8AC3CA203933 /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; + ECBE45806B470ACC5187B7D6 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; 7185A3C30E7FFC4F725769F1 /* BackgroundActivity.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9A771B316E19ED598CD4FB0B /* BackgroundActivity.swift */; }; 830BF933202D188E006DF9B1 /* HTTPURLRequest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 830BF932202D188E006DF9B1 /* HTTPURLRequest.swift */; }; 830DB3AC22380A3E00D65D25 /* HTTPHeadersSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 830DB3AB22380A3E00D65D25 /* HTTPHeadersSpec.swift */; }; @@ -378,6 +384,9 @@ AD5F10417C8E3269D4F701B3 /* JSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6F2A8C1D4E7B9053A1C4D8E6 /* JSONWriter.swift */; }; AFCE752CB1A75DEBABC23034 /* LDContextJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88DAB8D1864C14D9C7E06F11 /* LDContextJSONWriter.swift */; }; B0B1AD7CADE8DBB596C2DE9B /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; + DAA2C9B9C144F129217A7C55 /* OpenEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* OpenEventLog.swift */; }; + 44E0BE449714952A68628CEB /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; + CA2ADB3D20A229D70CA34F73 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; B40B419C249ADA6B00CD0726 /* DiagnosticCacheSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = B40B419B249ADA6B00CD0726 /* DiagnosticCacheSpec.swift */; }; B4265EB124E7390C001CFD2C /* TestUtil.swift in Sources */ = {isa = PBXBuildFile; fileRef = B4265EB024E7390C001CFD2C /* TestUtil.swift */; }; B468E71024B3C3AC00E0C883 /* ObjcLDEvaluationDetail.swift in Sources */ = {isa = PBXBuildFile; fileRef = B468E70F24B3C3AC00E0C883 /* ObjcLDEvaluationDetail.swift */; }; @@ -429,6 +438,9 @@ C443A41223186A4F00145710 /* ConnectionModeChangeObserver.swift in Sources */ = {isa = PBXBuildFile; fileRef = C443A40E23186A4F00145710 /* ConnectionModeChangeObserver.swift */; }; C4AF311E4DA8C3AB7040637D /* EventJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = B430884DBF2E541B919C08FA /* EventJSONWriter.swift */; }; CF9A1EB116D8AA3B7B5535B2 /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; + E2F8080AB2B3CD6E9A9C1FCC /* OpenEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* OpenEventLog.swift */; }; + F0F504E5EC32223905356098 /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; + 625ADAC0CCFE3BE324C45752 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; D7A500FCA04E3412F931003F /* ContextEncodingCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = D93B7F5F6CF5C3DE79FB4286 /* ContextEncodingCache.swift */; }; DAF5695D58E28B8CF9221213 /* LDContextJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88DAB8D1864C14D9C7E06F11 /* LDContextJSONWriter.swift */; }; EBFA09F4DFC59FCB6C0AE209 /* LDContextJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88DAB8D1864C14D9C7E06F11 /* LDContextJSONWriter.swift */; }; @@ -654,6 +666,9 @@ B4C9D4372489E20A004A9B03 /* DiagnosticReporter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DiagnosticReporter.swift; sourceTree = ""; }; B4F689132497B2FC004D3CE0 /* DiagnosticEventSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DiagnosticEventSpec.swift; sourceTree = ""; }; C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventStore.swift; sourceTree = ""; }; + E21BC7A0E0F8D272785D7985 /* OpenEventLog.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = OpenEventLog.swift; sourceTree = ""; }; + 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventLogFile.swift; sourceTree = ""; }; + B58551BB165F28784C785720 /* EventLogFormat.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventLogFormat.swift; sourceTree = ""; }; C408884623033B3600420721 /* ConnectionInformationStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConnectionInformationStore.swift; sourceTree = ""; }; C408884823033B7500420721 /* ConnectionInformation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConnectionInformation.swift; sourceTree = ""; }; C43C37E0236BA050003C1624 /* LDEvaluationDetail.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LDEvaluationDetail.swift; sourceTree = ""; }; @@ -1039,6 +1054,9 @@ D93B7F5F6CF5C3DE79FB4286 /* ContextEncodingCache.swift */, 9A771B316E19ED598CD4FB0B /* BackgroundActivity.swift */, C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */, + E21BC7A0E0F8D272785D7985 /* OpenEventLog.swift */, + 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */, + B58551BB165F28784C785720 /* EventLogFormat.swift */, ); path = ServiceObjects; sourceTree = ""; @@ -1566,6 +1584,9 @@ 40F4D07AD934AF976687011D /* ContextEncodingCache.swift in Sources */, A32FCDB22D53263851DDB727 /* BackgroundActivity.swift in Sources */, CF9A1EB116D8AA3B7B5535B2 /* EventStore.swift in Sources */, + E2F8080AB2B3CD6E9A9C1FCC /* OpenEventLog.swift in Sources */, + F0F504E5EC32223905356098 /* EventLogFile.swift in Sources */, + 625ADAC0CCFE3BE324C45752 /* EventLogFormat.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -1661,6 +1682,9 @@ 4882B223C23B63F30BC60941 /* ContextEncodingCache.swift in Sources */, 7185A3C30E7FFC4F725769F1 /* BackgroundActivity.swift in Sources */, 2CA9D36B3566D78850D6F0AF /* EventStore.swift in Sources */, + B8F228923BE11E444D1ECB8E /* OpenEventLog.swift in Sources */, + 3C55384C9E3485B493E1F86A /* EventLogFile.swift in Sources */, + AB4827D11084D58661E24B05 /* EventLogFormat.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -1755,6 +1779,9 @@ D7A500FCA04E3412F931003F /* ContextEncodingCache.swift in Sources */, A20146E7AF098560627C614E /* BackgroundActivity.swift in Sources */, 5AEA1F832DCA974314E07D21 /* EventStore.swift in Sources */, + D4225F1F05F2C55E9518941B /* OpenEventLog.swift in Sources */, + DDA788E3056F8AC3CA203933 /* EventLogFile.swift in Sources */, + ECBE45806B470ACC5187B7D6 /* EventLogFormat.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -1916,6 +1943,9 @@ BAE0B4F45562EBF4AC786B5D /* ContextEncodingCache.swift in Sources */, 3D34EEEF37158657D49335BB /* BackgroundActivity.swift in Sources */, B0B1AD7CADE8DBB596C2DE9B /* EventStore.swift in Sources */, + DAA2C9B9C144F129217A7C55 /* OpenEventLog.swift in Sources */, + 44E0BE449714952A68628CEB /* EventLogFile.swift in Sources */, + CA2ADB3D20A229D70CA34F73 /* EventLogFormat.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFile.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFile.swift new file mode 100644 index 000000000..67c0625cf --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFile.swift @@ -0,0 +1,138 @@ +import Foundation + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#elseif canImport(Musl) +import Musl +#endif + +/// The file operations an event log is made of, on descriptors rather than Foundation's file handles. +/// +/// Foundation's FileHandle is deliberately not used: it raises an Objective-C exception when a write fails, which is not +/// something a Swift caller can catch, and has taken host applications down with it when a device ran out of space. +enum EventLogFile { + /// Opens a file, creating it readable and writable by its owner alone. Nil where it could not, with `errno` saying + /// why. + static func openDescriptor(at url: URL, flags: Int32) -> Int32? { + let opened = url.withUnsafeFileSystemRepresentation { path -> Int32 in + guard let path = path + else { return -1 } + return openFile(path, flags, 0o600) + } + return opened >= 0 ? opened : nil + } + + /// Writes every byte, returning the errno it stopped on or nil where they all landed. + static func writeAll(_ descriptor: Int32, _ bytes: Data) -> Int32? { + var offset = 0 + return bytes.withUnsafeBytes { raw -> Int32? in + guard let base = raw.baseAddress + else { return nil } + while offset < raw.count { + let result = writeBytes(descriptor, base.advanced(by: offset), raw.count - offset) + if result > 0 { + offset += result + continue + } + // A signal interrupting the write is not a failure; anything else is. + if errno == EINTR { + continue + } + return errno + } + return nil + } + } + + static func isEmpty(_ descriptor: Int32) -> Bool { + lseek(descriptor, 0, SEEK_END) == 0 + } + + static func closeDescriptor(_ descriptor: Int32) { + closeFile(descriptor) + } + + static func excludeFromBackup(_ directory: URL) { + #if canImport(Darwin) + var url = directory + var values = URLResourceValues() + values.isExcludedFromBackup = true + try? url.setResourceValues(values) + #endif + } + + #if os(macOS) || targetEnvironment(macCatalyst) + /// Opens a file and takes its lock without waiting, or nil where another live process holds it. + /// + /// The lock has to be on the file that is still at `url`: one renamed or removed between the open and the lock, + /// by another process closing off or recovering it, is no longer a log anyone appends to. + static func openLocked(_ url: URL, flags: Int32) -> Int32? { + for _ in 0..<3 { + guard let opened = openDescriptor(at: url, flags: flags) + else { return nil } + guard lockFile(opened) + else { + closeFile(opened) + return nil + } + if isFile(opened, at: url) { + return opened + } + closeFile(opened) + } + return nil + } + #endif +} + +// The POSIX calls are wrapped so that the names cannot be shadowed by a member of the type using them, and so the +// platform differences stay in one place. +#if canImport(Darwin) +private func openFile(_ path: UnsafePointer, _ flags: Int32, _ mode: mode_t) -> Int32 { + Darwin.open(path, flags, mode) +} + +private func writeBytes(_ descriptor: Int32, _ bytes: UnsafeRawPointer, _ count: Int) -> Int { + Darwin.write(descriptor, bytes, count) +} + +private func closeFile(_ descriptor: Int32) { + _ = Darwin.close(descriptor) +} +#else +private func openFile(_ path: UnsafePointer, _ flags: Int32, _ mode: mode_t) -> Int32 { + open(path, flags, mode) +} + +private func writeBytes(_ descriptor: Int32, _ bytes: UnsafeRawPointer, _ count: Int) -> Int { + write(descriptor, bytes, count) +} + +private func closeFile(_ descriptor: Int32) { + _ = close(descriptor) +} +#endif + +#if os(macOS) || targetEnvironment(macCatalyst) +/// Takes an exclusive lock without waiting. Released by the kernel when the descriptor closes, including when the +/// process dies, so a crashed process never leaves one behind. +private func lockFile(_ descriptor: Int32) -> Bool { + flock(descriptor, LOCK_EX | LOCK_NB) == 0 +} + +/// Whether a descriptor is open on the file currently at `url`, rather than one since renamed or removed. +private func isFile(_ descriptor: Int32, at url: URL) -> Bool { + var opened = stat() + var named = stat() + guard fstat(descriptor, &opened) == 0 + else { return false } + let found = url.withUnsafeFileSystemRepresentation { path -> Bool in + guard let path = path + else { return false } + return stat(path, &named) == 0 + } + return found && opened.st_ino == named.st_ino && opened.st_dev == named.st_dev +} +#endif diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift new file mode 100644 index 000000000..e9e0af536 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift @@ -0,0 +1,118 @@ +import Foundation + +/// The on-disk shape of an event log. +/// +/// A log opens with a magic and a format version, and holds a sequence of frames: +/// +/// +- 4 bytes -+- 2 bytes -+ +- 2 bytes -+- 4 bytes -+- n bytes -+ +/// | LDEV | version | ... | type | length (n) | payload | +/// +-----------+-----------+ +-----------+-------------+-----------+ +/// +/// The length prefix is what makes a log recoverable: a reader can tell a frame the writer finished from one it did not, +/// without having to parse the event to find where it ends. The version is what makes an upgrade safe -- a log this +/// version does not understand is discarded rather than misread -- and the frame type leaves room for a later version to +/// write something new into a log this one still reads. +enum EventLogFormat { + /// Bump this whenever the framing or the meaning of a frame changes. + static let version: UInt16 = 1 + static let magic = Data("LDEV".utf8) + static let fileHeaderSize = 6 + static let frameHeaderSize = 6 + /// The only frame type written today; a reader skips a frame whose type it does not know. + static let eventFrame: UInt16 = 1 + /// A ceiling on a single frame, so a corrupt length cannot make recovery allocate wildly. + static let maxFrameSize = 8 * 1024 * 1024 + + static var fileHeader: Data { + var header = magic + header.append(bigEndian(version)) + return header + } + + static func frame(for encodedEvent: Data) -> Data { + var frame = Data(capacity: frameHeaderSize + encodedEvent.count) + frame.append(bigEndian(eventFrame)) + frame.append(bigEndian(UInt32(encodedEvent.count))) + frame.append(encodedEvent) + return frame + } + + /// Walks the frames of a log, stopping at the first one that is not wholly there. + /// + /// A process that died partway through a write leaves a torn frame at the end. Everything before it is intact, so + /// recovery keeps that and drops only the tail. Returns false when the file is not a log this version reads. + static func forEachFrame(in file: Data, _ visit: (UInt16, Data) -> Void) -> Bool { + guard file.count >= fileHeaderSize, + file.prefix(magic.count) == magic, + readUInt16(file, at: magic.count) == version + else { return false } + + var cursor = fileHeaderSize + while cursor + frameHeaderSize <= file.count { + let type = readUInt16(file, at: cursor) + let length = Int(readUInt32(file, at: cursor + 2)) + let start = cursor + frameHeaderSize + guard length > 0, length <= maxFrameSize, start + length <= file.count + else { break } + + visit(type, file.subdata(in: start..<(start + length))) + cursor = start + length + } + return true + } + + /// How many events a log holds, or nil where it is not one this version reads. + static func eventCount(in file: Data) -> Int? { + var events = 0 + guard forEachFrame(in: file, { type, _ in + if type == eventFrame { + events += 1 + } + }) + else { return nil } + return events + } + + /// Assembles the JSON array LaunchDarkly expects out of the frames, without parsing the events themselves: they were + /// serialized on the way in and are shipped exactly as they were recorded. + static func assembleBody(from file: Data) -> Data? { + var body = Data("[".utf8) + var isFirst = true + let readable = forEachFrame(in: file) { type, payload in + guard type == eventFrame + else { return } + if !isFirst { + body.append(UInt8(ascii: ",")) + } + body.append(payload) + isFirst = false + } + + guard readable, !isFirst + else { return nil } + + body.append(UInt8(ascii: "]")) + return body + } + + static func bigEndian(_ value: UInt16) -> Data { + Data([UInt8(truncatingIfNeeded: value >> 8), UInt8(truncatingIfNeeded: value)]) + } + + static func bigEndian(_ value: UInt32) -> Data { + Data([UInt8(truncatingIfNeeded: value >> 24), + UInt8(truncatingIfNeeded: value >> 16), + UInt8(truncatingIfNeeded: value >> 8), + UInt8(truncatingIfNeeded: value)]) + } + + static func readUInt16(_ data: Data, at offset: Int) -> UInt16 { + let start = data.startIndex + offset + return UInt16(data[start]) << 8 | UInt16(data[start + 1]) + } + + static func readUInt32(_ data: Data, at offset: Int) -> UInt32 { + let start = data.startIndex + offset + return UInt32(data[start]) << 24 | UInt32(data[start + 1]) << 16 | UInt32(data[start + 2]) << 8 | UInt32(data[start + 3]) + } +} diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index 8db83ed2c..9dffccb8e 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -82,8 +82,6 @@ protocol EventStoring { final class EventStore: EventStoring { /// Names a log that has been closed off for delivery; the rest of the name is the batch's payload ID. private static let batchPrefix = "ready-" - /// Names the log a single process appends to; the rest of the name is that process. - private static let openLogPrefix = "open-" /// How many staged bytes are allowed to accumulate before one of them pays for a write. /// /// This is what keeps a syscall off most recordings while bounding what a crash can take with it, for an @@ -95,9 +93,6 @@ final class EventStore: EventStoring { private let logger: OSLog /// How a stored log is read, injectable so a test can make a read fail while the file is intact. private let readFile: (URL) throws -> Data - /// How bytes reach the open log, injectable so a test can make a write fail partway through a session, as a full - /// disk does. Answers with the errno it stopped on, or nil where every byte landed. - private let writeLog: (Int32, Data) -> Int32? /// Where a commit runs when it was not asked for by a caller who needs it to have happened. /// /// Evaluating a flag must not put a write on whichever thread evaluated it, and that thread is usually the main one. @@ -109,7 +104,7 @@ final class EventStore: EventStoring { private let bufferLock = UnfairLock() /// Serializes commits so that two threads committing at once cannot interleave a frame or write frames out of /// order. Taken before `bufferLock`, never after it. - private let ioLock = UnfairLock() + private let ioLock: UnfairLock /// Only to be used while holding `bufferLock`. private var bufferData = Data() @@ -131,7 +126,7 @@ final class EventStore: EventStoring { private var isCommitScheduled = false /// Only to be used while holding `ioLock`. - private var descriptor: Int32 = -1 + private let openLog: OpenEventLog /// Whether a log left by a previous run has been dealt with. Only to be used while holding `ioLock`. private var hasRecoveredInterruptedLog = false @@ -154,39 +149,19 @@ final class EventStore: EventStoring { /// Only to be used while holding `ioLock`. private var eventCounts: [String: Int] = [:] - /// The log this process appends to. - /// - /// One per process rather than one per directory, so that only one process ever writes a given log and recovery - /// only ever touches its own. With one shared log, a second process starting up would close off, as if a previous - /// run had left it, a log the first is still appending to through its open descriptor, and the first's later - /// events would land in a batch already sent or deleted. Another process's log is left for that process to recover - /// the next time it runs. - /// - /// Named for the process, unless `locksOpenLog` found another live instance holding that name, in which case it is - /// named for this instance's process ID as well. Only to be used while holding `ioLock`. - private(set) var openLogUrl: URL - /// The readable, collision-proof form of the process name every log this process opens is named after. - private let processLogName: String - /// Whether the open log is locked while this store has it open, so that another live process can tell it is taken. - /// - /// Only where two live processes can run under one name, which a process name alone cannot keep apart: a Mac runs - /// a second instance of an application on request. Elsewhere a name is one live process, and a lock is avoided - /// rather than added for nothing, because iOS kills a suspended application that holds a lock on a file in a - /// shared container. - private let locksOpenLog: Bool - - /// Whether this platform can run two live instances of an application under one process name. True on a Mac, - /// which includes an iPhone or iPad application running on one, whether built with Mac Catalyst or not. - static var instancesCanShareProcessName: Bool { - #if os(macOS) - return true - #elseif canImport(Darwin) - return ProcessInfo.processInfo.isMacCatalystApp - #else - return false - #endif + /// Where the log this process appends to is, for a test or a diagnosis to look at. + var openLogUrl: URL { + ioLock.withLock { openLog.url } } + #if os(macOS) || targetEnvironment(macCatalyst) + /// Whether the open log is locked while it is open. Only to be changed before the store first touches the disk. + var locksOpenLog: Bool { + get { ioLock.withLock { openLog.isLocked } } + set { ioLock.withLock { openLog.isLocked = newValue } } + } + #endif + /// A batch is identified by its payload ID rather than by a path, so that a batch listed from the directory and the /// same batch as it was closed are one thing. private func url(of payloadId: String) -> URL { @@ -197,71 +172,24 @@ final class EventStore: EventStoring { capacity: Int, persistEvents: Bool = true, processName: String = ProcessInfo.processInfo.processName, - locksOpenLog: Bool = EventStore.instancesCanShareProcessName, logger: OSLog, commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.EventStore.commitQueue", qos: .userInitiated), readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }, - writeLog: @escaping (Int32, Data) -> Int32? = EventStore.writeAll) { + writeLog: @escaping (Int32, Data) -> Int32? = EventLogFile.writeAll) { self.directory = directory - self.processLogName = EventStore.logName(for: processName) - self.openLogUrl = directory.appendingPathComponent(EventStore.openLogPrefix + processLogName) - self.locksOpenLog = locksOpenLog + let ioLock = UnfairLock() + self.ioLock = ioLock + self.openLog = OpenEventLog(directory: directory, processName: processName, ioLock: ioLock, logger: logger, writeLog: writeLog) self.capacity = capacity self.logger = logger self.commitQueue = commitQueue self.readFile = readFile - self.writeLog = writeLog // An application that has not asked for persistence gets the same store running the same way it runs once a // write has failed: events are held, delivered, and lost only if the process dies. self.persistEvents = persistEvents self.readsDisk = persistEvents } - /// Names the log belonging to one process. - /// - /// The readable part is the process name with anything awkward in a filename replaced, which keeps a directory of - /// these diagnosable. That reduction is not one-to-one, and two processes sharing a log is the one thing this name - /// exists to prevent, so a digest of the original name is appended. - static func logName(for processName: String) -> String { - let trimmed = processName.trimmingCharacters(in: .whitespacesAndNewlines) - let name = trimmed.isEmpty ? "unknown" : trimmed - let readable = String(name.unicodeScalars.map { scalar -> Character in - switch scalar { - case "a"..."z", "A"..."Z", "0"..."9", "-", "_": - return Character(scalar) - default: - return "_" - } - }) - let digest = Util.sha256(name).prefix(4).map { String(format: "%02x", $0) }.joined() - return "\(readable)-\(digest)" - } - - /// The directory the store for a mobile key belongs in, or nil where the platform gave us nowhere to write. - /// - /// tvOS is the reason this is not simply Application Support: a tvOS application is only allowed to persist to - /// purgeable caches, so there the events are kept somewhere the system may reclaim, which is the most durability - /// the platform offers. - static func defaultDirectory(mobileKey: String) -> URL? { - #if os(tvOS) - let searchPath = FileManager.SearchPathDirectory.cachesDirectory - #else - let searchPath = FileManager.SearchPathDirectory.applicationSupportDirectory - #endif - - let root = FileManager.default.urls(for: searchPath, in: .userDomainMask).first - ?? FileManager.default.temporaryDirectory - - var keyComponent = Util.sha256(mobileKey).base64UrlEncodedString - if let bundleId = Bundle.main.bundleIdentifier { - keyComponent = "\(Util.sha256(bundleId).base64UrlEncodedString).\(keyComponent)" - } - - return root - .appendingPathComponent("com.launchdarkly.events", isDirectory: true) - .appendingPathComponent(keyComponent, isDirectory: true) - } - // MARK: Recording var pendingEventCount: Int { @@ -380,19 +308,15 @@ final class EventStore: EventStoring { /// Requires `ioLock`. private func closeOpenLog(_ events: Int) -> EventBatch? { ioLock.assertOwned() - // Closed after the rename rather than before it, so that a locked log is still locked while it moves. Another - // instance recovering unlocked logs could otherwise take it in between. - defer { closeDescriptor() } - let payloadId = UUID().uuidString do { - try FileManager.default.moveItem(at: openLogUrl, to: url(of: payloadId)) + try openLog.close(movingTo: url(of: payloadId)) } catch { os_log("%s could not close the event log: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) // The log is still there to try again with, unless it is the log that went missing -- a purged cache // directory on tvOS, say. Then its events are gone, and they have to stop counting against capacity or the // store would refuse events for the rest of the session. - if !FileManager.default.fileExists(atPath: openLogUrl.path) { + if !openLog.exists { bufferLock.lock() committedEvents = 0 bufferLock.unlock() @@ -555,24 +479,7 @@ final class EventStore: EventStoring { guard readsDisk, !hasRecoveredInterruptedLog else { return } hasRecoveredInterruptedLog = true - - guard locksOpenLog - else { - recover(logAt: openLogUrl) - return - } - - // A log that can be locked is one no live process has open, so whoever wrote it is gone. That includes a log a - // second instance opened under its process ID, which no later run would otherwise come back for by name. - let contents = (try? FileManager.default.contentsOfDirectory(at: directory, - includingPropertiesForKeys: nil, - options: [.skipsHiddenFiles])) ?? [] - for log in contents where log.lastPathComponent.hasPrefix(EventStore.openLogPrefix) { - guard let held = EventStore.openLocked(log, flags: O_RDONLY) - else { continue } - recover(logAt: log) - closeFile(held) - } + openLog.forEachLogLeftOpen { recover(logAt: $0) } } /// Closes off a log nobody is writing to any more. Requires `ioLock`. @@ -651,165 +558,26 @@ final class EventStore: EventStoring { /// Requires `ioLock`. private func append(_ bytes: Data) -> Bool { ioLock.assertOwned() - guard let descriptor = descriptorForAppending() + // Before the log is opened, because it may still be the one a previous run died writing. Appended to, this + // run's events would sit behind that run's torn last frame, where a reader can no longer find where they start. + if !openLog.isOpen { + recoverInterruptedLogOnce() + } + guard openLog.append(bytes) else { + // Never trap on a full disk. The SDK gives up on persistence for the rest of the session rather than taking + // the application down with it, which is how other SDKs have crashed their hosts. disablePersistence() return false } - return append(bytes, to: descriptor) - } - - /// Requires `ioLock`. - private func append(_ bytes: Data, to descriptor: Int32) -> Bool { - ioLock.assertOwned() - guard let failure = writeLog(descriptor, bytes) - else { return true } - - // Never trap on a full disk. The SDK gives up on persistence for the rest of the session rather than taking - // the application down with it, which is how other SDKs have crashed their hosts. - os_log("%s giving up on persisting events: errno %d", log: logger, type: .debug, typeName(and: #function), failure) - disablePersistence() - return false - } - - /// Writes every byte to the log, returning the errno it stopped on or nil where they all landed. - static func writeAll(_ descriptor: Int32, _ bytes: Data) -> Int32? { - var offset = 0 - return bytes.withUnsafeBytes { raw -> Int32? in - guard let base = raw.baseAddress - else { return nil } - while offset < raw.count { - let result = writeBytes(descriptor, base.advanced(by: offset), raw.count - offset) - if result > 0 { - offset += result - continue - } - // A signal interrupting the write is not a failure; anything else is. - if errno == EINTR { - continue - } - return errno - } - return nil - } - } - - /// Requires `ioLock`. - private func descriptorForAppending() -> Int32? { - ioLock.assertOwned() - if descriptor >= 0 { - return descriptor - } - - // Before opening, because the open log may still be the one a previous run died writing. Appended to, this run's - // events would sit behind that run's torn last frame, where a reader can no longer find where they start. - recoverInterruptedLogOnce() - - do { - try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) - } catch { - os_log("%s could not create the event directory: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) - return nil - } - excludeFromBackup() - - guard let opened = openLogForAppending() - else { - os_log("%s could not open the event log: errno %d", log: logger, type: .debug, typeName(and: #function), errno) - return nil - } - - descriptor = opened - - if lseek(opened, 0, SEEK_END) == 0 { - guard append(EventLogFormat.fileHeader, to: opened) - else { return nil } - } - - return descriptor - } - - /// Opens the log this process appends to, locked where logs are. Requires `ioLock`. - private func openLogForAppending() -> Int32? { - ioLock.assertOwned() - // O_APPEND is what makes each write land at the end of the file as one step, so that clients for several - // environments writing their own logs, or a commit racing a reader, cannot produce a spliced frame. - let flags = O_WRONLY | O_APPEND | O_CREAT - guard locksOpenLog - else { - let opened = EventStore.openDescriptor(at: openLogUrl, flags: flags) - return opened >= 0 ? opened : nil - } - - if let opened = EventStore.openLocked(openLogUrl, flags: flags) { - return opened - } - - // Another live instance of the application holds the log for this process name, so this one takes a log of its - // own. The process ID keeps it apart from every other live process, and the suffix from another store in this - // one, as a closed client's can be while it is still going away. Recovery finds the log by its lock rather than - // its name, so it is not orphaned when this instance ends. - let ownLog = directory.appendingPathComponent( - "\(EventStore.openLogPrefix)\(processLogName)-\(getpid())-\(UUID().uuidString.prefix(8))") - guard let opened = EventStore.openLocked(ownLog, flags: flags) - else { return nil } - openLogUrl = ownLog - return opened - } - - private static func openDescriptor(at url: URL, flags: Int32) -> Int32 { - url.withUnsafeFileSystemRepresentation { path -> Int32 in - guard let path = path - else { return -1 } - return openFile(path, flags, 0o600) - } - } - - /// Opens a file and takes its lock without waiting, or nil where another live process holds it. - /// - /// The lock has to be on the file that is still at `url`: one renamed or removed between the open and the lock, - /// by another process closing off or recovering it, is no longer a log anyone appends to. - private static func openLocked(_ url: URL, flags: Int32) -> Int32? { - for _ in 0..<3 { - let opened = openDescriptor(at: url, flags: flags) - guard opened >= 0 - else { return nil } - guard lockFile(opened) - else { - closeFile(opened) - return nil - } - if isFile(opened, at: url) { - return opened - } - closeFile(opened) - } - return nil - } - - /// Requires `ioLock`. - private func closeDescriptor() { - ioLock.assertOwned() - guard descriptor >= 0 - else { return } - closeFile(descriptor) - descriptor = -1 - } - - private func excludeFromBackup() { - #if canImport(Darwin) - var url = directory - var values = URLResourceValues() - values.isExcludedFromBackup = true - try? url.setResourceValues(values) - #endif + return true } deinit { // One acquisition: the lock is not recursive, and `commit()` would take it again. ioLock.lock() writeStagedBytes() - closeDescriptor() + openLog.closeDescriptor() ioLock.unlock() } } @@ -832,7 +600,7 @@ private extension EventStore { /// Requires `ioLock`. func disablePersistence() { ioLock.assertOwned() - closeDescriptor() + openLog.closeDescriptor() bufferLock.lock() persistEvents = false bufferLock.unlock() @@ -868,123 +636,6 @@ private extension EventStore { } } -/// The on-disk shape of an event log. -/// -/// A log opens with a magic and a format version, and holds a sequence of frames: -/// -/// +- 4 bytes -+- 2 bytes -+ +- 2 bytes -+- 4 bytes -+- n bytes -+ -/// | LDEV | version | ... | type | length (n) | payload | -/// +-----------+-----------+ +-----------+-------------+-----------+ -/// -/// The length prefix is what makes a log recoverable: a reader can tell a frame the writer finished from one it did not, -/// without having to parse the event to find where it ends. The version is what makes an upgrade safe -- a log this -/// version does not understand is discarded rather than misread -- and the frame type leaves room for a later version to -/// write something new into a log this one still reads. -enum EventLogFormat { - /// Bump this whenever the framing or the meaning of a frame changes. - static let version: UInt16 = 1 - static let magic = Data("LDEV".utf8) - static let fileHeaderSize = 6 - static let frameHeaderSize = 6 - /// The only frame type written today; a reader skips a frame whose type it does not know. - static let eventFrame: UInt16 = 1 - /// A ceiling on a single frame, so a corrupt length cannot make recovery allocate wildly. - static let maxFrameSize = 8 * 1024 * 1024 - - static var fileHeader: Data { - var header = magic - header.append(bigEndian(version)) - return header - } - - static func frame(for encodedEvent: Data) -> Data { - var frame = Data(capacity: frameHeaderSize + encodedEvent.count) - frame.append(bigEndian(eventFrame)) - frame.append(bigEndian(UInt32(encodedEvent.count))) - frame.append(encodedEvent) - return frame - } - - /// Walks the frames of a log, stopping at the first one that is not wholly there. - /// - /// A process that died partway through a write leaves a torn frame at the end. Everything before it is intact, so - /// recovery keeps that and drops only the tail. Returns false when the file is not a log this version reads. - static func forEachFrame(in file: Data, _ visit: (UInt16, Data) -> Void) -> Bool { - guard file.count >= fileHeaderSize, - file.prefix(magic.count) == magic, - readUInt16(file, at: magic.count) == version - else { return false } - - var cursor = fileHeaderSize - while cursor + frameHeaderSize <= file.count { - let type = readUInt16(file, at: cursor) - let length = Int(readUInt32(file, at: cursor + 2)) - let start = cursor + frameHeaderSize - guard length > 0, length <= maxFrameSize, start + length <= file.count - else { break } - - visit(type, file.subdata(in: start..<(start + length))) - cursor = start + length - } - return true - } - - /// How many events a log holds, or nil where it is not one this version reads. - static func eventCount(in file: Data) -> Int? { - var events = 0 - guard forEachFrame(in: file, { type, _ in - if type == eventFrame { - events += 1 - } - }) - else { return nil } - return events - } - - /// Assembles the JSON array LaunchDarkly expects out of the frames, without parsing the events themselves: they were - /// serialized on the way in and are shipped exactly as they were recorded. - static func assembleBody(from file: Data) -> Data? { - var body = Data("[".utf8) - var isFirst = true - let readable = forEachFrame(in: file) { type, payload in - guard type == eventFrame - else { return } - if !isFirst { - body.append(UInt8(ascii: ",")) - } - body.append(payload) - isFirst = false - } - - guard readable, !isFirst - else { return nil } - - body.append(UInt8(ascii: "]")) - return body - } - - static func bigEndian(_ value: UInt16) -> Data { - Data([UInt8(truncatingIfNeeded: value >> 8), UInt8(truncatingIfNeeded: value)]) - } - - static func bigEndian(_ value: UInt32) -> Data { - Data([UInt8(truncatingIfNeeded: value >> 24), - UInt8(truncatingIfNeeded: value >> 16), - UInt8(truncatingIfNeeded: value >> 8), - UInt8(truncatingIfNeeded: value)]) - } - - static func readUInt16(_ data: Data, at offset: Int) -> UInt16 { - let start = data.startIndex + offset - return UInt16(data[start]) << 8 | UInt16(data[start + 1]) - } - - static func readUInt32(_ data: Data, at offset: Int) -> UInt32 { - let start = data.startIndex + offset - return UInt32(data[start]) << 24 | UInt32(data[start + 1]) << 16 | UInt32(data[start + 2]) << 8 | UInt32(data[start + 3]) - } -} - extension EventStore { /// Every event the store is holding, encoded exactly as it will be sent. /// @@ -1003,8 +654,8 @@ extension EventStore { includingPropertiesForKeys: nil, options: [.skipsHiddenFiles])) ?? [] var logs = contents.filter { $0.lastPathComponent.hasPrefix(EventStore.batchPrefix) }.sorted { $0.path < $1.path } - if FileManager.default.fileExists(atPath: openLogUrl.path) { - logs.append(openLogUrl) + if openLog.exists { + logs.append(openLog.url) } var payloads: [Data] = [] @@ -1021,58 +672,31 @@ extension EventStore { } } -extension EventStore: TypeIdentifying { } - -// The POSIX calls are wrapped so that the names cannot be shadowed by a member of the type using them, and so the -// platform differences stay in one place. Foundation's FileHandle is deliberately not used: it raises an Objective-C -// exception when a write fails, which is not something a Swift caller can catch, and has taken host applications down -// with it when a device ran out of space. -#if canImport(Darwin) -private func openFile(_ path: UnsafePointer, _ flags: Int32, _ mode: mode_t) -> Int32 { - Darwin.open(path, flags, mode) -} - -private func writeBytes(_ descriptor: Int32, _ bytes: UnsafeRawPointer, _ count: Int) -> Int { - Darwin.write(descriptor, bytes, count) -} - -private func closeFile(_ descriptor: Int32) { - _ = Darwin.close(descriptor) -} - -/// Takes an exclusive lock without waiting. Released by the kernel when the descriptor closes, including when the -/// process dies, so a crashed process never leaves one behind. -private func lockFile(_ descriptor: Int32) -> Bool { - flock(descriptor, LOCK_EX | LOCK_NB) == 0 -} -#else -private func openFile(_ path: UnsafePointer, _ flags: Int32, _ mode: mode_t) -> Int32 { - open(path, flags, mode) -} - -private func writeBytes(_ descriptor: Int32, _ bytes: UnsafeRawPointer, _ count: Int) -> Int { - write(descriptor, bytes, count) -} +extension EventStore { + /// The directory the store for a mobile key belongs in, or nil where the platform gave us nowhere to write. + /// + /// tvOS is the reason this is not simply Application Support: a tvOS application is only allowed to persist to + /// purgeable caches, so there the events are kept somewhere the system may reclaim, which is the most durability + /// the platform offers. + static func defaultDirectory(mobileKey: String) -> URL? { + #if os(tvOS) + let searchPath = FileManager.SearchPathDirectory.cachesDirectory + #else + let searchPath = FileManager.SearchPathDirectory.applicationSupportDirectory + #endif -private func closeFile(_ descriptor: Int32) { - _ = close(descriptor) -} + let root = FileManager.default.urls(for: searchPath, in: .userDomainMask).first + ?? FileManager.default.temporaryDirectory -private func lockFile(_ descriptor: Int32) -> Bool { - flock(descriptor, LOCK_EX | LOCK_NB) == 0 -} -#endif + var keyComponent = Util.sha256(mobileKey).base64UrlEncodedString + if let bundleId = Bundle.main.bundleIdentifier { + keyComponent = "\(Util.sha256(bundleId).base64UrlEncodedString).\(keyComponent)" + } -/// Whether a descriptor is open on the file currently at `url`, rather than one since renamed or removed. -private func isFile(_ descriptor: Int32, at url: URL) -> Bool { - var opened = stat() - var named = stat() - guard fstat(descriptor, &opened) == 0 - else { return false } - let found = url.withUnsafeFileSystemRepresentation { path -> Bool in - guard let path = path - else { return false } - return stat(path, &named) == 0 + return root + .appendingPathComponent("com.launchdarkly.events", isDirectory: true) + .appendingPathComponent(keyComponent, isDirectory: true) } - return found && opened.st_ino == named.st_ino && opened.st_dev == named.st_dev } + +extension EventStore: TypeIdentifying { } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/OpenEventLog.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/OpenEventLog.swift new file mode 100644 index 000000000..00a1ab663 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/OpenEventLog.swift @@ -0,0 +1,233 @@ +import Foundation +import OSLog + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#elseif canImport(Musl) +import Musl +#endif + +/// The log this process appends to. +/// +/// One per process rather than one per directory, so that only one process ever writes a given log and recovery only +/// ever touches its own. With one shared log, a second process starting up would close off, as if a previous run had +/// left it, a log the first is still appending to through its open descriptor, and the first's later events would land +/// in a batch already sent or deleted. Another process's log is left for that process to recover the next time it runs. +/// +/// Belongs to an `EventStore`, which makes every call holding its `ioLock`. The lock is handed in so that each call can +/// check it is held, rather than this taking a lock of its own that would have to be ordered against the store's. +final class OpenEventLog { + /// Names a log a single process appends to; the rest of the name is that process. + static let prefix = "open-" + + private let directory: URL + private let ioLock: UnfairLock + private let logger: OSLog + /// How bytes reach the log, injectable so a test can make a write fail partway through a session, as a full disk + /// does. Answers with the errno it stopped on, or nil where every byte landed. + private let writeLog: (Int32, Data) -> Int32? + + /// Named for the process, unless on a Mac `isLocked` found another live instance holding that name, in which case + /// it is named for this instance's process ID as well. + private(set) var url: URL + private var descriptor: Int32 = -1 + + #if os(macOS) || targetEnvironment(macCatalyst) + /// The readable, collision-proof form of the process name every log this process opens is named after. + private let processLogName: String + /// Whether the log is locked while it is open, so that another live process can tell it is taken. Only to be + /// changed before the log is first opened or recovered. + /// + /// Only on a Mac, which runs a second instance of an application under the same process name on request, so that + /// a name alone cannot keep two live processes apart. Elsewhere a name is one live process, and a lock is not + /// compiled in at all: iOS kills a suspended application that holds a lock on a file in a shared container. + var isLocked = true + #endif + + init(directory: URL, processName: String, ioLock: UnfairLock, logger: OSLog, writeLog: @escaping (Int32, Data) -> Int32?) { + self.directory = directory + self.ioLock = ioLock + self.logger = logger + self.writeLog = writeLog + let processLogName = OpenEventLog.name(for: processName) + self.url = directory.appendingPathComponent(OpenEventLog.prefix + processLogName) + #if os(macOS) || targetEnvironment(macCatalyst) + self.processLogName = processLogName + #endif + } + + /// Names the log belonging to one process. + /// + /// The readable part is the process name with anything awkward in a filename replaced, which keeps a directory of + /// these diagnosable. That reduction is not one-to-one, and two processes sharing a log is the one thing this name + /// exists to prevent, so a digest of the original name is appended. + static func name(for processName: String) -> String { + let trimmed = processName.trimmingCharacters(in: .whitespacesAndNewlines) + let name = trimmed.isEmpty ? "unknown" : trimmed + let readable = String(name.unicodeScalars.map { scalar -> Character in + switch scalar { + case "a"..."z", "A"..."Z", "0"..."9", "-", "_": + return Character(scalar) + default: + return "_" + } + }) + let digest = Util.sha256(name).prefix(4).map { String(format: "%02x", $0) }.joined() + return "\(readable)-\(digest)" + } + + /// Requires `ioLock`. + var isOpen: Bool { + ioLock.assertOwned() + return descriptor >= 0 + } + + /// Requires `ioLock`. + var exists: Bool { + ioLock.assertOwned() + return FileManager.default.fileExists(atPath: url.path) + } + + /// Appends to the log, opening it first where it is not open. False where it could not be opened or written to. + /// + /// Requires `ioLock`. + func append(_ bytes: Data) -> Bool { + ioLock.assertOwned() + guard let descriptor = descriptorForAppending() + else { return false } + return write(bytes, to: descriptor) + } + + /// Moves the log to `destination` and closes it, so that the next append starts a new one. The descriptor is closed + /// whether or not the move succeeds. + /// + /// Requires `ioLock`. + func close(movingTo destination: URL) throws { + ioLock.assertOwned() + // Closed after the rename rather than before it, so that a locked log is still locked while it moves. Another + // instance recovering unlocked logs could otherwise take it in between. + defer { closeDescriptor() } + try FileManager.default.moveItem(at: url, to: destination) + } + + /// Closes the descriptor, leaving the log where it is. Requires `ioLock`. + func closeDescriptor() { + ioLock.assertOwned() + guard descriptor >= 0 + else { return } + EventLogFile.closeDescriptor(descriptor) + descriptor = -1 + } + + /// Hands `recover` each log left open by a process that is gone, which it has to move or remove. + /// + /// Only this process's own log, by name: another process's may still be open in a process that is alive. Where + /// logs are locked, any log no live process holds, whatever it is named. + /// + /// Requires `ioLock`. + func forEachLogLeftOpen(_ recover: (URL) -> Void) { + ioLock.assertOwned() + #if os(macOS) || targetEnvironment(macCatalyst) + if isLocked { + forEachUnheldLog(recover) + return + } + #endif + recover(url) + } + + /// Requires `ioLock`. + private func descriptorForAppending() -> Int32? { + ioLock.assertOwned() + if descriptor >= 0 { + return descriptor + } + + do { + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + } catch { + os_log("%s could not create the event directory: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) + return nil + } + EventLogFile.excludeFromBackup(directory) + + guard let opened = open() + else { + os_log("%s could not open the event log: errno %d", log: logger, type: .debug, typeName(and: #function), errno) + return nil + } + + descriptor = opened + + if EventLogFile.isEmpty(opened) { + guard write(EventLogFormat.fileHeader, to: opened) + else { return nil } + } + + return descriptor + } + + private func write(_ bytes: Data, to descriptor: Int32) -> Bool { + guard let failure = writeLog(descriptor, bytes) + else { return true } + os_log("%s could not write to the event log: errno %d", log: logger, type: .debug, typeName(and: #function), failure) + return false + } + + /// Opens the log, locked where logs are. Requires `ioLock`. + private func open() -> Int32? { + ioLock.assertOwned() + // O_APPEND is what makes each write land at the end of the file as one step, so that clients for several + // environments writing their own logs, or a commit racing a reader, cannot produce a spliced frame. + let flags = O_WRONLY | O_APPEND | O_CREAT + #if os(macOS) || targetEnvironment(macCatalyst) + if isLocked { + return openLocked(flags: flags) + } + #endif + return EventLogFile.openDescriptor(at: url, flags: flags) + } + + #if os(macOS) || targetEnvironment(macCatalyst) + /// Opens the log for this process name locked, or one of this instance's own where another live instance holds + /// it. Requires `ioLock`. + private func openLocked(flags: Int32) -> Int32? { + ioLock.assertOwned() + if let opened = EventLogFile.openLocked(url, flags: flags) { + return opened + } + + // Another live instance of the application holds the log for this process name, so this one takes a log of its + // own. The process ID keeps it apart from every other live process, and the suffix from another store in this + // one, as a closed client's can be while it is still going away. Recovery finds the log by its lock rather than + // its name, so it is not orphaned when this instance ends. + let ownLog = directory.appendingPathComponent( + "\(OpenEventLog.prefix)\(processLogName)-\(getpid())-\(UUID().uuidString.prefix(8))") + guard let opened = EventLogFile.openLocked(ownLog, flags: flags) + else { return nil } + url = ownLog + return opened + } + + /// Hands `recover` every open log no live process holds. Requires `ioLock`. + /// + /// A log that can be locked is one no live process has open, so whoever wrote it is gone. That includes a log a + /// second instance opened under its process ID, which no later run would otherwise come back for by name. + private func forEachUnheldLog(_ recover: (URL) -> Void) { + ioLock.assertOwned() + let contents = (try? FileManager.default.contentsOfDirectory(at: directory, + includingPropertiesForKeys: nil, + options: [.skipsHiddenFiles])) ?? [] + for log in contents where log.lastPathComponent.hasPrefix(OpenEventLog.prefix) { + guard let held = EventLogFile.openLocked(log, flags: O_RDONLY) + else { continue } + recover(log) + EventLogFile.closeDescriptor(held) + } + } + #endif +} + +extension OpenEventLog: TypeIdentifying { } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift index 2d83d2e83..ec89ea33a 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift @@ -1048,7 +1048,7 @@ extension EventReporterSpec { /// The events the next run of the application would find. func eventsLeftOnDisk() -> [LDValue] { - let reader = EventStore(directory: testContext.store.directory, capacity: 100, locksOpenLog: false, logger: .disabled) + let reader = EventStore.sharing(testContext.store.directory) return reader.pendingEventPayloads().compactMap { try? JSONDecoder().decode(LDValue.self, from: $0) } } @@ -1132,7 +1132,7 @@ extension EventReporterSpec { testContext.eventReporter.commitRecordedEvents() // Nothing delivered them and nothing closed the log, as would be the case had the process died here. - let recovered = EventStore(directory: testContext.store.directory, capacity: 100, locksOpenLog: false, logger: .disabled) + let recovered = EventStore.sharing(testContext.store.directory) let nextRun = EventReporter(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) nextRun.isOnline = true @@ -1154,7 +1154,7 @@ extension EventReporterSpec { testContext.recordEvents(2) testContext.eventReporter.commitRecordedEvents() - let recovered = EventStore(directory: testContext.store.directory, capacity: 100, locksOpenLog: false, logger: .disabled) + let recovered = EventStore.sharing(testContext.store.directory) let nextRun = EventReporter(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) // Nothing asks for a flush, and the report interval is several times longer than this waits, so the @@ -1264,7 +1264,7 @@ extension EventReporterSpec { writes += 1 writesLock.unlock() Thread.sleep(forTimeInterval: 0.05) - return EventStore.writeAll(descriptor, bytes) + return EventLogFile.writeAll(descriptor, bytes) }) testContext = TestContext(store: store, eventCapacity: 100) let reporter = testContext.eventReporter! @@ -1306,7 +1306,7 @@ extension EventReporterSpec { testContext.serviceMock.respondToEvents(with: 503) testContext.eventReporter.record(CustomEvent(key: "kept", context: testContext.context)) - let recovered = EventStore(directory: testContext.store.directory, capacity: 100, locksOpenLog: false, logger: .disabled) + let recovered = EventStore.sharing(testContext.store.directory) let nextRun = EventReporter(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) nextRun.isOnline = true defer { nextRun.isOnline = false } @@ -1329,7 +1329,7 @@ extension EventReporterSpec { // The same directory, as the application's next launch would find it. testContext.serviceMock.stubEventResponse(success: true) - let recovered = EventStore(directory: testContext.store.directory, capacity: 100, locksOpenLog: false, logger: .disabled) + let recovered = EventStore.sharing(testContext.store.directory) let nextRun = EventReporter(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) nextRun.isOnline = true defer { nextRun.isOnline = false } @@ -1642,7 +1642,7 @@ extension EventReporterSpec { let store = EventStore.temporary(writeLog: { descriptor, bytes in writes += 1 // The file header and the first event land; the device is full from then on. - return writes > 2 ? ENOSPC : EventStore.writeAll(descriptor, bytes) + return writes > 2 ? ENOSPC : EventLogFile.writeAll(descriptor, bytes) }) testContext = TestContext(store: store) testContext.eventReporter.record(CustomEvent(key: "written", context: LDContext.stub())) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 23d1c3038..85a1e2d51 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -13,26 +13,28 @@ final class EventStoreSpec: QuickSpec { fallbackSpec() optOutSpec() multiProcessSpec() + #if os(macOS) || targetEnvironment(macCatalyst) sameNameInstancesSpec() + #endif } + #if os(macOS) || targetEnvironment(macCatalyst) private func sameNameInstancesSpec() { - // As on a Mac, where a second instance of an application runs under the first one's process name. - describe("two live instances under one process name, where logs are locked") { + // A second instance of a Mac application runs under the first one's process name. + describe("two live instances under one process name") { var first: EventStore! afterEach { first.deleteEverything() } func instance() -> EventStore { - EventStore(directory: first.directory, capacity: 100, processName: "app", locksOpenLog: true, logger: .disabled) + EventStore(directory: first.directory, capacity: 100, processName: "app", logger: .disabled) } beforeEach { first = EventStore(directory: EventStore.temporary().directory, capacity: 100, processName: "app", - locksOpenLog: true, logger: .disabled) } @@ -79,7 +81,7 @@ final class EventStoreSpec: QuickSpec { first.commit() let directory = first.directory first = nil - first = EventStore(directory: directory, capacity: 100, processName: "app", locksOpenLog: true, logger: .disabled) + first = EventStore(directory: directory, capacity: 100, processName: "app", logger: .disabled) first.recoverInterruptedLog() let batches = first.pendingBatches() @@ -88,6 +90,7 @@ final class EventStoreSpec: QuickSpec { } } } + #endif private func multiProcessSpec() { describe("several processes sharing a directory") { @@ -99,11 +102,11 @@ final class EventStoreSpec: QuickSpec { /// Another process of the same application, using the same directory under its own name. Unlocked, as on an /// iPhone or iPad, where a process name is one live process and the name alone keeps the logs apart. func otherProcess(_ name: String = "widget") -> EventStore { - EventStore(directory: app.directory, capacity: 100, processName: name, locksOpenLog: false, logger: .disabled) + EventStore.sharing(app.directory, processName: name) } beforeEach { - app = EventStore(directory: EventStore.temporary().directory, capacity: 100, processName: "app", locksOpenLog: false, logger: .disabled) + app = EventStore.sharing(EventStore.temporary().directory, processName: "app") } it("leaves a log another process still has open where it is, so that process's later events reach it") { @@ -158,12 +161,12 @@ final class EventStoreSpec: QuickSpec { } it("names logs so that process names reducing to the same characters stay apart") { - let colon = EventStore.logName(for: "com.app:remote") - let dot = EventStore.logName(for: "com.app.remote") + let colon = OpenEventLog.name(for: "com.app:remote") + let dot = OpenEventLog.name(for: "com.app.remote") expect(colon) != dot expect(colon.hasPrefix("com_app_remote-")) == true expect(dot.hasPrefix("com_app_remote-")) == true - expect(EventStore.logName(for: " ").hasPrefix("unknown-")) == true + expect(OpenEventLog.name(for: " ").hasPrefix("unknown-")) == true } } } @@ -204,7 +207,7 @@ final class EventStoreSpec: QuickSpec { _ = persisting.stage(EventStoreSpec.payload("open")) persisting.commit() - let withoutPersistence = EventStore(directory: persisting.directory, capacity: 100, persistEvents: false, locksOpenLog: false, logger: .disabled) + let withoutPersistence = EventStore.sharing(persisting.directory, persistEvents: false) withoutPersistence.recoverInterruptedLog() expect(withoutPersistence.pendingBatches()).to(beEmpty()) expect(withoutPersistence.pendingEventCount) == 0 @@ -240,7 +243,7 @@ final class EventStoreSpec: QuickSpec { /// what actually reached the disk, which is the whole question a crash asks. private static func reader(sharing store: EventStore, readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }) -> EventStore { - EventStore(directory: store.directory, capacity: 100, locksOpenLog: false, logger: .disabled, readFile: readFile) + EventStore.sharing(store.directory, readFile: readFile) } /// What a process out of file descriptors gets for a file that is perfectly intact. @@ -710,7 +713,7 @@ final class EventStoreSpec: QuickSpec { let interrupted = EventStore.temporary(writeLog: { descriptor, bytes in writes += 1 // The file header and the first event land; the device is full from then on. - return writes > 2 ? ENOSPC : EventStore.writeAll(descriptor, bytes) + return writes > 2 ? ENOSPC : EventLogFile.writeAll(descriptor, bytes) }) defer { interrupted.deleteEverything() } diff --git a/LaunchDarkly/LaunchDarklyTests/TestUtil.swift b/LaunchDarkly/LaunchDarklyTests/TestUtil.swift index e67b2135c..55123f70f 100644 --- a/LaunchDarkly/LaunchDarklyTests/TestUtil.swift +++ b/LaunchDarkly/LaunchDarklyTests/TestUtil.swift @@ -53,27 +53,47 @@ extension EventStore { /// A store in a directory of its own under the temporary directory, so that tests neither see each other's events /// nor leave any in the directory a real client would use. /// - /// Unlocked unless asked, whatever the platform. Tests stand in for the next run of the application with a second - /// store while the first is still alive, and on a Mac a locked log would rightly be left to the live one. + /// Unlocked, like every store from `sharing`. static func temporary( capacity: Int = 100, persistEvents: Bool = true, - locksOpenLog: Bool = false, commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.tests.commitQueue"), readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }, - writeLog: @escaping (Int32, Data) -> Int32? = EventStore.writeAll + writeLog: @escaping (Int32, Data) -> Int32? = EventLogFile.writeAll ) -> EventStore { let directory = FileManager.default.temporaryDirectory .appendingPathComponent("com.launchdarkly.tests.events", isDirectory: true) .appendingPathComponent(UUID().uuidString, isDirectory: true) - return EventStore(directory: directory, - capacity: capacity, - persistEvents: persistEvents, - locksOpenLog: locksOpenLog, - logger: .disabled, - commitQueue: commitQueue, - readFile: readFile, - writeLog: writeLog) + return unlocked(EventStore(directory: directory, + capacity: capacity, + persistEvents: persistEvents, + logger: .disabled, + commitQueue: commitQueue, + readFile: readFile, + writeLog: writeLog)) + } + + /// Another store on `directory`, standing in for the next run of the application, or for another process. + /// + /// Unlocked on a Mac too. Tests open it while the first store is still alive, and a locked log would rightly be + /// left to the live instance holding it; the locking itself has tests of its own. + static func sharing(_ directory: URL, + persistEvents: Bool = true, + processName: String = ProcessInfo.processInfo.processName, + readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }) -> EventStore { + unlocked(EventStore(directory: directory, + capacity: 100, + persistEvents: persistEvents, + processName: processName, + logger: .disabled, + readFile: readFile)) + } + + private static func unlocked(_ store: EventStore) -> EventStore { + #if os(macOS) || targetEnvironment(macCatalyst) + store.locksOpenLog = false + #endif + return store } /// Removes the store's directory, including any batch still waiting to be delivered. From a76eda91751bcf426f5d46cd38484a164f1e70b6 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Sun, 11 Oct 2026 13:35:59 -0700 Subject: [PATCH 016/158] rename to recordLock --- .../ServiceObjects/EventReporter.swift | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift index 474bc4fdb..8de2a58b6 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift @@ -129,7 +129,7 @@ class EventReporter: EventReporting { /// Guards the last response date and the record of what has been lost. private let stateLock = UnfairLock() - /// Only to be used while holding `pendingLock`. + /// Only to be used while holding `recordLock`. private(set) var contextSummarizer: ContextSummarizer /// Only to be used while holding `stateLock`. private var responseDate: Date @@ -161,7 +161,7 @@ class EventReporter: EventReporting { /// event is on disk, whichever commit put it there. It also keeps two encoders from staging their runs in /// whichever order they happened to finish. /// - /// Taken before `pendingLock` and before anything the store locks, never after. + /// Taken before `recordLock` and before anything the store locks, never after. private let commitLock = UnfairLock() /// Guards everything one recording writes: the held events and the summarizer. @@ -172,9 +172,9 @@ class EventReporter: EventReporting { /// Held for the appends and the handover of the run, never across the encoder or a syscall. It is a lock rather /// than a queue because it is taken once per evaluation, and a `DispatchQueue.sync` costs enough at that rate to be /// worth avoiding: the same measurement that made `EvaluationExposureDeduper` a lock applies here. - private let pendingLock = UnfairLock() + private let recordLock = UnfairLock() - /// Full events recorded but not yet encoded. Only to be used while holding `pendingLock`. + /// Full events recorded but not yet encoded. Only to be used while holding `recordLock`. /// /// Held rather than encoded because encoding early would not make them durable: the store stages bytes into memory /// too, and only a commit reaches the file. Both forms are equally lost to a crash, so the encode may as well @@ -182,7 +182,7 @@ class EventReporter: EventReporting { private var pending: [Event] = [] /// Whether a commit is already queued, so a run of recordings past the threshold asks for one write rather than - /// one each. Only to be used while holding `pendingLock`. + /// one each. Only to be used while holding `recordLock`. private var isCommitScheduled = false /// How many events the SDK will hold in total, across `pending` and the store. @@ -267,10 +267,10 @@ class EventReporter: EventReporting { // MARK: Recording func record(_ event: Event) { - pendingLock.lock() + recordLock.lock() let dropped = !admitToPending(event) let needsCommit = pending.count >= EventReporter.pendingCommitThreshold - pendingLock.unlock() + recordLock.unlock() if dropped { reportDropped() @@ -296,13 +296,13 @@ class EventReporter: EventReporting { let debugEvent = recordingDebugEvent ? FeatureEvent(key: flagKey, context: context, value: value, defaultValue: defaultValue, featureFlag: featureFlag, includeReason: includeReason, isDebug: true) : nil var dropped = 0 - pendingLock.lock() + recordLock.lock() let counted = contextSummarizer.trackRequest(flagKey: flagKey, reportedValue: value, featureFlag: featureFlag, defaultValue: defaultValue, context: context) for event in [featureEvent, debugEvent].compactMap({ $0 }) where !admitToPending(event) { dropped += 1 } let needsCommit = pending.count >= EventReporter.pendingCommitThreshold - pendingLock.unlock() + recordLock.unlock() // A refused evaluation is a loss like a refused event, since nothing later reconstructs its counter. Its full // event is still decided by the event capacity alone. @@ -322,13 +322,13 @@ class EventReporter: EventReporting { } /// Adds an event to `pending` for the next commit to encode, returning false if the SDK is already full. Requires - /// `pendingLock`. + /// `recordLock`. /// /// Capacity is consulted before anything else, so an event that will not be kept is never encoded. That ordering is /// what bounds an application re-evaluating a tracked flag in a render loop: once the limit is reached an /// evaluation costs no more than its summary counter, however fast the loop runs. private func admitToPending(_ event: Event) -> Bool { - pendingLock.assertOwned() + recordLock.assertOwned() guard pending.count + store.pendingEventCount < capacity else { return false } @@ -385,23 +385,23 @@ class EventReporter: EventReporting { /// Queues a commit unless one is already queued, so a run of recordings asks for one write rather than one each. private func scheduleCommit() { - pendingLock.lock() + recordLock.lock() guard !isCommitScheduled else { - pendingLock.unlock() + recordLock.unlock() return } isCommitScheduled = true - pendingLock.unlock() + recordLock.unlock() commitQueue.async { [weak self] in guard let self else { return } // Cleared before the commit rather than after it, so that an event held while this one is in flight can ask // for another commit instead of finding one apparently already on its way. - self.pendingLock.lock() + self.recordLock.lock() self.isCommitScheduled = false - self.pendingLock.unlock() + self.recordLock.unlock() self.commitRecordedEvents() } @@ -431,7 +431,7 @@ class EventReporter: EventReporting { /// Requires `commitLock`. private func encodeAndCommitPending() { commitLock.assertOwned() - pendingLock.lock() + recordLock.lock() let run = pending pending = [] let summaries = contextSummarizer.hasLoggedRequests ? contextSummarizer.getSummaries() : [] @@ -441,7 +441,7 @@ class EventReporter: EventReporting { // Reserved in the same critical section the run leaves `pending` in, so capacity counts it in one place or the // other throughout the encode. store.reserve(run.count) - pendingLock.unlock() + recordLock.unlock() stage(run) store.releaseReservations() @@ -922,8 +922,8 @@ private final class NullEventStore: EventStoring { extension EventReporter { /// Full events accepted but not yet handed to a commit. var pendingEventsForTesting: [Event] { - pendingLock.lock() - defer { pendingLock.unlock() } + recordLock.lock() + defer { recordLock.unlock() } return pending } From a901782c7008bbc898a715f1e359f8af4fbc186e Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Sun, 11 Oct 2026 14:16:35 -0700 Subject: [PATCH 017/158] use withLock --- .../ServiceObjects/EventReporter.swift | 173 +++++++----------- .../ServiceObjects/EventStore.swift | 162 +++++++--------- 2 files changed, 135 insertions(+), 200 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift index 8de2a58b6..b900c19b6 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift @@ -119,9 +119,7 @@ class EventReporter: EventReporting { } var lastEventResponseDate: Date { - stateLock.lock() - defer { stateLock.unlock() } - return responseDate + stateLock.withLock { responseDate } } let service: DarklyServiceProvider @@ -267,10 +265,9 @@ class EventReporter: EventReporting { // MARK: Recording func record(_ event: Event) { - recordLock.lock() - let dropped = !admitToPending(event) - let needsCommit = pending.count >= EventReporter.pendingCommitThreshold - recordLock.unlock() + let (dropped, needsCommit) = recordLock.withLock { + (!admitToPending(event), pending.count >= EventReporter.pendingCommitThreshold) + } if dropped { reportDropped() @@ -296,13 +293,13 @@ class EventReporter: EventReporting { let debugEvent = recordingDebugEvent ? FeatureEvent(key: flagKey, context: context, value: value, defaultValue: defaultValue, featureFlag: featureFlag, includeReason: includeReason, isDebug: true) : nil var dropped = 0 - recordLock.lock() - let counted = contextSummarizer.trackRequest(flagKey: flagKey, reportedValue: value, featureFlag: featureFlag, defaultValue: defaultValue, context: context) - for event in [featureEvent, debugEvent].compactMap({ $0 }) where !admitToPending(event) { - dropped += 1 + let (counted, needsCommit) = recordLock.withLock { () -> (Bool, Bool) in + let counted = contextSummarizer.trackRequest(flagKey: flagKey, reportedValue: value, featureFlag: featureFlag, defaultValue: defaultValue, context: context) + for event in [featureEvent, debugEvent].compactMap({ $0 }) where !admitToPending(event) { + dropped += 1 + } + return (counted, pending.count >= EventReporter.pendingCommitThreshold) } - let needsCommit = pending.count >= EventReporter.pendingCommitThreshold - recordLock.unlock() // A refused evaluation is a loss like a refused event, since nothing later reconstructs its counter. Its full // event is still decided by the event capacity alone. @@ -345,18 +342,15 @@ class EventReporter: EventReporting { private func reportLost() { os_log("%s dropping an event the store would not take", log: service.config.logger, type: .error, typeName(and: #function)) service.diagnosticCache?.incrementDroppedEventCount() - stateLock.lock() - eventsLostSinceLastAnswer = true - stateLock.unlock() + stateLock.withLock { eventsLostSinceLastAnswer = true } } /// Whether anything has been lost since the last caller was told, which that caller is now the one to hear about. private func takeEventsLost() -> Bool { - stateLock.lock() - defer { stateLock.unlock() } - let lost = eventsLostSinceLastAnswer - eventsLostSinceLastAnswer = false - return lost + stateLock.withLock { + defer { eventsLostSinceLastAnswer = false } + return eventsLostSinceLastAnswer + } } /// Commits at a commit point, on the caller's thread or off it as the application asked. @@ -385,23 +379,19 @@ class EventReporter: EventReporting { /// Queues a commit unless one is already queued, so a run of recordings asks for one write rather than one each. private func scheduleCommit() { - recordLock.lock() - guard !isCommitScheduled - else { - recordLock.unlock() - return + let alreadyScheduled = recordLock.withLock { () -> Bool in + defer { isCommitScheduled = true } + return isCommitScheduled } - isCommitScheduled = true - recordLock.unlock() + guard !alreadyScheduled + else { return } commitQueue.async { [weak self] in guard let self else { return } // Cleared before the commit rather than after it, so that an event held while this one is in flight can ask // for another commit instead of finding one apparently already on its way. - self.recordLock.lock() - self.isCommitScheduled = false - self.recordLock.unlock() + self.recordLock.withLock { self.isCommitScheduled = false } self.commitRecordedEvents() } @@ -412,9 +402,7 @@ class EventReporter: EventReporting { /// The run and the counters are taken in one critical section, so that an evaluation's counter and its full event /// are staged by the same commit, and encoded outside it, so recording does not wait on the encoder. func commitRecordedEvents() { - commitLock.lock() - defer { commitLock.unlock() } - encodeAndCommitPending() + commitLock.withLock { encodeAndCommitPending() } } /// Commits what is held and closes it into a batch, as one step with respect to other commits. @@ -422,26 +410,27 @@ class EventReporter: EventReporting { /// Closing takes everything staged so far, and a commit stages its run one event at a time, so a commit running /// between the two would be closed partway: an evaluation's event in this batch and its summary in the next. private func commitAndCloseBatch() { - commitLock.lock() - defer { commitLock.unlock() } - encodeAndCommitPending() - _ = store.closeBatch() + commitLock.withLock { + encodeAndCommitPending() + _ = store.closeBatch() + } } /// Requires `commitLock`. private func encodeAndCommitPending() { commitLock.assertOwned() - recordLock.lock() - let run = pending - pending = [] - let summaries = contextSummarizer.hasLoggedRequests ? contextSummarizer.getSummaries() : [] - if !summaries.isEmpty { - contextSummarizer.clear() - } - // Reserved in the same critical section the run leaves `pending` in, so capacity counts it in one place or the - // other throughout the encode. - store.reserve(run.count) - recordLock.unlock() + let (run, summaries) = recordLock.withLock { () -> ([Event], [(tracker: FlagRequestTracker, context: LDContext)]) in + let run = pending + pending = [] + let summaries = contextSummarizer.hasLoggedRequests ? contextSummarizer.getSummaries() : [] + if !summaries.isEmpty { + contextSummarizer.clear() + } + // Reserved in the same critical section the run leaves `pending` in, so capacity counts it in one place or + // the other throughout the encode. + store.reserve(run.count) + return (run, summaries) + } stage(run) store.releaseReservations() @@ -649,19 +638,17 @@ class EventReporter: EventReporting { /// ago, which no earlier delivery could have taken. So a delivery that finds everything already claimed has /// nothing of its own to send, and `true` is the right answer for it. private func claimDeliverableBatches() -> [EventBatch] { - stateLock.lock() - defer { stateLock.unlock() } - let deliverable = store.pendingBatches().filter { !batchesInFlight.contains($0.payloadId) } - batchesInFlight.formUnion(deliverable.map { $0.payloadId }) - return deliverable + stateLock.withLock { + let deliverable = store.pendingBatches().filter { !batchesInFlight.contains($0.payloadId) } + batchesInFlight.formUnion(deliverable.map { $0.payloadId }) + return deliverable + } } /// Releases the claim once the delivery has an answer, whether or not every batch was sent: `deliver` stops at the /// first retryable failure, and what it did not reach has to be deliverable again. private func releaseClaim(on batches: [EventBatch]) { - stateLock.lock() - batchesInFlight.subtract(batches.map { $0.payloadId }) - stateLock.unlock() + stateLock.withLock { batchesInFlight.subtract(batches.map { $0.payloadId }) } } private func deliver(_ batches: [EventBatch], _ completion: FlushOutcomeClosure?) { @@ -756,11 +743,11 @@ class EventReporter: EventReporting { private func outcome(sentEvents: Int, response: HTTPURLResponse?, error: Error?, isRetry: Bool) -> DeliveryOutcome { if error == nil && (200..<300).contains(response?.statusCode ?? 0) { let serverTime = response?.headerDate - stateLock.lock() - if let serverTime = serverTime, serverTime > responseDate { - responseDate = serverTime + stateLock.withLock { + if let serverTime = serverTime, serverTime > responseDate { + responseDate = serverTime + } } - stateLock.unlock() os_log("%s Completed sending %d event(s)", log: service.config.logger, type: .debug, typeName(and: #function), sentEvents) self.reportSyncComplete(nil) @@ -833,68 +820,54 @@ private final class NullEventStore: EventStoring { private var closed: [String: [Data]] = [:] var pendingEventCount: Int { - lock.lock() - defer { lock.unlock() } - return reserved + staged.count + closed.values.reduce(0) { $0 + $1.count } + lock.withLock { reserved + staged.count + closed.values.reduce(0) { $0 + $1.count } } } var isPersisting: Bool { false } func stage(_ encodedEvent: Data, bypassingCapacity: Bool) -> Bool { - lock.lock() - defer { lock.unlock() } - staged.append(encodedEvent) + lock.withLock { staged.append(encodedEvent) } return true } func reserve(_ events: Int) { - lock.lock() - defer { lock.unlock() } - reserved += events + lock.withLock { reserved += events } } func stageReserved(_ encodedEvent: Data) -> Bool { - lock.lock() - defer { lock.unlock() } - reserved = max(0, reserved - 1) - staged.append(encodedEvent) + lock.withLock { + reserved = max(0, reserved - 1) + staged.append(encodedEvent) + } return true } func releaseReservations() { - lock.lock() - defer { lock.unlock() } - reserved = 0 + lock.withLock { reserved = 0 } } func commit() { } func closeBatch() -> EventBatch? { - lock.lock() - defer { lock.unlock() } + lock.withLock { + guard !staged.isEmpty + else { return nil } - guard !staged.isEmpty - else { return nil } - - let payloadId = UUID().uuidString - let events = staged - staged = [] - closed[payloadId] = events - return EventBatch(payloadId: payloadId, eventCount: events.count) + let payloadId = UUID().uuidString + let events = staged + staged = [] + closed[payloadId] = events + return EventBatch(payloadId: payloadId, eventCount: events.count) + } } func pendingBatches() -> [EventBatch] { - lock.lock() - defer { lock.unlock() } - return closed.map { EventBatch(payloadId: $0.key, eventCount: $0.value.count) } + lock.withLock { closed.map { EventBatch(payloadId: $0.key, eventCount: $0.value.count) } } } func body(of batch: EventBatch) -> Data? { - lock.lock() - defer { lock.unlock() } - - guard let events = closed[batch.payloadId] + guard let events = lock.withLock({ closed[batch.payloadId] }) else { return nil } var body = Data("[".utf8) @@ -909,9 +882,7 @@ private final class NullEventStore: EventStoring { } func remove(_ batch: EventBatch) { - lock.lock() - defer { lock.unlock() } - closed[batch.payloadId] = nil + lock.withLock { closed[batch.payloadId] = nil } } func recoverInterruptedLog() { @@ -922,15 +893,11 @@ private final class NullEventStore: EventStoring { extension EventReporter { /// Full events accepted but not yet handed to a commit. var pendingEventsForTesting: [Event] { - recordLock.lock() - defer { recordLock.unlock() } - return pending + recordLock.withLock { pending } } func setLastEventResponseDate(_ date: Date) { - stateLock.lock() - responseDate = date - stateLock.unlock() + stateLock.withLock { responseDate = date } } /// Occupies the queue a delivery runs on, so a test can see what one that has not started yet collects. diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index 9dffccb8e..34c7f6100 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -193,27 +193,19 @@ final class EventStore: EventStoring { // MARK: Recording var pendingEventCount: Int { - bufferLock.lock() - defer { bufferLock.unlock() } - return reservedEvents + bufferedEventCount + committedEvents + closedEvents + bufferLock.withLock { reservedEvents + bufferedEventCount + committedEvents + closedEvents } } var isPersisting: Bool { - bufferLock.lock() - defer { bufferLock.unlock() } - return persistEvents + bufferLock.withLock { persistEvents } } func reserve(_ events: Int) { - bufferLock.lock() - reservedEvents += events - bufferLock.unlock() + bufferLock.withLock { reservedEvents += events } } func releaseReservations() { - bufferLock.lock() - reservedEvents = 0 - bufferLock.unlock() + bufferLock.withLock { reservedEvents = 0 } } func stage(_ encodedEvent: Data, bypassingCapacity: Bool = false) -> Bool { @@ -229,34 +221,32 @@ final class EventStore: EventStoring { else { os_log("%s dropping an event larger than a log frame allows", log: logger, type: .debug, typeName(and: #function)) if usingReservation { - bufferLock.lock() - reservedEvents = max(0, reservedEvents - 1) - bufferLock.unlock() + bufferLock.withLock { reservedEvents = max(0, reservedEvents - 1) } } return false } - bufferLock.lock() - - guard bypassingCapacity || reservedEvents + bufferedEventCount + committedEvents + closedEvents < capacity - else { - bufferLock.unlock() - return false - } + // Whether the staged bytes are due a write, or nil where the event was refused for capacity. + let needsCommit = bufferLock.withLock { () -> Bool? in + guard bypassingCapacity || reservedEvents + bufferedEventCount + committedEvents + closedEvents < capacity + else { return nil } - // In the same critical section as the append, so the event is never counted twice or not at all. - if usingReservation { - reservedEvents = max(0, reservedEvents - 1) - } - bufferData.append(EventLogFormat.frame(for: encodedEvent)) - bufferedEventCount += 1 - // Nothing to schedule once persistence has been given up on: there is nowhere for a commit to put these bytes, - // so they stay in memory until a delivery closes them into a batch. - let needsCommit = persistEvents && bufferData.count >= EventStore.stagingThreshold && !isCommitScheduled - if needsCommit { - isCommitScheduled = true + // In the same critical section as the append, so the event is never counted twice or not at all. + if usingReservation { + reservedEvents = max(0, reservedEvents - 1) + } + bufferData.append(EventLogFormat.frame(for: encodedEvent)) + bufferedEventCount += 1 + // Nothing to schedule once persistence has been given up on: there is nowhere for a commit to put these + // bytes, so they stay in memory until a delivery closes them into a batch. + let due = persistEvents && bufferData.count >= EventStore.stagingThreshold && !isCommitScheduled + if due { + isCommitScheduled = true + } + return due } - bufferLock.unlock() + guard let needsCommit + else { return false } // Handed to another thread rather than done here. Recording an event is something a flag evaluation does, and an // evaluation is expected to be a memory operation: the caller is usually the main thread, where a write that @@ -272,17 +262,13 @@ final class EventStore: EventStoring { private func commitScheduled() { // Cleared before the commit rather than after it, so that an event staged while this write is in flight can ask // for another one instead of finding a commit apparently already on its way and waiting for a commit point. - bufferLock.lock() - isCommitScheduled = false - bufferLock.unlock() + bufferLock.withLock { isCommitScheduled = false } commit() } func commit() { - ioLock.lock() - defer { ioLock.unlock() } - writeStagedBytes() + ioLock.withLock { writeStagedBytes() } } func closeBatch() -> EventBatch? { @@ -291,10 +277,7 @@ final class EventStore: EventStoring { writeStagedBytes() - bufferLock.lock() - let events = committedEvents - let stillBuffered = bufferedEventCount - bufferLock.unlock() + let (events, stillBuffered) = bufferLock.withLock { (committedEvents, bufferedEventCount) } let fromLog = events > 0 ? closeOpenLog(events) : nil // Closed after the log, so the older events are delivered first. Both can hold events at once: a write that @@ -317,19 +300,17 @@ final class EventStore: EventStoring { // directory on tvOS, say. Then its events are gone, and they have to stop counting against capacity or the // store would refuse events for the rest of the session. if !openLog.exists { - bufferLock.lock() - committedEvents = 0 - bufferLock.unlock() + bufferLock.withLock { committedEvents = 0 } } return nil } eventCounts[payloadId] = events - bufferLock.lock() - committedEvents = 0 - closedEvents += events - bufferLock.unlock() + bufferLock.withLock { + committedEvents = 0 + closedEvents += events + } return EventBatch(payloadId: payloadId, eventCount: events) } @@ -367,9 +348,7 @@ final class EventStore: EventStoring { EventBatch(payloadId: $0.payloadId, eventCount: $0.eventCount) }) - bufferLock.lock() - closedEvents = batches.reduce(0) { $0 + $1.eventCount } - bufferLock.unlock() + bufferLock.withLock { closedEvents = batches.reduce(0) { $0 + $1.eventCount } } return batches } @@ -457,9 +436,7 @@ final class EventStore: EventStoring { eventCounts[batch.payloadId] = nil } - bufferLock.lock() - closedEvents = max(0, closedEvents - batch.eventCount) - bufferLock.unlock() + bufferLock.withLock { closedEvents = max(0, closedEvents - batch.eventCount) } } /// Closes any log left open by a previous run of this process, so its events join the batches to deliver. @@ -468,9 +445,7 @@ final class EventStore: EventStoring { /// process's own log is touched: another process's may still be open in a process that is alive. Where logs are /// locked, any log no live process holds is recovered, whatever it is named. func recoverInterruptedLog() { - ioLock.lock() - defer { ioLock.unlock() } - recoverInterruptedLogOnce() + ioLock.withLock { recoverInterruptedLogOnce() } } /// Requires `ioLock`. Does its work once per store: after that, the open log is one this run opened. @@ -525,33 +500,29 @@ final class EventStore: EventStoring { /// Requires `ioLock`. private func writeStagedBytes() { ioLock.assertOwned() - bufferLock.lock() - guard persistEvents, bufferedEventCount > 0 - else { + let taken = bufferLock.withLock { () -> (bytes: Data, events: Int)? in // Staged bytes are left where they are. With nowhere durable to put them, memory is better than dropping // them: a delivery can still close them into a batch and send them. - bufferLock.unlock() - return + guard persistEvents, bufferedEventCount > 0 + else { return nil } + defer { + bufferData = Data() + bufferedEventCount = 0 + } + return (bufferData, bufferedEventCount) } - let bytes = bufferData - let events = bufferedEventCount - bufferData = Data() - bufferedEventCount = 0 - bufferLock.unlock() - - if append(bytes) { - bufferLock.lock() - committedEvents += events - bufferLock.unlock() + guard let taken + else { return } + + if append(taken.bytes) { + bufferLock.withLock { committedEvents += taken.events } } else { // Put back, to be delivered from memory rather than lost. Ahead of anything staged while the write was in // flight, so the events keep the order they were recorded in. - bufferLock.lock() - let laterFrames = bufferData - bufferData = bytes - bufferData.append(laterFrames) - bufferedEventCount += events - bufferLock.unlock() + bufferLock.withLock { + bufferData = taken.bytes + bufferData + bufferedEventCount += taken.events + } } } @@ -575,10 +546,10 @@ final class EventStore: EventStoring { deinit { // One acquisition: the lock is not recursive, and `commit()` would take it again. - ioLock.lock() - writeStagedBytes() - openLog.closeDescriptor() - ioLock.unlock() + ioLock.withLock { + writeStagedBytes() + openLog.closeDescriptor() + } } } @@ -601,9 +572,7 @@ private extension EventStore { func disablePersistence() { ioLock.assertOwned() openLog.closeDescriptor() - bufferLock.lock() - persistEvents = false - bufferLock.unlock() + bufferLock.withLock { persistEvents = false } } /// Closes whatever is staged into a batch that never reaches a file. @@ -611,12 +580,13 @@ private extension EventStore { /// Requires `ioLock`. func closeInMemoryBatch() -> EventBatch? { ioLock.assertOwned() - bufferLock.lock() - let frames = bufferData - let events = bufferedEventCount - bufferData = Data() - bufferedEventCount = 0 - bufferLock.unlock() + let (frames, events) = bufferLock.withLock { () -> (Data, Int) in + defer { + bufferData = Data() + bufferedEventCount = 0 + } + return (bufferData, bufferedEventCount) + } // Assembled through the same reader a batch on disk goes through, so a held batch and a written one produce // byte-identical request bodies. @@ -628,9 +598,7 @@ private extension EventStore { let payloadId = UUID().uuidString inMemoryBatches.append(HeldBatch(payloadId: payloadId, body: body, eventCount: events)) - bufferLock.lock() - closedEvents += events - bufferLock.unlock() + bufferLock.withLock { closedEvents += events } return EventBatch(payloadId: payloadId, eventCount: events) } From 8bffc8347079d505dc635b121c028686f96b49c7 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Sun, 11 Oct 2026 15:47:10 -0700 Subject: [PATCH 018/158] Keep the batch paths off the disk, and out of ioLock, when persistence is off Co-authored-by: Cursor --- .../ServiceObjects/EventStore.swift | 86 ++++++++++++------- .../ServiceObjects/EventStoreSpec.swift | 24 ++++++ 2 files changed, 78 insertions(+), 32 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index 34c7f6100..9a2813cc8 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -136,9 +136,16 @@ final class EventStore: EventStoring { /// it kept a log: hold the events, deliver them, lose them only if the process dies. Kept in the order they were /// closed, so delivery still goes oldest first. /// - /// Only to be used while holding `ioLock`. + /// Only to be used while holding `heldLock`. private var inMemoryBatches: [HeldBatch] = [] + /// Guards `inMemoryBatches`. + /// + /// Its own lock rather than `ioLock`, so that a store the application told to keep off the disk -- where held + /// batches are the only ones there are -- never waits on the lock disk I/O is serialized under. Taken inside + /// `ioLock` where there is a disk, never the other way round. + private let heldLock = UnfairLock() + /// How many events are in each batch this process closed or recovered, so that listing them does not have to read /// them back. /// @@ -268,10 +275,16 @@ final class EventStore: EventStoring { } func commit() { + // Where the application asked for no disk there is nothing for a commit to reach, and so nothing to serialize. + guard readsDisk + else { return } ioLock.withLock { writeStagedBytes() } } func closeBatch() -> EventBatch? { + guard readsDisk + else { return closeInMemoryBatch() } + ioLock.lock() defer { ioLock.unlock() } @@ -337,22 +350,23 @@ final class EventStore: EventStoring { } func pendingBatches() -> [EventBatch] { - ioLock.lock() - defer { ioLock.unlock() } - - var batches = readsDisk ? batchesOnDisk() : [] - - // After the files, since anything held in memory was closed by this run and so is newer than whatever a - // previous run left on the disk. - batches.append(contentsOf: inMemoryBatches.map { - EventBatch(payloadId: $0.payloadId, eventCount: $0.eventCount) - }) + // Listed in one critical section where there is a disk, so that the files and what is held cannot be counted + // either side of a close that moves events between them. + let batches = readsDisk ? ioLock.withLock { batchesOnDisk() + heldBatches() } : heldBatches() bufferLock.withLock { closedEvents = batches.reduce(0) { $0 + $1.eventCount } } return batches } + /// The batches the filesystem would not take, after the files: anything held was closed by this run and so is + /// newer than whatever a previous run left on the disk. + private func heldBatches() -> [EventBatch] { + heldLock.withLock { + inMemoryBatches.map { EventBatch(payloadId: $0.payloadId, eventCount: $0.eventCount) } + } + } + /// The batch files in the directory, oldest first. /// /// Requires `ioLock`. @@ -411,29 +425,37 @@ final class EventStore: EventStoring { } func body(of batch: EventBatch) throws -> Data? { - ioLock.lock() - defer { ioLock.unlock() } - - if let held = inMemoryBatches.first(where: { $0.payloadId == batch.payloadId }) { + if let held = heldLock.withLock({ inMemoryBatches.first { $0.payloadId == batch.payloadId } }) { return held.body } - switch read(url(of: batch.payloadId)) { - case .missing: return nil - case .failed(let error): throw error - case .bytes(let file): return EventLogFormat.assembleBody(from: file) + // A batch is either held or on the disk, never both and never moved between them, so a batch that is not held + // in a store with no disk does not exist. + guard readsDisk + else { return nil } + + return try ioLock.withLock { + switch read(url(of: batch.payloadId)) { + case .missing: return nil + case .failed(let error): throw error + case .bytes(let file): return EventLogFormat.assembleBody(from: file) + } } } func remove(_ batch: EventBatch) { - ioLock.lock() - defer { ioLock.unlock() } - - if let index = inMemoryBatches.firstIndex(where: { $0.payloadId == batch.payloadId }) { + let wasHeld = heldLock.withLock { () -> Bool in + guard let index = inMemoryBatches.firstIndex(where: { $0.payloadId == batch.payloadId }) + else { return false } inMemoryBatches.remove(at: index) - } else { - try? FileManager.default.removeItem(at: url(of: batch.payloadId)) - eventCounts[batch.payloadId] = nil + return true + } + + if !wasHeld, readsDisk { + ioLock.withLock { + try? FileManager.default.removeItem(at: url(of: batch.payloadId)) + eventCounts[batch.payloadId] = nil + } } bufferLock.withLock { closedEvents = max(0, closedEvents - batch.eventCount) } @@ -577,9 +599,9 @@ private extension EventStore { /// Closes whatever is staged into a batch that never reaches a file. /// - /// Requires `ioLock`. + /// Called under `ioLock` where there is a disk, so that it and the log are closed as one step, and on its own + /// where there is not. func closeInMemoryBatch() -> EventBatch? { - ioLock.assertOwned() let (frames, events) = bufferLock.withLock { () -> (Data, Int) in defer { bufferData = Data() @@ -596,7 +618,7 @@ private extension EventStore { else { return nil } let payloadId = UUID().uuidString - inMemoryBatches.append(HeldBatch(payloadId: payloadId, body: body, eventCount: events)) + heldLock.withLock { inMemoryBatches.append(HeldBatch(payloadId: payloadId, body: body, eventCount: events)) } bufferLock.withLock { closedEvents += events } @@ -612,12 +634,12 @@ extension EventStore { func pendingEventPayloads() -> [Data] { commit() - ioLock.lock() - defer { ioLock.unlock() } - guard readsDisk else { return [] } + ioLock.lock() + defer { ioLock.unlock() } + let contents = (try? FileManager.default.contentsOfDirectory(at: directory, includingPropertiesForKeys: nil, options: [.skipsHiddenFiles])) ?? [] diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 85a1e2d51..4cf2b2df1 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -198,6 +198,30 @@ final class EventStoreSpec: QuickSpec { expect(try store.body(of: batch)) == Data("[{\"kind\":\"custom\",\"key\":\"held\"}]".utf8) } + it("asks the filesystem nothing, even about a batch it does not hold") { + var reads = 0 + let store = EventStore.temporary(persistEvents: false, readFile: { _ in + reads += 1 + return Data() + }) + defer { store.deleteEverything() } + + _ = store.stage(EventStoreSpec.payload("held")) + let batch = store.closeBatch() + expect(store.pendingBatches().map(\.payloadId)) == [batch?.payloadId] + expect(EventStoreSpec.keys(ofBody: batch.flatMap { try? store.body(of: $0) })) == ["held"] + batch.map(store.remove) + + // A batch this store never closed, which is what sends the paths that form a file URL down the branch + // that would otherwise ask the filesystem about it. + let stranger = EventBatch(payloadId: UUID().uuidString, eventCount: 1) + expect(try store.body(of: stranger)).to(beNil()) + store.remove(stranger) + + expect(reads) == 0 + expect(FileManager.default.fileExists(atPath: store.directory.path)) == false + } + it("leaves alone what a run that did persist left behind, for the next run that persists") { let persisting = EventStore.temporary() defer { persisting.deleteEverything() } From 81a7856328a872799777ac7a245dbcdb65dd2b11 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Sun, 11 Oct 2026 15:47:10 -0700 Subject: [PATCH 019/158] Answer a flush for the batch another delivery is holding A commit only stages events; whichever delivery runs next is what closes them into a batch. So a flush could find everything claimed, conclude it had nothing of its own left to send, and report success for events that were about to fail with somebody else's delivery. batchesInFlight becomes a map to a Claim holding the batch's outcome and the deliveries waiting to hear it, and a DeliveryAnswer gathers this delivery's own outcome with every claim it skipped. Only the answer waits; the batches it closed for itself go out straight away. The loss flag becomes a counter for the same reason. Each delivery takes the count as it closes its batch and answers for the losses up to that mark, so a loss reaches the caller whose delivery it happened before rather than whichever response arrived first. Claims also move off stateLock onto a claimLock of their own, so an evaluating thread reading lastEventResponseDate never waits on a listing of the store. Co-authored-by: Cursor --- LaunchDarkly.xcodeproj/project.pbxproj | 10 + .../ServiceObjects/DeliveryClaims.swift | 47 ++++ .../ServiceObjects/EventReporter.swift | 154 ++++++++--- .../ServiceObjects/EventReporterSpec.swift | 243 ++++++++++++++---- 4 files changed, 368 insertions(+), 86 deletions(-) create mode 100644 LaunchDarkly/LaunchDarkly/ServiceObjects/DeliveryClaims.swift diff --git a/LaunchDarkly.xcodeproj/project.pbxproj b/LaunchDarkly.xcodeproj/project.pbxproj index 78f70e577..ad6d58112 100644 --- a/LaunchDarkly.xcodeproj/project.pbxproj +++ b/LaunchDarkly.xcodeproj/project.pbxproj @@ -19,6 +19,7 @@ 2CA9D36B3566D78850D6F0AF /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; B8F228923BE11E444D1ECB8E /* OpenEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* OpenEventLog.swift */; }; 3C55384C9E3485B493E1F86A /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; + C3F7E502BD916C40AE238F64 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; AB4827D11084D58661E24B05 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; 3D2406142E0D90E000F91253 /* Plugin.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3D2406102E0D90E000F91253 /* Plugin.swift */; }; 3D2406152E0D90E000F91253 /* PluginMetadata.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3D2406112E0D90E000F91253 /* PluginMetadata.swift */; }; @@ -54,6 +55,7 @@ 5AEA1F832DCA974314E07D21 /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; D4225F1F05F2C55E9518941B /* OpenEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* OpenEventLog.swift */; }; DDA788E3056F8AC3CA203933 /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; + D4080613CEA27D51BF349075 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; ECBE45806B470ACC5187B7D6 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; 7185A3C30E7FFC4F725769F1 /* BackgroundActivity.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9A771B316E19ED598CD4FB0B /* BackgroundActivity.swift */; }; 830BF933202D188E006DF9B1 /* HTTPURLRequest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 830BF932202D188E006DF9B1 /* HTTPURLRequest.swift */; }; @@ -386,6 +388,7 @@ B0B1AD7CADE8DBB596C2DE9B /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; DAA2C9B9C144F129217A7C55 /* OpenEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* OpenEventLog.swift */; }; 44E0BE449714952A68628CEB /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; + E5191724DFB38E62C0451086 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; CA2ADB3D20A229D70CA34F73 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; B40B419C249ADA6B00CD0726 /* DiagnosticCacheSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = B40B419B249ADA6B00CD0726 /* DiagnosticCacheSpec.swift */; }; B4265EB124E7390C001CFD2C /* TestUtil.swift in Sources */ = {isa = PBXBuildFile; fileRef = B4265EB024E7390C001CFD2C /* TestUtil.swift */; }; @@ -440,6 +443,7 @@ CF9A1EB116D8AA3B7B5535B2 /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; E2F8080AB2B3CD6E9A9C1FCC /* OpenEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* OpenEventLog.swift */; }; F0F504E5EC32223905356098 /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; + B2E6D4F1AC805B3F9D127E53 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; 625ADAC0CCFE3BE324C45752 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; D7A500FCA04E3412F931003F /* ContextEncodingCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = D93B7F5F6CF5C3DE79FB4286 /* ContextEncodingCache.swift */; }; DAF5695D58E28B8CF9221213 /* LDContextJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88DAB8D1864C14D9C7E06F11 /* LDContextJSONWriter.swift */; }; @@ -668,6 +672,7 @@ C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventStore.swift; sourceTree = ""; }; E21BC7A0E0F8D272785D7985 /* OpenEventLog.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = OpenEventLog.swift; sourceTree = ""; }; 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventLogFile.swift; sourceTree = ""; }; + A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = DeliveryClaims.swift; sourceTree = ""; }; B58551BB165F28784C785720 /* EventLogFormat.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventLogFormat.swift; sourceTree = ""; }; C408884623033B3600420721 /* ConnectionInformationStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConnectionInformationStore.swift; sourceTree = ""; }; C408884823033B7500420721 /* ConnectionInformation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConnectionInformation.swift; sourceTree = ""; }; @@ -1056,6 +1061,7 @@ C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */, E21BC7A0E0F8D272785D7985 /* OpenEventLog.swift */, 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */, + A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */, B58551BB165F28784C785720 /* EventLogFormat.swift */, ); path = ServiceObjects; @@ -1586,6 +1592,7 @@ CF9A1EB116D8AA3B7B5535B2 /* EventStore.swift in Sources */, E2F8080AB2B3CD6E9A9C1FCC /* OpenEventLog.swift in Sources */, F0F504E5EC32223905356098 /* EventLogFile.swift in Sources */, + B2E6D4F1AC805B3F9D127E53 /* DeliveryClaims.swift in Sources */, 625ADAC0CCFE3BE324C45752 /* EventLogFormat.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; @@ -1684,6 +1691,7 @@ 2CA9D36B3566D78850D6F0AF /* EventStore.swift in Sources */, B8F228923BE11E444D1ECB8E /* OpenEventLog.swift in Sources */, 3C55384C9E3485B493E1F86A /* EventLogFile.swift in Sources */, + C3F7E502BD916C40AE238F64 /* DeliveryClaims.swift in Sources */, AB4827D11084D58661E24B05 /* EventLogFormat.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; @@ -1781,6 +1789,7 @@ 5AEA1F832DCA974314E07D21 /* EventStore.swift in Sources */, D4225F1F05F2C55E9518941B /* OpenEventLog.swift in Sources */, DDA788E3056F8AC3CA203933 /* EventLogFile.swift in Sources */, + D4080613CEA27D51BF349075 /* DeliveryClaims.swift in Sources */, ECBE45806B470ACC5187B7D6 /* EventLogFormat.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; @@ -1945,6 +1954,7 @@ B0B1AD7CADE8DBB596C2DE9B /* EventStore.swift in Sources */, DAA2C9B9C144F129217A7C55 /* OpenEventLog.swift in Sources */, 44E0BE449714952A68628CEB /* EventLogFile.swift in Sources */, + E5191724DFB38E62C0451086 /* DeliveryClaims.swift in Sources */, CA2ADB3D20A229D70CA34F73 /* EventLogFormat.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DeliveryClaims.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DeliveryClaims.swift new file mode 100644 index 000000000..161cc2f79 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DeliveryClaims.swift @@ -0,0 +1,47 @@ +import Foundation + +/// What one delivery's answer is made of, where deliveries overlap. +/// +/// A commit only stages events; whichever delivery runs next is what closes them into a batch. So the batch holding a +/// caller's events may belong to a delivery already under way, and answering that caller means hearing from it. These +/// are the two small pieces that arrange it: a `Claim` is a batch somebody is waiting on, and a `DeliveryAnswer` +/// gathers the claims one delivery's answer depends on. +extension EventReporter { + /// A batch one delivery has taken, which another delivery's answer may depend on. + final class Claim { + /// This batch's result, or nil while the delivery holding it is still waiting. Only to be used while holding + /// the reporter's `claimLock`. + var outcome: Bool? + /// Deliveries whose answer waits on this batch. Only to be used while holding the reporter's `claimLock`. + var waiters: [FlushOutcomeClosure] = [] + } + + /// Gathers the outcomes one delivery's answer depends on, and reports success only where all of them succeeded. + /// + /// How many there are is settled before the first can arrive, so this needs no notion of being complete beyond + /// counting down. + final class DeliveryAnswer { + private let lock = UnfairLock() + /// Outcomes still to come, and the answer to make once there are none. Only to be used while holding `lock`. + private var outstanding: Int + private var delivered = true + private var answer: FlushOutcomeClosure? + + init(outstanding: Int, _ answer: @escaping FlushOutcomeClosure) { + self.outstanding = outstanding + self.answer = answer + } + + func report(_ delivered: Bool) { + let (answer, outcome) = lock.withLock { () -> (FlushOutcomeClosure?, Bool) in + self.delivered = self.delivered && delivered + outstanding -= 1 + guard outstanding <= 0 + else { return (nil, self.delivered) } + defer { self.answer = nil } + return (self.answer, self.delivered) + } + answer?(outcome) + } + } +} diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift index b900c19b6..22d745075 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift @@ -142,14 +142,21 @@ class EventReporter: EventReporting { private var deliveryRunCount = 0 #endif - /// Whether events the SDK accepted were dropped for good, cleared once a caller who hears an outcome has been - /// told so. Only to be used while holding `stateLock`. + /// How many events the SDK accepted and then dropped for good. Only to be used while holding `stateLock`. /// /// A batch LaunchDarkly refuses in a way that may pass is kept and retried, so it is not a loss. What is lost is /// what the store never received: an event or a summary that could not be serialized when it was committed, or /// that serialized too large to store. A delivery that finds nothing to send cannot tell those apart from events /// that arrived, so without this it would report success for them. - private var eventsLostSinceLastAnswer = false + /// + /// A count rather than a flag because deliveries overlap. Each takes the count as it closes its batch, and + /// answers for the losses up to that mark, so a loss is told to the caller whose delivery it happened before + /// rather than to whichever response arrived first. + private var lostEvents: UInt64 = 0 + + /// The mark up to which losses have already been reported to a caller, so that none is told twice. Only to be + /// used while holding `stateLock`. + private var reportedLostEvents: UInt64 = 0 /// Held for the whole of a commit, so that only one runs at a time. /// @@ -202,7 +209,14 @@ class EventReporter: EventReporting { /// question and the delivery that acts on it are both enqueued there, in that order. private var hasEventsFromPreviousRun = false - /// Payload IDs a delivery has taken and not yet had an answer for. Only to be used while holding `stateLock`. + /// Guards `batchesInFlight`. + /// + /// Its own lock rather than `stateLock`, which an evaluating thread takes to read `lastEventResponseDate`: a claim + /// is made next to a listing of the store, and nothing a flag evaluation waits on may be held near the disk. + private let claimLock = UnfairLock() + + /// The batches a delivery has taken and not yet had an answer for, by payload ID. Only to be used while holding + /// `claimLock`. /// /// `deliveryQueue` serializes the *start* of a delivery but not the round trip it waits on, and a batch stays on /// disk until its response arrives. Without this, a delivery beginning inside that window would list the same batch @@ -210,8 +224,9 @@ class EventReporter: EventReporting { /// requests arriving at once are the same one. /// /// This bounds the overlap rather than preventing it, which is the point: a later delivery still runs, and still - /// sends what it closed for itself. Only the batches already on their way are left out of it. - private var batchesInFlight: Set = [] + /// sends what it closed for itself. Only the batches already on their way are left out of it, and a caller waiting + /// on that later delivery hears their outcome too. + private var batchesInFlight: [String: Claim] = [:] private let onSyncComplete: EventSyncCompleteClosure? @@ -342,14 +357,20 @@ class EventReporter: EventReporting { private func reportLost() { os_log("%s dropping an event the store would not take", log: service.config.logger, type: .error, typeName(and: #function)) service.diagnosticCache?.incrementDroppedEventCount() - stateLock.withLock { eventsLostSinceLastAnswer = true } + stateLock.withLock { lostEvents += 1 } } - /// Whether anything has been lost since the last caller was told, which that caller is now the one to hear about. - private func takeEventsLost() -> Bool { + /// Whether anything was lost up to `mark` that no caller has been told about, which this caller is now the one to + /// hear about. + /// + /// Marks it reported, so the next caller is not told about the same loss. A delivery that nobody is waiting on + /// never asks, so a scheduled delivery does not consume the answer a later caller is owed. + private func takeEventsLost(upTo mark: UInt64) -> Bool { stateLock.withLock { - defer { eventsLostSinceLastAnswer = false } - return eventsLostSinceLastAnswer + guard mark > reportedLostEvents + else { return false } + reportedLostEvents = mark + return true } } @@ -409,10 +430,14 @@ class EventReporter: EventReporting { /// /// Closing takes everything staged so far, and a commit stages its run one event at a time, so a commit running /// between the two would be closed partway: an evaluation's event in this batch and its summary in the next. - private func commitAndCloseBatch() { + /// + /// - Returns: the loss count as of the close, which is the mark this delivery answers for. Taken under the same + /// lock, so it covers every loss the closed batch could have suffered and nothing a later commit suffers. + private func commitAndCloseBatch() -> UInt64 { commitLock.withLock { encodeAndCommitPending() _ = store.closeBatch() + return stateLock.withLock { lostEvents } } } @@ -540,8 +565,9 @@ class EventReporter: EventReporting { /// Queues a delivery the way a caller asking for one expects, which is as a commit point. /// /// At `.immediate` everything accepted before the call is on disk by the time it returns, even where the delivery - /// itself cannot run because the client is offline. It is also what makes the answer cover the caller's own - /// events: they are closed into a batch here, before any delivery can take one. + /// itself cannot run because the client is offline. What it does not do is close those events into a batch: only + /// a delivery does that, so a delivery already under way may be the one holding them. That is why the answer + /// waits on the claims it skips rather than only on what it sent itself. private func commitThenQueueDelivery(_ completion: FlushOutcomeClosure?) { commitAtCommitPoint() queueDelivery(completion) @@ -587,16 +613,15 @@ class EventReporter: EventReporting { #endif deliveryLock.unlock() - reportEvents { delivered in + reportEvents { delivered, lossMark in guard !waiters.isEmpty else { return } // Taken once for the delivery rather than once per waiter, so that every caller joined to it hears about - // a loss instead of only whichever happened to be answered first. Left alone where nobody is waiting, so - // a scheduled delivery does not consume the answer a later caller is owed. - let kept = !self.takeEventsLost() + // a loss instead of only whichever happened to be answered first. + let kept = !self.takeEventsLost(upTo: lossMark) // Answered away from `deliveryQueue`, and from whichever thread the delivery ended on, so that every - // completion reaches its caller the same way. A completion is free to record or flush again: it is never - // running on the queue those wait for. + // completion reaches its caller the same way. A completion is free to flush again: it is never running + // on the queue the delivery it asks for would be queued on. DispatchQueue.global().async { for waiter in waiters { waiter(delivered && kept) @@ -605,50 +630,103 @@ class EventReporter: EventReporting { } } - private func reportEvents(completion: FlushOutcomeClosure?) { + private func reportEvents(completion: ((_ delivered: Bool, _ lossMark: UInt64) -> Void)?) { guard isOnline else { os_log("%s aborted. EventReporter is offline", log: service.config.logger, type: .debug, typeName(and: #function)) reportSyncComplete(.isOffline) - completion?(false) + completion?(false, 0) return } - commitAndCloseBatch() + let lossMark = commitAndCloseBatch() + + let (batches, waitingOn) = claimDeliverableBatches() + // This delivery's own outcome, plus that of every batch another delivery is already sending. The caller's + // events may be in either: a delivery that started after they were recorded closed them into its batch. + let answer = DeliveryAnswer(outstanding: waitingOn.count + 1) { delivered in + completion?(delivered, lossMark) + } + for claim in waitingOn { + whenSettled(claim, answer.report) + } - let batches = claimDeliverableBatches() guard !batches.isEmpty else { os_log("%s aborted. Event store is empty", log: service.config.logger, type: .debug, typeName(and: #function)) reportSyncComplete(nil) - completion?(true) + answer.report(true) return } os_log("%s starting", log: service.config.logger, type: .debug, typeName(and: #function)) deliver(batches) { [weak self] delivered in - self?.releaseClaim(on: batches) - completion?(delivered) + self?.releaseClaim(on: batches, delivered: delivered) + answer.report(delivered) } } /// Takes the batches this delivery will send, leaving out any another delivery is still waiting on a response for. /// - /// The caller's own events are never among those left out: they were closed into a batch of their own a moment - /// ago, which no earlier delivery could have taken. So a delivery that finds everything already claimed has - /// nothing of its own to send, and `true` is the right answer for it. - private func claimDeliverableBatches() -> [EventBatch] { - stateLock.withLock { - let deliverable = store.pendingBatches().filter { !batchesInFlight.contains($0.payloadId) } - batchesInFlight.formUnion(deliverable.map { $0.payloadId }) - return deliverable + /// - Returns: the batches to send, and the claims on the ones left out. A batch left out may hold this caller's + /// events -- `commitRecordedEvents` only stages them, and whichever delivery runs next is what closes them into + /// a batch -- so the answer has to wait for those claims rather than assume they are someone else's business. + private func claimDeliverableBatches() -> (deliverable: [EventBatch], waitingOn: [Claim]) { + // Listed before the lock is taken, because listing reads the disk. Taking what appeared after it is the next + // delivery's business; all this has to be atomic is deciding which of these batches are this delivery's. + let pending = store.pendingBatches() + return claimLock.withLock { + var deliverable: [EventBatch] = [] + var waitingOn: [Claim] = [] + for batch in pending { + if let claimed = batchesInFlight[batch.payloadId] { + waitingOn.append(claimed) + } else { + batchesInFlight[batch.payloadId] = Claim() + deliverable.append(batch) + } + } + return (deliverable, waitingOn) } } - /// Releases the claim once the delivery has an answer, whether or not every batch was sent: `deliver` stops at the + /// Releases the claims once the delivery has an answer, whether or not every batch was sent: `deliver` stops at the /// first retryable failure, and what it did not reach has to be deliverable again. - private func releaseClaim(on batches: [EventBatch]) { - stateLock.withLock { batchesInFlight.subtract(batches.map { $0.payloadId }) } + /// + /// Every batch in the pass is settled with the pass's own result, rather than its own. A pass that stopped partway + /// did not hear back about what it never reached, so the batches it is giving up are not known to have arrived. + private func releaseClaim(on batches: [EventBatch], delivered: Bool) { + let waiters = claimLock.withLock { () -> [FlushOutcomeClosure] in + batches.flatMap { batch -> [FlushOutcomeClosure] in + guard let claimed = batchesInFlight.removeValue(forKey: batch.payloadId) + else { return [] } + claimed.outcome = delivered + return claimed.waiters + } + } + + for waiter in waiters { + waiter(delivered) + } + } + + /// Calls back with a claimed batch's outcome, once there is one. + /// + /// Settling and registering both happen under `claimLock`, so a claim released between the listing and this call + /// answers from what it recorded rather than leaving a waiter nobody will ever come back to. + private func whenSettled(_ claim: Claim, _ completion: @escaping FlushOutcomeClosure) { + let settled = claimLock.withLock { () -> Bool? in + guard let outcome = claim.outcome + else { + claim.waiters.append(completion) + return nil + } + return outcome + } + + if let settled { + completion(settled) + } } private func deliver(_ batches: [EventBatch], _ completion: FlushOutcomeClosure?) { diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift index ec89ea33a..eb65fb4f2 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift @@ -82,25 +82,23 @@ final class EventReporterSpec: QuickSpec { events.compactMap { encodeToLDValue($0) } } - /// Flushes, and answers whether recording from the completion got anywhere. + /// Flushes, and answers whether a flush made from the completion got anywhere. /// - /// Recording waits on the reporter's queue, so a completion run on that queue cannot record. The recording - /// happens on another thread against a deadline, so that a reporter answering on its own queue fails this - /// rather than hanging the test process on a deadlock it can never leave. - func recordedFromACompletion() -> Bool { - var recorded = false + /// A completion is free to flush again, which only works where it is not running on the queue deliveries go + /// through: from there the delivery it asks for would be queued behind the completion asking for it. Waited + /// on against a deadline, so a reporter answering on its own queue fails this rather than hanging the test + /// process on a deadlock it can never leave. + func flushedFromACompletion() -> Bool { + var flushed = false waitUntil(timeout: .seconds(5)) { done in eventReporter.flushReportingOutcome { _ in let finished = DispatchSemaphore(value: 0) - DispatchQueue.global().async { - eventReporter.record(CustomEvent(key: "recorded from a completion", context: LDContext.stub())) - finished.signal() - } - recorded = finished.wait(timeout: .now() + 1) == .success + eventReporter.flush { finished.signal() } + flushed = finished.wait(timeout: .now() + 1) == .success done() } } - return recorded + return flushed } /// Flushes and returns what the flush answered. @@ -1205,12 +1203,13 @@ extension EventReporterSpec { // the window in which a second delivery would find that batch and send it again. testContext.serviceMock.holdsEventCompletions = true - var firstFinished = false - testContext.eventReporter.flush { firstFinished = true } + // Answered from the delivery's own threads and polled from this one. + let first = Answer() + testContext.eventReporter.flush { first.record(true) } expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) - var secondFinished = false - testContext.eventReporter.flush { secondFinished = true } + let second = Answer() + testContext.eventReporter.flush { second.record(true) } // Nothing more goes out while the first delivery is unanswered. Thread.sleep(forTimeInterval: 0.3) expect(testContext.serviceMock.publishEventDataCallCount) == 1 @@ -1218,14 +1217,40 @@ extension EventReporterSpec { testContext.serviceMock.holdsEventCompletions = false testContext.serviceMock.releaseHeldEventCompletions() - // The second caller is answered by the pass that runs once the first delivery is done. It finds an - // empty store, so the batch is sent once in total. - expect(firstFinished).toEventually(beTrue()) - expect(secondFinished).toEventually(beTrue()) + // The second caller finds the only batch claimed by the first delivery, so it waits on that claim + // rather than sending anything. The batch goes out once in total. + expect(first.value()).toEventually(beTrue()) + expect(second.value()).toEventually(beTrue()) expect(testContext.serviceMock.publishEventDataCallCount) == 1 expect(testContext.store.pendingEventCount) == 0 } + it("answers an evaluating thread while a delivery is listing the store") { + testContext = TestContext() + // Standing in for the listing a delivery makes on a device whose disk is busy, which on the first one + // after launch reads every batch a previous run left behind. + let store = SlowListingStore(testContext.store) + defer { store.release.signal() } + let reporter = EventReporter(service: testContext.serviceMock, onSyncComplete: nil, store: store) + reporter.isOnline = true + + reporter.record(CustomEvent(key: "tracked", context: testContext.context)) + reporter.flush(completion: nil) + expect(store.listing.wait(timeout: .now() + 5)) == .success + + // Read on the evaluating thread for every flag with debug events turned on, so it must not end up + // behind whatever the delivery is holding while it lists. + let answered = DispatchSemaphore(value: 0) + DispatchQueue.global().async { + _ = reporter.lastEventResponseDate + answered.signal() + } + expect(answered.wait(timeout: .now() + 2)) == .success + + store.release.signal() + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) + } + it("writes a deferred track while a delivery holds the delivery queue") { testContext = TestContext(eventPersistence: .deferred) // Stands in for a post to a network that does not answer, which holds the queue for as long as its @@ -1602,10 +1627,11 @@ extension EventReporterSpec { } it("reports false and keeps the batch when it could not be read, and delivers it once it can be") { - var failing = false + // Set from this thread and read on the reporter's, which reads files from the moment it is built. + let failing = Answer() let store = EventStore.temporary(readFile: { url in // What a process out of file descriptors gets for a file that is perfectly intact. - guard !failing + guard failing.value() != true else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(EMFILE)) } return try Data(contentsOf: url) }) @@ -1613,7 +1639,7 @@ extension EventReporterSpec { testContext.recordEvents(1) testContext.eventReporter.isOnline = true - failing = true + failing.record(true) var delivered: Bool? waitUntil { done in testContext.eventReporter.flushReportingOutcome { result in @@ -1625,7 +1651,7 @@ extension EventReporterSpec { expect(testContext.serviceMock.publishEventDataCallCount) == 0 expect(store.pendingBatches().count) == 1 - failing = false + failing.record(false) waitUntil { done in testContext.eventReporter.flushReportingOutcome { result in delivered = result @@ -1768,7 +1794,7 @@ extension EventReporterSpec { testContext.serviceMock.releaseHeldEventCompletions() } - it("reports true to a caller with nothing of its own while a delivery is in flight") { + it("inherits the outcome of a delivery holding the batch it would have sent") { testContext = TestContext(stubResponseSuccess: false) testContext.recordEvents(1) testContext.eventReporter.isOnline = true @@ -1777,33 +1803,65 @@ extension EventReporterSpec { testContext.eventReporter.flushReportingOutcome { _ in } expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) - // Everything on disk belongs to the delivery already running, and this caller recorded nothing, so - // there is nothing of its own left unsent. It is not made to wait on, or inherit, a batch that is - // not its own. - var delivered: Bool? - testContext.eventReporter.flushReportingOutcome { result in delivered = result } - expect(delivered).toEventually(beTrue()) + // Everything on disk belongs to the delivery already running, and that may include this caller's + // events: a commit only stages them, and whichever delivery runs next is what closes them into a + // batch. So this caller is answered by that delivery rather than told its events arrived. + let answer = Answer() + testContext.eventReporter.flushReportingOutcome(completion: answer.record) + Thread.sleep(forTimeInterval: 0.3) + expect(answer.value()).to(beNil()) expect(testContext.serviceMock.publishEventDataCallCount) == 1 testContext.serviceMock.holdsEventCompletions = false testContext.serviceMock.releaseHeldEventCompletions() + // Allows for the one-second retry, after which the batch stays on disk and nobody was told it went. + expect(answer.value()).toEventually(beFalse(), timeout: .seconds(10)) } - it("answers away from the queue a recorded event waits on, after a delivery") { + it("tells the caller whose delivery the loss happened before, not whichever answers first") { testContext = TestContext() testContext.recordEvents(1) testContext.eventReporter.isOnline = true + testContext.serviceMock.holdsEventCompletions = true + + // A closes a batch of its own and waits on it. Nothing has been lost at the point it closed. + let first = Answer() + testContext.eventReporter.flushReportingOutcome(completion: first.record) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) + + // B closes a batch of its own, dropping an event the store will never take as it does, so the loss + // falls between the two closes. B has a post of its own outstanding, which is what leaves A to be + // answered first. + testContext.eventReporter.record(CustomEvent(key: "poison", context: LDContext.stub(), metricValue: .nan)) + testContext.recordEvents(1) + let second = Answer() + testContext.eventReporter.flushReportingOutcome(completion: second.record) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(2)) - expect(testContext.recordedFromACompletion()) == true + // Responses land in the order they were made, so A is answered while B's post is still out. With + // one shared flag A would take B's loss here, and B would report success for the event it dropped. + testContext.serviceMock.holdsEventCompletions = false + testContext.serviceMock.releaseHeldEventCompletions() + + expect(first.value()).toEventually(beTrue()) + expect(second.value()).toEventually(beFalse()) + } + + it("answers away from the queue a delivery runs on, after a delivery") { + testContext = TestContext() + testContext.recordEvents(1) + testContext.eventReporter.isOnline = true + + expect(testContext.flushedFromACompletion()) == true } - it("answers away from the queue a recorded event waits on, with nothing to send") { + it("answers away from the queue a delivery runs on, with nothing to send") { testContext = TestContext() testContext.eventReporter.isOnline = true // The path that answers without ever leaving the reporter's queue, so it is the one that would // deadlock while the delivery path, answering from the response, did not. - expect(testContext.recordedFromACompletion()) == true + expect(testContext.flushedFromACompletion()) == true } it("sends one delivery for the flushes that arrive before it starts, and answers them all") { @@ -1820,29 +1878,43 @@ extension EventReporterSpec { } expect(testContext.eventReporter.queuedDeliveryWaiterCount) == 0 - var delivered: [Bool] = [] - let answers = UnfairLock() // Both completions run at once, and the poll below reads from a third thread. - let answered: () -> [Bool] = { - answers.lock() - defer { answers.unlock() } - return delivered - } + let answers = Answers() for _ in 0..<2 { - testContext.eventReporter.flushReportingOutcome { result in - answers.lock() - delivered.append(result) - answers.unlock() - } + testContext.eventReporter.flushReportingOutcome(completion: answers.record) } expect(testContext.eventReporter.queuedDeliveryWaiterCount) == 2 released.signal() // Allows for the one-second retry, after which the batch stays on disk. - expect(answered()).toEventually(equal([false, false]), timeout: .seconds(10)) + expect(answers.value()).toEventually(equal([false, false]), timeout: .seconds(10)) expect(testContext.eventReporter.startedDeliveryCount) == 1 expect(testContext.serviceMock.publishEventDataCallCount) == 2 // the first attempt and its retry } + + it("answers every flush that arrived during a refused delivery with its outcome") { + testContext = TestContext(stubResponseSuccess: false) + testContext.recordEvents(1) + testContext.eventReporter.isOnline = true + testContext.serviceMock.holdsEventCompletions = true + + let answers = Answers() + testContext.eventReporter.flushReportingOutcome(completion: answers.record) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) + + // Held on the first post, each of these finds the only batch claimed by it. None of them is told + // its events arrived on the strength of having nothing left of its own to send. + for _ in 0..<20 { + testContext.eventReporter.flushReportingOutcome(completion: answers.record) + } + + testContext.serviceMock.holdsEventCompletions = false + testContext.serviceMock.releaseHeldEventCompletions() + + // Allows for the one-second retry the second delivery makes. + expect(answers.value().count).toEventually(equal(21), timeout: .seconds(10)) + expect(answers.value().contains(true)) == false + } } } @@ -2082,6 +2154,81 @@ private final class ClosingHookStore: EventStoring { } } +/// What a flush was told, written from whichever thread answered it and read from the test's. +private final class Answer { + private let lock = UnfairLock() + private var answered: Bool? + + /// Pass as a `FlushOutcomeClosure`. + func record(_ delivered: Bool) { + lock.withLock { answered = delivered } + } + + /// Nil until the flush has been answered. + func value() -> Bool? { + lock.withLock { answered } + } +} + +/// What a run of flushes were told, written from whichever threads answered them and read from the test's. +private final class Answers { + private let lock = UnfairLock() + private var answered: [Bool] = [] + + /// Pass as a `FlushOutcomeClosure`. + func record(_ delivered: Bool) { + lock.withLock { answered.append(delivered) } + } + + /// In the order the flushes were answered, which is not the order they were asked in. + func value() -> [Bool] { + lock.withLock { answered } + } +} + +/// A store that holds a listing open, so a test can see what else is waiting while one is under way. +private final class SlowListingStore: EventStoring { + let inner: EventStoring + /// Signalled once a listing is being held. + let listing = DispatchSemaphore(value: 0) + /// Signal to let it finish. + let release = DispatchSemaphore(value: 0) + + private let lock = UnfairLock() + private var listings = 0 + + init(_ inner: EventStoring) { + self.inner = inner + } + + var pendingEventCount: Int { inner.pendingEventCount } + var isPersisting: Bool { inner.isPersisting } + func stage(_ encodedEvent: Data, bypassingCapacity: Bool) -> Bool { inner.stage(encodedEvent, bypassingCapacity: bypassingCapacity) } + func reserve(_ events: Int) { inner.reserve(events) } + func stageReserved(_ encodedEvent: Data) -> Bool { inner.stageReserved(encodedEvent) } + func releaseReservations() { inner.releaseReservations() } + func commit() { inner.commit() } + func closeBatch() -> EventBatch? { inner.closeBatch() } + func body(of batch: EventBatch) throws -> Data? { try inner.body(of: batch) } + func remove(_ batch: EventBatch) { inner.remove(batch) } + func recoverInterruptedLog() { inner.recoverInterruptedLog() } + + /// Every listing but the first is held, the first being the one the reporter makes as it starts up. + func pendingBatches() -> [EventBatch] { + let held = lock.withLock { () -> Bool in + listings += 1 + return listings > 1 + } + if held { + listing.signal() + _ = release.wait(timeout: .now() + 5) + // Once opened the gate stays open. + release.signal() + } + return inner.pendingBatches() + } +} + /// Waits while letting the main run loop turn, since that is where the report timer fires. private func waitLettingTimersFire(_ seconds: TimeInterval) { RunLoop.current.run(until: Date().addingTimeInterval(seconds)) From 425aefbf8dee1a4b6abe0eeb3df778eada56166c Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Sun, 11 Oct 2026 17:29:35 -0700 Subject: [PATCH 020/158] simplify --- .../ServiceObjects/EventStore.swift | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index 9a2813cc8..3390e7c44 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -527,11 +527,10 @@ final class EventStore: EventStoring { // them: a delivery can still close them into a batch and send them. guard persistEvents, bufferedEventCount > 0 else { return nil } - defer { - bufferData = Data() - bufferedEventCount = 0 - } - return (bufferData, bufferedEventCount) + let staged = (bytes: bufferData, events: bufferedEventCount) + bufferData = Data() + bufferedEventCount = 0 + return staged } guard let taken else { return } @@ -603,11 +602,10 @@ private extension EventStore { /// where there is not. func closeInMemoryBatch() -> EventBatch? { let (frames, events) = bufferLock.withLock { () -> (Data, Int) in - defer { - bufferData = Data() - bufferedEventCount = 0 - } - return (bufferData, bufferedEventCount) + let staged = (bufferData, bufferedEventCount) + bufferData = Data() + bufferedEventCount = 0 + return staged } // Assembled through the same reader a batch on disk goes through, so a held batch and a written one produce From 12cc8f0d0b796eea577b4700e865a5be9ec02d73 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Sun, 11 Oct 2026 18:08:22 -0700 Subject: [PATCH 021/158] Take the frames a failed write left back off the log, so they are not sent twice A write that fails partway can leave whole frames in the log while the store keeps every byte of it in memory to deliver from there. The landed frames were then sent twice: beside the memory copy when the log was closed this session, or by recovery on the next launch. The log is now cut back to where the write started before persistence is given up on. Co-authored-by: Cursor --- .../ServiceObjects/EventLogFile.swift | 19 +++++++ .../ServiceObjects/OpenEventLog.swift | 17 ++++++- .../ServiceObjects/EventStoreSpec.swift | 51 +++++++++++++++++++ 3 files changed, 85 insertions(+), 2 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFile.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFile.swift index 67c0625cf..5d6acd7b9 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFile.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFile.swift @@ -50,6 +50,17 @@ enum EventLogFile { lseek(descriptor, 0, SEEK_END) == 0 } + /// How long the file is, or nil where that could not be found. + static func size(_ descriptor: Int32) -> off_t? { + let end = lseek(descriptor, 0, SEEK_END) + return end >= 0 ? end : nil + } + + /// Cuts the file back to `length`. False where it could not, with `errno` saying why. + static func truncate(_ descriptor: Int32, to length: off_t) -> Bool { + truncateFile(descriptor, length) == 0 + } + static func closeDescriptor(_ descriptor: Int32) { closeFile(descriptor) } @@ -98,6 +109,10 @@ private func writeBytes(_ descriptor: Int32, _ bytes: UnsafeRawPointer, _ count: Darwin.write(descriptor, bytes, count) } +private func truncateFile(_ descriptor: Int32, _ length: off_t) -> Int32 { + Darwin.ftruncate(descriptor, length) +} + private func closeFile(_ descriptor: Int32) { _ = Darwin.close(descriptor) } @@ -110,6 +125,10 @@ private func writeBytes(_ descriptor: Int32, _ bytes: UnsafeRawPointer, _ count: write(descriptor, bytes, count) } +private func truncateFile(_ descriptor: Int32, _ length: off_t) -> Int32 { + ftruncate(descriptor, length) +} + private func closeFile(_ descriptor: Int32) { _ = close(descriptor) } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/OpenEventLog.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/OpenEventLog.swift index 00a1ab663..27dfc8e7b 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/OpenEventLog.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/OpenEventLog.swift @@ -90,14 +90,27 @@ final class OpenEventLog { return FileManager.default.fileExists(atPath: url.path) } - /// Appends to the log, opening it first where it is not open. False where it could not be opened or written to. + /// Appends to the log, opening it first where it is not open. False where it could not be opened or written to, in + /// which case none of `bytes` is left in the log. /// /// Requires `ioLock`. func append(_ bytes: Data) -> Bool { ioLock.assertOwned() guard let descriptor = descriptorForAppending() else { return false } - return write(bytes, to: descriptor) + + let start = EventLogFile.size(descriptor) + guard write(bytes, to: descriptor) + else { + // A write that fails partway can leave whole frames behind. The caller keeps every one of `bytes` in memory + // to deliver from there, so frames left here as well would be sent twice: beside the memory copy when the + // log is closed this session, or by recovery on the next launch. + if let start, !EventLogFile.truncate(descriptor, to: start) { + os_log("%s could not cut back a failed write: errno %d", log: logger, type: .debug, typeName(and: #function), errno) + } + return false + } + return true } /// Moves the log to `destination` and closes it, so that the next append starts a new one. The descriptor is closed diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 4cf2b2df1..8f93691e9 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -756,6 +756,57 @@ final class EventStoreSpec: QuickSpec { expect(interrupted.pendingEventCount) == 2 } + it("sends the frames a write landed before it failed once, not from the log and memory both") { + var writes = 0 + let interrupted = EventStore.temporary(writeLog: { descriptor, bytes in + writes += 1 + // The file header and the first event land. The next write gets a whole frame and part of another + // onto the disk before the device is full. + guard writes > 2 + else { return EventLogFile.writeAll(descriptor, bytes) } + _ = EventLogFile.writeAll(descriptor, bytes.dropLast(3)) + return ENOSPC + }) + defer { interrupted.deleteEverything() } + + _ = interrupted.stage(EventStoreSpec.payload("written")) + interrupted.commit() + _ = interrupted.stage(EventStoreSpec.payload("landed")) + _ = interrupted.stage(EventStoreSpec.payload("torn")) + interrupted.commit() + expect(interrupted.isPersisting) == false + + _ = interrupted.closeBatch() + + let bodies = interrupted.pendingBatches().map { EventStoreSpec.keys(ofBody: try? interrupted.body(of: $0)) } + expect(bodies) == [["written"], ["landed", "torn"]] + expect(interrupted.pendingEventCount) == 3 + } + + it("leaves the next run nothing to recover from a first write that failed partway") { + var writes = 0 + let interrupted = EventStore.temporary(writeLog: { descriptor, bytes in + writes += 1 + // The file header lands, and then a whole frame and part of another before the device is full. + guard writes > 1 + else { return EventLogFile.writeAll(descriptor, bytes) } + _ = EventLogFile.writeAll(descriptor, bytes.dropLast(3)) + return ENOSPC + }) + defer { interrupted.deleteEverything() } + + _ = interrupted.stage(EventStoreSpec.payload("landed")) + _ = interrupted.stage(EventStoreSpec.payload("torn")) + interrupted.commit() + let fromMemory = interrupted.closeBatch() + expect(EventStoreSpec.keys(ofBody: fromMemory.flatMap { try? interrupted.body(of: $0) })) == ["landed", "torn"] + + // Delivered from memory this run, so the next must not find any of it again in the log. + let next = EventStoreSpec.reader(sharing: interrupted) + next.recoverInterruptedLog() + expect(next.pendingBatches()).to(beEmpty()) + } + it("holds a batch until it is removed, so a failed delivery can be retried") { _ = store.stage(EventStoreSpec.payload("retry-me")) let batch = store.closeBatch()! From a65b8fe92d46f4c388f1ec92d74a1e2a4429d863 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Sun, 11 Oct 2026 18:27:53 -0700 Subject: [PATCH 022/158] Rename OpenEventLog to DiskEventLog, and make EventLogFile the open file it writes through DiskEventLog reads apart from diagnostic logging; the store's property becomes diskLog. File names on disk are unchanged, so a log left by an earlier version is still recovered. EventLogFile was a namespace of functions over a raw descriptor, with -1 standing for closed. It is now a value owning the descriptor, with writing, sizing, truncating and closing as its methods, and DiskEventLog holds it as an optional. The write hook tests use takes the file rather than a descriptor. Co-authored-by: Cursor --- LaunchDarkly.xcodeproj/project.pbxproj | 20 ++--- ...{OpenEventLog.swift => DiskEventLog.swift} | 61 +++++++------ .../ServiceObjects/EventLogFile.swift | 86 ++++++++++--------- .../ServiceObjects/EventStore.swift | 30 +++---- .../ServiceObjects/EventReporterSpec.swift | 8 +- .../ServiceObjects/EventStoreSpec.swift | 22 ++--- LaunchDarkly/LaunchDarklyTests/TestUtil.swift | 2 +- 7 files changed, 116 insertions(+), 113 deletions(-) rename LaunchDarkly/LaunchDarkly/ServiceObjects/{OpenEventLog.swift => DiskEventLog.swift} (87%) diff --git a/LaunchDarkly.xcodeproj/project.pbxproj b/LaunchDarkly.xcodeproj/project.pbxproj index ad6d58112..f994bbcf0 100644 --- a/LaunchDarkly.xcodeproj/project.pbxproj +++ b/LaunchDarkly.xcodeproj/project.pbxproj @@ -17,7 +17,7 @@ 29FE129A280413D4008CC918 /* Util.swift in Sources */ = {isa = PBXBuildFile; fileRef = 29FE1297280413D4008CC918 /* Util.swift */; }; 29FE129B280413D4008CC918 /* Util.swift in Sources */ = {isa = PBXBuildFile; fileRef = 29FE1297280413D4008CC918 /* Util.swift */; }; 2CA9D36B3566D78850D6F0AF /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; - B8F228923BE11E444D1ECB8E /* OpenEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* OpenEventLog.swift */; }; + B8F228923BE11E444D1ECB8E /* DiskEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */; }; 3C55384C9E3485B493E1F86A /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; C3F7E502BD916C40AE238F64 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; AB4827D11084D58661E24B05 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; @@ -53,7 +53,7 @@ 50EE85C52EA0487F007CC662 /* TimeoutExecutor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50EE85C12EA0487F007CC662 /* TimeoutExecutor.swift */; }; 50EE85C72EA0749C007CC662 /* TimeoutExecutorSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50EE85C62EA0749C007CC662 /* TimeoutExecutorSpec.swift */; }; 5AEA1F832DCA974314E07D21 /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; - D4225F1F05F2C55E9518941B /* OpenEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* OpenEventLog.swift */; }; + D4225F1F05F2C55E9518941B /* DiskEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */; }; DDA788E3056F8AC3CA203933 /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; D4080613CEA27D51BF349075 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; ECBE45806B470ACC5187B7D6 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; @@ -386,7 +386,7 @@ AD5F10417C8E3269D4F701B3 /* JSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6F2A8C1D4E7B9053A1C4D8E6 /* JSONWriter.swift */; }; AFCE752CB1A75DEBABC23034 /* LDContextJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88DAB8D1864C14D9C7E06F11 /* LDContextJSONWriter.swift */; }; B0B1AD7CADE8DBB596C2DE9B /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; - DAA2C9B9C144F129217A7C55 /* OpenEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* OpenEventLog.swift */; }; + DAA2C9B9C144F129217A7C55 /* DiskEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */; }; 44E0BE449714952A68628CEB /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; E5191724DFB38E62C0451086 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; CA2ADB3D20A229D70CA34F73 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; @@ -441,7 +441,7 @@ C443A41223186A4F00145710 /* ConnectionModeChangeObserver.swift in Sources */ = {isa = PBXBuildFile; fileRef = C443A40E23186A4F00145710 /* ConnectionModeChangeObserver.swift */; }; C4AF311E4DA8C3AB7040637D /* EventJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = B430884DBF2E541B919C08FA /* EventJSONWriter.swift */; }; CF9A1EB116D8AA3B7B5535B2 /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; - E2F8080AB2B3CD6E9A9C1FCC /* OpenEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* OpenEventLog.swift */; }; + E2F8080AB2B3CD6E9A9C1FCC /* DiskEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */; }; F0F504E5EC32223905356098 /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; B2E6D4F1AC805B3F9D127E53 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; 625ADAC0CCFE3BE324C45752 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; @@ -670,7 +670,7 @@ B4C9D4372489E20A004A9B03 /* DiagnosticReporter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DiagnosticReporter.swift; sourceTree = ""; }; B4F689132497B2FC004D3CE0 /* DiagnosticEventSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DiagnosticEventSpec.swift; sourceTree = ""; }; C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventStore.swift; sourceTree = ""; }; - E21BC7A0E0F8D272785D7985 /* OpenEventLog.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = OpenEventLog.swift; sourceTree = ""; }; + E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = DiskEventLog.swift; sourceTree = ""; }; 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventLogFile.swift; sourceTree = ""; }; A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = DeliveryClaims.swift; sourceTree = ""; }; B58551BB165F28784C785720 /* EventLogFormat.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventLogFormat.swift; sourceTree = ""; }; @@ -1059,7 +1059,7 @@ D93B7F5F6CF5C3DE79FB4286 /* ContextEncodingCache.swift */, 9A771B316E19ED598CD4FB0B /* BackgroundActivity.swift */, C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */, - E21BC7A0E0F8D272785D7985 /* OpenEventLog.swift */, + E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */, 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */, A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */, B58551BB165F28784C785720 /* EventLogFormat.swift */, @@ -1590,7 +1590,7 @@ 40F4D07AD934AF976687011D /* ContextEncodingCache.swift in Sources */, A32FCDB22D53263851DDB727 /* BackgroundActivity.swift in Sources */, CF9A1EB116D8AA3B7B5535B2 /* EventStore.swift in Sources */, - E2F8080AB2B3CD6E9A9C1FCC /* OpenEventLog.swift in Sources */, + E2F8080AB2B3CD6E9A9C1FCC /* DiskEventLog.swift in Sources */, F0F504E5EC32223905356098 /* EventLogFile.swift in Sources */, B2E6D4F1AC805B3F9D127E53 /* DeliveryClaims.swift in Sources */, 625ADAC0CCFE3BE324C45752 /* EventLogFormat.swift in Sources */, @@ -1689,7 +1689,7 @@ 4882B223C23B63F30BC60941 /* ContextEncodingCache.swift in Sources */, 7185A3C30E7FFC4F725769F1 /* BackgroundActivity.swift in Sources */, 2CA9D36B3566D78850D6F0AF /* EventStore.swift in Sources */, - B8F228923BE11E444D1ECB8E /* OpenEventLog.swift in Sources */, + B8F228923BE11E444D1ECB8E /* DiskEventLog.swift in Sources */, 3C55384C9E3485B493E1F86A /* EventLogFile.swift in Sources */, C3F7E502BD916C40AE238F64 /* DeliveryClaims.swift in Sources */, AB4827D11084D58661E24B05 /* EventLogFormat.swift in Sources */, @@ -1787,7 +1787,7 @@ D7A500FCA04E3412F931003F /* ContextEncodingCache.swift in Sources */, A20146E7AF098560627C614E /* BackgroundActivity.swift in Sources */, 5AEA1F832DCA974314E07D21 /* EventStore.swift in Sources */, - D4225F1F05F2C55E9518941B /* OpenEventLog.swift in Sources */, + D4225F1F05F2C55E9518941B /* DiskEventLog.swift in Sources */, DDA788E3056F8AC3CA203933 /* EventLogFile.swift in Sources */, D4080613CEA27D51BF349075 /* DeliveryClaims.swift in Sources */, ECBE45806B470ACC5187B7D6 /* EventLogFormat.swift in Sources */, @@ -1952,7 +1952,7 @@ BAE0B4F45562EBF4AC786B5D /* ContextEncodingCache.swift in Sources */, 3D34EEEF37158657D49335BB /* BackgroundActivity.swift in Sources */, B0B1AD7CADE8DBB596C2DE9B /* EventStore.swift in Sources */, - DAA2C9B9C144F129217A7C55 /* OpenEventLog.swift in Sources */, + DAA2C9B9C144F129217A7C55 /* DiskEventLog.swift in Sources */, 44E0BE449714952A68628CEB /* EventLogFile.swift in Sources */, E5191724DFB38E62C0451086 /* DeliveryClaims.swift in Sources */, CA2ADB3D20A229D70CA34F73 /* EventLogFormat.swift in Sources */, diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/OpenEventLog.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift similarity index 87% rename from LaunchDarkly/LaunchDarkly/ServiceObjects/OpenEventLog.swift rename to LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift index 27dfc8e7b..60124b2f1 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/OpenEventLog.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift @@ -9,7 +9,7 @@ import Glibc import Musl #endif -/// The log this process appends to. +/// The file of events on disk this process appends to, not to be confused with diagnostic logging. /// /// One per process rather than one per directory, so that only one process ever writes a given log and recovery only /// ever touches its own. With one shared log, a second process starting up would close off, as if a previous run had @@ -18,7 +18,7 @@ import Musl /// /// Belongs to an `EventStore`, which makes every call holding its `ioLock`. The lock is handed in so that each call can /// check it is held, rather than this taking a lock of its own that would have to be ordered against the store's. -final class OpenEventLog { +final class DiskEventLog { /// Names a log a single process appends to; the rest of the name is that process. static let prefix = "open-" @@ -27,12 +27,13 @@ final class OpenEventLog { private let logger: OSLog /// How bytes reach the log, injectable so a test can make a write fail partway through a session, as a full disk /// does. Answers with the errno it stopped on, or nil where every byte landed. - private let writeLog: (Int32, Data) -> Int32? + private let writeLog: (EventLogFile, Data) -> Int32? /// Named for the process, unless on a Mac `isLocked` found another live instance holding that name, in which case /// it is named for this instance's process ID as well. private(set) var url: URL - private var descriptor: Int32 = -1 + /// Nil until the first append opens it, and again once it is closed. + private var file: EventLogFile? #if os(macOS) || targetEnvironment(macCatalyst) /// The readable, collision-proof form of the process name every log this process opens is named after. @@ -46,13 +47,13 @@ final class OpenEventLog { var isLocked = true #endif - init(directory: URL, processName: String, ioLock: UnfairLock, logger: OSLog, writeLog: @escaping (Int32, Data) -> Int32?) { + init(directory: URL, processName: String, ioLock: UnfairLock, logger: OSLog, writeLog: @escaping (EventLogFile, Data) -> Int32?) { self.directory = directory self.ioLock = ioLock self.logger = logger self.writeLog = writeLog - let processLogName = OpenEventLog.name(for: processName) - self.url = directory.appendingPathComponent(OpenEventLog.prefix + processLogName) + let processLogName = DiskEventLog.name(for: processName) + self.url = directory.appendingPathComponent(DiskEventLog.prefix + processLogName) #if os(macOS) || targetEnvironment(macCatalyst) self.processLogName = processLogName #endif @@ -81,7 +82,7 @@ final class OpenEventLog { /// Requires `ioLock`. var isOpen: Bool { ioLock.assertOwned() - return descriptor >= 0 + return file != nil } /// Requires `ioLock`. @@ -96,16 +97,16 @@ final class OpenEventLog { /// Requires `ioLock`. func append(_ bytes: Data) -> Bool { ioLock.assertOwned() - guard let descriptor = descriptorForAppending() + guard let file = fileForAppending() else { return false } - let start = EventLogFile.size(descriptor) - guard write(bytes, to: descriptor) + let start = file.size + guard write(bytes, to: file) else { // A write that fails partway can leave whole frames behind. The caller keeps every one of `bytes` in memory // to deliver from there, so frames left here as well would be sent twice: beside the memory copy when the // log is closed this session, or by recovery on the next launch. - if let start, !EventLogFile.truncate(descriptor, to: start) { + if let start, !file.truncate(to: start) { os_log("%s could not cut back a failed write: errno %d", log: logger, type: .debug, typeName(and: #function), errno) } return false @@ -128,10 +129,8 @@ final class OpenEventLog { /// Closes the descriptor, leaving the log where it is. Requires `ioLock`. func closeDescriptor() { ioLock.assertOwned() - guard descriptor >= 0 - else { return } - EventLogFile.closeDescriptor(descriptor) - descriptor = -1 + file?.close() + file = nil } /// Hands `recover` each log left open by a process that is gone, which it has to move or remove. @@ -152,10 +151,10 @@ final class OpenEventLog { } /// Requires `ioLock`. - private func descriptorForAppending() -> Int32? { + private func fileForAppending() -> EventLogFile? { ioLock.assertOwned() - if descriptor >= 0 { - return descriptor + if let file { + return file } do { @@ -172,25 +171,25 @@ final class OpenEventLog { return nil } - descriptor = opened + file = opened - if EventLogFile.isEmpty(opened) { + if opened.isEmpty { guard write(EventLogFormat.fileHeader, to: opened) else { return nil } } - return descriptor + return opened } - private func write(_ bytes: Data, to descriptor: Int32) -> Bool { - guard let failure = writeLog(descriptor, bytes) + private func write(_ bytes: Data, to file: EventLogFile) -> Bool { + guard let failure = writeLog(file, bytes) else { return true } os_log("%s could not write to the event log: errno %d", log: logger, type: .debug, typeName(and: #function), failure) return false } /// Opens the log, locked where logs are. Requires `ioLock`. - private func open() -> Int32? { + private func open() -> EventLogFile? { ioLock.assertOwned() // O_APPEND is what makes each write land at the end of the file as one step, so that clients for several // environments writing their own logs, or a commit racing a reader, cannot produce a spliced frame. @@ -200,13 +199,13 @@ final class OpenEventLog { return openLocked(flags: flags) } #endif - return EventLogFile.openDescriptor(at: url, flags: flags) + return EventLogFile.open(at: url, flags: flags) } #if os(macOS) || targetEnvironment(macCatalyst) /// Opens the log for this process name locked, or one of this instance's own where another live instance holds /// it. Requires `ioLock`. - private func openLocked(flags: Int32) -> Int32? { + private func openLocked(flags: Int32) -> EventLogFile? { ioLock.assertOwned() if let opened = EventLogFile.openLocked(url, flags: flags) { return opened @@ -217,7 +216,7 @@ final class OpenEventLog { // one, as a closed client's can be while it is still going away. Recovery finds the log by its lock rather than // its name, so it is not orphaned when this instance ends. let ownLog = directory.appendingPathComponent( - "\(OpenEventLog.prefix)\(processLogName)-\(getpid())-\(UUID().uuidString.prefix(8))") + "\(DiskEventLog.prefix)\(processLogName)-\(getpid())-\(UUID().uuidString.prefix(8))") guard let opened = EventLogFile.openLocked(ownLog, flags: flags) else { return nil } url = ownLog @@ -233,14 +232,14 @@ final class OpenEventLog { let contents = (try? FileManager.default.contentsOfDirectory(at: directory, includingPropertiesForKeys: nil, options: [.skipsHiddenFiles])) ?? [] - for log in contents where log.lastPathComponent.hasPrefix(OpenEventLog.prefix) { + for log in contents where log.lastPathComponent.hasPrefix(DiskEventLog.prefix) { guard let held = EventLogFile.openLocked(log, flags: O_RDONLY) else { continue } recover(log) - EventLogFile.closeDescriptor(held) + held.close() } } #endif } -extension OpenEventLog: TypeIdentifying { } +extension DiskEventLog: TypeIdentifying { } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFile.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFile.swift index 5d6acd7b9..87e49acfb 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFile.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFile.swift @@ -8,24 +8,60 @@ import Glibc import Musl #endif -/// The file operations an event log is made of, on descriptors rather than Foundation's file handles. +/// An open file an event log is written through, on a descriptor rather than Foundation's file handles. /// /// Foundation's FileHandle is deliberately not used: it raises an Objective-C exception when a write fails, which is not /// something a Swift caller can catch, and has taken host applications down with it when a device ran out of space. -enum EventLogFile { +/// +/// A value, so a copy names the same open file. Whoever opened it closes it once, and uses no copy after that. +struct EventLogFile { + let descriptor: Int32 + /// Opens a file, creating it readable and writable by its owner alone. Nil where it could not, with `errno` saying /// why. - static func openDescriptor(at url: URL, flags: Int32) -> Int32? { + static func open(at url: URL, flags: Int32) -> EventLogFile? { let opened = url.withUnsafeFileSystemRepresentation { path -> Int32 in guard let path = path else { return -1 } return openFile(path, flags, 0o600) } - return opened >= 0 ? opened : nil + return opened >= 0 ? EventLogFile(descriptor: opened) : nil + } + + #if os(macOS) || targetEnvironment(macCatalyst) + /// Opens a file and takes its lock without waiting, or nil where another live process holds it. + /// + /// The lock has to be on the file that is still at `url`: one renamed or removed between the open and the lock, + /// by another process closing off or recovering it, is no longer a log anyone appends to. + static func openLocked(_ url: URL, flags: Int32) -> EventLogFile? { + for _ in 0..<3 { + guard let opened = open(at: url, flags: flags) + else { return nil } + guard lockFile(opened.descriptor) + else { + opened.close() + return nil + } + if isFile(opened.descriptor, at: url) { + return opened + } + opened.close() + } + return nil + } + #endif + + static func excludeFromBackup(_ directory: URL) { + #if canImport(Darwin) + var url = directory + var values = URLResourceValues() + values.isExcludedFromBackup = true + try? url.setResourceValues(values) + #endif } /// Writes every byte, returning the errno it stopped on or nil where they all landed. - static func writeAll(_ descriptor: Int32, _ bytes: Data) -> Int32? { + func writeAll(_ bytes: Data) -> Int32? { var offset = 0 return bytes.withUnsafeBytes { raw -> Int32? in guard let base = raw.baseAddress @@ -46,56 +82,24 @@ enum EventLogFile { } } - static func isEmpty(_ descriptor: Int32) -> Bool { + var isEmpty: Bool { lseek(descriptor, 0, SEEK_END) == 0 } /// How long the file is, or nil where that could not be found. - static func size(_ descriptor: Int32) -> off_t? { + var size: off_t? { let end = lseek(descriptor, 0, SEEK_END) return end >= 0 ? end : nil } /// Cuts the file back to `length`. False where it could not, with `errno` saying why. - static func truncate(_ descriptor: Int32, to length: off_t) -> Bool { + func truncate(to length: off_t) -> Bool { truncateFile(descriptor, length) == 0 } - static func closeDescriptor(_ descriptor: Int32) { + func close() { closeFile(descriptor) } - - static func excludeFromBackup(_ directory: URL) { - #if canImport(Darwin) - var url = directory - var values = URLResourceValues() - values.isExcludedFromBackup = true - try? url.setResourceValues(values) - #endif - } - - #if os(macOS) || targetEnvironment(macCatalyst) - /// Opens a file and takes its lock without waiting, or nil where another live process holds it. - /// - /// The lock has to be on the file that is still at `url`: one renamed or removed between the open and the lock, - /// by another process closing off or recovering it, is no longer a log anyone appends to. - static func openLocked(_ url: URL, flags: Int32) -> Int32? { - for _ in 0..<3 { - guard let opened = openDescriptor(at: url, flags: flags) - else { return nil } - guard lockFile(opened) - else { - closeFile(opened) - return nil - } - if isFile(opened, at: url) { - return opened - } - closeFile(opened) - } - return nil - } - #endif } // The POSIX calls are wrapped so that the names cannot be shadowed by a member of the type using them, and so the diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index 3390e7c44..adf388644 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -126,7 +126,7 @@ final class EventStore: EventStoring { private var isCommitScheduled = false /// Only to be used while holding `ioLock`. - private let openLog: OpenEventLog + private let diskLog: DiskEventLog /// Whether a log left by a previous run has been dealt with. Only to be used while holding `ioLock`. private var hasRecoveredInterruptedLog = false @@ -158,14 +158,14 @@ final class EventStore: EventStoring { /// Where the log this process appends to is, for a test or a diagnosis to look at. var openLogUrl: URL { - ioLock.withLock { openLog.url } + ioLock.withLock { diskLog.url } } #if os(macOS) || targetEnvironment(macCatalyst) /// Whether the open log is locked while it is open. Only to be changed before the store first touches the disk. var locksOpenLog: Bool { - get { ioLock.withLock { openLog.isLocked } } - set { ioLock.withLock { openLog.isLocked = newValue } } + get { ioLock.withLock { diskLog.isLocked } } + set { ioLock.withLock { diskLog.isLocked = newValue } } } #endif @@ -182,11 +182,11 @@ final class EventStore: EventStoring { logger: OSLog, commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.EventStore.commitQueue", qos: .userInitiated), readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }, - writeLog: @escaping (Int32, Data) -> Int32? = EventLogFile.writeAll) { + writeLog: @escaping (EventLogFile, Data) -> Int32? = { $0.writeAll($1) }) { self.directory = directory let ioLock = UnfairLock() self.ioLock = ioLock - self.openLog = OpenEventLog(directory: directory, processName: processName, ioLock: ioLock, logger: logger, writeLog: writeLog) + self.diskLog = DiskEventLog(directory: directory, processName: processName, ioLock: ioLock, logger: logger, writeLog: writeLog) self.capacity = capacity self.logger = logger self.commitQueue = commitQueue @@ -306,13 +306,13 @@ final class EventStore: EventStoring { ioLock.assertOwned() let payloadId = UUID().uuidString do { - try openLog.close(movingTo: url(of: payloadId)) + try diskLog.close(movingTo: url(of: payloadId)) } catch { os_log("%s could not close the event log: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) // The log is still there to try again with, unless it is the log that went missing -- a purged cache // directory on tvOS, say. Then its events are gone, and they have to stop counting against capacity or the // store would refuse events for the rest of the session. - if !openLog.exists { + if !diskLog.exists { bufferLock.withLock { committedEvents = 0 } } return nil @@ -476,7 +476,7 @@ final class EventStore: EventStoring { guard readsDisk, !hasRecoveredInterruptedLog else { return } hasRecoveredInterruptedLog = true - openLog.forEachLogLeftOpen { recover(logAt: $0) } + diskLog.forEachLogLeftOpen { recover(logAt: $0) } } /// Closes off a log nobody is writing to any more. Requires `ioLock`. @@ -552,10 +552,10 @@ final class EventStore: EventStoring { ioLock.assertOwned() // Before the log is opened, because it may still be the one a previous run died writing. Appended to, this // run's events would sit behind that run's torn last frame, where a reader can no longer find where they start. - if !openLog.isOpen { + if !diskLog.isOpen { recoverInterruptedLogOnce() } - guard openLog.append(bytes) + guard diskLog.append(bytes) else { // Never trap on a full disk. The SDK gives up on persistence for the rest of the session rather than taking // the application down with it, which is how other SDKs have crashed their hosts. @@ -569,7 +569,7 @@ final class EventStore: EventStoring { // One acquisition: the lock is not recursive, and `commit()` would take it again. ioLock.withLock { writeStagedBytes() - openLog.closeDescriptor() + diskLog.closeDescriptor() } } } @@ -592,7 +592,7 @@ private extension EventStore { /// Requires `ioLock`. func disablePersistence() { ioLock.assertOwned() - openLog.closeDescriptor() + diskLog.closeDescriptor() bufferLock.withLock { persistEvents = false } } @@ -642,8 +642,8 @@ extension EventStore { includingPropertiesForKeys: nil, options: [.skipsHiddenFiles])) ?? [] var logs = contents.filter { $0.lastPathComponent.hasPrefix(EventStore.batchPrefix) }.sorted { $0.path < $1.path } - if openLog.exists { - logs.append(openLog.url) + if diskLog.exists { + logs.append(diskLog.url) } var payloads: [Data] = [] diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift index eb65fb4f2..6512c9db5 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift @@ -1284,12 +1284,12 @@ extension EventReporterSpec { let writesLock = NSLock() var writes = 0 // A slow disk, so that the other tracks arrive while a write is under way. - let store = EventStore.temporary(writeLog: { descriptor, bytes in + let store = EventStore.temporary(writeLog: { file, bytes in writesLock.lock() writes += 1 writesLock.unlock() Thread.sleep(forTimeInterval: 0.05) - return EventLogFile.writeAll(descriptor, bytes) + return file.writeAll(bytes) }) testContext = TestContext(store: store, eventCapacity: 100) let reporter = testContext.eventReporter! @@ -1665,10 +1665,10 @@ extension EventReporterSpec { it("delivers what memory was left holding beside what the disk took, and reports true") { var writes = 0 - let store = EventStore.temporary(writeLog: { descriptor, bytes in + let store = EventStore.temporary(writeLog: { file, bytes in writes += 1 // The file header and the first event land; the device is full from then on. - return writes > 2 ? ENOSPC : EventLogFile.writeAll(descriptor, bytes) + return writes > 2 ? ENOSPC : file.writeAll(bytes) }) testContext = TestContext(store: store) testContext.eventReporter.record(CustomEvent(key: "written", context: LDContext.stub())) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 8f93691e9..1cd5891de 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -161,12 +161,12 @@ final class EventStoreSpec: QuickSpec { } it("names logs so that process names reducing to the same characters stay apart") { - let colon = OpenEventLog.name(for: "com.app:remote") - let dot = OpenEventLog.name(for: "com.app.remote") + let colon = DiskEventLog.name(for: "com.app:remote") + let dot = DiskEventLog.name(for: "com.app.remote") expect(colon) != dot expect(colon.hasPrefix("com_app_remote-")) == true expect(dot.hasPrefix("com_app_remote-")) == true - expect(OpenEventLog.name(for: " ").hasPrefix("unknown-")) == true + expect(DiskEventLog.name(for: " ").hasPrefix("unknown-")) == true } } } @@ -734,10 +734,10 @@ final class EventStoreSpec: QuickSpec { it("delivers what memory was left holding beside what the disk already took") { var writes = 0 - let interrupted = EventStore.temporary(writeLog: { descriptor, bytes in + let interrupted = EventStore.temporary(writeLog: { file, bytes in writes += 1 // The file header and the first event land; the device is full from then on. - return writes > 2 ? ENOSPC : EventLogFile.writeAll(descriptor, bytes) + return writes > 2 ? ENOSPC : file.writeAll(bytes) }) defer { interrupted.deleteEverything() } @@ -758,13 +758,13 @@ final class EventStoreSpec: QuickSpec { it("sends the frames a write landed before it failed once, not from the log and memory both") { var writes = 0 - let interrupted = EventStore.temporary(writeLog: { descriptor, bytes in + let interrupted = EventStore.temporary(writeLog: { file, bytes in writes += 1 // The file header and the first event land. The next write gets a whole frame and part of another // onto the disk before the device is full. guard writes > 2 - else { return EventLogFile.writeAll(descriptor, bytes) } - _ = EventLogFile.writeAll(descriptor, bytes.dropLast(3)) + else { return file.writeAll(bytes) } + _ = file.writeAll(bytes.dropLast(3)) return ENOSPC }) defer { interrupted.deleteEverything() } @@ -785,12 +785,12 @@ final class EventStoreSpec: QuickSpec { it("leaves the next run nothing to recover from a first write that failed partway") { var writes = 0 - let interrupted = EventStore.temporary(writeLog: { descriptor, bytes in + let interrupted = EventStore.temporary(writeLog: { file, bytes in writes += 1 // The file header lands, and then a whole frame and part of another before the device is full. guard writes > 1 - else { return EventLogFile.writeAll(descriptor, bytes) } - _ = EventLogFile.writeAll(descriptor, bytes.dropLast(3)) + else { return file.writeAll(bytes) } + _ = file.writeAll(bytes.dropLast(3)) return ENOSPC }) defer { interrupted.deleteEverything() } diff --git a/LaunchDarkly/LaunchDarklyTests/TestUtil.swift b/LaunchDarkly/LaunchDarklyTests/TestUtil.swift index 55123f70f..51703dd6a 100644 --- a/LaunchDarkly/LaunchDarklyTests/TestUtil.swift +++ b/LaunchDarkly/LaunchDarklyTests/TestUtil.swift @@ -59,7 +59,7 @@ extension EventStore { persistEvents: Bool = true, commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.tests.commitQueue"), readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }, - writeLog: @escaping (Int32, Data) -> Int32? = EventLogFile.writeAll + writeLog: @escaping (EventLogFile, Data) -> Int32? = { $0.writeAll($1) } ) -> EventStore { let directory = FileManager.default.temporaryDirectory .appendingPathComponent("com.launchdarkly.tests.events", isDirectory: true) From ea815e25dca5fda84b1fafa58acd143eabe31289 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Sun, 11 Oct 2026 19:36:56 -0700 Subject: [PATCH 023/158] Count an evaluation through the default subscript, so a context is looked up once Co-authored-by: Cursor --- .../ServiceObjects/ContextSummarizer.swift | 30 ++++++------------- 1 file changed, 9 insertions(+), 21 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/ContextSummarizer.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/ContextSummarizer.swift index 08afa1bd1..53f42e230 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/ContextSummarizer.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/ContextSummarizer.swift @@ -70,18 +70,7 @@ class ContextSummarizer { @discardableResult func trackRequest(flagKey: LDFlagKey, reportedValue: LDValue, featureFlag: FeatureFlag?, defaultValue: LDValue, context: LDContext) -> Bool { let key = key(for: context) - if let index = trackers.index(forKey: key) { - trackers.values[index].tracker.trackRequest( - flagKey: flagKey, - reportedValue: reportedValue, - featureFlag: featureFlag, - defaultValue: defaultValue, - context: context - ) - return true - } - - guard trackers.count < maxContexts + guard trackers.count < maxContexts || trackers[key] != nil else { if !hasLoggedContextsExceeded { hasLoggedContextsExceeded = true @@ -90,16 +79,15 @@ class ContextSummarizer { return false } - var tracker = FlagRequestTracker(logger: logger) - tracker.trackRequest( - flagKey: flagKey, - reportedValue: reportedValue, - featureFlag: featureFlag, - defaultValue: defaultValue, - context: context - ) // Redaction is applied when the summary is encoded; see `Event.redactsAnonymousAttributes`. - trackers[key] = TrackerWithContext(tracker: tracker, context: context) + trackers[key, default: TrackerWithContext(tracker: FlagRequestTracker(logger: logger), context: context)] + .tracker.trackRequest( + flagKey: flagKey, + reportedValue: reportedValue, + featureFlag: featureFlag, + defaultValue: defaultValue, + context: context + ) return true } From 35ae12e0ca2aaa9f10f06f22507d9929b6e98958 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Sun, 11 Oct 2026 19:41:41 -0700 Subject: [PATCH 024/158] Move NullEventStore into a file of its own Co-authored-by: Cursor --- LaunchDarkly.xcodeproj/project.pbxproj | 10 +++ .../ServiceObjects/EventReporter.swift | 78 ------------------ .../ServiceObjects/NullEventStore.swift | 79 +++++++++++++++++++ 3 files changed, 89 insertions(+), 78 deletions(-) create mode 100644 LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventStore.swift diff --git a/LaunchDarkly.xcodeproj/project.pbxproj b/LaunchDarkly.xcodeproj/project.pbxproj index f994bbcf0..64eb103bf 100644 --- a/LaunchDarkly.xcodeproj/project.pbxproj +++ b/LaunchDarkly.xcodeproj/project.pbxproj @@ -18,6 +18,7 @@ 29FE129B280413D4008CC918 /* Util.swift in Sources */ = {isa = PBXBuildFile; fileRef = 29FE1297280413D4008CC918 /* Util.swift */; }; 2CA9D36B3566D78850D6F0AF /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; B8F228923BE11E444D1ECB8E /* DiskEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */; }; + 513F2EA8711D6DA3DA7AE41B /* NullEventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = BA3ED412A285F54822108659 /* NullEventStore.swift */; }; 3C55384C9E3485B493E1F86A /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; C3F7E502BD916C40AE238F64 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; AB4827D11084D58661E24B05 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; @@ -54,6 +55,7 @@ 50EE85C72EA0749C007CC662 /* TimeoutExecutorSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50EE85C62EA0749C007CC662 /* TimeoutExecutorSpec.swift */; }; 5AEA1F832DCA974314E07D21 /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; D4225F1F05F2C55E9518941B /* DiskEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */; }; + BCB7C8017479FD22963B3BB2 /* NullEventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = BA3ED412A285F54822108659 /* NullEventStore.swift */; }; DDA788E3056F8AC3CA203933 /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; D4080613CEA27D51BF349075 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; ECBE45806B470ACC5187B7D6 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; @@ -387,6 +389,7 @@ AFCE752CB1A75DEBABC23034 /* LDContextJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88DAB8D1864C14D9C7E06F11 /* LDContextJSONWriter.swift */; }; B0B1AD7CADE8DBB596C2DE9B /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; DAA2C9B9C144F129217A7C55 /* DiskEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */; }; + 99050D62DBD6A90685F6E28C /* NullEventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = BA3ED412A285F54822108659 /* NullEventStore.swift */; }; 44E0BE449714952A68628CEB /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; E5191724DFB38E62C0451086 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; CA2ADB3D20A229D70CA34F73 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; @@ -442,6 +445,7 @@ C4AF311E4DA8C3AB7040637D /* EventJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = B430884DBF2E541B919C08FA /* EventJSONWriter.swift */; }; CF9A1EB116D8AA3B7B5535B2 /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; E2F8080AB2B3CD6E9A9C1FCC /* DiskEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */; }; + 9DAA86F524C6D802D69B8553 /* NullEventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = BA3ED412A285F54822108659 /* NullEventStore.swift */; }; F0F504E5EC32223905356098 /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; B2E6D4F1AC805B3F9D127E53 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; 625ADAC0CCFE3BE324C45752 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; @@ -671,6 +675,7 @@ B4F689132497B2FC004D3CE0 /* DiagnosticEventSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DiagnosticEventSpec.swift; sourceTree = ""; }; C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventStore.swift; sourceTree = ""; }; E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = DiskEventLog.swift; sourceTree = ""; }; + BA3ED412A285F54822108659 /* NullEventStore.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = NullEventStore.swift; sourceTree = ""; }; 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventLogFile.swift; sourceTree = ""; }; A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = DeliveryClaims.swift; sourceTree = ""; }; B58551BB165F28784C785720 /* EventLogFormat.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventLogFormat.swift; sourceTree = ""; }; @@ -1060,6 +1065,7 @@ 9A771B316E19ED598CD4FB0B /* BackgroundActivity.swift */, C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */, E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */, + BA3ED412A285F54822108659 /* NullEventStore.swift */, 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */, A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */, B58551BB165F28784C785720 /* EventLogFormat.swift */, @@ -1591,6 +1597,7 @@ A32FCDB22D53263851DDB727 /* BackgroundActivity.swift in Sources */, CF9A1EB116D8AA3B7B5535B2 /* EventStore.swift in Sources */, E2F8080AB2B3CD6E9A9C1FCC /* DiskEventLog.swift in Sources */, + 9DAA86F524C6D802D69B8553 /* NullEventStore.swift in Sources */, F0F504E5EC32223905356098 /* EventLogFile.swift in Sources */, B2E6D4F1AC805B3F9D127E53 /* DeliveryClaims.swift in Sources */, 625ADAC0CCFE3BE324C45752 /* EventLogFormat.swift in Sources */, @@ -1690,6 +1697,7 @@ 7185A3C30E7FFC4F725769F1 /* BackgroundActivity.swift in Sources */, 2CA9D36B3566D78850D6F0AF /* EventStore.swift in Sources */, B8F228923BE11E444D1ECB8E /* DiskEventLog.swift in Sources */, + 513F2EA8711D6DA3DA7AE41B /* NullEventStore.swift in Sources */, 3C55384C9E3485B493E1F86A /* EventLogFile.swift in Sources */, C3F7E502BD916C40AE238F64 /* DeliveryClaims.swift in Sources */, AB4827D11084D58661E24B05 /* EventLogFormat.swift in Sources */, @@ -1788,6 +1796,7 @@ A20146E7AF098560627C614E /* BackgroundActivity.swift in Sources */, 5AEA1F832DCA974314E07D21 /* EventStore.swift in Sources */, D4225F1F05F2C55E9518941B /* DiskEventLog.swift in Sources */, + BCB7C8017479FD22963B3BB2 /* NullEventStore.swift in Sources */, DDA788E3056F8AC3CA203933 /* EventLogFile.swift in Sources */, D4080613CEA27D51BF349075 /* DeliveryClaims.swift in Sources */, ECBE45806B470ACC5187B7D6 /* EventLogFormat.swift in Sources */, @@ -1953,6 +1962,7 @@ 3D34EEEF37158657D49335BB /* BackgroundActivity.swift in Sources */, B0B1AD7CADE8DBB596C2DE9B /* EventStore.swift in Sources */, DAA2C9B9C144F129217A7C55 /* DiskEventLog.swift in Sources */, + 99050D62DBD6A90685F6E28C /* NullEventStore.swift in Sources */, 44E0BE449714952A68628CEB /* EventLogFile.swift in Sources */, E5191724DFB38E62C0451086 /* DeliveryClaims.swift in Sources */, CA2ADB3D20A229D70CA34F73 /* EventLogFormat.swift in Sources */, diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift index 22d745075..bfcbb2e2e 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift @@ -889,84 +889,6 @@ extension EventReporter { extension EventReporter: TypeIdentifying { } -/// Stands in where the platform gave the SDK nowhere to write. Events are recorded and delivered from memory for the -/// life of the process, which is the behavior the SDK had before it kept a log. -private final class NullEventStore: EventStoring { - private let lock = UnfairLock() - private var staged: [Data] = [] - private var reserved = 0 - private var closed: [String: [Data]] = [:] - - var pendingEventCount: Int { - lock.withLock { reserved + staged.count + closed.values.reduce(0) { $0 + $1.count } } - } - - var isPersisting: Bool { false } - - func stage(_ encodedEvent: Data, bypassingCapacity: Bool) -> Bool { - lock.withLock { staged.append(encodedEvent) } - return true - } - - func reserve(_ events: Int) { - lock.withLock { reserved += events } - } - - func stageReserved(_ encodedEvent: Data) -> Bool { - lock.withLock { - reserved = max(0, reserved - 1) - staged.append(encodedEvent) - } - return true - } - - func releaseReservations() { - lock.withLock { reserved = 0 } - } - - func commit() { - } - - func closeBatch() -> EventBatch? { - lock.withLock { - guard !staged.isEmpty - else { return nil } - - let payloadId = UUID().uuidString - let events = staged - staged = [] - closed[payloadId] = events - return EventBatch(payloadId: payloadId, eventCount: events.count) - } - } - - func pendingBatches() -> [EventBatch] { - lock.withLock { closed.map { EventBatch(payloadId: $0.key, eventCount: $0.value.count) } } - } - - func body(of batch: EventBatch) -> Data? { - guard let events = lock.withLock({ closed[batch.payloadId] }) - else { return nil } - - var body = Data("[".utf8) - for (index, event) in events.enumerated() { - if index > 0 { - body.append(UInt8(ascii: ",")) - } - body.append(event) - } - body.append(UInt8(ascii: "]")) - return body - } - - func remove(_ batch: EventBatch) { - lock.withLock { closed[batch.payloadId] = nil } - } - - func recoverInterruptedLog() { - } -} - #if DEBUG extension EventReporter { /// Full events accepted but not yet handed to a commit. diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventStore.swift new file mode 100644 index 000000000..5f74cfc61 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventStore.swift @@ -0,0 +1,79 @@ +import Foundation + +/// Stands in where the platform gave the SDK nowhere to write. Events are recorded and delivered from memory for the +/// life of the process, which is the behavior the SDK had before it kept a log. +final class NullEventStore: EventStoring { + private let lock = UnfairLock() + private var staged: [Data] = [] + private var reserved = 0 + private var closed: [String: [Data]] = [:] + + var pendingEventCount: Int { + lock.withLock { reserved + staged.count + closed.values.reduce(0) { $0 + $1.count } } + } + + var isPersisting: Bool { false } + + func stage(_ encodedEvent: Data, bypassingCapacity: Bool) -> Bool { + lock.withLock { staged.append(encodedEvent) } + return true + } + + func reserve(_ events: Int) { + lock.withLock { reserved += events } + } + + func stageReserved(_ encodedEvent: Data) -> Bool { + lock.withLock { + reserved = max(0, reserved - 1) + staged.append(encodedEvent) + } + return true + } + + func releaseReservations() { + lock.withLock { reserved = 0 } + } + + func commit() { + } + + func closeBatch() -> EventBatch? { + lock.withLock { + guard !staged.isEmpty + else { return nil } + + let payloadId = UUID().uuidString + let events = staged + staged = [] + closed[payloadId] = events + return EventBatch(payloadId: payloadId, eventCount: events.count) + } + } + + func pendingBatches() -> [EventBatch] { + lock.withLock { closed.map { EventBatch(payloadId: $0.key, eventCount: $0.value.count) } } + } + + func body(of batch: EventBatch) -> Data? { + guard let events = lock.withLock({ closed[batch.payloadId] }) + else { return nil } + + var body = Data("[".utf8) + for (index, event) in events.enumerated() { + if index > 0 { + body.append(UInt8(ascii: ",")) + } + body.append(event) + } + body.append(UInt8(ascii: "]")) + return body + } + + func remove(_ batch: EventBatch) { + lock.withLock { closed[batch.payloadId] = nil } + } + + func recoverInterruptedLog() { + } +} From 4fcc3e097ed938c89f8db285f08fcf205f4e438f Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Sun, 11 Oct 2026 19:44:53 -0700 Subject: [PATCH 025/158] Group the EventReporter fields by the lock or queue that guards them Co-authored-by: Cursor --- .../ServiceObjects/EventReporter.swift | 134 ++++++++++-------- 1 file changed, 76 insertions(+), 58 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift index bfcbb2e2e..46c56ba65 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift @@ -113,50 +113,25 @@ class EventReporter: EventReporting { /// against the number of writes. private static let pendingCommitThreshold = 32 - var isOnline: Bool { - get { timerQueue.sync { eventReportTimer != nil } } - set { timerQueue.sync { newValue ? startReporting() : stopReporting() } } - } - - var lastEventResponseDate: Date { - stateLock.withLock { responseDate } - } + // MARK: Set in init and never changed let service: DarklyServiceProvider let store: EventStoring - /// Guards the last response date and the record of what has been lost. - private let stateLock = UnfairLock() - /// Only to be used while holding `recordLock`. - private(set) var contextSummarizer: ContextSummarizer - /// Only to be used while holding `stateLock`. - private var responseDate: Date + /// How many events the SDK will hold in total, across `pending` and the store. + private let capacity: Int - /// Guards `queuedDelivery`. Taken on a caller's thread and on `deliveryQueue`, and never held across either. - private let deliveryLock = UnfairLock() - /// Callers waiting on a delivery that is queued and has not taken the batches yet, or nil where none is queued. - private var queuedDelivery: [FlushOutcomeClosure]? + /// Where a commit runs when no caller is waiting on it. + private let commitQueue: DispatchQueue - #if DEBUG - /// How many deliveries have started, which is fewer than the flushes asked for where they were joined. - private var deliveryRunCount = 0 - #endif + private let onSyncComplete: EventSyncCompleteClosure? - /// How many events the SDK accepted and then dropped for good. Only to be used while holding `stateLock`. - /// - /// A batch LaunchDarkly refuses in a way that may pass is kept and retried, so it is not a loss. What is lost is - /// what the store never received: an event or a summary that could not be serialized when it was committed, or - /// that serialized too large to store. A delivery that finds nothing to send cannot tell those apart from events - /// that arrived, so without this it would report success for them. - /// - /// A count rather than a flag because deliveries overlap. Each takes the count as it closes its batch, and - /// answers for the losses up to that mark, so a loss is told to the caller whose delivery it happened before - /// rather than to whichever response arrived first. - private var lostEvents: UInt64 = 0 + /// Shared by the threads recording events, so it must not be mutated after `init`. + private let encoder: JSONEncoder - /// The mark up to which losses have already been reported to a caller, so that none is told twice. Only to be - /// used while holding `stateLock`. - private var reportedLostEvents: UInt64 = 0 + let encoding: Encoding + + // MARK: Guarded by commitLock /// Held for the whole of a commit, so that only one runs at a time. /// @@ -169,6 +144,14 @@ class EventReporter: EventReporting { /// Taken before `recordLock` and before anything the store locks, never after. private let commitLock = UnfairLock() + /// Outlives a commit rather than being built per commit, so that its buffer and context cache span them. + /// + /// A commit point writes a single event, so a cache that lived for one commit would never be read. It needs no + /// lock of its own because it is only ever reached from `encode(_:)`, which runs under `commitLock`. + fileprivate let eventJSONWriter: EventJSONWriter + + // MARK: Guarded by recordLock + /// Guards everything one recording writes: the held events and the summarizer. /// /// One evaluation can produce a counter, a full event and a debug event, and the three have to land together. Taken @@ -179,6 +162,9 @@ class EventReporter: EventReporting { /// worth avoiding: the same measurement that made `EvaluationExposureDeduper` a lock applies here. private let recordLock = UnfairLock() + /// Only to be used while holding `recordLock`. + private(set) var contextSummarizer: ContextSummarizer + /// Full events recorded but not yet encoded. Only to be used while holding `recordLock`. /// /// Held rather than encoded because encoding early would not make them durable: the store stages bytes into memory @@ -190,24 +176,46 @@ class EventReporter: EventReporting { /// one each. Only to be used while holding `recordLock`. private var isCommitScheduled = false - /// How many events the SDK will hold in total, across `pending` and the store. - private let capacity: Int - - /// Where a commit runs when no caller is waiting on it. - private let commitQueue: DispatchQueue + // MARK: Guarded by stateLock - private var timerQueue = DispatchQueue(label: "com.launchdarkly.EventReporter.timerQueue") - private var eventReportTimer: TimeResponding? - var isReportingActive: Bool { eventReportTimer != nil } + var lastEventResponseDate: Date { + stateLock.withLock { responseDate } + } - /// Deliveries run here, off whatever thread recorded an event. - private let deliveryQueue = DispatchQueue(label: "com.launchdarkly.eventSyncQueue", qos: .userInitiated) + /// Guards the last response date and the record of what has been lost. + private let stateLock = UnfairLock() + /// Only to be used while holding `stateLock`. + private var responseDate: Date - /// Whether the store held events from a previous run of the application when this reporter started. + /// How many events the SDK accepted and then dropped for good. Only to be used while holding `stateLock`. /// - /// Only to be used on `deliveryQueue`, which is what makes it safe without a lock: the recovery that answers the - /// question and the delivery that acts on it are both enqueued there, in that order. - private var hasEventsFromPreviousRun = false + /// A batch LaunchDarkly refuses in a way that may pass is kept and retried, so it is not a loss. What is lost is + /// what the store never received: an event or a summary that could not be serialized when it was committed, or + /// that serialized too large to store. A delivery that finds nothing to send cannot tell those apart from events + /// that arrived, so without this it would report success for them. + /// + /// A count rather than a flag because deliveries overlap. Each takes the count as it closes its batch, and + /// answers for the losses up to that mark, so a loss is told to the caller whose delivery it happened before + /// rather than to whichever response arrived first. + private var lostEvents: UInt64 = 0 + + /// The mark up to which losses have already been reported to a caller, so that none is told twice. Only to be + /// used while holding `stateLock`. + private var reportedLostEvents: UInt64 = 0 + + // MARK: Guarded by deliveryLock + + /// Guards `queuedDelivery`. Taken on a caller's thread and on `deliveryQueue`, and never held across either. + private let deliveryLock = UnfairLock() + /// Callers waiting on a delivery that is queued and has not taken the batches yet, or nil where none is queued. + private var queuedDelivery: [FlushOutcomeClosure]? + + #if DEBUG + /// How many deliveries have started, which is fewer than the flushes asked for where they were joined. + private var deliveryRunCount = 0 + #endif + + // MARK: Guarded by claimLock /// Guards `batchesInFlight`. /// @@ -228,17 +236,27 @@ class EventReporter: EventReporting { /// on that later delivery hears their outcome too. private var batchesInFlight: [String: Claim] = [:] - private let onSyncComplete: EventSyncCompleteClosure? + // MARK: Confined to timerQueue - /// Shared by the threads recording events, so it must not be mutated after `init`. - private let encoder: JSONEncoder + var isOnline: Bool { + get { timerQueue.sync { eventReportTimer != nil } } + set { timerQueue.sync { newValue ? startReporting() : stopReporting() } } + } - let encoding: Encoding - /// Outlives a commit rather than being built per commit, so that its buffer and context cache span them. + private var timerQueue = DispatchQueue(label: "com.launchdarkly.EventReporter.timerQueue") + private var eventReportTimer: TimeResponding? + var isReportingActive: Bool { eventReportTimer != nil } + + // MARK: Confined to deliveryQueue + + /// Deliveries run here, off whatever thread recorded an event. + private let deliveryQueue = DispatchQueue(label: "com.launchdarkly.eventSyncQueue", qos: .userInitiated) + + /// Whether the store held events from a previous run of the application when this reporter started. /// - /// A commit point writes a single event, so a cache that lived for one commit would never be read. It needs no - /// lock of its own because it is only ever reached from `encode(_:)`, which runs under `commitLock`. - fileprivate let eventJSONWriter: EventJSONWriter + /// Only to be used on `deliveryQueue`, which is what makes it safe without a lock: the recovery that answers the + /// question and the delivery that acts on it are both enqueued there, in that order. + private var hasEventsFromPreviousRun = false init(service: DarklyServiceProvider, onSyncComplete: EventSyncCompleteClosure?, From d32c6693cbba1b0352b77f69c67510940e3db49e Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Sun, 11 Oct 2026 19:50:12 -0700 Subject: [PATCH 026/158] Look a context up once even at the limit, through its index there Co-authored-by: Cursor --- .../ServiceObjects/ContextSummarizer.swift | 29 ++++++++++++------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/ContextSummarizer.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/ContextSummarizer.swift index 53f42e230..207afa20e 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/ContextSummarizer.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/ContextSummarizer.swift @@ -69,8 +69,24 @@ class ContextSummarizer { /// already are. A context already being counted is never turned away. @discardableResult func trackRequest(flagKey: LDFlagKey, reportedValue: LDValue, featureFlag: FeatureFlag?, defaultValue: LDValue, context: LDContext) -> Bool { + func count(in entry: inout TrackerWithContext) { + entry.tracker.trackRequest( + flagKey: flagKey, + reportedValue: reportedValue, + featureFlag: featureFlag, + defaultValue: defaultValue, + context: context + ) + } + let key = key(for: context) - guard trackers.count < maxContexts || trackers[key] != nil + if trackers.count < maxContexts { + // Redaction is applied when the summary is encoded; see `Event.redactsAnonymousAttributes`. + count(in: &trackers[key, default: TrackerWithContext(tracker: FlagRequestTracker(logger: logger), context: context)]) + return true + } + + guard let index = trackers.index(forKey: key) else { if !hasLoggedContextsExceeded { hasLoggedContextsExceeded = true @@ -78,16 +94,7 @@ class ContextSummarizer { } return false } - - // Redaction is applied when the summary is encoded; see `Event.redactsAnonymousAttributes`. - trackers[key, default: TrackerWithContext(tracker: FlagRequestTracker(logger: logger), context: context)] - .tracker.trackRequest( - flagKey: flagKey, - reportedValue: reportedValue, - featureFlag: featureFlag, - defaultValue: defaultValue, - context: context - ) + count(in: &trackers.values[index]) return true } From 40ab485f2647e46422a1a545567c223395db03a6 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Sun, 11 Oct 2026 21:17:32 -0700 Subject: [PATCH 027/158] Go back to looking a context up through its index, which reads more plainly Co-authored-by: Cursor --- .../ServiceObjects/ContextSummarizer.swift | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/ContextSummarizer.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/ContextSummarizer.swift index 207afa20e..08afa1bd1 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/ContextSummarizer.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/ContextSummarizer.swift @@ -69,24 +69,19 @@ class ContextSummarizer { /// already are. A context already being counted is never turned away. @discardableResult func trackRequest(flagKey: LDFlagKey, reportedValue: LDValue, featureFlag: FeatureFlag?, defaultValue: LDValue, context: LDContext) -> Bool { - func count(in entry: inout TrackerWithContext) { - entry.tracker.trackRequest( + let key = key(for: context) + if let index = trackers.index(forKey: key) { + trackers.values[index].tracker.trackRequest( flagKey: flagKey, reportedValue: reportedValue, featureFlag: featureFlag, defaultValue: defaultValue, context: context ) - } - - let key = key(for: context) - if trackers.count < maxContexts { - // Redaction is applied when the summary is encoded; see `Event.redactsAnonymousAttributes`. - count(in: &trackers[key, default: TrackerWithContext(tracker: FlagRequestTracker(logger: logger), context: context)]) return true } - guard let index = trackers.index(forKey: key) + guard trackers.count < maxContexts else { if !hasLoggedContextsExceeded { hasLoggedContextsExceeded = true @@ -94,7 +89,17 @@ class ContextSummarizer { } return false } - count(in: &trackers.values[index]) + + var tracker = FlagRequestTracker(logger: logger) + tracker.trackRequest( + flagKey: flagKey, + reportedValue: reportedValue, + featureFlag: featureFlag, + defaultValue: defaultValue, + context: context + ) + // Redaction is applied when the summary is encoded; see `Event.redactsAnonymousAttributes`. + trackers[key] = TrackerWithContext(tracker: tracker, context: context) return true } From 20203d99fb5f544017f0895983023a6f22c29a74 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Sun, 11 Oct 2026 21:17:32 -0700 Subject: [PATCH 028/158] Admit an evaluation's events without building an array for them Co-authored-by: Cursor --- LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift index 46c56ba65..94dfbd2c4 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift @@ -328,9 +328,8 @@ class EventReporter: EventReporting { var dropped = 0 let (counted, needsCommit) = recordLock.withLock { () -> (Bool, Bool) in let counted = contextSummarizer.trackRequest(flagKey: flagKey, reportedValue: value, featureFlag: featureFlag, defaultValue: defaultValue, context: context) - for event in [featureEvent, debugEvent].compactMap({ $0 }) where !admitToPending(event) { - dropped += 1 - } + if let featureEvent, !admitToPending(featureEvent) { dropped += 1 } + if let debugEvent, !admitToPending(debugEvent) { dropped += 1 } return (counted, pending.count >= EventReporter.pendingCommitThreshold) } From 4ab5f5d2afe2053f5c990a4fc967f0b702a29dcb Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Sun, 11 Oct 2026 21:20:44 -0700 Subject: [PATCH 029/158] Rename EventReporter to DirectEventProcessor, as Android names it, with the protocol and null object in files of their own Co-authored-by: Cursor --- LaunchDarkly.xcodeproj/project.pbxproj | 48 +- .../GeneratedCode/mocks.generated.swift | 4 +- LaunchDarkly/LaunchDarkly/LDClient.swift | 18 +- LaunchDarkly/LaunchDarkly/LDClientFlush.swift | 6 +- .../LaunchDarkly/LDClientVariation.swift | 2 +- .../ServiceObjects/ClientServiceFactory.swift | 14 +- .../ServiceObjects/DeliveryClaims.swift | 2 +- ...orter.swift => DirectEventProcessor.swift} | 105 +---- .../ServiceObjects/EventProcessor.swift | 45 ++ .../ServiceObjects/NullEventProcessor.swift | 37 ++ .../EventJSONWriterTests.swift | 4 +- .../LDClientEvaluationHookSpec.swift | 2 +- .../LaunchDarklyTests/LDClientSpec.swift | 234 +++++----- .../Mocks/ClientServiceMockFactory.swift | 20 +- ...c.swift => DirectEventProcessorSpec.swift} | 420 +++++++++--------- .../LaunchDarklyTests/TestContext.swift | 6 +- 16 files changed, 495 insertions(+), 472 deletions(-) rename LaunchDarkly/LaunchDarkly/ServiceObjects/{EventReporter.swift => DirectEventProcessor.swift} (89%) create mode 100644 LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift create mode 100644 LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventProcessor.swift rename LaunchDarkly/LaunchDarklyTests/ServiceObjects/{EventReporterSpec.swift => DirectEventProcessorSpec.swift} (85%) diff --git a/LaunchDarkly.xcodeproj/project.pbxproj b/LaunchDarkly.xcodeproj/project.pbxproj index 64eb103bf..2e3ba05bb 100644 --- a/LaunchDarkly.xcodeproj/project.pbxproj +++ b/LaunchDarkly.xcodeproj/project.pbxproj @@ -79,7 +79,9 @@ 831188522113ADF700D77CB5 /* KeyedValueCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83D559731FD87CC9002D10C8 /* KeyedValueCache.swift */; }; 831188562113AE0800D77CB5 /* FlagSynchronizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */; }; 831188572113AE0B00D77CB5 /* FlagChangeNotifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8358F25F1F476AD800ECE1AF /* FlagChangeNotifier.swift */; }; - 831188582113AE0F00D77CB5 /* EventReporter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* EventReporter.swift */; }; + 831188582113AE0F00D77CB5 /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; + 014901E06BE1F2B0FAAD6AA6 /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; + 86B71C91DB78D0F17DBE0E69 /* EventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 293A30078FEAA25BB4C3D26C /* EventProcessor.swift */; }; 831188592113AE1200D77CB5 /* FlagStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831D8B731F72994600ED65E8 /* FlagStore.swift */; }; 8311885A2113AE1500D77CB5 /* Log.swift in Sources */ = {isa = PBXBuildFile; fileRef = 837EF3732059C237009D628A /* Log.swift */; }; 8311885B2113AE1D00D77CB5 /* Throttler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831AAE2B20A9E4F600B46DBA /* Throttler.swift */; }; @@ -120,7 +122,9 @@ 831EF35120655E730001C643 /* KeyedValueCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83D559731FD87CC9002D10C8 /* KeyedValueCache.swift */; }; 831EF35520655E730001C643 /* FlagSynchronizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */; }; 831EF35620655E730001C643 /* FlagChangeNotifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8358F25F1F476AD800ECE1AF /* FlagChangeNotifier.swift */; }; - 831EF35720655E730001C643 /* EventReporter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* EventReporter.swift */; }; + 831EF35720655E730001C643 /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; + CFF357D68BBF2DC8395930F2 /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; + 850C49C9054E642CC006E730 /* EventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 293A30078FEAA25BB4C3D26C /* EventProcessor.swift */; }; 831EF35820655E730001C643 /* FlagStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831D8B731F72994600ED65E8 /* FlagStore.swift */; }; 831EF35920655E730001C643 /* Log.swift in Sources */ = {isa = PBXBuildFile; fileRef = 837EF3732059C237009D628A /* Log.swift */; }; 831EF35A20655E730001C643 /* HTTPHeaders.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831D8B6E1F71532300ED65E8 /* HTTPHeaders.swift */; }; @@ -187,7 +191,7 @@ 83B8C2471FE4071F0082B8A9 /* HTTPURLResponse.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83B8C2461FE4071F0082B8A9 /* HTTPURLResponse.swift */; }; 83B9A080204F56F4000C3F17 /* FlagChangeObserverSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83B9A07F204F56F4000C3F17 /* FlagChangeObserverSpec.swift */; }; 83B9A082204F6022000C3F17 /* FlagsUnchangedObserver.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83B9A081204F6022000C3F17 /* FlagsUnchangedObserver.swift */; }; - 83CFE7CE1F7AD81D0010544E /* EventReporterSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83CFE7CD1F7AD81D0010544E /* EventReporterSpec.swift */; }; + 83CFE7CE1F7AD81D0010544E /* DirectEventProcessorSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83CFE7CD1F7AD81D0010544E /* DirectEventProcessorSpec.swift */; }; 83CFE7D11F7AD8DC0010544E /* DarklyServiceMock.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83CFE7D01F7AD8DC0010544E /* DarklyServiceMock.swift */; }; 83D559741FD87CC9002D10C8 /* KeyedValueCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83D559731FD87CC9002D10C8 /* KeyedValueCache.swift */; }; 83D9EC752062DEAB004D7FA6 /* LDCommon.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83B6C4B51F4DE7630055351C /* LDCommon.swift */; }; @@ -202,7 +206,9 @@ 83D9EC832062DEAB004D7FA6 /* KeyedValueCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83D559731FD87CC9002D10C8 /* KeyedValueCache.swift */; }; 83D9EC872062DEAB004D7FA6 /* FlagSynchronizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */; }; 83D9EC882062DEAB004D7FA6 /* FlagChangeNotifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8358F25F1F476AD800ECE1AF /* FlagChangeNotifier.swift */; }; - 83D9EC892062DEAB004D7FA6 /* EventReporter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* EventReporter.swift */; }; + 83D9EC892062DEAB004D7FA6 /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; + 54745AEF1A43D165F5A6E08F /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; + 7F4004579B9094737F3C21D1 /* EventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 293A30078FEAA25BB4C3D26C /* EventProcessor.swift */; }; 83D9EC8A2062DEAB004D7FA6 /* FlagStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831D8B731F72994600ED65E8 /* FlagStore.swift */; }; 83D9EC8B2062DEAB004D7FA6 /* Log.swift in Sources */ = {isa = PBXBuildFile; fileRef = 837EF3732059C237009D628A /* Log.swift */; }; 83D9EC8C2062DEAB004D7FA6 /* HTTPHeaders.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831D8B6E1F71532300ED65E8 /* HTTPHeaders.swift */; }; @@ -226,7 +232,9 @@ 83EF67931F9945E800403126 /* EventSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83EF67921F9945E800403126 /* EventSpec.swift */; }; 83F0A5641FB5F33800550A95 /* LDConfigSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83F0A5631FB5F33800550A95 /* LDConfigSpec.swift */; }; 83FEF8DD1F266742001CF12C /* FlagSynchronizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */; }; - 83FEF8DF1F2667E4001CF12C /* EventReporter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* EventReporter.swift */; }; + 83FEF8DF1F2667E4001CF12C /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; + 9FA0EE3A87678F58F4406B24 /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; + 7B38D33B59821BD9BCFFA731 /* EventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 293A30078FEAA25BB4C3D26C /* EventProcessor.swift */; }; 8B3D9E2F5A6C1047B2D5E9F1 /* JSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6F2A8C1D4E7B9053A1C4D8E6 /* JSONWriter.swift */; }; 8E38AA8C306308E500C4C163 /* RetryState.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8E38AA8B306308E500C4C163 /* RetryState.swift */; }; 8E38AA8D306308E500C4C163 /* RetryState.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8E38AA8B306308E500C4C163 /* RetryState.swift */; }; @@ -595,7 +603,7 @@ 83B8C2461FE4071F0082B8A9 /* HTTPURLResponse.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HTTPURLResponse.swift; sourceTree = ""; }; 83B9A07F204F56F4000C3F17 /* FlagChangeObserverSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FlagChangeObserverSpec.swift; sourceTree = ""; }; 83B9A081204F6022000C3F17 /* FlagsUnchangedObserver.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FlagsUnchangedObserver.swift; sourceTree = ""; }; - 83CFE7CD1F7AD81D0010544E /* EventReporterSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EventReporterSpec.swift; sourceTree = ""; }; + 83CFE7CD1F7AD81D0010544E /* DirectEventProcessorSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DirectEventProcessorSpec.swift; sourceTree = ""; }; 83CFE7D01F7AD8DC0010544E /* DarklyServiceMock.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DarklyServiceMock.swift; sourceTree = ""; }; 83D559731FD87CC9002D10C8 /* KeyedValueCache.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeyedValueCache.swift; sourceTree = ""; }; 83D9EC6B2062DBB7004D7FA6 /* LaunchDarkly_watchOS.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = LaunchDarkly_watchOS.framework; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -608,7 +616,9 @@ 83EF67921F9945E800403126 /* EventSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EventSpec.swift; sourceTree = ""; }; 83F0A5631FB5F33800550A95 /* LDConfigSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LDConfigSpec.swift; sourceTree = ""; }; 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FlagSynchronizer.swift; sourceTree = ""; }; - 83FEF8DE1F2667E4001CF12C /* EventReporter.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = EventReporter.swift; sourceTree = ""; }; + 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DirectEventProcessor.swift; sourceTree = ""; }; + BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NullEventProcessor.swift; sourceTree = ""; }; + 293A30078FEAA25BB4C3D26C /* EventProcessor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = EventProcessor.swift; sourceTree = ""; }; 88DAB8D1864C14D9C7E06F11 /* LDContextJSONWriter.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = LDContextJSONWriter.swift; sourceTree = ""; }; 8E38AA8B306308E500C4C163 /* RetryState.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RetryState.swift; sourceTree = ""; }; 8E38AA903063090600C4C163 /* RetryStateSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RetryStateSpec.swift; sourceTree = ""; }; @@ -774,7 +784,7 @@ 50EE85C62EA0749C007CC662 /* TimeoutExecutorSpec.swift */, A3047D5B2A606A0000F568E0 /* EnvironmentReporting */, B46F344025E6DB7D0078D45F /* DiagnosticReporterSpec.swift */, - 83CFE7CD1F7AD81D0010544E /* EventReporterSpec.swift */, + 83CFE7CD1F7AD81D0010544E /* DirectEventProcessorSpec.swift */, 83B8C2441FE360CF0082B8A9 /* FlagChangeNotifierSpec.swift */, 83DDBEFF1FA2589900E428B6 /* FlagStoreSpec.swift */, 831D8B761F72A4B400ED65E8 /* FlagSynchronizerSpec.swift */, @@ -1051,7 +1061,9 @@ 83B1D7C82073F354006D1B1C /* CwlSysctl.swift */, B4C9D4372489E20A004A9B03 /* DiagnosticReporter.swift */, 831425B0206B030100F2EF36 /* EnvironmentReporter.swift */, - 83FEF8DE1F2667E4001CF12C /* EventReporter.swift */, + 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */, + BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */, + 293A30078FEAA25BB4C3D26C /* EventProcessor.swift */, 8358F25F1F476AD800ECE1AF /* FlagChangeNotifier.swift */, 831D8B731F72994600ED65E8 /* FlagStore.swift */, 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */, @@ -1530,7 +1542,9 @@ A310881A2837DC0400184942 /* Reference.swift in Sources */, 3D3AB9462A4F16FE003AECF1 /* ReportingConsts.swift in Sources */, 831188522113ADF700D77CB5 /* KeyedValueCache.swift in Sources */, - 831188582113AE0F00D77CB5 /* EventReporter.swift in Sources */, + 831188582113AE0F00D77CB5 /* DirectEventProcessor.swift in Sources */, + 014901E06BE1F2B0FAAD6AA6 /* NullEventProcessor.swift in Sources */, + 86B71C91DB78D0F17DBE0E69 /* EventProcessor.swift in Sources */, 50EE85C42EA0487F007CC662 /* TimeoutExecutor.swift in Sources */, A350386E2F4F4F610032BA9F /* IdentifySeriesContext.swift in Sources */, FED0000000000000000000C1 /* TrackSeriesContext.swift in Sources */, @@ -1651,7 +1665,9 @@ A3BA7CF52BD05A280000DB28 /* EvaluationSeriesContext.swift in Sources */, 831EF35620655E730001C643 /* FlagChangeNotifier.swift in Sources */, A358D6D92A4DE6A500270C60 /* ApplicationInfoEnvironmentReporter.swift in Sources */, - 831EF35720655E730001C643 /* EventReporter.swift in Sources */, + 831EF35720655E730001C643 /* DirectEventProcessor.swift in Sources */, + CFF357D68BBF2DC8395930F2 /* NullEventProcessor.swift in Sources */, + 850C49C9054E642CC006E730 /* EventProcessor.swift in Sources */, A350386F2F4F4F610032BA9F /* IdentifySeriesContext.swift in Sources */, FED0000000000000000000C2 /* TrackSeriesContext.swift in Sources */, 831EF35820655E730001C643 /* FlagStore.swift in Sources */, @@ -1757,7 +1773,9 @@ 29F9D19E2812E005008D12C0 /* ObjcLDValue.swift in Sources */, A3599E882A4B4AD400DB5C67 /* Modifier.swift in Sources */, 8358F2601F476AD800ECE1AF /* FlagChangeNotifier.swift in Sources */, - 83FEF8DF1F2667E4001CF12C /* EventReporter.swift in Sources */, + 83FEF8DF1F2667E4001CF12C /* DirectEventProcessor.swift in Sources */, + 9FA0EE3A87678F58F4406B24 /* NullEventProcessor.swift in Sources */, + 7B38D33B59821BD9BCFFA731 /* EventProcessor.swift in Sources */, 831D2AAF2061AAA000B4AC3C /* Thread.swift in Sources */, A358D6DD2A4DE7D600270C60 /* IOSEnvironmentReporter.swift in Sources */, 83B9A082204F6022000C3F17 /* FlagsUnchangedObserver.swift in Sources */, @@ -1810,7 +1828,7 @@ 3D2406252E0D90EA00F91253 /* LDClientPluginsSpec.swift in Sources */, A3047D662A606B6000F568E0 /* EnvironmentReporterChainBaseSpec.swift in Sources */, A31088272837DCA900184942 /* LDContextSpec.swift in Sources */, - 83CFE7CE1F7AD81D0010544E /* EventReporterSpec.swift in Sources */, + 83CFE7CE1F7AD81D0010544E /* DirectEventProcessorSpec.swift in Sources */, 8392FFA32033565700320914 /* HTTPURLResponse.swift in Sources */, 83411A5F1FABDA8700E5CF39 /* mocks.generated.swift in Sources */, 83DDBF001FA2589900E428B6 /* FlagStoreSpec.swift in Sources */, @@ -1919,7 +1937,9 @@ 9AE1D0D42F4A1B000001FD9C /* UnfairLock.swift in Sources */, 83D9EC882062DEAB004D7FA6 /* FlagChangeNotifier.swift in Sources */, C443A40B2315AA4D00145710 /* NetworkReporter.swift in Sources */, - 83D9EC892062DEAB004D7FA6 /* EventReporter.swift in Sources */, + 83D9EC892062DEAB004D7FA6 /* DirectEventProcessor.swift in Sources */, + 54745AEF1A43D165F5A6E08F /* NullEventProcessor.swift in Sources */, + 7F4004579B9094737F3C21D1 /* EventProcessor.swift in Sources */, 83D9EC8A2062DEAB004D7FA6 /* FlagStore.swift in Sources */, 29F9D19F2812E005008D12C0 /* ObjcLDValue.swift in Sources */, A3599E892A4B4AD400DB5C67 /* Modifier.swift in Sources */, diff --git a/LaunchDarkly/GeneratedCode/mocks.generated.swift b/LaunchDarkly/GeneratedCode/mocks.generated.swift index dc3757e6a..aca360bd5 100644 --- a/LaunchDarkly/GeneratedCode/mocks.generated.swift +++ b/LaunchDarkly/GeneratedCode/mocks.generated.swift @@ -174,8 +174,8 @@ final class EnvironmentReportingMock: EnvironmentReporting { } } -// MARK: - EventReportingMock -final class EventReportingMock: EventReporting { +// MARK: - EventProcessorMock +final class EventProcessorMock: EventProcessor { var isOnlineSetCount = 0 var setIsOnlineCallback: (() throws -> Void)? diff --git a/LaunchDarkly/LaunchDarkly/LDClient.swift b/LaunchDarkly/LaunchDarkly/LDClient.swift index 64cd82bc4..516e54002 100644 --- a/LaunchDarkly/LaunchDarkly/LDClient.swift +++ b/LaunchDarkly/LaunchDarkly/LDClient.swift @@ -66,7 +66,7 @@ public class LDClient { let oldValue = _isOnline _isOnline = newValue flagSynchronizer.isOnline = _isOnline - eventReporter.isOnline = _isOnline + eventProcessor.isOnline = _isOnline diagnosticReporter.setMode(runMode, online: _isOnline) if _isOnline != oldValue { connectionInformation = ConnectionInformation.onlineSetCheck(connectionInformation: connectionInformation, ldClient: self, config: config, online: _isOnline) @@ -265,7 +265,7 @@ public class LDClient { os_log("%s", log: config.logger, type: .debug, typeName(and: #function)) // The OS may kill a backgrounded application without warning, so make its events durable now. On every // platform: macOS does not suspend an application that loses focus, but it can still be quit or killed. - eventReporter.commitAtCommitPoint() + eventProcessor.commitAtCommitPoint() // A backgrounded process is suspended once idle, so deliver inside an activity assertion. Only where suspension // is real: macOS posts this notification whenever the application loses focus, and never suspends it for that. // What does not get out is on disk for the next launch. @@ -276,7 +276,7 @@ public class LDClient { finished() return } - self.eventReporter.flushReportingOutcome { delivered in + self.eventProcessor.flushReportingOutcome { delivered in if !delivered { os_log("%s events did not reach LaunchDarkly before suspension; they remain on disk", log: self.config.logger, type: .debug, self.typeName(and: #function)) } @@ -479,7 +479,7 @@ public class LDClient { internalIdentifyQueue.sync { if self.context == updatedContext { - self.eventReporter.record(IdentifyEvent(context: self.context)) + self.eventProcessor.record(IdentifyEvent(context: self.context)) completion?() return } @@ -529,7 +529,7 @@ public class LDClient { onSyncComplete: self.onFlagSyncComplete) if self.hasStarted { - self.eventReporter.record(IdentifyEvent(context: self.context)) + self.eventProcessor.record(IdentifyEvent(context: self.context)) } if self.isOnline && self.isInSupportedRunMode { @@ -773,7 +773,7 @@ public class LDClient { key, String(describing: data), String(describing: metricValue)) - eventReporter.record(event) + eventProcessor.record(event) executeAfterTrackHooks(key: key, data: data, metricValue: metricValue) } @@ -979,7 +979,7 @@ public class LDClient { private(set) var flagCache: FeatureFlagCaching private(set) var flagSynchronizer: LDFlagSynchronizing var flagChangeNotifier: FlagChangeNotifying - private(set) var eventReporter: EventReporting + private(set) var eventProcessor: EventProcessor private(set) var environmentReporter: EnvironmentReporting private(set) var throttler: Throttling private(set) var diagnosticReporter: DiagnosticReporting @@ -1053,7 +1053,7 @@ public class LDClient { // Reports over the logger rather than over this client, which is not usable until every stored property // below has a value, so that the reporter can be built once rather than built and replaced. let logger = configuration.logger - eventReporter = self.serviceFactory.makeEventReporter(config: config, service: service) { result in + eventProcessor = self.serviceFactory.makeEventProcessor(config: config, service: service) { result in LDClient.logEventSyncComplete(result, logger) } connectionInformation = self.serviceFactory.makeConnectionInformation() @@ -1082,7 +1082,7 @@ public class LDClient { flagStore.replaceStore(newStoredItems: cachedFlags) } - eventReporter.record(IdentifyEvent(context: context)) + eventProcessor.record(IdentifyEvent(context: context)) self.connectionInformation = ConnectionInformation.uncacheConnectionInformation(config: config, ldClient: self, clientServiceFactory: self.serviceFactory) internalSetOnline(configuration.startOnline) { diff --git a/LaunchDarkly/LaunchDarkly/LDClientFlush.swift b/LaunchDarkly/LaunchDarkly/LDClientFlush.swift index 8ab4abce6..be9d10a7b 100644 --- a/LaunchDarkly/LaunchDarkly/LDClientFlush.swift +++ b/LaunchDarkly/LaunchDarkly/LDClientFlush.swift @@ -58,7 +58,7 @@ extension LDClient { var outstanding = clients.count for client in clients { - client.eventReporter.flushReportingOutcome { result in + client.eventProcessor.flushReportingOutcome { result in lock.lock() delivered = delivered && result outstanding -= 1 @@ -138,7 +138,7 @@ extension LDClient { // one before starting the next would spend the caller's budget on deliveries that could have been running all // along, and leave the last environment with none of it. for client in clients { - client.eventReporter.flushReportingOutcome { result in + client.eventProcessor.flushReportingOutcome { result in lock.lock() delivered = delivered && result lock.unlock() @@ -169,6 +169,6 @@ extension LDClient { } func internalFlush() { - eventReporter.flush(completion: nil) + eventProcessor.flush(completion: nil) } } diff --git a/LaunchDarkly/LaunchDarkly/LDClientVariation.swift b/LaunchDarkly/LaunchDarkly/LDClientVariation.swift index 77bd60f85..a3dd71352 100644 --- a/LaunchDarkly/LaunchDarkly/LDClientVariation.swift +++ b/LaunchDarkly/LaunchDarkly/LDClientVariation.swift @@ -220,7 +220,7 @@ extension LDClient { result = LDEvaluationDetail(value: defaultValue, variationIndex: nil, reason: ["kind": "ERROR", "errorKind": "FLAG_NOT_FOUND"]) } - eventReporter.recordFlagEvaluationEvents(flagKey: flagKey, + eventProcessor.recordFlagEvaluationEvents(flagKey: flagKey, value: result.value.toLDValue(), defaultValue: defaultValue.toLDValue(), featureFlag: featureFlag, diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/ClientServiceFactory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/ClientServiceFactory.swift index 79b162f5e..1027d9ab5 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/ClientServiceFactory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/ClientServiceFactory.swift @@ -15,8 +15,8 @@ protocol ClientServiceCreating { service: DarklyServiceProvider, onSyncComplete: FlagSyncCompleteClosure?) -> LDFlagSynchronizing func makeFlagChangeNotifier() -> FlagChangeNotifying - func makeEventReporter(config: LDConfig, service: DarklyServiceProvider) -> EventReporting - func makeEventReporter(config: LDConfig, service: DarklyServiceProvider, onSyncComplete: EventSyncCompleteClosure?) -> EventReporting + func makeEventProcessor(config: LDConfig, service: DarklyServiceProvider) -> EventProcessor + func makeEventProcessor(config: LDConfig, service: DarklyServiceProvider, onSyncComplete: EventSyncCompleteClosure?) -> EventProcessor func makeStreamingProvider(url: URL, httpHeaders: [String: String], connectMethod: String, connectBody: Data?, handler: EventHandler, delegate: RequestHeaderTransform?, errorHandler: ConnectionErrorHandler?) -> DarklyStreamingProvider func makeEnvironmentReporter(config: LDConfig) -> EnvironmentReporting func makeThrottler(environmentReporter: EnvironmentReporting) -> Throttling @@ -66,15 +66,15 @@ final class ClientServiceFactory: ClientServiceCreating { FlagChangeNotifier(logger: logger) } - func makeEventReporter(config: LDConfig, service: DarklyServiceProvider) -> EventReporting { - makeEventReporter(config: config, service: service, onSyncComplete: nil) + func makeEventProcessor(config: LDConfig, service: DarklyServiceProvider) -> EventProcessor { + makeEventProcessor(config: config, service: service, onSyncComplete: nil) } - func makeEventReporter(config: LDConfig, service: DarklyServiceProvider, onSyncComplete: EventSyncCompleteClosure? = nil) -> EventReporting { + func makeEventProcessor(config: LDConfig, service: DarklyServiceProvider, onSyncComplete: EventSyncCompleteClosure? = nil) -> EventProcessor { if config.sendEvents { - return EventReporter(service: service, onSyncComplete: onSyncComplete) + return DirectEventProcessor(service: service, onSyncComplete: onSyncComplete) } else { - return NullEventReporter() + return NullEventProcessor() } } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DeliveryClaims.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DeliveryClaims.swift index 161cc2f79..18ebd386a 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DeliveryClaims.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DeliveryClaims.swift @@ -6,7 +6,7 @@ import Foundation /// caller's events may belong to a delivery already under way, and answering that caller means hearing from it. These /// are the two small pieces that arrange it: a `Claim` is a batch somebody is waiting on, and a `DeliveryAnswer` /// gathers the claims one delivery's answer depends on. -extension EventReporter { +extension DirectEventProcessor { /// A batch one delivery has taken, which another delivery's answer may depend on. final class Claim { /// This batch's result, or nil while the delivery holding it is still waiting. Only to be used while holding diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift similarity index 89% rename from LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift rename to LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 94dfbd2c4..b553c14f1 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -1,85 +1,6 @@ import Foundation import OSLog -typealias EventSyncCompleteClosure = ((SynchronizingError?) -> Void) -/// Reports whether LaunchDarkly accepted every event a flush covered. -typealias FlushOutcomeClosure = (Bool) -> Void -// sourcery: autoMockable -protocol EventReporting { - // sourcery: defaultMockValue = false - var isOnline: Bool { get set } - // sourcery: defaultMockValue = Date.distantPast - var lastEventResponseDate: Date { get } - - func record(_ event: Event) - // swiftlint:disable:next function_parameter_count - func recordFlagEvaluationEvents(flagKey: LDFlagKey, - value: LDValue, - defaultValue: LDValue, - featureFlag: FeatureFlag?, - context: LDContext, - includeReason: Bool) - func flush(completion: CompletionClosure?) - - /// Like `flush`. Reports `true` if LaunchDarkly accepted the pending batches, or there were none to send. Reports - /// `false` if the reporter is offline, a batch was refused for good, a retryable failure left batches on disk, or - /// an event the SDK had accepted could not be kept to send at all. - /// - /// Flushes arriving faster than a delivery completes join the one that is queued, since a delivery that has not - /// started yet will send their events too. A delivery that has already started is not joined, so an answer is - /// always about the batch the caller's own delivery took, never one an earlier delivery lost. - /// - /// The completion, like `flush`'s, runs on a background queue, and never on the reporter's own: a completion is - /// free to record events or flush again without waiting on the queue it is running on. It is not the caller's - /// thread either, and two completions may run at once, so anything it touches needs to expect that. - func flushReportingOutcome(completion: @escaping FlushOutcomeClosure) - - /// Makes everything recorded so far outlive the process, without waiting for a delivery. - /// - /// The SDK does this itself at the points where an application is most likely to be about to die. It is worth - /// calling directly before deliberately terminating the process. - func commitRecordedEvents() - - /// Commits as a commit point does: before returning at `.immediate`, on a background queue at `.deferred`, and not - /// at all where nothing is persisted, where a commit would only move the encode earlier. - func commitAtCommitPoint() -} - -class NullEventReporter: EventReporting { - var isOnline: Bool = true - var lastEventResponseDate: Date = Date() - - func record(_ event: Event) { - } - - func recordFlagEvaluationEvents(flagKey: LDFlagKey, - value: LDValue, - defaultValue: LDValue, - featureFlag: FeatureFlag?, - context: LDContext, - includeReason: Bool) { - } - - // There is never anything to send, but a completion still answers on a background queue, as `EventReporter`'s - // does. Were it answered inline, the threading a caller sees would depend on whether events are turned on, and a - // caller doing what the contract allows -- taking a lock it already holds, say -- would deadlock only there. - func flush(completion: CompletionClosure?) { - guard let completion - else { return } - DispatchQueue.global().async { completion() } - } - - func flushReportingOutcome(completion: @escaping FlushOutcomeClosure) { - DispatchQueue.global().async { completion(true) } - } - - func commitRecordedEvents() { - } - - func commitAtCommitPoint() { - } -} - /// Records analytics events and delivers them to LaunchDarkly. /// /// Recorded events end up in an `EventStore`, an on-disk log, rather than held in memory until a delivery succeeds. @@ -100,7 +21,7 @@ class NullEventReporter: EventReporting { /// Whether a commit point runs on the caller's thread is the application's choice, through /// `LDConfig.eventPersistence`. Only at `.immediate` can `track` promise the event is on disk by the time it returns. /// At `.disabled` nothing commits early: the held run waits for the delivery, which encodes it. -class EventReporter: EventReporting { +class DirectEventProcessor: EventProcessor { /// How many full events may be held unencoded before one of them pays for a commit. /// /// It sets two things at once. The first is how many evaluations a termination can take -- not everything a crash @@ -243,7 +164,7 @@ class EventReporter: EventReporting { set { timerQueue.sync { newValue ? startReporting() : stopReporting() } } } - private var timerQueue = DispatchQueue(label: "com.launchdarkly.EventReporter.timerQueue") + private var timerQueue = DispatchQueue(label: "com.launchdarkly.DirectEventProcessor.timerQueue") private var eventReportTimer: TimeResponding? var isReportingActive: Bool { eventReportTimer != nil } @@ -262,7 +183,7 @@ class EventReporter: EventReporting { onSyncComplete: EventSyncCompleteClosure?, store: EventStoring? = nil, encoding: Encoding = .handWritten, - commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.EventReporter.commitQueue", qos: .userInitiated)) { + commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.DirectEventProcessor.commitQueue", qos: .userInitiated)) { self.eventJSONWriter = EventJSONWriter(config: service.config) self.service = service self.onSyncComplete = onSyncComplete @@ -270,9 +191,9 @@ class EventReporter: EventReporting { self.encoding = encoding self.capacity = service.config.eventCapacity self.commitQueue = commitQueue - self.encoder = EventReporter.makeEncoder(config: service.config) + self.encoder = DirectEventProcessor.makeEncoder(config: service.config) self.contextSummarizer = ContextSummarizer(logger: service.config.logger, maxContexts: service.config.eventCapacity) - self.store = store ?? EventReporter.makeStore(config: service.config) + self.store = store ?? DirectEventProcessor.makeStore(config: service.config) // A log left open by a previous run has to be closed before it can be delivered, but nothing waits on that: // doing it on the caller's thread would put file I/O in the way of the client starting up. @@ -299,13 +220,13 @@ class EventReporter: EventReporting { func record(_ event: Event) { let (dropped, needsCommit) = recordLock.withLock { - (!admitToPending(event), pending.count >= EventReporter.pendingCommitThreshold) + (!admitToPending(event), pending.count >= DirectEventProcessor.pendingCommitThreshold) } if dropped { reportDropped() } - if EventReporter.isCommitPoint(event.kind) { + if DirectEventProcessor.isCommitPoint(event.kind) { commitAtCommitPoint() } else if needsCommit { scheduleCommitWherePersisting() @@ -330,7 +251,7 @@ class EventReporter: EventReporting { let counted = contextSummarizer.trackRequest(flagKey: flagKey, reportedValue: value, featureFlag: featureFlag, defaultValue: defaultValue, context: context) if let featureEvent, !admitToPending(featureEvent) { dropped += 1 } if let debugEvent, !admitToPending(debugEvent) { dropped += 1 } - return (counted, pending.count >= EventReporter.pendingCommitThreshold) + return (counted, pending.count >= DirectEventProcessor.pendingCommitThreshold) } // A refused evaluation is a loss like a refused event, since nothing later reconstructs its counter. Its full @@ -650,7 +571,7 @@ class EventReporter: EventReporting { private func reportEvents(completion: ((_ delivered: Bool, _ lossMark: UInt64) -> Void)?) { guard isOnline else { - os_log("%s aborted. EventReporter is offline", log: service.config.logger, type: .debug, typeName(and: #function)) + os_log("%s aborted. DirectEventProcessor is offline", log: service.config.logger, type: .debug, typeName(and: #function)) reportSyncComplete(.isOffline) completion?(false, 0) return @@ -871,7 +792,7 @@ class EventReporter: EventReporting { } private func reportSyncComplete(_ result: SynchronizingError?) { - // The eventReporter is created when the LDClient singleton is created, and kept for the app's lifetime. So while the use of self in the async block does setup a retain cycle, it's not going to cause a memory leak + // The eventProcessor is created when the LDClient singleton is created, and kept for the app's lifetime. So while the use of self in the async block does setup a retain cycle, it's not going to cause a memory leak guard let onSyncComplete = onSyncComplete else { return } DispatchQueue.main.async { @@ -880,7 +801,7 @@ class EventReporter: EventReporting { } } -extension EventReporter { +extension DirectEventProcessor { /// Which encoder recorded events go through. /// /// `.codable` is the reference `.handWritten` is tested against. The two produce equal JSON but not identical @@ -904,10 +825,10 @@ extension EventReporter { } } -extension EventReporter: TypeIdentifying { } +extension DirectEventProcessor: TypeIdentifying { } #if DEBUG - extension EventReporter { + extension DirectEventProcessor { /// Full events accepted but not yet handed to a commit. var pendingEventsForTesting: [Event] { recordLock.withLock { pending } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift new file mode 100644 index 000000000..2f2d4ede5 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift @@ -0,0 +1,45 @@ +import Foundation + +typealias EventSyncCompleteClosure = ((SynchronizingError?) -> Void) +/// Reports whether LaunchDarkly accepted every event a flush covered. +typealias FlushOutcomeClosure = (Bool) -> Void +// sourcery: autoMockable +protocol EventProcessor { + // sourcery: defaultMockValue = false + var isOnline: Bool { get set } + // sourcery: defaultMockValue = Date.distantPast + var lastEventResponseDate: Date { get } + + func record(_ event: Event) + // swiftlint:disable:next function_parameter_count + func recordFlagEvaluationEvents(flagKey: LDFlagKey, + value: LDValue, + defaultValue: LDValue, + featureFlag: FeatureFlag?, + context: LDContext, + includeReason: Bool) + func flush(completion: CompletionClosure?) + + /// Like `flush`. Reports `true` if LaunchDarkly accepted the pending batches, or there were none to send. Reports + /// `false` if the reporter is offline, a batch was refused for good, a retryable failure left batches on disk, or + /// an event the SDK had accepted could not be kept to send at all. + /// + /// Flushes arriving faster than a delivery completes join the one that is queued, since a delivery that has not + /// started yet will send their events too. A delivery that has already started is not joined, so an answer is + /// always about the batch the caller's own delivery took, never one an earlier delivery lost. + /// + /// The completion, like `flush`'s, runs on a background queue, and never on the reporter's own: a completion is + /// free to record events or flush again without waiting on the queue it is running on. It is not the caller's + /// thread either, and two completions may run at once, so anything it touches needs to expect that. + func flushReportingOutcome(completion: @escaping FlushOutcomeClosure) + + /// Makes everything recorded so far outlive the process, without waiting for a delivery. + /// + /// The SDK does this itself at the points where an application is most likely to be about to die. It is worth + /// calling directly before deliberately terminating the process. + func commitRecordedEvents() + + /// Commits as a commit point does: before returning at `.immediate`, on a background queue at `.deferred`, and not + /// at all where nothing is persisted, where a commit would only move the encode earlier. + func commitAtCommitPoint() +} diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventProcessor.swift new file mode 100644 index 000000000..fa3658500 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventProcessor.swift @@ -0,0 +1,37 @@ +import Foundation + +class NullEventProcessor: EventProcessor { + var isOnline: Bool = true + var lastEventResponseDate: Date = Date() + + func record(_ event: Event) { + } + + func recordFlagEvaluationEvents(flagKey: LDFlagKey, + value: LDValue, + defaultValue: LDValue, + featureFlag: FeatureFlag?, + context: LDContext, + includeReason: Bool) { + } + + // There is never anything to send, but a completion still answers on a background queue, as + // `DirectEventProcessor`'s does. Were it answered inline, the threading a caller sees would depend on whether events + // are turned on, and a caller doing what the contract allows -- taking a lock it already holds, say -- would + // deadlock only there. + func flush(completion: CompletionClosure?) { + guard let completion + else { return } + DispatchQueue.global().async { completion() } + } + + func flushReportingOutcome(completion: @escaping FlushOutcomeClosure) { + DispatchQueue.global().async { completion(true) } + } + + func commitRecordedEvents() { + } + + func commitAtCommitPoint() { + } +} diff --git a/LaunchDarkly/LaunchDarklyTests/EventJSONWriterTests.swift b/LaunchDarkly/LaunchDarklyTests/EventJSONWriterTests.swift index 05868c036..7d268b147 100644 --- a/LaunchDarkly/LaunchDarklyTests/EventJSONWriterTests.swift +++ b/LaunchDarkly/LaunchDarklyTests/EventJSONWriterTests.swift @@ -534,8 +534,8 @@ final class EventJSONWriterTests: XCTestCase { let service = DarklyServiceMock() service.config = config - let codableReporter = EventReporter(service: service, onSyncComplete: nil, store: codableStore, encoding: .codable) - let handWrittenReporter = EventReporter(service: service, onSyncComplete: nil, store: handWrittenStore, encoding: .handWritten) + let codableReporter = DirectEventProcessor(service: service, onSyncComplete: nil, store: codableStore, encoding: .codable) + let handWrittenReporter = DirectEventProcessor(service: service, onSyncComplete: nil, store: handWrittenStore, encoding: .handWritten) codableReporter.record(event) handWrittenReporter.record(event) diff --git a/LaunchDarkly/LaunchDarklyTests/LDClientEvaluationHookSpec.swift b/LaunchDarkly/LaunchDarklyTests/LDClientEvaluationHookSpec.swift index 95d7066d5..c54528d5a 100644 --- a/LaunchDarkly/LaunchDarklyTests/LDClientEvaluationHookSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/LDClientEvaluationHookSpec.swift @@ -219,7 +219,7 @@ final class LDClientEvaluationHookSpec: XCTestCase { } // Deduplication applies to hooks only, so analytics events are unaffected. - XCTAssertEqual(testContext.eventReporterMock.recordFlagEvaluationEventsCallCount, 3) + XCTAssertEqual(testContext.eventProcessorMock.recordFlagEvaluationEventsCallCount, 3) } func testEvaluationsOfDifferentFlagsReachHooksSeparately() { diff --git a/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift b/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift index 2c896aae4..efea986d4 100644 --- a/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift @@ -67,7 +67,7 @@ final class LDClientSpec: QuickSpec { let testContext = TestContext(startOnline: true) testContext.start() - expect(testContext.serviceFactoryMock.makeEventReporterCallCount) == 1 + expect(testContext.serviceFactoryMock.makeEventProcessorCallCount) == 1 expect(testContext.serviceFactoryMock.makeFlagSynchronizerCallCount) == 1 } @@ -91,10 +91,10 @@ final class LDClientSpec: QuickSpec { it("takes the client and service objects online") { expect(testContext.subject.isOnline) == true expect(testContext.subject.flagSynchronizer.isOnline) == testContext.subject.isOnline - expect(testContext.subject.eventReporter.isOnline) == testContext.subject.isOnline + expect(testContext.subject.eventProcessor.isOnline) == testContext.subject.isOnline } it("creates one event reporter, so one store owns the event directory") { - expect(testContext.serviceFactoryMock.makeEventReporterCallCount) == 1 + expect(testContext.serviceFactoryMock.makeEventProcessorCallCount) == 1 expect(testContext.serviceFactoryMock.onEventSyncComplete).toNot(beNil()) } it("saves the config") { @@ -102,7 +102,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.subject.service.config) == testContext.config expect(testContext.makeFlagSynchronizerStreamingMode) == testContext.config.streamingMode expect(testContext.makeFlagSynchronizerPollingInterval) == testContext.config.flagPollingInterval(runMode: testContext.subject.runMode) - expect(testContext.serviceFactoryMock.makeEventReporterReceivedService?.config) == testContext.config + expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.config) == testContext.config } it("saves the context") { expect(testContext.subject.context) == testContext.context @@ -111,14 +111,14 @@ final class LDClientSpec: QuickSpec { if let makeFlagSynchronizerReceivedParameters = testContext.serviceFactoryMock.makeFlagSynchronizerReceivedParameters { expect(makeFlagSynchronizerReceivedParameters.service) === testContext.subject.service } - expect(testContext.serviceFactoryMock.makeEventReporterReceivedService?.context) == testContext.context + expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.context) == testContext.context } it("uncaches the new contexts flags") { expect(testContext.featureFlagCachingMock.getCachedDataCallCount) == 1 expect(testContext.featureFlagCachingMock.getCachedDataReceivedArguments?.cacheKey) == testContext.context.fullyQualifiedHashedKey() } it("records an identify event") { - expect(testContext.eventReporterMock.recordCallCount) == 1 + expect(testContext.eventProcessorMock.recordCallCount) == 1 expect((testContext.recordedEvent as? IdentifyEvent)?.context) == testContext.context } it("converts cached data") { @@ -138,14 +138,14 @@ final class LDClientSpec: QuickSpec { it("leaves the client and service objects offline") { expect(testContext.subject.isOnline) == false expect(testContext.subject.flagSynchronizer.isOnline) == testContext.subject.isOnline - expect(testContext.subject.eventReporter.isOnline) == testContext.subject.isOnline + expect(testContext.subject.eventProcessor.isOnline) == testContext.subject.isOnline } it("saves the config") { expect(testContext.subject.config) == testContext.config expect(testContext.subject.service.config) == testContext.config expect(testContext.makeFlagSynchronizerStreamingMode) == testContext.config.streamingMode expect(testContext.makeFlagSynchronizerPollingInterval) == testContext.config.flagPollingInterval(runMode: testContext.subject.runMode) - expect(testContext.serviceFactoryMock.makeEventReporterReceivedService?.config) == testContext.config + expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.config) == testContext.config } it("saves the context") { expect(testContext.subject.context) == testContext.context @@ -154,14 +154,14 @@ final class LDClientSpec: QuickSpec { if let makeFlagSynchronizerReceivedParameters = testContext.serviceFactoryMock.makeFlagSynchronizerReceivedParameters { expect(makeFlagSynchronizerReceivedParameters.service) === testContext.subject.service } - expect(testContext.serviceFactoryMock.makeEventReporterReceivedService?.context) == testContext.context + expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.context) == testContext.context } it("uncaches the new contexts flags") { expect(testContext.featureFlagCachingMock.getCachedDataCallCount) == 1 expect(testContext.featureFlagCachingMock.getCachedDataReceivedArguments?.cacheKey) == testContext.context.fullyQualifiedHashedKey() } it("records an identify event") { - expect(testContext.eventReporterMock.recordCallCount) == 1 + expect(testContext.eventProcessorMock.recordCallCount) == 1 expect((testContext.recordedEvent as? IdentifyEvent)?.context) == testContext.context } it("converts cached data") { @@ -187,7 +187,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.subject.service.config) == testContext.config expect(testContext.makeFlagSynchronizerStreamingMode) == testContext.config.streamingMode expect(testContext.makeFlagSynchronizerPollingInterval) == testContext.config.flagPollingInterval(runMode: testContext.subject.runMode) - expect(testContext.serviceFactoryMock.makeEventReporterReceivedService?.config) == testContext.config + expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.config) == testContext.config } it("saves the context") { expect(testContext.subject.context) == testContext.context @@ -196,14 +196,14 @@ final class LDClientSpec: QuickSpec { if let makeFlagSynchronizerReceivedParameters = testContext.serviceFactoryMock.makeFlagSynchronizerReceivedParameters { expect(makeFlagSynchronizerReceivedParameters.service.context) == testContext.context } - expect(testContext.serviceFactoryMock.makeEventReporterReceivedService?.context) == testContext.context + expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.context) == testContext.context } it("uncaches the new contexts flags") { expect(testContext.featureFlagCachingMock.getCachedDataCallCount) == 2 // called on init and subsequent identify expect(testContext.featureFlagCachingMock.getCachedDataReceivedArguments?.cacheKey) == testContext.context.fullyQualifiedHashedKey() } it("records an identify event") { - expect(testContext.eventReporterMock.recordCallCount) == 2 // both start and internalIdentify + expect(testContext.eventProcessorMock.recordCallCount) == 2 // both start and internalIdentify expect((testContext.recordedEvent as? IdentifyEvent)?.context) == testContext.context } it("converts cached data") { @@ -222,20 +222,20 @@ final class LDClientSpec: QuickSpec { expect(testContext.subject.service.config) == testContext.config expect(testContext.makeFlagSynchronizerStreamingMode) == testContext.config.streamingMode expect(testContext.makeFlagSynchronizerPollingInterval) == testContext.config.flagPollingInterval(runMode: testContext.subject.runMode) - expect(testContext.serviceFactoryMock.makeEventReporterReceivedService?.config) == testContext.config + expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.config) == testContext.config } it("uses anonymous context") { expect(testContext.subject.context.fullyQualifiedKey()) == LDContext.defaultKey(kind: testContext.subject.context.kind) expect(testContext.subject.service.context) == testContext.subject.context expect(testContext.makeFlagSynchronizerService?.context) == testContext.subject.context - expect(testContext.serviceFactoryMock.makeEventReporterReceivedService?.context) == testContext.subject.context + expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.context) == testContext.subject.context } it("uncaches the new contexts flags") { expect(testContext.featureFlagCachingMock.getCachedDataCallCount) == 1 expect(testContext.featureFlagCachingMock.getCachedDataReceivedArguments?.cacheKey) == testContext.subject.context.fullyQualifiedHashedKey() } it("records an identify event") { - expect(testContext.eventReporterMock.recordCallCount) == 1 + expect(testContext.eventProcessorMock.recordCallCount) == 1 expect((testContext.recordedEvent as? IdentifyEvent)?.context) == testContext.subject.context } it("converts cached data") { @@ -383,7 +383,7 @@ final class LDClientSpec: QuickSpec { if os == .iOS && os == SystemCapabilities.operatingSystem { expect(testContext.subject.flagSynchronizer.isOnline) == os.isBackgroundEnabled } - expect(testContext.subject.eventReporter.isOnline) == testContext.subject.isOnline + expect(testContext.subject.eventProcessor.isOnline) == testContext.subject.isOnline } it("saves the config") { expect(testContext.subject.service.config) == testContext.config @@ -392,7 +392,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.makeFlagSynchronizerStreamingMode) == os.backgroundStreamingMode } expect(testContext.makeFlagSynchronizerPollingInterval) == testContext.config.flagPollingInterval(runMode: .background) - expect(testContext.serviceFactoryMock.makeEventReporterReceivedService?.config) == testContext.config + expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.config) == testContext.config } it("saves the context") { expect(testContext.subject.context) == testContext.context @@ -401,14 +401,14 @@ final class LDClientSpec: QuickSpec { if let makeFlagSynchronizerReceivedParameters = testContext.serviceFactoryMock.makeFlagSynchronizerReceivedParameters { expect(makeFlagSynchronizerReceivedParameters.service) === testContext.subject.service } - expect(testContext.serviceFactoryMock.makeEventReporterReceivedService?.context) == testContext.context + expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.context) == testContext.context } it("uncaches the new contexts flags") { expect(testContext.featureFlagCachingMock.getCachedDataCallCount) == 1 expect(testContext.featureFlagCachingMock.getCachedDataReceivedArguments?.cacheKey) == testContext.context.fullyQualifiedHashedKey() } it("records an identify event") { - expect(testContext.eventReporterMock.recordCallCount) == 1 + expect(testContext.eventProcessorMock.recordCallCount) == 1 expect((testContext.recordedEvent as? IdentifyEvent)?.context) == testContext.context } } @@ -425,13 +425,13 @@ final class LDClientSpec: QuickSpec { it("leaves the client and service objects offline") { expect(testContext.subject.isOnline) == true expect(testContext.subject.flagSynchronizer.isOnline) == false - expect(testContext.subject.eventReporter.isOnline) == true + expect(testContext.subject.eventProcessor.isOnline) == true } it("saves the config") { expect(testContext.subject.service.config) == testContext.config expect(testContext.makeFlagSynchronizerStreamingMode) == LDStreamingMode.polling expect(testContext.makeFlagSynchronizerPollingInterval) == testContext.config.flagPollingInterval(runMode: .background) - expect(testContext.serviceFactoryMock.makeEventReporterReceivedService?.config) == testContext.config + expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.config) == testContext.config } it("saves the context") { expect(testContext.subject.context) == testContext.context @@ -440,14 +440,14 @@ final class LDClientSpec: QuickSpec { if let makeFlagSynchronizerReceivedParameters = testContext.serviceFactoryMock.makeFlagSynchronizerReceivedParameters { expect(makeFlagSynchronizerReceivedParameters.service.context) == testContext.context } - expect(testContext.serviceFactoryMock.makeEventReporterReceivedService?.context) == testContext.context + expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.context) == testContext.context } it("uncaches the new contexts flags") { expect(testContext.featureFlagCachingMock.getCachedDataCallCount) == 1 expect(testContext.featureFlagCachingMock.getCachedDataReceivedArguments?.cacheKey) == testContext.context.fullyQualifiedHashedKey() } it("records an identify event") { - expect(testContext.eventReporterMock.recordCallCount) == 1 + expect(testContext.eventProcessorMock.recordCallCount) == 1 expect((testContext.recordedEvent as? IdentifyEvent)?.context) == testContext.context } } @@ -472,13 +472,13 @@ final class LDClientSpec: QuickSpec { expect(testContext.makeFlagSynchronizerService?.context) == newContext expect(testContext.subject.isOnline) == true - expect(testContext.subject.eventReporter.isOnline) == true + expect(testContext.subject.eventProcessor.isOnline) == true expect(testContext.subject.flagSynchronizer.isOnline) == true expect(testContext.featureFlagCachingMock.getCachedDataCallCount) == 1 expect(testContext.featureFlagCachingMock.getCachedDataReceivedArguments?.cacheKey) == newContext.fullyQualifiedHashedKey() - expect(testContext.eventReporterMock.recordReceivedEvent?.kind == .identify).to(beTrue()) + expect(testContext.eventProcessorMock.recordReceivedEvent?.kind == .identify).to(beTrue()) } it("when the client is offline") { let testContext = TestContext() @@ -494,13 +494,13 @@ final class LDClientSpec: QuickSpec { expect(testContext.makeFlagSynchronizerService?.context) == newContext expect(testContext.subject.isOnline) == false - expect(testContext.subject.eventReporter.isOnline) == false + expect(testContext.subject.eventProcessor.isOnline) == false expect(testContext.subject.flagSynchronizer.isOnline) == false expect(testContext.featureFlagCachingMock.getCachedDataCallCount) == 1 expect(testContext.featureFlagCachingMock.getCachedDataReceivedArguments?.cacheKey) == newContext.fullyQualifiedHashedKey() - expect(testContext.eventReporterMock.recordReceivedEvent?.kind == .identify).to(beTrue()) + expect(testContext.eventProcessorMock.recordReceivedEvent?.kind == .identify).to(beTrue()) } it("when the new context has cached feature flags") { let stubFlags = FlagMaintainingMock.stubStoredItems() @@ -547,9 +547,9 @@ final class LDClientSpec: QuickSpec { expect(testContext.makeFlagSynchronizerService?.context) == testContext.context expect(testContext.subject.isOnline) == true - expect(testContext.subject.eventReporter.isOnline) == true + expect(testContext.subject.eventProcessor.isOnline) == true expect(testContext.subject.flagSynchronizer.isOnline) == true - expect(testContext.eventReporterMock.recordReceivedEvent?.kind == .identify).to(beTrue()) + expect(testContext.eventProcessorMock.recordReceivedEvent?.kind == .identify).to(beTrue()) } it("no cache requires no store interaction") { @@ -563,9 +563,9 @@ final class LDClientSpec: QuickSpec { expect(testContext.makeFlagSynchronizerService?.context) == testContext.context expect(testContext.subject.isOnline) == true - expect(testContext.subject.eventReporter.isOnline) == true + expect(testContext.subject.eventProcessor.isOnline) == true expect(testContext.subject.flagSynchronizer.isOnline) == true - expect(testContext.eventReporterMock.recordReceivedEvent?.kind == .identify).to(beTrue()) + expect(testContext.eventProcessorMock.recordReceivedEvent?.kind == .identify).to(beTrue()) } it("ifAvailable requires no store information on cache miss") { @@ -579,9 +579,9 @@ final class LDClientSpec: QuickSpec { expect(testContext.makeFlagSynchronizerService?.context) == testContext.context expect(testContext.subject.isOnline) == true - expect(testContext.subject.eventReporter.isOnline) == true + expect(testContext.subject.eventProcessor.isOnline) == true expect(testContext.subject.flagSynchronizer.isOnline) == true - expect(testContext.eventReporterMock.recordReceivedEvent?.kind == .identify).to(beTrue()) + expect(testContext.eventProcessorMock.recordReceivedEvent?.kind == .identify).to(beTrue()) } it("ifAvailable updates store when cache is present") { @@ -628,7 +628,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.subject.isOnline) == true expect(testContext.flagSynchronizerMock.isOnline) == false - expect(testContext.subject.eventReporter.isOnline) == true + expect(testContext.subject.eventProcessor.isOnline) == true expect(testContext.subject.isInitialized) == false expect(completed).toEventually(beTrue()) } @@ -692,17 +692,17 @@ final class LDClientSpec: QuickSpec { testContext.start() let original = testContext.featureFlagCachingMock.getCachedDataCallback var isOnlineDuringSwap: Bool? - var eventReporterOnlineDuringSwap: Bool? + var eventProcessorOnlineDuringSwap: Bool? testContext.featureFlagCachingMock.getCachedDataCallback = { try original?() isOnlineDuringSwap = testContext.subject.isOnline - eventReporterOnlineDuringSwap = testContext.subject.eventReporter.isOnline + eventProcessorOnlineDuringSwap = testContext.subject.eventProcessor.isOnline } testContext.subject.internalIdentify(newContext: LDContext.stub(), useCache: .no) expect(isOnlineDuringSwap) == true - expect(eventReporterOnlineDuringSwap) == true + expect(eventProcessorOnlineDuringSwap) == true } it("recovers when the app backgrounds midway through an identify") { let testContext = TestContext(startOnline: true, enableBackgroundUpdates: false) @@ -807,7 +807,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.throttlerMock?.runThrottledCallCount) == 1 expect(testContext.subject.isOnline) == true expect(testContext.subject.flagSynchronizer.isOnline) == testContext.subject.isOnline - expect(testContext.subject.eventReporter.isOnline) == testContext.subject.isOnline + expect(testContext.subject.eventProcessor.isOnline) == testContext.subject.isOnline } it("set offline when the client is online") { let testContext = TestContext(startOnline: true) @@ -818,7 +818,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.throttlerMock?.runThrottledCallCount) == 0 expect(testContext.subject.isOnline) == false expect(testContext.subject.flagSynchronizer.isOnline) == testContext.subject.isOnline - expect(testContext.subject.eventReporter.isOnline) == testContext.subject.isOnline + expect(testContext.subject.eventProcessor.isOnline) == testContext.subject.isOnline } context("set online when the client runs in the background") { OperatingSystem.allOperatingSystems.forEach { os in @@ -838,7 +838,7 @@ final class LDClientSpec: QuickSpec { } expect(testContext.makeFlagSynchronizerPollingInterval) == testContext.config.flagPollingInterval(runMode: testContext.subject.runMode) - expect(testContext.subject.eventReporter.isOnline) == testContext.subject.isOnline + expect(testContext.subject.eventProcessor.isOnline) == testContext.subject.isOnline } it("while configured to disable background updates") { let testContext = TestContext(enableBackgroundUpdates: false) @@ -850,7 +850,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.subject.flagSynchronizer.isOnline) == testContext.subject.isOnline expect(testContext.makeFlagSynchronizerStreamingMode) == LDStreamingMode.polling expect(testContext.makeFlagSynchronizerPollingInterval) == testContext.config.flagPollingInterval(runMode: .background) - expect(testContext.subject.eventReporter.isOnline) == testContext.subject.isOnline + expect(testContext.subject.eventProcessor.isOnline) == testContext.subject.isOnline } } } @@ -863,7 +863,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.throttlerMock?.runThrottledCallCount) == 0 expect(testContext.subject.isOnline) == false expect(testContext.subject.flagSynchronizer.isOnline) == testContext.subject.isOnline - expect(testContext.subject.eventReporter.isOnline) == testContext.subject.isOnline + expect(testContext.subject.eventProcessor.isOnline) == testContext.subject.isOnline } } } @@ -876,7 +876,7 @@ final class LDClientSpec: QuickSpec { testContext.subject.close() expect(testContext.subject.isOnline) == false - expect(testContext.eventReporterMock.flushCallCount) == 1 + expect(testContext.eventProcessorMock.flushCallCount) == 1 } it("when started and offline") { let testContext = TestContext() @@ -884,7 +884,7 @@ final class LDClientSpec: QuickSpec { testContext.subject.close() expect(testContext.subject.isOnline) == false - expect(testContext.eventReporterMock.flushCallCount) == 1 + expect(testContext.eventProcessorMock.flushCallCount) == 1 } it("when already stopped") { let testContext = TestContext() @@ -893,7 +893,7 @@ final class LDClientSpec: QuickSpec { testContext.subject.close() expect(testContext.subject.isOnline) == false - expect(testContext.eventReporterMock.flushCallCount) == 1 + expect(testContext.eventProcessorMock.flushCallCount) == 1 } } } @@ -904,7 +904,7 @@ final class LDClientSpec: QuickSpec { let testContext = TestContext() testContext.start() testContext.subject.track(key: "customEvent", data: "abc", metricValue: 5.0) - let receivedEvent = testContext.eventReporterMock.recordReceivedEvent as? CustomEvent + let receivedEvent = testContext.eventProcessorMock.recordReceivedEvent as? CustomEvent expect(receivedEvent?.key) == "customEvent" expect(receivedEvent?.context) == testContext.context expect(receivedEvent?.data) == "abc" @@ -915,17 +915,17 @@ final class LDClientSpec: QuickSpec { config.sendEvents = false let testContext = TestContext(newConfig: config) testContext.start() - let priorRecordedEvents = testContext.eventReporterMock.recordCallCount + let priorRecordedEvents = testContext.eventProcessorMock.recordCallCount testContext.subject.track(key: "customEvent", data: "abc", metricValue: 5.0) - expect(testContext.eventReporterMock.recordCallCount) == priorRecordedEvents + expect(testContext.eventProcessorMock.recordCallCount) == priorRecordedEvents } context("does not record when client was stopped") { let testContext = TestContext() testContext.start() testContext.subject.close() - let priorRecordedEvents = testContext.eventReporterMock.recordCallCount + let priorRecordedEvents = testContext.eventProcessorMock.recordCallCount testContext.subject.track(key: "abc") - expect(testContext.eventReporterMock.recordCallCount) == priorRecordedEvents + expect(testContext.eventProcessorMock.recordCallCount) == priorRecordedEvents } } } @@ -954,12 +954,12 @@ final class LDClientSpec: QuickSpec { } it("records a flag evaluation event") { _ = testContext.subject.boolVariation(forKey: DarklyServiceMock.FlagKeys.bool, defaultValue: DefaultFlagValues.bool) - expect(testContext.eventReporterMock.recordFlagEvaluationEventsCallCount) == 1 - expect(testContext.eventReporterMock.recordFlagEvaluationEventsReceivedArguments?.flagKey) == DarklyServiceMock.FlagKeys.bool - expect(testContext.eventReporterMock.recordFlagEvaluationEventsReceivedArguments?.value) == DarklyServiceMock.FlagValues.bool - expect(testContext.eventReporterMock.recordFlagEvaluationEventsReceivedArguments?.defaultValue) == .bool(DefaultFlagValues.bool) - expect(testContext.eventReporterMock.recordFlagEvaluationEventsReceivedArguments?.featureFlag) == testContext.flagStoreMock.storedItems.featureFlags[DarklyServiceMock.FlagKeys.bool] - expect(testContext.eventReporterMock.recordFlagEvaluationEventsReceivedArguments?.context) == testContext.context + expect(testContext.eventProcessorMock.recordFlagEvaluationEventsCallCount) == 1 + expect(testContext.eventProcessorMock.recordFlagEvaluationEventsReceivedArguments?.flagKey) == DarklyServiceMock.FlagKeys.bool + expect(testContext.eventProcessorMock.recordFlagEvaluationEventsReceivedArguments?.value) == DarklyServiceMock.FlagValues.bool + expect(testContext.eventProcessorMock.recordFlagEvaluationEventsReceivedArguments?.defaultValue) == .bool(DefaultFlagValues.bool) + expect(testContext.eventProcessorMock.recordFlagEvaluationEventsReceivedArguments?.featureFlag) == testContext.flagStoreMock.storedItems.featureFlags[DarklyServiceMock.FlagKeys.bool] + expect(testContext.eventProcessorMock.recordFlagEvaluationEventsReceivedArguments?.context) == testContext.context } } } @@ -975,12 +975,12 @@ final class LDClientSpec: QuickSpec { } it("records a flag evaluation event") { _ = testContext.subject.boolVariation(forKey: DarklyServiceMock.FlagKeys.bool, defaultValue: DefaultFlagValues.bool) - expect(testContext.eventReporterMock.recordFlagEvaluationEventsCallCount) == 1 - expect(testContext.eventReporterMock.recordFlagEvaluationEventsReceivedArguments?.flagKey) == DarklyServiceMock.FlagKeys.bool - expect(testContext.eventReporterMock.recordFlagEvaluationEventsReceivedArguments?.value) == .bool(DefaultFlagValues.bool) - expect(testContext.eventReporterMock.recordFlagEvaluationEventsReceivedArguments?.defaultValue) == .bool(DefaultFlagValues.bool) - expect(testContext.eventReporterMock.recordFlagEvaluationEventsReceivedArguments?.featureFlag).to(beNil()) - expect(testContext.eventReporterMock.recordFlagEvaluationEventsReceivedArguments?.context) == testContext.context + expect(testContext.eventProcessorMock.recordFlagEvaluationEventsCallCount) == 1 + expect(testContext.eventProcessorMock.recordFlagEvaluationEventsReceivedArguments?.flagKey) == DarklyServiceMock.FlagKeys.bool + expect(testContext.eventProcessorMock.recordFlagEvaluationEventsReceivedArguments?.value) == .bool(DefaultFlagValues.bool) + expect(testContext.eventProcessorMock.recordFlagEvaluationEventsReceivedArguments?.defaultValue) == .bool(DefaultFlagValues.bool) + expect(testContext.eventProcessorMock.recordFlagEvaluationEventsReceivedArguments?.featureFlag).to(beNil()) + expect(testContext.eventProcessorMock.recordFlagEvaluationEventsReceivedArguments?.context) == testContext.context } } } @@ -994,8 +994,8 @@ final class LDClientSpec: QuickSpec { let storedItems = StoredItems(items: flags) testContext.flagStoreMock.replaceStore(newStoredItems: storedItems) var events = [FeatureEvent]() - testContext.eventReporterMock.recordFlagEvaluationEventsCallback = { - let args = testContext.eventReporterMock.recordFlagEvaluationEventsReceivedArguments! + testContext.eventProcessorMock.recordFlagEvaluationEventsCallback = { + let args = testContext.eventProcessorMock.recordFlagEvaluationEventsReceivedArguments! let event = FeatureEvent(key: args.flagKey, context: args.context, value: args.value, defaultValue: args.defaultValue, featureFlag: args.featureFlag, includeReason: args.includeReason, isDebug: false) events.append(event) } @@ -1024,8 +1024,8 @@ final class LDClientSpec: QuickSpec { var events = [FeatureEvent]() beforeEach { events = [] - testContext.eventReporterMock.recordFlagEvaluationEventsCallback = { - let args = testContext.eventReporterMock.recordFlagEvaluationEventsReceivedArguments! + testContext.eventProcessorMock.recordFlagEvaluationEventsCallback = { + let args = testContext.eventProcessorMock.recordFlagEvaluationEventsReceivedArguments! events.append(FeatureEvent(key: args.flagKey, context: args.context, value: args.value, defaultValue: args.defaultValue, featureFlag: args.featureFlag, includeReason: args.includeReason, isDebug: false)) } } @@ -1305,7 +1305,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.subject.getConnectionInformation().currentConnectionMode) == .establishingStreamingConnection expect(testContext.subject.isInitialized).to(beFalse()) expect(testContext.subject.isOnline).to(beTrue()) - expect(testContext.eventReporterMock.isOnline).to(beTrue()) + expect(testContext.eventProcessorMock.isOnline).to(beTrue()) } it("keeps the data source running on a recoverable flag error") { let testContext = TestContext(startOnline: true) @@ -1369,7 +1369,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.subject.isOnline) == true expect(testContext.subject.runMode) == LDClientRunMode.background - expect(testContext.eventReporterMock.isOnline) == true + expect(testContext.eventProcessorMock.isOnline) == true expect(testContext.flagSynchronizerMock.isOnline) == false } it("tries to deliver what it has before the process is suspended, where it is suspended") { @@ -1378,11 +1378,11 @@ final class LDClientSpec: QuickSpec { NotificationCenter.default.post(name: SystemCapabilities.backgroundNotification!, object: self) #if os(iOS) || os(tvOS) - expect(testContext.eventReporterMock.flushReportingOutcomeCallCount).toEventually(equal(1)) + expect(testContext.eventProcessorMock.flushReportingOutcomeCallCount).toEventually(equal(1)) #else // macOS posts this on every loss of focus, which suspends nothing. expect(testContext.subject.runMode).toEventually(equal(LDClientRunMode.background)) - expect(testContext.eventReporterMock.flushReportingOutcomeCallCount) == 0 + expect(testContext.eventProcessorMock.flushReportingOutcomeCallCount) == 0 #endif } it("background updates enabled") { @@ -1396,7 +1396,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.subject.isOnline) == true expect(testContext.subject.runMode) == LDClientRunMode.background - expect(testContext.eventReporterMock.isOnline) == true + expect(testContext.eventProcessorMock.isOnline) == true // TODO(os-tests): We need to expand this to the other OSs if os == .iOS && os == SystemCapabilities.operatingSystem { @@ -1414,7 +1414,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.subject.isOnline) == false expect(testContext.subject.runMode) == LDClientRunMode.background - expect(testContext.eventReporterMock.isOnline) == false + expect(testContext.eventProcessorMock.isOnline) == false expect(testContext.flagSynchronizerMock.isOnline) == false } } @@ -1431,7 +1431,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.subject.isOnline) == true expect(testContext.subject.runMode) == LDClientRunMode.foreground - expect(testContext.eventReporterMock.isOnline) == true + expect(testContext.eventProcessorMock.isOnline) == true expect(testContext.flagSynchronizerMock.isOnline) == true } it("when offline at foreground notification") { @@ -1441,7 +1441,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.subject.isOnline) == false expect(testContext.subject.runMode) == LDClientRunMode.foreground - expect(testContext.eventReporterMock.isOnline) == false + expect(testContext.eventProcessorMock.isOnline) == false expect(testContext.flagSynchronizerMock.isOnline) == false } } @@ -1461,7 +1461,7 @@ final class LDClientSpec: QuickSpec { testContext.start() testContext.subject.setRunMode(.background) - expect(testContext.eventReporterMock.isOnline) == true + expect(testContext.eventProcessorMock.isOnline) == true expect(testContext.flagSynchronizerMock.isOnline) == true expect(testContext.flagSynchronizerMock.streamingMode) == LDStreamingMode.streaming } @@ -1470,7 +1470,7 @@ final class LDClientSpec: QuickSpec { testContext.start() testContext.subject.setRunMode(.background) - expect(testContext.eventReporterMock.isOnline) == true + expect(testContext.eventProcessorMock.isOnline) == true expect(testContext.flagSynchronizerMock.isOnline) == true expect(testContext.flagSynchronizerMock.streamingMode) == LDStreamingMode.polling expect(testContext.flagSynchronizerMock.pollingInterval) == testContext.config.flagPollingInterval(runMode: .background) @@ -1481,7 +1481,7 @@ final class LDClientSpec: QuickSpec { testContext.start() testContext.subject.setRunMode(.background) - expect(testContext.eventReporterMock.isOnline) == true + expect(testContext.eventProcessorMock.isOnline) == true expect(testContext.flagSynchronizerMock.isOnline) == false expect(testContext.flagSynchronizerMock.streamingMode) == LDStreamingMode.polling expect(testContext.flagSynchronizerMock.pollingInterval) == testContext.config.flagPollingInterval(runMode: .background) @@ -1490,13 +1490,13 @@ final class LDClientSpec: QuickSpec { it("set foreground") { let testContext = TestContext(startOnline: true) testContext.start() - let eventReporterIsOnlineSetCount = testContext.eventReporterMock.isOnlineSetCount + let eventProcessorIsOnlineSetCount = testContext.eventProcessorMock.isOnlineSetCount let flagSynchronizerIsOnlineSetCount = testContext.flagSynchronizerMock.isOnlineSetCount let makeFlagSynchronizerCallCount = testContext.serviceFactoryMock.makeFlagSynchronizerCallCount testContext.subject.setRunMode(.foreground) - expect(testContext.eventReporterMock.isOnline) == true - expect(testContext.eventReporterMock.isOnlineSetCount) == eventReporterIsOnlineSetCount + expect(testContext.eventProcessorMock.isOnline) == true + expect(testContext.eventProcessorMock.isOnlineSetCount) == eventProcessorIsOnlineSetCount expect(testContext.flagSynchronizerMock.isOnline) == true expect(testContext.flagSynchronizerMock.isOnlineSetCount) == flagSynchronizerIsOnlineSetCount expect(testContext.serviceFactoryMock.makeFlagSynchronizerCallCount) == makeFlagSynchronizerCallCount @@ -1507,13 +1507,13 @@ final class LDClientSpec: QuickSpec { let testContext = TestContext(startOnline: true) testContext.start() testContext.subject.setRunMode(.background) - let eventReporterIsOnlineSetCount = testContext.eventReporterMock.isOnlineSetCount + let eventProcessorIsOnlineSetCount = testContext.eventProcessorMock.isOnlineSetCount let flagSynchronizerIsOnlineSetCount = testContext.flagSynchronizerMock.isOnlineSetCount let makeFlagSynchronizerCallCount = testContext.serviceFactoryMock.makeFlagSynchronizerCallCount testContext.subject.setRunMode(.background) - expect(testContext.eventReporterMock.isOnline) == true - expect(testContext.eventReporterMock.isOnlineSetCount) == eventReporterIsOnlineSetCount + expect(testContext.eventProcessorMock.isOnline) == true + expect(testContext.eventProcessorMock.isOnlineSetCount) == eventProcessorIsOnlineSetCount expect(testContext.flagSynchronizerMock.isOnline) == true expect(testContext.flagSynchronizerMock.isOnlineSetCount) == flagSynchronizerIsOnlineSetCount expect(testContext.serviceFactoryMock.makeFlagSynchronizerCallCount) == makeFlagSynchronizerCallCount @@ -1524,7 +1524,7 @@ final class LDClientSpec: QuickSpec { testContext.start(runMode: .background) testContext.subject.setRunMode(.foreground) - expect(testContext.eventReporterMock.isOnline) == true + expect(testContext.eventProcessorMock.isOnline) == true expect(testContext.flagSynchronizerMock.isOnline) == true expect(testContext.flagSynchronizerMock.streamingMode) == LDStreamingMode.streaming } @@ -1533,7 +1533,7 @@ final class LDClientSpec: QuickSpec { testContext.start(runMode: .background) testContext.subject.setRunMode(.foreground) - expect(testContext.eventReporterMock.isOnline) == true + expect(testContext.eventProcessorMock.isOnline) == true expect(testContext.flagSynchronizerMock.isOnline) == true expect(testContext.flagSynchronizerMock.streamingMode) == LDStreamingMode.polling expect(testContext.flagSynchronizerMock.pollingInterval) == testContext.config.flagPollingInterval(runMode: .foreground) @@ -1550,7 +1550,7 @@ final class LDClientSpec: QuickSpec { testContext.subject.setRunMode(.background) - expect(testContext.eventReporterMock.isOnline) == false + expect(testContext.eventProcessorMock.isOnline) == false expect(testContext.flagSynchronizerMock.isOnline) == false expect(testContext.flagSynchronizerMock.streamingMode) == LDStreamingMode.streaming } @@ -1560,7 +1560,7 @@ final class LDClientSpec: QuickSpec { testContext.subject.setRunMode(.background) - expect(testContext.eventReporterMock.isOnline) == false + expect(testContext.eventProcessorMock.isOnline) == false expect(testContext.flagSynchronizerMock.isOnline) == false expect(testContext.flagSynchronizerMock.streamingMode) == LDStreamingMode.polling expect(testContext.flagSynchronizerMock.pollingInterval) == testContext.config.flagPollingInterval(runMode: .background) @@ -1570,13 +1570,13 @@ final class LDClientSpec: QuickSpec { let testContext = TestContext() waitUntil { testContext.start(completion: $0) } - let eventReporterIsOnlineSetCount = testContext.eventReporterMock.isOnlineSetCount + let eventProcessorIsOnlineSetCount = testContext.eventProcessorMock.isOnlineSetCount let flagSynchronizerIsOnlineSetCount = testContext.flagSynchronizerMock.isOnlineSetCount let makeFlagSynchronizerCallCount = testContext.serviceFactoryMock.makeFlagSynchronizerCallCount testContext.subject.setRunMode(.foreground) - expect(testContext.eventReporterMock.isOnline) == false - expect(testContext.eventReporterMock.isOnlineSetCount) == eventReporterIsOnlineSetCount + expect(testContext.eventProcessorMock.isOnline) == false + expect(testContext.eventProcessorMock.isOnlineSetCount) == eventProcessorIsOnlineSetCount expect(testContext.flagSynchronizerMock.isOnline) == false expect(testContext.flagSynchronizerMock.isOnlineSetCount) == flagSynchronizerIsOnlineSetCount expect(testContext.serviceFactoryMock.makeFlagSynchronizerCallCount) == makeFlagSynchronizerCallCount @@ -1587,13 +1587,13 @@ final class LDClientSpec: QuickSpec { let testContext = TestContext() waitUntil { testContext.start(runMode: .background, completion: $0) } - let eventReporterIsOnlineSetCount = testContext.eventReporterMock.isOnlineSetCount + let eventProcessorIsOnlineSetCount = testContext.eventProcessorMock.isOnlineSetCount let flagSynchronizerIsOnlineSetCount = testContext.flagSynchronizerMock.isOnlineSetCount let makeFlagSynchronizerCallCount = testContext.serviceFactoryMock.makeFlagSynchronizerCallCount testContext.subject.setRunMode(.background) - expect(testContext.eventReporterMock.isOnline) == false - expect(testContext.eventReporterMock.isOnlineSetCount) == eventReporterIsOnlineSetCount + expect(testContext.eventProcessorMock.isOnline) == false + expect(testContext.eventProcessorMock.isOnlineSetCount) == eventProcessorIsOnlineSetCount expect(testContext.flagSynchronizerMock.isOnline) == false expect(testContext.flagSynchronizerMock.isOnlineSetCount) == flagSynchronizerIsOnlineSetCount expect(testContext.serviceFactoryMock.makeFlagSynchronizerCallCount) == makeFlagSynchronizerCallCount @@ -1605,7 +1605,7 @@ final class LDClientSpec: QuickSpec { testContext.subject.setRunMode(.foreground) - expect(testContext.eventReporterMock.isOnline) == false + expect(testContext.eventProcessorMock.isOnline) == false expect(testContext.flagSynchronizerMock.isOnline) == false expect(testContext.flagSynchronizerMock.streamingMode) == LDStreamingMode.streaming } @@ -1615,7 +1615,7 @@ final class LDClientSpec: QuickSpec { testContext.subject.setRunMode(.foreground) - expect(testContext.eventReporterMock.isOnline) == false + expect(testContext.eventProcessorMock.isOnline) == false expect(testContext.flagSynchronizerMock.isOnline) == false expect(testContext.flagSynchronizerMock.streamingMode) == LDStreamingMode.polling expect(testContext.flagSynchronizerMock.pollingInterval) == testContext.config.flagPollingInterval(runMode: .foreground) @@ -1652,13 +1652,13 @@ final class LDClientSpec: QuickSpec { let testContext = TestContext() testContext.start() testContext.subject.flush() - expect(testContext.eventReporterMock.flushCallCount) == 1 + expect(testContext.eventProcessorMock.flushCallCount) == 1 } } describe("flush with a completion") { /// Answers from another thread, because this call does not block and must not be answered inside it. - func answer(_ mock: EventReportingMock, with delivered: Bool, after delay: TimeInterval = 0.05) { + func answer(_ mock: EventProcessorMock, with delivered: Bool, after delay: TimeInterval = 0.05) { mock.flushReportingOutcomeCallback = { [weak mock] in let completion = mock?.flushReportingOutcomeReceivedCompletion DispatchQueue.global().asyncAfter(deadline: .now() + delay) { completion?(delivered) } @@ -1668,7 +1668,7 @@ final class LDClientSpec: QuickSpec { it("reports true when delivery succeeds") { let testContext = TestContext() testContext.start() - answer(testContext.eventReporterMock, with: true) + answer(testContext.eventProcessorMock, with: true) var delivered: Bool? testContext.subject.flush { delivered = $0 } @@ -1680,7 +1680,7 @@ final class LDClientSpec: QuickSpec { it("reports false when delivery cannot finish") { let testContext = TestContext() testContext.start() - answer(testContext.eventReporterMock, with: false) + answer(testContext.eventProcessorMock, with: false) var delivered: Bool? testContext.subject.flush { delivered = $0 } @@ -1693,12 +1693,12 @@ final class LDClientSpec: QuickSpec { stale.subject.close() let current = TestContext() current.start() - answer(current.eventReporterMock, with: true) + answer(current.eventProcessorMock, with: true) var delivered: Bool? stale.subject.flush { delivered = $0 } expect(delivered).toEventually(beFalse()) - expect(current.eventReporterMock.flushReportingOutcomeCallCount) == 0 + expect(current.eventProcessorMock.flushReportingOutcomeCallCount) == 0 } it("answers a closed client off the caller's thread, as it does one with a delivery to wait for") { @@ -1723,10 +1723,10 @@ final class LDClientSpec: QuickSpec { let testContext = TestContext() try testContext.config.setSecondaryMobileKeys(["alternate": Constants.alternateMockMobileKey]) testContext.start() - let answering = LDClient.get(environment: "alternate")?.eventReporter as? EventReportingMock + let answering = LDClient.get(environment: "alternate")?.eventProcessor as? EventProcessorMock expect(answering).toNot(beNil()) // The one that fails answers last, so reporting before every environment is in would say true. - answer(testContext.eventReporterMock, with: true, after: 0.05) + answer(testContext.eventProcessorMock, with: true, after: 0.05) answer(answering!, with: false, after: 0.2) var answers: [Bool] = [] @@ -1739,7 +1739,7 @@ final class LDClientSpec: QuickSpec { describe("flushAndWait") { /// Answers inside the call, because `flushAndWait` blocks the test thread. - func answer(_ mock: EventReportingMock, with delivered: Bool) { + func answer(_ mock: EventProcessorMock, with delivered: Bool) { mock.flushReportingOutcomeCallback = { [weak mock] in mock?.flushReportingOutcomeReceivedCompletion?(delivered) } @@ -1748,16 +1748,16 @@ final class LDClientSpec: QuickSpec { it("reports true when delivery succeeds") { let testContext = TestContext() testContext.start() - answer(testContext.eventReporterMock, with: true) + answer(testContext.eventProcessorMock, with: true) expect(testContext.subject.flushAndWait(timeout: 1.0)) == true - expect(testContext.eventReporterMock.flushReportingOutcomeCallCount) == 1 + expect(testContext.eventProcessorMock.flushReportingOutcomeCallCount) == 1 } it("reports false when delivery cannot finish") { let testContext = TestContext() testContext.start() - answer(testContext.eventReporterMock, with: false) + answer(testContext.eventProcessorMock, with: false) expect(testContext.subject.flushAndWait(timeout: 1.0)) == false } @@ -1765,7 +1765,7 @@ final class LDClientSpec: QuickSpec { it("reports false when the budget expires first") { let testContext = TestContext() testContext.start() - testContext.eventReporterMock.flushReportingOutcomeCallback = nil + testContext.eventProcessorMock.flushReportingOutcomeCallback = nil expect(testContext.subject.flushAndWait(timeout: 0.05)) == false } @@ -1774,17 +1774,17 @@ final class LDClientSpec: QuickSpec { let testContext = TestContext() testContext.start() // Answers inside the call, so a zero budget that waited at all would report true. - answer(testContext.eventReporterMock, with: true) + answer(testContext.eventProcessorMock, with: true) expect(testContext.subject.flushAndWait(timeout: 0)) == false // Still worth calling: at `.immediate` the events are on disk by the time the delivery is queued. - expect(testContext.eventReporterMock.flushReportingOutcomeCallCount) == 1 + expect(testContext.eventProcessorMock.flushReportingOutcomeCallCount) == 1 } it("reports false once closed") { let testContext = TestContext() testContext.start() - answer(testContext.eventReporterMock, with: true) + answer(testContext.eventProcessorMock, with: true) testContext.subject.close() expect(testContext.subject.flushAndWait(timeout: 1.0)) == false @@ -1796,11 +1796,11 @@ final class LDClientSpec: QuickSpec { stale.subject.close() let current = TestContext() current.start() - answer(current.eventReporterMock, with: true) + answer(current.eventProcessorMock, with: true) // Its own events were never attempted, so the outcome of another client's delivery says nothing of them. expect(stale.subject.flushAndWait(timeout: 1.0)) == false - expect(current.eventReporterMock.flushReportingOutcomeCallCount) == 0 + expect(current.eventProcessorMock.flushReportingOutcomeCallCount) == 0 } it("gives every environment the whole budget rather than what the one before it left") { @@ -1811,7 +1811,7 @@ final class LDClientSpec: QuickSpec { // Each takes most of the budget, so delivering them one after another would leave the second // environment past the deadline before its own delivery had begun. for environment in ["alternate", LDConfig.Constants.primaryEnvironmentName] { - let reporter = LDClient.get(environment: environment)?.eventReporter as? EventReportingMock + let reporter = LDClient.get(environment: environment)?.eventProcessor as? EventProcessorMock expect(reporter).toNot(beNil()) reporter?.flushReportingOutcomeCallback = { [weak reporter] in let completion = reporter?.flushReportingOutcomeReceivedCompletion @@ -1828,13 +1828,13 @@ final class LDClientSpec: QuickSpec { testContext.start() // One answers most of the way through the budget and the other never does, so the budget the first // spent is the budget the second has left. Giving each its own copy of it would take twice as long. - let answering = LDClient.get(environment: "alternate")?.eventReporter as? EventReportingMock + let answering = LDClient.get(environment: "alternate")?.eventProcessor as? EventProcessorMock expect(answering).toNot(beNil()) answering?.flushReportingOutcomeCallback = { [weak answering] in let completion = answering?.flushReportingOutcomeReceivedCompletion DispatchQueue.global().asyncAfter(deadline: .now() + 0.5) { completion?(true) } } - testContext.eventReporterMock.flushReportingOutcomeCallback = nil + testContext.eventProcessorMock.flushReportingOutcomeCallback = nil // Answering at 0.5s into a 0.6s budget leaves the second wait 0.1s, against the 0.6s a fresh copy // would give it, so the two outcomes are 0.6s and 1.1s with room on either side of the threshold. diff --git a/LaunchDarkly/LaunchDarklyTests/Mocks/ClientServiceMockFactory.swift b/LaunchDarkly/LaunchDarklyTests/Mocks/ClientServiceMockFactory.swift index 1a0fe0c0e..b4c859ebb 100644 --- a/LaunchDarkly/LaunchDarklyTests/Mocks/ClientServiceMockFactory.swift +++ b/LaunchDarkly/LaunchDarklyTests/Mocks/ClientServiceMockFactory.swift @@ -72,17 +72,17 @@ final class ClientServiceMockFactory: ClientServiceCreating { return makeFlagChangeNotifierReturnValue } - var makeEventReporterCallCount = 0 - var makeEventReporterReceivedConfig: LDConfig? = nil - var makeEventReporterReceivedService: DarklyServiceProvider? = nil + var makeEventProcessorCallCount = 0 + var makeEventProcessorReceivedConfig: LDConfig? = nil + var makeEventProcessorReceivedService: DarklyServiceProvider? = nil var onEventSyncComplete: EventSyncCompleteClosure? = nil - func makeEventReporter(config: LDConfig, service: DarklyServiceProvider, onSyncComplete: EventSyncCompleteClosure?) -> EventReporting { - makeEventReporterCallCount += 1 - makeEventReporterReceivedConfig = config - makeEventReporterReceivedService = service + func makeEventProcessor(config: LDConfig, service: DarklyServiceProvider, onSyncComplete: EventSyncCompleteClosure?) -> EventProcessor { + makeEventProcessorCallCount += 1 + makeEventProcessorReceivedConfig = config + makeEventProcessorReceivedService = service onEventSyncComplete = onSyncComplete - let mock = EventReportingMock() + let mock = EventProcessorMock() // Answer by default so backgrounding does not hold its activity assertion until timeout. mock.flushReportingOutcomeCallback = { [weak mock] in mock?.flushReportingOutcomeReceivedCompletion?(true) @@ -90,8 +90,8 @@ final class ClientServiceMockFactory: ClientServiceCreating { return mock } - func makeEventReporter(config: LDConfig, service: DarklyServiceProvider) -> EventReporting { - return makeEventReporter(config: config, service: service, onSyncComplete: nil) + func makeEventProcessor(config: LDConfig, service: DarklyServiceProvider) -> EventProcessor { + return makeEventProcessor(config: config, service: service, onSyncComplete: nil) } var makeStreamingProviderCallCount = 0 diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift similarity index 85% rename from LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift rename to LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 6512c9db5..9e4915b2c 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -3,14 +3,14 @@ import Quick import Nimble @testable import LaunchDarkly -final class EventReporterSpec: QuickSpec { +final class DirectEventProcessorSpec: QuickSpec { struct Constants { static let eventFlushInterval: TimeInterval = 10.0 static let eventFlushIntervalHalfSecond: TimeInterval = 0.5 } struct TestContext { - var eventReporter: EventReporter! + var eventProcessor: DirectEventProcessor! var config: LDConfig! var context: LDContext! var serviceMock: DarklyServiceMock! @@ -32,7 +32,7 @@ final class EventReporterSpec: QuickSpec { store: EventStore? = nil, eventCapacity: Int = Event.Kind.allKinds.count, eventPersistence: EventPersistence = .immediate, - commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.tests.EventReporter.commitQueue")) { + commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.tests.DirectEventProcessor.commitQueue")) { config = LDConfig.stub config.eventCapacity = eventCapacity @@ -53,27 +53,27 @@ final class EventReporterSpec: QuickSpec { self.lastEventResponseDate = lastEventResponseDate.adjustedForHttpUrlHeaderUse self.store = store ?? EventStore.temporary(capacity: config.eventCapacity) - eventReporter = EventReporter(service: serviceMock, onSyncComplete: onSyncComplete, + eventProcessor = DirectEventProcessor(service: serviceMock, onSyncComplete: onSyncComplete, store: self.store, commitQueue: commitQueue) (0.. [LDValue] { - eventReporter.commitRecordedEvents() + eventProcessor.commitRecordedEvents() return store.pendingEventPayloads().compactMap { try? JSONDecoder().decode(LDValue.self, from: $0) } } @@ -91,9 +91,9 @@ final class EventReporterSpec: QuickSpec { func flushedFromACompletion() -> Bool { var flushed = false waitUntil(timeout: .seconds(5)) { done in - eventReporter.flushReportingOutcome { _ in + eventProcessor.flushReportingOutcome { _ in let finished = DispatchSemaphore(value: 0) - eventReporter.flush { finished.signal() } + eventProcessor.flush { finished.signal() } flushed = finished.wait(timeout: .now() + 1) == .success done() } @@ -106,7 +106,7 @@ final class EventReporterSpec: QuickSpec { var delivered: Bool? // Long enough for a refused post and its retry a second later. waitUntil(timeout: .seconds(10)) { done in - eventReporter.flushReportingOutcome { result in + eventProcessor.flushReportingOutcome { result in delivered = result done() } @@ -115,7 +115,7 @@ final class EventReporterSpec: QuickSpec { } func cleanUp() { - eventReporter.isOnline = false + eventProcessor.isOnline = false store.deleteEverything() } } @@ -132,13 +132,13 @@ final class EventReporterSpec: QuickSpec { commitSpec() flushReportingOutcomeSpec() outageSpec() - nullEventReporterSpec() + nullEventProcessorSpec() } - private func nullEventReporterSpec() { - describe("NullEventReporter") { + private func nullEventProcessorSpec() { + describe("NullEventProcessor") { it("answers every flush off the caller's thread") { - let reporter = NullEventReporter() + let reporter = NullEventProcessor() let caller = Thread.current let lock = UnfairLock() var answers: [(delivered: Bool, onCaller: Bool)] = [] @@ -165,12 +165,12 @@ final class EventReporterSpec: QuickSpec { var testContext: TestContext! beforeEach { testContext = TestContext() - testContext.eventReporter = EventReporter(service: testContext.serviceMock, onSyncComplete: { _ in }, store: testContext.store) + testContext.eventProcessor = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: { _ in }, store: testContext.store) } it("starts offline without reporting events") { - expect(testContext.eventReporter.service) === testContext.serviceMock - expect(testContext.eventReporter.isOnline) == false - expect(testContext.eventReporter.isReportingActive) == false + expect(testContext.eventProcessor.service) === testContext.serviceMock + expect(testContext.eventProcessor.isOnline) == false + expect(testContext.eventProcessor.isReportingActive) == false expect(testContext.serviceMock.publishEventDataCallCount) == 0 } } @@ -187,13 +187,13 @@ final class EventReporterSpec: QuickSpec { } context("online to offline") { beforeEach { - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true - testContext.eventReporter.isOnline = false + testContext.eventProcessor.isOnline = false } it("goes offline and stops reporting") { - expect(testContext.eventReporter.isOnline) == false - expect(testContext.eventReporter.isReportingActive) == false + expect(testContext.eventProcessor.isOnline) == false + expect(testContext.eventProcessor.isReportingActive) == false expect(testContext.serviceMock.publishEventDataCallCount) == 0 } } @@ -202,11 +202,11 @@ final class EventReporterSpec: QuickSpec { beforeEach { testContext = TestContext(eventCount: Event.Kind.allKinds.count) - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true } it("goes online and starts reporting") { - expect(testContext.eventReporter.isOnline) == true - expect(testContext.eventReporter.isReportingActive) == true + expect(testContext.eventProcessor.isOnline) == true + expect(testContext.eventProcessor.isReportingActive) == true expect(testContext.serviceMock.publishEventDataCallCount) == 0 } } @@ -214,11 +214,11 @@ final class EventReporterSpec: QuickSpec { beforeEach { testContext = TestContext() - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true } it("goes online and starts reporting") { - expect(testContext.eventReporter.isOnline) == true - expect(testContext.eventReporter.isReportingActive) == true + expect(testContext.eventProcessor.isOnline) == true + expect(testContext.eventProcessor.isReportingActive) == true expect(testContext.serviceMock.publishEventDataCallCount) == 0 } } @@ -226,13 +226,13 @@ final class EventReporterSpec: QuickSpec { context("online to online") { beforeEach { testContext = TestContext() - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true } it("stays online and continues reporting") { - expect(testContext.eventReporter.isOnline) == true - expect(testContext.eventReporter.isReportingActive) == true + expect(testContext.eventProcessor.isOnline) == true + expect(testContext.eventProcessor.isReportingActive) == true expect(testContext.serviceMock.publishEventDataCallCount) == 0 } } @@ -240,11 +240,11 @@ final class EventReporterSpec: QuickSpec { beforeEach { testContext = TestContext(eventCount: Event.Kind.allKinds.count) - testContext.eventReporter.isOnline = false + testContext.eventProcessor.isOnline = false } it("stays offline and does not start reporting") { - expect(testContext.eventReporter.isOnline) == false - expect(testContext.eventReporter.isReportingActive) == false + expect(testContext.eventProcessor.isOnline) == false + expect(testContext.eventProcessor.isReportingActive) == false expect(testContext.serviceMock.publishEventDataCallCount) == 0 expect(testContext.committedEvents()) == testContext.recordedEventsAsJSON() } @@ -258,11 +258,11 @@ final class EventReporterSpec: QuickSpec { context("event store empty") { beforeEach { testContext = TestContext() - testContext.recordEvents(Event.Kind.allKinds.count) // Stub events, call testContext.eventReporter.recordEvent, and keeps them in testContext.events + testContext.recordEvents(Event.Kind.allKinds.count) // Stub events, call testContext.eventProcessor.recordEvent, and keeps them in testContext.events } it("records events up to event capacity") { - expect(testContext.eventReporter.isOnline) == false - expect(testContext.eventReporter.isReportingActive) == false + expect(testContext.eventProcessor.isOnline) == false + expect(testContext.eventProcessor.isReportingActive) == false expect(testContext.serviceMock.publishEventDataCallCount) == 0 expect(testContext.committedEvents()) == testContext.recordedEventsAsJSON() } @@ -276,11 +276,11 @@ final class EventReporterSpec: QuickSpec { testContext = TestContext(eventCount: Event.Kind.allKinds.count) extraEvent = Event.stub(.feature, with: testContext.context) - testContext.eventReporter.record(extraEvent) + testContext.eventProcessor.record(extraEvent) } it("doesn't record any more events") { - expect(testContext.eventReporter.isOnline) == false - expect(testContext.eventReporter.isReportingActive) == false + expect(testContext.eventProcessor.isOnline) == false + expect(testContext.eventProcessor.isReportingActive) == false expect(testContext.serviceMock.publishEventDataCallCount) == 0 expect(testContext.committedEvents()) == testContext.recordedEventsAsJSON() } @@ -297,10 +297,10 @@ final class EventReporterSpec: QuickSpec { describe("unserializable events") { var serviceMock: DarklyServiceMock! var ldContext: LDContext! - var reporter: EventReporter! + var reporter: DirectEventProcessor! var store: EventStore! - func makeReporter(encoding: EventReporter.Encoding) -> EventReporter { + func makeReporter(encoding: DirectEventProcessor.Encoding) -> DirectEventProcessor { var config = LDConfig.stub config.eventCapacity = 8 ldContext = LDContext.stub() @@ -308,14 +308,14 @@ final class EventReporterSpec: QuickSpec { serviceMock.config = config serviceMock.stubEventResponse(success: true) store = EventStore.temporary(capacity: config.eventCapacity) - return EventReporter(service: serviceMock, onSyncComplete: nil, store: store, encoding: encoding) + return DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: encoding) } afterEach { reporter?.isOnline = false } - for encoding in [EventReporter.Encoding.codable, .handWritten] { + for encoding in [DirectEventProcessor.Encoding.codable, .handWritten] { context("\(encoding): when an event holds a non-finite metric") { beforeEach { reporter = makeReporter(encoding: encoding) @@ -449,8 +449,8 @@ final class EventReporterSpec: QuickSpec { testContext.cleanUp() } let erOnline = { - expect(testContext.eventReporter.isOnline) == true - expect(testContext.eventReporter.isReportingActive) == true + expect(testContext.eventProcessor.isOnline) == true + expect(testContext.eventProcessor.isReportingActive) == true } context("online") { context("success") { @@ -461,10 +461,10 @@ final class EventReporterSpec: QuickSpec { testContext.syncResult = result syncComplete() }) - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true testContext.recordEvents(Event.Kind.nonSummaryKinds.count) // Track some flag evaluations to populate the contextSummarizer - testContext.eventReporter.recordFlagEvaluationEvents( + testContext.eventProcessor.recordFlagEvaluationEvents( flagKey: "test-flag", value: .bool(true), defaultValue: .bool(false), @@ -472,7 +472,7 @@ final class EventReporterSpec: QuickSpec { context: testContext.context, includeReason: false ) - testContext.eventReporter.flush(completion: nil) + testContext.eventProcessor.flush(completion: nil) } } it("reports events and a summary event") { @@ -489,8 +489,8 @@ final class EventReporterSpec: QuickSpec { expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 expect(testContext.store.pendingEventCount) == 0 - expect(testContext.eventReporter.lastEventResponseDate) == testContext.eventStubResponseDate - expect(testContext.eventReporter.contextSummarizer.hasLoggedRequests) == false + expect(testContext.eventProcessor.lastEventResponseDate) == testContext.eventStubResponseDate + expect(testContext.eventProcessor.contextSummarizer.hasLoggedRequests) == false expect(testContext.syncResult).to(beNil()) } } @@ -501,9 +501,9 @@ final class EventReporterSpec: QuickSpec { testContext.syncResult = result syncComplete() }) - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true testContext.recordEvents(Event.Kind.nonSummaryKinds.count) - testContext.eventReporter.flush(completion: nil) + testContext.eventProcessor.flush(completion: nil) } } it("reports events without a summary event") { @@ -514,8 +514,8 @@ final class EventReporterSpec: QuickSpec { expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == testContext.events.count expect(testContext.store.pendingEventCount) == 0 - expect(testContext.eventReporter.lastEventResponseDate) == testContext.eventStubResponseDate - expect(testContext.eventReporter.contextSummarizer.hasLoggedRequests) == false + expect(testContext.eventProcessor.lastEventResponseDate) == testContext.eventStubResponseDate + expect(testContext.eventProcessor.contextSummarizer.hasLoggedRequests) == false expect(testContext.syncResult).to(beNil()) } } @@ -526,9 +526,9 @@ final class EventReporterSpec: QuickSpec { testContext.syncResult = result syncComplete() }) - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true // Track some flag evaluations to populate the contextSummarizer - testContext.eventReporter.recordFlagEvaluationEvents( + testContext.eventProcessor.recordFlagEvaluationEvents( flagKey: "test-flag", value: .bool(true), defaultValue: .bool(false), @@ -536,7 +536,7 @@ final class EventReporterSpec: QuickSpec { context: testContext.context, includeReason: false ) - testContext.eventReporter.flush(completion: nil) + testContext.eventProcessor.flush(completion: nil) } } it("reports only a summary event") { @@ -552,8 +552,8 @@ final class EventReporterSpec: QuickSpec { expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == 1 expect(testContext.store.pendingEventCount) == 0 - expect(testContext.eventReporter.lastEventResponseDate) == testContext.eventStubResponseDate - expect(testContext.eventReporter.contextSummarizer.hasLoggedRequests) == false + expect(testContext.eventProcessor.lastEventResponseDate) == testContext.eventStubResponseDate + expect(testContext.eventProcessor.contextSummarizer.hasLoggedRequests) == false expect(testContext.syncResult).to(beNil()) } } @@ -564,8 +564,8 @@ final class EventReporterSpec: QuickSpec { testContext.syncResult = result syncComplete() }) - testContext.eventReporter.isOnline = true - testContext.eventReporter.flush(completion: nil) + testContext.eventProcessor.isOnline = true + testContext.eventProcessor.flush(completion: nil) } } it("does not report events") { @@ -573,8 +573,8 @@ final class EventReporterSpec: QuickSpec { expect(testContext.serviceMock.publishEventDataCallCount) == 0 expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 0 expect(testContext.store.pendingEventCount) == 0 - expect(testContext.eventReporter.lastEventResponseDate) == Date.distantPast - expect(testContext.eventReporter.contextSummarizer.hasLoggedRequests) == false + expect(testContext.eventProcessor.lastEventResponseDate) == Date.distantPast + expect(testContext.eventProcessor.contextSummarizer.hasLoggedRequests) == false expect(testContext.syncResult).to(beNil()) } } @@ -587,10 +587,10 @@ final class EventReporterSpec: QuickSpec { testContext.syncResult = result syncComplete() }) - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true testContext.recordEvents(Event.Kind.nonSummaryKinds.count) // Track some flag evaluations to populate the contextSummarizer - testContext.eventReporter.recordFlagEvaluationEvents( + testContext.eventProcessor.recordFlagEvaluationEvents( flagKey: "test-flag", value: .bool(true), defaultValue: .bool(false), @@ -598,7 +598,7 @@ final class EventReporterSpec: QuickSpec { context: testContext.context, includeReason: false ) - testContext.eventReporter.flush(completion: nil) + testContext.eventProcessor.flush(completion: nil) } } it("keeps the events to retry after the failure") { @@ -616,8 +616,8 @@ final class EventReporterSpec: QuickSpec { } expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 - expect(testContext.eventReporter.lastEventResponseDate) == Date.distantPast - expect(testContext.eventReporter.contextSummarizer.hasLoggedRequests) == false + expect(testContext.eventProcessor.lastEventResponseDate) == Date.distantPast + expect(testContext.eventProcessor.contextSummarizer.hasLoggedRequests) == false guard case let .request(error) = testContext.syncResult else { fail("Expected error result for event send") @@ -633,10 +633,10 @@ final class EventReporterSpec: QuickSpec { testContext.syncResult = result syncComplete() }) - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true testContext.recordEvents(Event.Kind.nonSummaryKinds.count) // Track some flag evaluations to populate the contextSummarizer - testContext.eventReporter.recordFlagEvaluationEvents( + testContext.eventProcessor.recordFlagEvaluationEvents( flagKey: "test-flag", value: .bool(true), defaultValue: .bool(false), @@ -644,7 +644,7 @@ final class EventReporterSpec: QuickSpec { context: testContext.context, includeReason: false ) - testContext.eventReporter.flush(completion: nil) + testContext.eventProcessor.flush(completion: nil) } } it("keeps the events to retry after the failure") { @@ -662,8 +662,8 @@ final class EventReporterSpec: QuickSpec { } expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 - expect(testContext.eventReporter.lastEventResponseDate) == Date.distantPast - expect(testContext.eventReporter.contextSummarizer.hasLoggedRequests) == false + expect(testContext.eventProcessor.lastEventResponseDate) == Date.distantPast + expect(testContext.eventProcessor.contextSummarizer.hasLoggedRequests) == false let expectedError = testContext.serviceMock.errorEventHTTPURLResponse guard case let .response(error) = testContext.syncResult else { @@ -682,10 +682,10 @@ final class EventReporterSpec: QuickSpec { testContext.syncResult = result syncComplete() }) - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true testContext.recordEvents(Event.Kind.nonSummaryKinds.count) // Track some flag evaluations to populate the contextSummarizer - testContext.eventReporter.recordFlagEvaluationEvents( + testContext.eventProcessor.recordFlagEvaluationEvents( flagKey: "test-flag", value: .bool(true), defaultValue: .bool(false), @@ -693,7 +693,7 @@ final class EventReporterSpec: QuickSpec { context: testContext.context, includeReason: false ) - testContext.eventReporter.flush(completion: nil) + testContext.eventProcessor.flush(completion: nil) } } it("keeps the events to retry after the failure") { @@ -711,8 +711,8 @@ final class EventReporterSpec: QuickSpec { } expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 - expect(testContext.eventReporter.lastEventResponseDate) == Date.distantPast - expect(testContext.eventReporter.contextSummarizer.hasLoggedRequests) == false + expect(testContext.eventProcessor.lastEventResponseDate) == Date.distantPast + expect(testContext.eventProcessor.contextSummarizer.hasLoggedRequests) == false guard case let .request(error) = testContext.syncResult else { fail("Expected error result for event send") @@ -733,16 +733,16 @@ final class EventReporterSpec: QuickSpec { httpVersion: DarklyServiceMock.Constants.httpVersion, headerFields: nil) testContext.serviceMock.stubbedEventResponse = (nil, unauthorized, nil, nil) - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true testContext.recordEvents(Event.Kind.nonSummaryKinds.count) - testContext.eventReporter.flush(completion: nil) + testContext.eventProcessor.flush(completion: nil) } } it("drops events and stops reporting") { expect(testContext.serviceMock.publishEventDataCallCount) == 1 expect(testContext.store.pendingEventCount).toEventually(equal(0)) - expect(testContext.eventReporter.isOnline).to(beFalse()) - expect(testContext.eventReporter.isReportingActive).to(beFalse()) + expect(testContext.eventProcessor.isOnline).to(beFalse()) + expect(testContext.eventProcessor.isReportingActive).to(beFalse()) if case let .response(error) = testContext.syncResult { expect((error as? HTTPURLResponse)?.statusCode) == HTTPURLResponse.StatusCodes.unauthorized } else { @@ -750,9 +750,9 @@ final class EventReporterSpec: QuickSpec { } } it("can be brought back online") { - testContext.eventReporter.isOnline = true - expect(testContext.eventReporter.isOnline).to(beTrue()) - expect(testContext.eventReporter.isReportingActive).to(beTrue()) + testContext.eventProcessor.isOnline = true + expect(testContext.eventProcessor.isOnline).to(beTrue()) + expect(testContext.eventProcessor.isReportingActive).to(beTrue()) } } } @@ -766,7 +766,7 @@ final class EventReporterSpec: QuickSpec { }) testContext.recordEvents(Event.Kind.nonSummaryKinds.count) // Track some flag evaluations to populate the contextSummarizer - testContext.eventReporter.recordFlagEvaluationEvents( + testContext.eventProcessor.recordFlagEvaluationEvents( flagKey: "test-flag", value: .bool(true), defaultValue: .bool(false), @@ -774,22 +774,22 @@ final class EventReporterSpec: QuickSpec { context: testContext.context, includeReason: false ) - testContext.eventReporter.flush(completion: nil) + testContext.eventProcessor.flush(completion: nil) } } it("doesn't report events, but persists them") { - expect(testContext.eventReporter.isOnline) == false - expect(testContext.eventReporter.isReportingActive) == false + expect(testContext.eventProcessor.isOnline) == false + expect(testContext.eventProcessor.isReportingActive) == false expect(testContext.serviceMock.publishEventDataCallCount) == 0 expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 0 - expect(testContext.eventReporter.lastEventResponseDate) == Date.distantPast + expect(testContext.eventProcessor.lastEventResponseDate) == Date.distantPast // Being offline delays delivery, not durability: a flush is a commit point, so the recorded events and // the evaluations counted so far are on disk even though nothing can be sent. let pending = testContext.committedEvents() expect(pending.dropLast().map { $0 }) == testContext.recordedEventsAsJSON() expect(pending.last?.kindField) == "summary" - expect(testContext.eventReporter.contextSummarizer.hasLoggedRequests) == false + expect(testContext.eventProcessor.contextSummarizer.hasLoggedRequests) == false guard case .isOffline = testContext.syncResult else { fail("Expected error .isOffline result for event send") @@ -805,9 +805,9 @@ final class EventReporterSpec: QuickSpec { let serviceMock = DarklyServiceMock() /// A reporter writing to a store of its own, and that store, so a test can read back what was recorded. - func makeReporter() -> (EventReporter, EventStore) { + func makeReporter() -> (DirectEventProcessor, EventStore) { let store = EventStore.temporary() - return (EventReporter(service: serviceMock, onSyncComplete: nil, store: store), store) + return (DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store), store) } describe("recordFlagEvaluationEvents") { @@ -820,7 +820,7 @@ final class EventReporterSpec: QuickSpec { service.diagnosticCache = diagnosticCache let store = EventStore.temporary(capacity: config.eventCapacity) defer { store.deleteEverything() } - let reporter = EventReporter(service: service, onSyncComplete: nil, store: store) + let reporter = DirectEventProcessor(service: service, onSyncComplete: nil, store: store) let untracked = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: false) let tracked = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: true) @@ -989,7 +989,7 @@ final class EventReporterSpec: QuickSpec { context("with events") { beforeEach { testContext = TestContext(eventFlushInterval: Constants.eventFlushIntervalHalfSecond) - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true testContext.recordEvents(Event.Kind.allKinds.count) } it("reports events") { @@ -1007,7 +1007,7 @@ final class EventReporterSpec: QuickSpec { context("on later intervals") { beforeEach { testContext = TestContext(eventFlushInterval: Constants.eventFlushIntervalHalfSecond) - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true testContext.recordEvents(Event.Kind.nonSummaryKinds.count) } it("publishes events recorded after the first fire") { @@ -1019,7 +1019,7 @@ final class EventReporterSpec: QuickSpec { } it("without events") { testContext = TestContext(eventFlushInterval: Constants.eventFlushIntervalHalfSecond) - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true waitUntil { done in DispatchQueue.main.asyncAfter(deadline: DispatchTime.now() + Constants.eventFlushIntervalHalfSecond) { @@ -1032,7 +1032,7 @@ final class EventReporterSpec: QuickSpec { } } -extension EventReporterSpec { +extension DirectEventProcessorSpec { /// What the SDK is able to report after the process it was recording in is gone. /// /// These are written against a second store reading the same directory, which is what the next run of the @@ -1054,7 +1054,7 @@ extension EventReporterSpec { testContext = TestContext() // The sequence that loses events today: something is tracked and the process ends immediately after. - testContext.eventReporter.record(CustomEvent(key: "fatal-error", context: testContext.context, data: ["message": "boom"])) + testContext.eventProcessor.record(CustomEvent(key: "fatal-error", context: testContext.context, data: ["message": "boom"])) let onDisk = eventsLeftOnDisk() expect(onDisk.count) == 1 @@ -1064,7 +1064,7 @@ extension EventReporterSpec { it("still writes a tracked event when the write is not on the caller's thread") { testContext = TestContext(eventPersistence: .deferred) - testContext.eventReporter.record(CustomEvent(key: "fatal-error", context: testContext.context, data: ["message": "boom"])) + testContext.eventProcessor.record(CustomEvent(key: "fatal-error", context: testContext.context, data: ["message": "boom"])) // The promise is weaker by exactly one scheduling hop: the event reaches the disk, just not before // recording it returned. @@ -1079,8 +1079,8 @@ extension EventReporterSpec { // An untracked flag is only ever reported as a summary, so this is the case where the exposure exists // nowhere but in memory until something forces it out. - testContext.eventReporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: testContext.context, includeReason: false) - testContext.eventReporter.record(CustomEvent(key: "fatal-error", context: testContext.context, data: nil)) + testContext.eventProcessor.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: testContext.context, includeReason: false) + testContext.eventProcessor.record(CustomEvent(key: "fatal-error", context: testContext.context, data: nil)) expect(eventsLeftOnDisk().compactMap { $0.kindField }) == ["custom", "summary"] } @@ -1091,10 +1091,10 @@ extension EventReporterSpec { // The deliberate trade: an evaluation does not pay for a write, so it is only staged until something // else commits it. - testContext.eventReporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: testContext.context, includeReason: false) + testContext.eventProcessor.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: testContext.context, includeReason: false) expect(eventsLeftOnDisk()).to(beEmpty()) - testContext.eventReporter.flush(completion: nil) + testContext.eventProcessor.flush(completion: nil) expect(eventsLeftOnDisk().compactMap { $0.kindField }) == ["feature", "summary"] } @@ -1113,7 +1113,7 @@ extension EventReporterSpec { expect(Thread.isMainThread) == true // Far past the pending threshold, so one coalesced commit is waiting. for _ in 0..<400 { - testContext.eventReporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: testContext.context, includeReason: false) + testContext.eventProcessor.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: testContext.context, includeReason: false) } expect(eventsLeftOnDisk()).to(beEmpty()) @@ -1127,11 +1127,11 @@ extension EventReporterSpec { it("reports the events a previous run left behind") { testContext = TestContext() testContext.recordEvents(2) - testContext.eventReporter.commitRecordedEvents() + testContext.eventProcessor.commitRecordedEvents() // Nothing delivered them and nothing closed the log, as would be the case had the process died here. let recovered = EventStore.sharing(testContext.store.directory) - let nextRun = EventReporter(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) + let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) nextRun.isOnline = true waitUntil { done in @@ -1150,10 +1150,10 @@ extension EventReporterSpec { it("delivers what a previous run left behind as soon as it is online, without waiting for a report interval") { testContext = TestContext() testContext.recordEvents(2) - testContext.eventReporter.commitRecordedEvents() + testContext.eventProcessor.commitRecordedEvents() let recovered = EventStore.sharing(testContext.store.directory) - let nextRun = EventReporter(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) + let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) // Nothing asks for a flush, and the report interval is several times longer than this waits, so the // only thing that can deliver these is the reporter coming online. @@ -1167,7 +1167,7 @@ extension EventReporterSpec { it("does not bring a delivery forward when a previous run left nothing behind") { testContext = TestContext() - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true // Coming online is not itself a reason to send: an ordinary start has nothing waiting, and the events // this run records keep to the report interval. @@ -1181,8 +1181,8 @@ extension EventReporterSpec { testContext.recordEvents(1) waitUntil(timeout: .seconds(10)) { done in - testContext.eventReporter.isOnline = true - testContext.eventReporter.flush(completion: done) + testContext.eventProcessor.isOnline = true + testContext.eventProcessor.flush(completion: done) } let kept = testContext.store.pendingBatches() @@ -1198,18 +1198,18 @@ extension EventReporterSpec { it("does not send a batch again when a delivery is asked for while one is in flight") { testContext = TestContext() testContext.recordEvents(1) - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true // The response is held back, so the first delivery stays in flight and its batch stays on disk -- // the window in which a second delivery would find that batch and send it again. testContext.serviceMock.holdsEventCompletions = true // Answered from the delivery's own threads and polled from this one. let first = Answer() - testContext.eventReporter.flush { first.record(true) } + testContext.eventProcessor.flush { first.record(true) } expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) let second = Answer() - testContext.eventReporter.flush { second.record(true) } + testContext.eventProcessor.flush { second.record(true) } // Nothing more goes out while the first delivery is unanswered. Thread.sleep(forTimeInterval: 0.3) expect(testContext.serviceMock.publishEventDataCallCount) == 1 @@ -1231,7 +1231,7 @@ extension EventReporterSpec { // after launch reads every batch a previous run left behind. let store = SlowListingStore(testContext.store) defer { store.release.signal() } - let reporter = EventReporter(service: testContext.serviceMock, onSyncComplete: nil, store: store) + let reporter = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: store) reporter.isOnline = true reporter.record(CustomEvent(key: "tracked", context: testContext.context)) @@ -1257,9 +1257,9 @@ extension EventReporterSpec { // timeouts allow. let released = DispatchSemaphore(value: 0) defer { released.signal() } - testContext.eventReporter.occupyQueue(until: released) + testContext.eventProcessor.occupyQueue(until: released) - testContext.eventReporter.record(CustomEvent(key: "tracked", context: testContext.context)) + testContext.eventProcessor.record(CustomEvent(key: "tracked", context: testContext.context)) expect(eventsLeftOnDisk().compactMap { $0.kindField }).toEventually(equal(["custom"])) } @@ -1268,13 +1268,13 @@ extension EventReporterSpec { testContext = TestContext(lastEventResponseDate: Date()) let released = DispatchSemaphore(value: 0) defer { released.signal() } - testContext.eventReporter.occupyQueue(until: released) + testContext.eventProcessor.occupyQueue(until: released) let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: true) - testContext.eventReporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: testContext.context, includeReason: false) + testContext.eventProcessor.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: testContext.context, includeReason: false) // What `LDClient.close()` relies on: the write is the caller's, so a delivery that never finishes // cannot keep it from happening. - testContext.eventReporter.flush(completion: nil) + testContext.eventProcessor.flush(completion: nil) expect(eventsLeftOnDisk().compactMap { $0.kindField }) == ["feature", "summary"] } @@ -1292,7 +1292,7 @@ extension EventReporterSpec { return file.writeAll(bytes) }) testContext = TestContext(store: store, eventCapacity: 100) - let reporter = testContext.eventReporter! + let reporter = testContext.eventProcessor! let context = testContext.context! let start = DispatchSemaphore(value: 0) @@ -1329,10 +1329,10 @@ extension EventReporterSpec { it("tries a batch a previous run left once at start, with its retry, and then waits while it is still refused") { testContext = TestContext() testContext.serviceMock.respondToEvents(with: 503) - testContext.eventReporter.record(CustomEvent(key: "kept", context: testContext.context)) + testContext.eventProcessor.record(CustomEvent(key: "kept", context: testContext.context)) let recovered = EventStore.sharing(testContext.store.directory) - let nextRun = EventReporter(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) + let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) nextRun.isOnline = true defer { nextRun.isOnline = false } @@ -1346,16 +1346,16 @@ extension EventReporterSpec { it("leaves a batch still refused when the run ended for the next run to deliver, under the same payload id") { testContext = TestContext() testContext.serviceMock.respondToEvents(with: 503) - testContext.eventReporter.record(CustomEvent(key: "kept", context: testContext.context)) - testContext.eventReporter.isOnline = true + testContext.eventProcessor.record(CustomEvent(key: "kept", context: testContext.context)) + testContext.eventProcessor.isOnline = true expect(testContext.flushOutcome()) == false let payloadId = testContext.serviceMock.publishedPayloadIds.first - testContext.eventReporter.isOnline = false + testContext.eventProcessor.isOnline = false // The same directory, as the application's next launch would find it. testContext.serviceMock.stubEventResponse(success: true) let recovered = EventStore.sharing(testContext.store.directory) - let nextRun = EventReporter(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) + let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) nextRun.isOnline = true defer { nextRun.isOnline = false } @@ -1386,18 +1386,18 @@ extension EventReporterSpec { // Past the pending threshold and then a commit point: either would commit where events are persisted. for _ in 0..<100 { - testContext.eventReporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: testContext.context, includeReason: false) + testContext.eventProcessor.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: testContext.context, includeReason: false) } - testContext.eventReporter.record(CustomEvent(key: "custom", context: testContext.context)) - testContext.eventReporter.flush(completion: nil) + testContext.eventProcessor.record(CustomEvent(key: "custom", context: testContext.context)) + testContext.eventProcessor.flush(completion: nil) commits.sync { } // Encoding them early would buy nothing, since a commit here writes nothing to disk. expect(testContext.store.pendingEventCount) == 0 - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true waitUntil { done in - testContext.eventReporter.flush(completion: done) + testContext.eventProcessor.flush(completion: done) } let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) valueIsArray(published) { events in @@ -1411,7 +1411,7 @@ extension EventReporterSpec { store: EventStore.temporary(capacity: .max), eventCapacity: .max) let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: true) - let reporter = testContext.eventReporter! + let reporter = testContext.eventProcessor! let context = testContext.context! let recorded = DispatchSemaphore(value: 0) @@ -1459,7 +1459,7 @@ extension EventReporterSpec { let commits = DispatchQueue(label: "com.launchdarkly.tests.suspended") commits.suspend() defer { commits.resume() } - let reporter = EventReporter(service: serviceMock, onSyncComplete: nil, store: store, encoding: .codable, commitQueue: commits) + let reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: .codable, commitQueue: commits) let slow = EncodingGate(key: "slow", context: LDContext.stub()) reporter.record(slow) @@ -1493,7 +1493,7 @@ extension EventReporterSpec { let commits = DispatchQueue(label: "com.launchdarkly.tests.suspended") commits.suspend() defer { commits.resume() } - let reporter = EventReporter(service: serviceMock, onSyncComplete: nil, store: store, encoding: .codable, commitQueue: commits) + let reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: .codable, commitQueue: commits) let slow = EncodingGate(key: "slow", context: LDContext.stub()) let competingCommit = DispatchSemaphore(value: 0) @@ -1535,7 +1535,7 @@ extension EventReporterSpec { let commits = DispatchQueue(label: "com.launchdarkly.tests.suspended") commits.suspend() defer { commits.resume() } - let reporter = EventReporter(service: serviceMock, onSyncComplete: nil, store: store, encoding: .codable, commitQueue: commits) + let reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: .codable, commitQueue: commits) let slow = EncodingGate(key: "slow", context: LDContext.stub()) reporter.record(CustomEvent(key: "first", context: LDContext.stub())) @@ -1568,8 +1568,8 @@ extension EventReporterSpec { let contexts = 6 for index in 0.. 2 ? ENOSPC : file.writeAll(bytes) }) testContext = TestContext(store: store) - testContext.eventReporter.record(CustomEvent(key: "written", context: LDContext.stub())) + testContext.eventProcessor.record(CustomEvent(key: "written", context: LDContext.stub())) // Its write fails, so persistence is given up with "written" in the log and this in memory. - testContext.eventReporter.record(CustomEvent(key: "held", context: LDContext.stub())) + testContext.eventProcessor.record(CustomEvent(key: "held", context: LDContext.stub())) expect(store.isPersisting) == false - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true var delivered: Bool? waitUntil { done in - testContext.eventReporter.flushReportingOutcome { result in + testContext.eventProcessor.flushReportingOutcome { result in delivered = result done() } @@ -1692,12 +1692,12 @@ extension EventReporterSpec { it("reports false when an event could not be serialized, though the rest were accepted") { testContext = TestContext() - testContext.eventReporter.record(CustomEvent(key: "poison", context: LDContext.stub(), metricValue: .nan)) + testContext.eventProcessor.record(CustomEvent(key: "poison", context: LDContext.stub(), metricValue: .nan)) testContext.recordEvents(1) - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true var delivered: Bool? waitUntil { done in - testContext.eventReporter.flushReportingOutcome { result in + testContext.eventProcessor.flushReportingOutcome { result in delivered = result done() } @@ -1707,7 +1707,7 @@ extension EventReporterSpec { // Told once, to the caller who was there to hear it. waitUntil { done in - testContext.eventReporter.flushReportingOutcome { result in + testContext.eventProcessor.flushReportingOutcome { result in delivered = result done() } @@ -1719,12 +1719,12 @@ extension EventReporterSpec { testContext = TestContext() // Larger than a log frame holds, so the store refuses it however much room is left. let oversized = String(repeating: "x", count: EventLogFormat.maxFrameSize) - testContext.eventReporter.record(CustomEvent(key: oversized, context: LDContext.stub())) - testContext.eventReporter.record(CustomEvent(key: "fine", context: LDContext.stub())) - testContext.eventReporter.isOnline = true + testContext.eventProcessor.record(CustomEvent(key: oversized, context: LDContext.stub())) + testContext.eventProcessor.record(CustomEvent(key: "fine", context: LDContext.stub())) + testContext.eventProcessor.isOnline = true var delivered: Bool? waitUntil { done in - testContext.eventReporter.flushReportingOutcome { result in + testContext.eventProcessor.flushReportingOutcome { result in delivered = result done() } @@ -1737,10 +1737,10 @@ extension EventReporterSpec { it("reports false when a retryable failure leaves the batch on disk") { testContext = TestContext(stubResponseSuccess: false) testContext.recordEvents(1) - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true var delivered: Bool? waitUntil(timeout: .seconds(10)) { done in - testContext.eventReporter.flushReportingOutcome { result in + testContext.eventProcessor.flushReportingOutcome { result in delivered = result done() } @@ -1757,16 +1757,16 @@ extension EventReporterSpec { headerFields: nil) testContext.serviceMock.stubbedEventResponse = (nil, unauthorized, nil, nil) testContext.recordEvents(1) - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true var delivered: Bool? waitUntil { done in - testContext.eventReporter.flushReportingOutcome { result in + testContext.eventProcessor.flushReportingOutcome { result in delivered = result done() } } expect(delivered) == false - expect(testContext.eventReporter.isOnline) == false + expect(testContext.eventProcessor.isOnline) == false // Kept off the device, since no later delivery would fare better. expect(testContext.store.pendingEventCount) == 0 } @@ -1774,16 +1774,16 @@ extension EventReporterSpec { it("keeps draining while a delivery is in flight, and sends each batch once") { testContext = TestContext() testContext.recordEvents(1) - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true testContext.serviceMock.holdsEventCompletions = true - testContext.eventReporter.flushReportingOutcome { _ in } + testContext.eventProcessor.flushReportingOutcome { _ in } expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) // Holding this back until the first delivery answers would let the store fill through a retry, which // can run for twice the connection timeout, and events recorded past the capacity are lost. testContext.recordEvents(1) - testContext.eventReporter.flushReportingOutcome { _ in } + testContext.eventProcessor.flushReportingOutcome { _ in } expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(2)) // The first batch is still awaiting its response, so the second delivery left it alone rather than @@ -1797,17 +1797,17 @@ extension EventReporterSpec { it("inherits the outcome of a delivery holding the batch it would have sent") { testContext = TestContext(stubResponseSuccess: false) testContext.recordEvents(1) - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true testContext.serviceMock.holdsEventCompletions = true - testContext.eventReporter.flushReportingOutcome { _ in } + testContext.eventProcessor.flushReportingOutcome { _ in } expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) // Everything on disk belongs to the delivery already running, and that may include this caller's // events: a commit only stages them, and whichever delivery runs next is what closes them into a // batch. So this caller is answered by that delivery rather than told its events arrived. let answer = Answer() - testContext.eventReporter.flushReportingOutcome(completion: answer.record) + testContext.eventProcessor.flushReportingOutcome(completion: answer.record) Thread.sleep(forTimeInterval: 0.3) expect(answer.value()).to(beNil()) expect(testContext.serviceMock.publishEventDataCallCount) == 1 @@ -1821,21 +1821,21 @@ extension EventReporterSpec { it("tells the caller whose delivery the loss happened before, not whichever answers first") { testContext = TestContext() testContext.recordEvents(1) - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true testContext.serviceMock.holdsEventCompletions = true // A closes a batch of its own and waits on it. Nothing has been lost at the point it closed. let first = Answer() - testContext.eventReporter.flushReportingOutcome(completion: first.record) + testContext.eventProcessor.flushReportingOutcome(completion: first.record) expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) // B closes a batch of its own, dropping an event the store will never take as it does, so the loss // falls between the two closes. B has a post of its own outstanding, which is what leaves A to be // answered first. - testContext.eventReporter.record(CustomEvent(key: "poison", context: LDContext.stub(), metricValue: .nan)) + testContext.eventProcessor.record(CustomEvent(key: "poison", context: LDContext.stub(), metricValue: .nan)) testContext.recordEvents(1) let second = Answer() - testContext.eventReporter.flushReportingOutcome(completion: second.record) + testContext.eventProcessor.flushReportingOutcome(completion: second.record) expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(2)) // Responses land in the order they were made, so A is answered while B's post is still out. With @@ -1850,14 +1850,14 @@ extension EventReporterSpec { it("answers away from the queue a delivery runs on, after a delivery") { testContext = TestContext() testContext.recordEvents(1) - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true expect(testContext.flushedFromACompletion()) == true } it("answers away from the queue a delivery runs on, with nothing to send") { testContext = TestContext() - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true // The path that answers without ever leaving the reporter's queue, so it is the one that would // deadlock while the delivery path, answering from the response, did not. @@ -1867,45 +1867,45 @@ extension EventReporterSpec { it("sends one delivery for the flushes that arrive before it starts, and answers them all") { testContext = TestContext(stubResponseSuccess: false) testContext.recordEvents(1) - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true let released = DispatchSemaphore(value: 0) - testContext.eventReporter.occupyQueue(until: released) + testContext.eventProcessor.occupyQueue(until: released) // An application flushing in a loop would otherwise queue a delivery each time, leaving the flush // that matters waiting behind all of them, and growing the waiter list without bound. for _ in 0..<100 { - testContext.eventReporter.flush(completion: nil) + testContext.eventProcessor.flush(completion: nil) } - expect(testContext.eventReporter.queuedDeliveryWaiterCount) == 0 + expect(testContext.eventProcessor.queuedDeliveryWaiterCount) == 0 // Both completions run at once, and the poll below reads from a third thread. let answers = Answers() for _ in 0..<2 { - testContext.eventReporter.flushReportingOutcome(completion: answers.record) + testContext.eventProcessor.flushReportingOutcome(completion: answers.record) } - expect(testContext.eventReporter.queuedDeliveryWaiterCount) == 2 + expect(testContext.eventProcessor.queuedDeliveryWaiterCount) == 2 released.signal() // Allows for the one-second retry, after which the batch stays on disk. expect(answers.value()).toEventually(equal([false, false]), timeout: .seconds(10)) - expect(testContext.eventReporter.startedDeliveryCount) == 1 + expect(testContext.eventProcessor.startedDeliveryCount) == 1 expect(testContext.serviceMock.publishEventDataCallCount) == 2 // the first attempt and its retry } it("answers every flush that arrived during a refused delivery with its outcome") { testContext = TestContext(stubResponseSuccess: false) testContext.recordEvents(1) - testContext.eventReporter.isOnline = true + testContext.eventProcessor.isOnline = true testContext.serviceMock.holdsEventCompletions = true let answers = Answers() - testContext.eventReporter.flushReportingOutcome(completion: answers.record) + testContext.eventProcessor.flushReportingOutcome(completion: answers.record) expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) // Held on the first post, each of these finds the only batch claimed by it. None of them is told // its events arrived on the strength of having nothing left of its own to send. for _ in 0..<20 { - testContext.eventReporter.flushReportingOutcome(completion: answers.record) + testContext.eventProcessor.flushReportingOutcome(completion: answers.record) } testContext.serviceMock.holdsEventCompletions = false @@ -1933,8 +1933,8 @@ extension EventReporterSpec { testContext = TestContext(store: EventStore.temporary(persistEvents: persistence != .disabled), eventPersistence: persistence) testContext.serviceMock.respondToEvents(with: status) - testContext.eventReporter.record(CustomEvent(key: "kept", context: testContext.context)) - testContext.eventReporter.isOnline = true + testContext.eventProcessor.record(CustomEvent(key: "kept", context: testContext.context)) + testContext.eventProcessor.isOnline = true for flush in 0.. Date: Sun, 11 Oct 2026 21:21:00 -0700 Subject: [PATCH 030/158] Name the processor's capacity eventCapacity, after the setting it comes from Co-authored-by: Cursor --- .../LaunchDarkly/ServiceObjects/DirectEventProcessor.swift | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index b553c14f1..b13ddf42c 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -40,7 +40,7 @@ class DirectEventProcessor: EventProcessor { let store: EventStoring /// How many events the SDK will hold in total, across `pending` and the store. - private let capacity: Int + private let eventCapacity: Int /// Where a commit runs when no caller is waiting on it. private let commitQueue: DispatchQueue @@ -189,7 +189,7 @@ class DirectEventProcessor: EventProcessor { self.onSyncComplete = onSyncComplete self.responseDate = Date() self.encoding = encoding - self.capacity = service.config.eventCapacity + self.eventCapacity = service.config.eventCapacity self.commitQueue = commitQueue self.encoder = DirectEventProcessor.makeEncoder(config: service.config) self.contextSummarizer = ContextSummarizer(logger: service.config.logger, maxContexts: service.config.eventCapacity) @@ -279,7 +279,7 @@ class DirectEventProcessor: EventProcessor { /// evaluation costs no more than its summary counter, however fast the loop runs. private func admitToPending(_ event: Event) -> Bool { recordLock.assertOwned() - guard pending.count + store.pendingEventCount < capacity + guard pending.count + store.pendingEventCount < eventCapacity else { return false } pending.append(event) From 65b2bbf8fee8732613f5b49a12483e365925b566 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Sun, 11 Oct 2026 21:24:13 -0700 Subject: [PATCH 031/158] Move summarizing and encoding into OutboundEventBuffer, as Android has it Co-authored-by: Cursor --- LaunchDarkly.xcodeproj/project.pbxproj | 10 ++ .../ServiceObjects/DirectEventProcessor.swift | 68 ++------------ .../ServiceObjects/OutboundEventBuffer.swift | 91 +++++++++++++++++++ .../DirectEventProcessorSpec.swift | 56 ++++++------ 4 files changed, 137 insertions(+), 88 deletions(-) create mode 100644 LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift diff --git a/LaunchDarkly.xcodeproj/project.pbxproj b/LaunchDarkly.xcodeproj/project.pbxproj index 2e3ba05bb..7d868a762 100644 --- a/LaunchDarkly.xcodeproj/project.pbxproj +++ b/LaunchDarkly.xcodeproj/project.pbxproj @@ -80,6 +80,7 @@ 831188562113AE0800D77CB5 /* FlagSynchronizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */; }; 831188572113AE0B00D77CB5 /* FlagChangeNotifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8358F25F1F476AD800ECE1AF /* FlagChangeNotifier.swift */; }; 831188582113AE0F00D77CB5 /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; + 629D5A9FE152DBC3E20D6DFC /* OutboundEventBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */; }; 014901E06BE1F2B0FAAD6AA6 /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; 86B71C91DB78D0F17DBE0E69 /* EventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 293A30078FEAA25BB4C3D26C /* EventProcessor.swift */; }; 831188592113AE1200D77CB5 /* FlagStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831D8B731F72994600ED65E8 /* FlagStore.swift */; }; @@ -123,6 +124,7 @@ 831EF35520655E730001C643 /* FlagSynchronizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */; }; 831EF35620655E730001C643 /* FlagChangeNotifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8358F25F1F476AD800ECE1AF /* FlagChangeNotifier.swift */; }; 831EF35720655E730001C643 /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; + 5A89648200D2566D3F8409BA /* OutboundEventBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */; }; CFF357D68BBF2DC8395930F2 /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; 850C49C9054E642CC006E730 /* EventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 293A30078FEAA25BB4C3D26C /* EventProcessor.swift */; }; 831EF35820655E730001C643 /* FlagStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831D8B731F72994600ED65E8 /* FlagStore.swift */; }; @@ -207,6 +209,7 @@ 83D9EC872062DEAB004D7FA6 /* FlagSynchronizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */; }; 83D9EC882062DEAB004D7FA6 /* FlagChangeNotifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8358F25F1F476AD800ECE1AF /* FlagChangeNotifier.swift */; }; 83D9EC892062DEAB004D7FA6 /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; + 73E23C0FA009141BB4CBC24C /* OutboundEventBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */; }; 54745AEF1A43D165F5A6E08F /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; 7F4004579B9094737F3C21D1 /* EventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 293A30078FEAA25BB4C3D26C /* EventProcessor.swift */; }; 83D9EC8A2062DEAB004D7FA6 /* FlagStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831D8B731F72994600ED65E8 /* FlagStore.swift */; }; @@ -233,6 +236,7 @@ 83F0A5641FB5F33800550A95 /* LDConfigSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83F0A5631FB5F33800550A95 /* LDConfigSpec.swift */; }; 83FEF8DD1F266742001CF12C /* FlagSynchronizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */; }; 83FEF8DF1F2667E4001CF12C /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; + 7D37AD3ECAA9D8E2927228C5 /* OutboundEventBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */; }; 9FA0EE3A87678F58F4406B24 /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; 7B38D33B59821BD9BCFFA731 /* EventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 293A30078FEAA25BB4C3D26C /* EventProcessor.swift */; }; 8B3D9E2F5A6C1047B2D5E9F1 /* JSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6F2A8C1D4E7B9053A1C4D8E6 /* JSONWriter.swift */; }; @@ -617,6 +621,7 @@ 83F0A5631FB5F33800550A95 /* LDConfigSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LDConfigSpec.swift; sourceTree = ""; }; 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FlagSynchronizer.swift; sourceTree = ""; }; 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DirectEventProcessor.swift; sourceTree = ""; }; + 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OutboundEventBuffer.swift; sourceTree = ""; }; BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NullEventProcessor.swift; sourceTree = ""; }; 293A30078FEAA25BB4C3D26C /* EventProcessor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = EventProcessor.swift; sourceTree = ""; }; 88DAB8D1864C14D9C7E06F11 /* LDContextJSONWriter.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = LDContextJSONWriter.swift; sourceTree = ""; }; @@ -1062,6 +1067,7 @@ B4C9D4372489E20A004A9B03 /* DiagnosticReporter.swift */, 831425B0206B030100F2EF36 /* EnvironmentReporter.swift */, 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */, + 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */, BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */, 293A30078FEAA25BB4C3D26C /* EventProcessor.swift */, 8358F25F1F476AD800ECE1AF /* FlagChangeNotifier.swift */, @@ -1543,6 +1549,7 @@ 3D3AB9462A4F16FE003AECF1 /* ReportingConsts.swift in Sources */, 831188522113ADF700D77CB5 /* KeyedValueCache.swift in Sources */, 831188582113AE0F00D77CB5 /* DirectEventProcessor.swift in Sources */, + 629D5A9FE152DBC3E20D6DFC /* OutboundEventBuffer.swift in Sources */, 014901E06BE1F2B0FAAD6AA6 /* NullEventProcessor.swift in Sources */, 86B71C91DB78D0F17DBE0E69 /* EventProcessor.swift in Sources */, 50EE85C42EA0487F007CC662 /* TimeoutExecutor.swift in Sources */, @@ -1666,6 +1673,7 @@ 831EF35620655E730001C643 /* FlagChangeNotifier.swift in Sources */, A358D6D92A4DE6A500270C60 /* ApplicationInfoEnvironmentReporter.swift in Sources */, 831EF35720655E730001C643 /* DirectEventProcessor.swift in Sources */, + 5A89648200D2566D3F8409BA /* OutboundEventBuffer.swift in Sources */, CFF357D68BBF2DC8395930F2 /* NullEventProcessor.swift in Sources */, 850C49C9054E642CC006E730 /* EventProcessor.swift in Sources */, A350386F2F4F4F610032BA9F /* IdentifySeriesContext.swift in Sources */, @@ -1774,6 +1782,7 @@ A3599E882A4B4AD400DB5C67 /* Modifier.swift in Sources */, 8358F2601F476AD800ECE1AF /* FlagChangeNotifier.swift in Sources */, 83FEF8DF1F2667E4001CF12C /* DirectEventProcessor.swift in Sources */, + 7D37AD3ECAA9D8E2927228C5 /* OutboundEventBuffer.swift in Sources */, 9FA0EE3A87678F58F4406B24 /* NullEventProcessor.swift in Sources */, 7B38D33B59821BD9BCFFA731 /* EventProcessor.swift in Sources */, 831D2AAF2061AAA000B4AC3C /* Thread.swift in Sources */, @@ -1938,6 +1947,7 @@ 83D9EC882062DEAB004D7FA6 /* FlagChangeNotifier.swift in Sources */, C443A40B2315AA4D00145710 /* NetworkReporter.swift in Sources */, 83D9EC892062DEAB004D7FA6 /* DirectEventProcessor.swift in Sources */, + 73E23C0FA009141BB4CBC24C /* OutboundEventBuffer.swift in Sources */, 54745AEF1A43D165F5A6E08F /* NullEventProcessor.swift in Sources */, 7F4004579B9094737F3C21D1 /* EventProcessor.swift in Sources */, 83D9EC8A2062DEAB004D7FA6 /* FlagStore.swift in Sources */, diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index b13ddf42c..0faa6c1c5 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -47,10 +47,8 @@ class DirectEventProcessor: EventProcessor { private let onSyncComplete: EventSyncCompleteClosure? - /// Shared by the threads recording events, so it must not be mutated after `init`. - private let encoder: JSONEncoder - - let encoding: Encoding + /// Its summarizer is guarded by `recordLock`, and its encoders are reached only under `commitLock`. + let eventBuffer: OutboundEventBuffer // MARK: Guarded by commitLock @@ -65,12 +63,6 @@ class DirectEventProcessor: EventProcessor { /// Taken before `recordLock` and before anything the store locks, never after. private let commitLock = UnfairLock() - /// Outlives a commit rather than being built per commit, so that its buffer and context cache span them. - /// - /// A commit point writes a single event, so a cache that lived for one commit would never be read. It needs no - /// lock of its own because it is only ever reached from `encode(_:)`, which runs under `commitLock`. - fileprivate let eventJSONWriter: EventJSONWriter - // MARK: Guarded by recordLock /// Guards everything one recording writes: the held events and the summarizer. @@ -83,9 +75,6 @@ class DirectEventProcessor: EventProcessor { /// worth avoiding: the same measurement that made `EvaluationExposureDeduper` a lock applies here. private let recordLock = UnfairLock() - /// Only to be used while holding `recordLock`. - private(set) var contextSummarizer: ContextSummarizer - /// Full events recorded but not yet encoded. Only to be used while holding `recordLock`. /// /// Held rather than encoded because encoding early would not make them durable: the store stages bytes into memory @@ -182,17 +171,14 @@ class DirectEventProcessor: EventProcessor { init(service: DarklyServiceProvider, onSyncComplete: EventSyncCompleteClosure?, store: EventStoring? = nil, - encoding: Encoding = .handWritten, + encoding: OutboundEventBuffer.Encoding = .handWritten, commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.DirectEventProcessor.commitQueue", qos: .userInitiated)) { - self.eventJSONWriter = EventJSONWriter(config: service.config) self.service = service self.onSyncComplete = onSyncComplete self.responseDate = Date() - self.encoding = encoding self.eventCapacity = service.config.eventCapacity self.commitQueue = commitQueue - self.encoder = DirectEventProcessor.makeEncoder(config: service.config) - self.contextSummarizer = ContextSummarizer(logger: service.config.logger, maxContexts: service.config.eventCapacity) + self.eventBuffer = OutboundEventBuffer(config: service.config, encoding: encoding) self.store = store ?? DirectEventProcessor.makeStore(config: service.config) // A log left open by a previous run has to be closed before it can be delivered, but nothing waits on that: @@ -248,7 +234,7 @@ class DirectEventProcessor: EventProcessor { var dropped = 0 let (counted, needsCommit) = recordLock.withLock { () -> (Bool, Bool) in - let counted = contextSummarizer.trackRequest(flagKey: flagKey, reportedValue: value, featureFlag: featureFlag, defaultValue: defaultValue, context: context) + let counted = eventBuffer.summarize(flagKey: flagKey, reportedValue: value, featureFlag: featureFlag, defaultValue: defaultValue, context: context) if let featureEvent, !admitToPending(featureEvent) { dropped += 1 } if let debugEvent, !admitToPending(debugEvent) { dropped += 1 } return (counted, pending.count >= DirectEventProcessor.pendingCommitThreshold) @@ -385,10 +371,7 @@ class DirectEventProcessor: EventProcessor { let (run, summaries) = recordLock.withLock { () -> ([Event], [(tracker: FlagRequestTracker, context: LDContext)]) in let run = pending pending = [] - let summaries = contextSummarizer.hasLoggedRequests ? contextSummarizer.getSummaries() : [] - if !summaries.isEmpty { - contextSummarizer.clear() - } + let summaries = eventBuffer.takeSummaries() // Reserved in the same critical section the run leaves `pending` in, so capacity counts it in one place or // the other throughout the encode. store.reserve(run.count) @@ -412,7 +395,7 @@ class DirectEventProcessor: EventProcessor { private func stage(_ run: [Event]) { commitLock.assertOwned() for event in run { - guard let encoded = encode(event) + guard let encoded = eventBuffer.encode(event) else { reportLost() continue @@ -443,8 +426,7 @@ class DirectEventProcessor: EventProcessor { /// counted the same as one that produced a single summary covering the same evaluations. private func stageSummaries(_ summaries: [(tracker: FlagRequestTracker, context: LDContext)]) { for summary in summaries { - let summaryEvent = SummaryEvent(flagRequestTracker: summary.tracker, context: summary.context) - guard let encoded = encode(summaryEvent) + guard let encoded = eventBuffer.encodeSummary(summary) else { reportLost() continue @@ -457,16 +439,6 @@ class DirectEventProcessor: EventProcessor { } } - private func encode(_ event: Event) -> Data? { - let encoded: Data? = encoding == .codable ? try? encoder.encode(event) : eventJSONWriter.encode(event) - guard let encoded = encoded - else { - os_log("%s Failed to serialize event for publication: %s", log: service.config.logger, type: .error, typeName(and: #function), String(describing: event)) - return nil - } - return encoded - } - // MARK: Reporting private func startReporting() { @@ -801,30 +773,6 @@ class DirectEventProcessor: EventProcessor { } } -extension DirectEventProcessor { - /// Which encoder recorded events go through. - /// - /// `.codable` is the reference `.handWritten` is tested against. The two produce equal JSON but not identical - /// bytes: key order differs, as do `/` against `\/` and `0` against `-0`. - enum Encoding { - case codable - case handWritten - } - - fileprivate static func makeEncoder(config: LDConfig) -> JSONEncoder { - let encoder = JSONEncoder() - encoder.userInfo = [ - LDContext.UserInfoKeys.allAttributesPrivate: config.allContextAttributesPrivate, - LDContext.UserInfoKeys.globalPrivateAttributes: config.privateContextAttributes.map { $0 } - ] - encoder.dateEncodingStrategy = .custom { date, encoder in - var container = encoder.singleValueContainer() - try container.encode(date.millisSince1970) - } - return encoder - } -} - extension DirectEventProcessor: TypeIdentifying { } #if DEBUG diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift new file mode 100644 index 000000000..5f5996bb9 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift @@ -0,0 +1,91 @@ +import Foundation +import OSLog + +/// Turns what `DirectEventProcessor` records into what it stages: evaluations into summary counters, and events into the +/// JSON the events endpoint accepts. Android's `OutboundEventBuffer` plays the same part. +/// +/// It has no lock of its own. The summarizer is only to be used while holding the processor's `recordLock`, and the +/// encoders are only reached under its `commitLock`. +final class OutboundEventBuffer { + /// Which encoder recorded events go through. + /// + /// `.codable` is the reference `.handWritten` is tested against. The two produce equal JSON but not identical + /// bytes: key order differs, as do `/` against `\/` and `0` against `-0`. + enum Encoding { + case codable + case handWritten + } + + let encoding: Encoding + private let logger: OSLog + + /// Only to be used while holding the processor's `recordLock`. + private(set) var contextSummarizer: ContextSummarizer + + /// Shared by the threads recording events, so it must not be mutated after `init`. + private let encoder: JSONEncoder + + /// Outlives a commit rather than being built per commit, so that its buffer and context cache span them. + /// + /// A commit point writes a single event, so a cache that lived for one commit would never be read. It needs no + /// lock of its own because it is only ever reached from `encode(_:)`, which runs under the processor's + /// `commitLock`. + private let eventJSONWriter: EventJSONWriter + + init(config: LDConfig, encoding: Encoding) { + self.encoding = encoding + self.logger = config.logger + self.contextSummarizer = ContextSummarizer(logger: config.logger, maxContexts: config.eventCapacity) + self.encoder = OutboundEventBuffer.makeEncoder(config: config) + self.eventJSONWriter = EventJSONWriter(config: config) + } + + /// Counts an evaluation into its context's summary. Requires the processor's `recordLock`. + /// + /// - Returns: false if the evaluation was not counted, because its context is not counted yet and as many contexts + /// as the summarizer allows already are. + func summarize(flagKey: LDFlagKey, reportedValue: LDValue, featureFlag: FeatureFlag?, defaultValue: LDValue, context: LDContext) -> Bool { + contextSummarizer.trackRequest(flagKey: flagKey, reportedValue: reportedValue, featureFlag: featureFlag, defaultValue: defaultValue, context: context) + } + + /// Takes the counters gathered so far, leaving the summarizer empty, or nothing if nothing was counted. Requires the + /// processor's `recordLock`. + func takeSummaries() -> [(tracker: FlagRequestTracker, context: LDContext)] { + let summaries = contextSummarizer.hasLoggedRequests ? contextSummarizer.getSummaries() : [] + if !summaries.isEmpty { + contextSummarizer.clear() + } + return summaries + } + + /// Requires the processor's `commitLock`. + func encode(_ event: Event) -> Data? { + let encoded: Data? = encoding == .codable ? try? encoder.encode(event) : eventJSONWriter.encode(event) + guard let encoded = encoded + else { + os_log("%s Failed to serialize event for publication: %s", log: logger, type: .error, typeName(and: #function), String(describing: event)) + return nil + } + return encoded + } + + /// Encodes the summary of one context's counters. Requires the processor's `commitLock`. + func encodeSummary(_ summary: (tracker: FlagRequestTracker, context: LDContext)) -> Data? { + encode(SummaryEvent(flagRequestTracker: summary.tracker, context: summary.context)) + } + + private static func makeEncoder(config: LDConfig) -> JSONEncoder { + let encoder = JSONEncoder() + encoder.userInfo = [ + LDContext.UserInfoKeys.allAttributesPrivate: config.allContextAttributesPrivate, + LDContext.UserInfoKeys.globalPrivateAttributes: config.privateContextAttributes.map { $0 } + ] + encoder.dateEncodingStrategy = .custom { date, encoder in + var container = encoder.singleValueContainer() + try container.encode(date.millisSince1970) + } + return encoder + } +} + +extension OutboundEventBuffer: TypeIdentifying { } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 9e4915b2c..10e2bc551 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -300,7 +300,7 @@ final class DirectEventProcessorSpec: QuickSpec { var reporter: DirectEventProcessor! var store: EventStore! - func makeReporter(encoding: DirectEventProcessor.Encoding) -> DirectEventProcessor { + func makeReporter(encoding: OutboundEventBuffer.Encoding) -> DirectEventProcessor { var config = LDConfig.stub config.eventCapacity = 8 ldContext = LDContext.stub() @@ -315,7 +315,7 @@ final class DirectEventProcessorSpec: QuickSpec { reporter?.isOnline = false } - for encoding in [DirectEventProcessor.Encoding.codable, .handWritten] { + for encoding in [OutboundEventBuffer.Encoding.codable, .handWritten] { context("\(encoding): when an event holds a non-finite metric") { beforeEach { reporter = makeReporter(encoding: encoding) @@ -410,7 +410,7 @@ final class DirectEventProcessorSpec: QuickSpec { } expect(serviceMock.publishEventDataCallCount) == 0 expect(store.pendingEventCount) == 0 - expect(reporter.contextSummarizer.hasLoggedRequests) == false + expect(reporter.eventBuffer.contextSummarizer.hasLoggedRequests) == false waitUntil { done in reporter.recordFlagEvaluationEvents(flagKey: "after-poison", @@ -490,7 +490,7 @@ final class DirectEventProcessorSpec: QuickSpec { expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 expect(testContext.store.pendingEventCount) == 0 expect(testContext.eventProcessor.lastEventResponseDate) == testContext.eventStubResponseDate - expect(testContext.eventProcessor.contextSummarizer.hasLoggedRequests) == false + expect(testContext.eventProcessor.eventBuffer.contextSummarizer.hasLoggedRequests) == false expect(testContext.syncResult).to(beNil()) } } @@ -515,7 +515,7 @@ final class DirectEventProcessorSpec: QuickSpec { expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == testContext.events.count expect(testContext.store.pendingEventCount) == 0 expect(testContext.eventProcessor.lastEventResponseDate) == testContext.eventStubResponseDate - expect(testContext.eventProcessor.contextSummarizer.hasLoggedRequests) == false + expect(testContext.eventProcessor.eventBuffer.contextSummarizer.hasLoggedRequests) == false expect(testContext.syncResult).to(beNil()) } } @@ -553,7 +553,7 @@ final class DirectEventProcessorSpec: QuickSpec { expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == 1 expect(testContext.store.pendingEventCount) == 0 expect(testContext.eventProcessor.lastEventResponseDate) == testContext.eventStubResponseDate - expect(testContext.eventProcessor.contextSummarizer.hasLoggedRequests) == false + expect(testContext.eventProcessor.eventBuffer.contextSummarizer.hasLoggedRequests) == false expect(testContext.syncResult).to(beNil()) } } @@ -574,7 +574,7 @@ final class DirectEventProcessorSpec: QuickSpec { expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 0 expect(testContext.store.pendingEventCount) == 0 expect(testContext.eventProcessor.lastEventResponseDate) == Date.distantPast - expect(testContext.eventProcessor.contextSummarizer.hasLoggedRequests) == false + expect(testContext.eventProcessor.eventBuffer.contextSummarizer.hasLoggedRequests) == false expect(testContext.syncResult).to(beNil()) } } @@ -617,7 +617,7 @@ final class DirectEventProcessorSpec: QuickSpec { expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 expect(testContext.eventProcessor.lastEventResponseDate) == Date.distantPast - expect(testContext.eventProcessor.contextSummarizer.hasLoggedRequests) == false + expect(testContext.eventProcessor.eventBuffer.contextSummarizer.hasLoggedRequests) == false guard case let .request(error) = testContext.syncResult else { fail("Expected error result for event send") @@ -663,7 +663,7 @@ final class DirectEventProcessorSpec: QuickSpec { expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 expect(testContext.eventProcessor.lastEventResponseDate) == Date.distantPast - expect(testContext.eventProcessor.contextSummarizer.hasLoggedRequests) == false + expect(testContext.eventProcessor.eventBuffer.contextSummarizer.hasLoggedRequests) == false let expectedError = testContext.serviceMock.errorEventHTTPURLResponse guard case let .response(error) = testContext.syncResult else { @@ -712,7 +712,7 @@ final class DirectEventProcessorSpec: QuickSpec { expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 expect(testContext.eventProcessor.lastEventResponseDate) == Date.distantPast - expect(testContext.eventProcessor.contextSummarizer.hasLoggedRequests) == false + expect(testContext.eventProcessor.eventBuffer.contextSummarizer.hasLoggedRequests) == false guard case let .request(error) = testContext.syncResult else { fail("Expected error result for event send") @@ -789,7 +789,7 @@ final class DirectEventProcessorSpec: QuickSpec { let pending = testContext.committedEvents() expect(pending.dropLast().map { $0 }) == testContext.recordedEventsAsJSON() expect(pending.last?.kindField) == "summary" - expect(testContext.eventProcessor.contextSummarizer.hasLoggedRequests) == false + expect(testContext.eventProcessor.eventBuffer.contextSummarizer.hasLoggedRequests) == false guard case .isOffline = testContext.syncResult else { fail("Expected error .isOffline result for event send") @@ -833,7 +833,7 @@ final class DirectEventProcessorSpec: QuickSpec { reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: tracked, context: LDContext.stub(key: "c"), includeReason: false) expect(diagnosticCache.incrementDroppedEventCountCallCount) == 1 - expect(reporter.contextSummarizer.getSummaries().count) == 2 + expect(reporter.eventBuffer.contextSummarizer.getSummaries().count) == 2 reporter.commitRecordedEvents() expect(recordedEvents(store).compactMap { $0.kindField }) == ["feature", "summary", "summary"] } @@ -842,8 +842,8 @@ final class DirectEventProcessorSpec: QuickSpec { defer { store.deleteEverything() } reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: nil, context: context, includeReason: true) expect(recordedEvents(store)).to(beEmpty()) - expect(reporter.contextSummarizer.hasLoggedRequests) == true - let summaries = reporter.contextSummarizer.getSummaries() + expect(reporter.eventBuffer.contextSummarizer.hasLoggedRequests) == true + let summaries = reporter.eventBuffer.contextSummarizer.getSummaries() expect(summaries.count) == 1 let tracker = summaries[0].tracker expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" @@ -857,8 +857,8 @@ final class DirectEventProcessorSpec: QuickSpec { let flag = FeatureFlag(flagKey: "unused", value: nil, variation: 1, flagVersion: 2, trackEvents: false) reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: true) expect(recordedEvents(store)).to(beEmpty()) - expect(reporter.contextSummarizer.hasLoggedRequests) == true - let summaries = reporter.contextSummarizer.getSummaries() + expect(reporter.eventBuffer.contextSummarizer.hasLoggedRequests) == true + let summaries = reporter.eventBuffer.contextSummarizer.getSummaries() expect(summaries.count) == 1 let tracker = summaries[0].tracker expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" @@ -874,8 +874,8 @@ final class DirectEventProcessorSpec: QuickSpec { let expected = FeatureEvent(key: "flag-key", context: context, value: "a", defaultValue: "b", featureFlag: flag, includeReason: true, isDebug: false) expect(expectedEvents(reporter.pendingEventsForTesting)) == expectedEvents([expected]) expect(recordedEvents(store)).to(beEmpty()) - expect(reporter.contextSummarizer.hasLoggedRequests) == true - let summaries = reporter.contextSummarizer.getSummaries() + expect(reporter.eventBuffer.contextSummarizer.hasLoggedRequests) == true + let summaries = reporter.eventBuffer.contextSummarizer.getSummaries() expect(summaries.count) == 1 let tracker = summaries[0].tracker expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" @@ -889,8 +889,8 @@ final class DirectEventProcessorSpec: QuickSpec { let flag = FeatureFlag(flagKey: "unused", value: nil, variation: 1, flagVersion: 2, trackEvents: false, debugEventsUntilDate: Date().addingTimeInterval(-1.0)) reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: true) expect(recordedEvents(store)).to(beEmpty()) - expect(reporter.contextSummarizer.hasLoggedRequests) == true - let summaries = reporter.contextSummarizer.getSummaries() + expect(reporter.eventBuffer.contextSummarizer.hasLoggedRequests) == true + let summaries = reporter.eventBuffer.contextSummarizer.getSummaries() expect(summaries.count) == 1 let tracker = summaries[0].tracker expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" @@ -906,8 +906,8 @@ final class DirectEventProcessorSpec: QuickSpec { let expected = FeatureEvent(key: "flag-key", context: context, value: "a", defaultValue: "b", featureFlag: flag, includeReason: false, isDebug: true) expect(expectedEvents(reporter.pendingEventsForTesting)) == expectedEvents([expected]) expect(recordedEvents(store)).to(beEmpty()) - expect(reporter.contextSummarizer.hasLoggedRequests) == true - let summaries = reporter.contextSummarizer.getSummaries() + expect(reporter.eventBuffer.contextSummarizer.hasLoggedRequests) == true + let summaries = reporter.eventBuffer.contextSummarizer.getSummaries() expect(summaries.count) == 1 let tracker = summaries[0].tracker expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" @@ -922,8 +922,8 @@ final class DirectEventProcessorSpec: QuickSpec { let flag = FeatureFlag(flagKey: "unused", value: nil, variation: 1, flagVersion: 2, trackEvents: false, debugEventsUntilDate: Date().addingTimeInterval(3.0)) reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: true) expect(recordedEvents(store)).to(beEmpty()) - expect(reporter.contextSummarizer.hasLoggedRequests) == true - let summaries = reporter.contextSummarizer.getSummaries() + expect(reporter.eventBuffer.contextSummarizer.hasLoggedRequests) == true + let summaries = reporter.eventBuffer.contextSummarizer.getSummaries() expect(summaries.count) == 1 let tracker = summaries[0].tracker expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" @@ -941,8 +941,8 @@ final class DirectEventProcessorSpec: QuickSpec { let expectedDebug = FeatureEvent(key: "flag-key", context: context, value: "a", defaultValue: "b", featureFlag: flag, includeReason: false, isDebug: true) expect(expectedEvents(reporter.pendingEventsForTesting)) == expectedEvents([expectedFeature, expectedDebug]) expect(recordedEvents(store)).to(beEmpty()) - expect(reporter.contextSummarizer.hasLoggedRequests) == true - let summaries = reporter.contextSummarizer.getSummaries() + expect(reporter.eventBuffer.contextSummarizer.hasLoggedRequests) == true + let summaries = reporter.eventBuffer.contextSummarizer.getSummaries() expect(summaries.count) == 1 let tracker = summaries[0].tracker expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" @@ -968,8 +968,8 @@ final class DirectEventProcessorSpec: QuickSpec { expect(held.filter { $0.kindField == "feature" }.count) == 10 expect(held.filter { $0.kindField == "debug" }.count) == 10 expect(recordedEvents(store)).to(beEmpty()) - expect(reporter.contextSummarizer.hasLoggedRequests) == true - let summaries = reporter.contextSummarizer.getSummaries() + expect(reporter.eventBuffer.contextSummarizer.hasLoggedRequests) == true + let summaries = reporter.eventBuffer.contextSummarizer.getSummaries() expect(summaries.count) == 1 let tracker = summaries[0].tracker expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" From 79ddb323091b973ab7d1137d80106f4f542f9c1b Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Sun, 11 Oct 2026 21:27:14 -0700 Subject: [PATCH 032/158] Move posting a batch and reading the response into AnalyticsEventSender, as Android has it Co-authored-by: Cursor --- LaunchDarkly.xcodeproj/project.pbxproj | 10 +++ .../ServiceObjects/AnalyticsEventSender.swift | 64 ++++++++++++++ .../ServiceObjects/DirectEventProcessor.swift | 86 +++++-------------- 3 files changed, 96 insertions(+), 64 deletions(-) create mode 100644 LaunchDarkly/LaunchDarkly/ServiceObjects/AnalyticsEventSender.swift diff --git a/LaunchDarkly.xcodeproj/project.pbxproj b/LaunchDarkly.xcodeproj/project.pbxproj index 7d868a762..0c0cd2618 100644 --- a/LaunchDarkly.xcodeproj/project.pbxproj +++ b/LaunchDarkly.xcodeproj/project.pbxproj @@ -80,6 +80,7 @@ 831188562113AE0800D77CB5 /* FlagSynchronizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */; }; 831188572113AE0B00D77CB5 /* FlagChangeNotifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8358F25F1F476AD800ECE1AF /* FlagChangeNotifier.swift */; }; 831188582113AE0F00D77CB5 /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; + B66ECB2014E0FC95C23DA0A2 /* AnalyticsEventSender.swift in Sources */ = {isa = PBXBuildFile; fileRef = 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */; }; 629D5A9FE152DBC3E20D6DFC /* OutboundEventBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */; }; 014901E06BE1F2B0FAAD6AA6 /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; 86B71C91DB78D0F17DBE0E69 /* EventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 293A30078FEAA25BB4C3D26C /* EventProcessor.swift */; }; @@ -124,6 +125,7 @@ 831EF35520655E730001C643 /* FlagSynchronizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */; }; 831EF35620655E730001C643 /* FlagChangeNotifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8358F25F1F476AD800ECE1AF /* FlagChangeNotifier.swift */; }; 831EF35720655E730001C643 /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; + 367D374B5CD1B2A8596C0738 /* AnalyticsEventSender.swift in Sources */ = {isa = PBXBuildFile; fileRef = 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */; }; 5A89648200D2566D3F8409BA /* OutboundEventBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */; }; CFF357D68BBF2DC8395930F2 /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; 850C49C9054E642CC006E730 /* EventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 293A30078FEAA25BB4C3D26C /* EventProcessor.swift */; }; @@ -209,6 +211,7 @@ 83D9EC872062DEAB004D7FA6 /* FlagSynchronizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */; }; 83D9EC882062DEAB004D7FA6 /* FlagChangeNotifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8358F25F1F476AD800ECE1AF /* FlagChangeNotifier.swift */; }; 83D9EC892062DEAB004D7FA6 /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; + 321659EE1824B83BD0D0F22D /* AnalyticsEventSender.swift in Sources */ = {isa = PBXBuildFile; fileRef = 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */; }; 73E23C0FA009141BB4CBC24C /* OutboundEventBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */; }; 54745AEF1A43D165F5A6E08F /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; 7F4004579B9094737F3C21D1 /* EventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 293A30078FEAA25BB4C3D26C /* EventProcessor.swift */; }; @@ -236,6 +239,7 @@ 83F0A5641FB5F33800550A95 /* LDConfigSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83F0A5631FB5F33800550A95 /* LDConfigSpec.swift */; }; 83FEF8DD1F266742001CF12C /* FlagSynchronizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */; }; 83FEF8DF1F2667E4001CF12C /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; + E91F5DCAEEE5FC4AC408BA67 /* AnalyticsEventSender.swift in Sources */ = {isa = PBXBuildFile; fileRef = 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */; }; 7D37AD3ECAA9D8E2927228C5 /* OutboundEventBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */; }; 9FA0EE3A87678F58F4406B24 /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; 7B38D33B59821BD9BCFFA731 /* EventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 293A30078FEAA25BB4C3D26C /* EventProcessor.swift */; }; @@ -621,6 +625,7 @@ 83F0A5631FB5F33800550A95 /* LDConfigSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LDConfigSpec.swift; sourceTree = ""; }; 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FlagSynchronizer.swift; sourceTree = ""; }; 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DirectEventProcessor.swift; sourceTree = ""; }; + 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AnalyticsEventSender.swift; sourceTree = ""; }; 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OutboundEventBuffer.swift; sourceTree = ""; }; BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NullEventProcessor.swift; sourceTree = ""; }; 293A30078FEAA25BB4C3D26C /* EventProcessor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = EventProcessor.swift; sourceTree = ""; }; @@ -1067,6 +1072,7 @@ B4C9D4372489E20A004A9B03 /* DiagnosticReporter.swift */, 831425B0206B030100F2EF36 /* EnvironmentReporter.swift */, 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */, + 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */, 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */, BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */, 293A30078FEAA25BB4C3D26C /* EventProcessor.swift */, @@ -1549,6 +1555,7 @@ 3D3AB9462A4F16FE003AECF1 /* ReportingConsts.swift in Sources */, 831188522113ADF700D77CB5 /* KeyedValueCache.swift in Sources */, 831188582113AE0F00D77CB5 /* DirectEventProcessor.swift in Sources */, + B66ECB2014E0FC95C23DA0A2 /* AnalyticsEventSender.swift in Sources */, 629D5A9FE152DBC3E20D6DFC /* OutboundEventBuffer.swift in Sources */, 014901E06BE1F2B0FAAD6AA6 /* NullEventProcessor.swift in Sources */, 86B71C91DB78D0F17DBE0E69 /* EventProcessor.swift in Sources */, @@ -1673,6 +1680,7 @@ 831EF35620655E730001C643 /* FlagChangeNotifier.swift in Sources */, A358D6D92A4DE6A500270C60 /* ApplicationInfoEnvironmentReporter.swift in Sources */, 831EF35720655E730001C643 /* DirectEventProcessor.swift in Sources */, + 367D374B5CD1B2A8596C0738 /* AnalyticsEventSender.swift in Sources */, 5A89648200D2566D3F8409BA /* OutboundEventBuffer.swift in Sources */, CFF357D68BBF2DC8395930F2 /* NullEventProcessor.swift in Sources */, 850C49C9054E642CC006E730 /* EventProcessor.swift in Sources */, @@ -1782,6 +1790,7 @@ A3599E882A4B4AD400DB5C67 /* Modifier.swift in Sources */, 8358F2601F476AD800ECE1AF /* FlagChangeNotifier.swift in Sources */, 83FEF8DF1F2667E4001CF12C /* DirectEventProcessor.swift in Sources */, + E91F5DCAEEE5FC4AC408BA67 /* AnalyticsEventSender.swift in Sources */, 7D37AD3ECAA9D8E2927228C5 /* OutboundEventBuffer.swift in Sources */, 9FA0EE3A87678F58F4406B24 /* NullEventProcessor.swift in Sources */, 7B38D33B59821BD9BCFFA731 /* EventProcessor.swift in Sources */, @@ -1947,6 +1956,7 @@ 83D9EC882062DEAB004D7FA6 /* FlagChangeNotifier.swift in Sources */, C443A40B2315AA4D00145710 /* NetworkReporter.swift in Sources */, 83D9EC892062DEAB004D7FA6 /* DirectEventProcessor.swift in Sources */, + 321659EE1824B83BD0D0F22D /* AnalyticsEventSender.swift in Sources */, 73E23C0FA009141BB4CBC24C /* OutboundEventBuffer.swift in Sources */, 54745AEF1A43D165F5A6E08F /* NullEventProcessor.swift in Sources */, 7F4004579B9094737F3C21D1 /* EventProcessor.swift in Sources */, diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/AnalyticsEventSender.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/AnalyticsEventSender.swift new file mode 100644 index 000000000..6b27e2786 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/AnalyticsEventSender.swift @@ -0,0 +1,64 @@ +import Foundation +import OSLog + +/// Posts a batch of events to LaunchDarkly, retrying once a second later where the failure may pass, and says what the +/// response means. Android's `AnalyticsEventSender` plays the same part. +/// +/// It keeps no state. What an outcome means for the SDK -- the server's time, going offline, removing the batch -- is +/// left to `DirectEventProcessor`, which owns that state. +final class AnalyticsEventSender { + enum Outcome { + /// LaunchDarkly took the events. Carries the time the response gave, if it gave one. + case accepted(serverTime: Date?) + /// LaunchDarkly will never take these events, so keeping them only wastes space. + case refused(HTTPURLResponse?) + /// The events may still be deliverable. Carries why the retry failed. + case retryable(SynchronizingError) + } + + private let service: DarklyServiceProvider + + init(service: DarklyServiceProvider) { + self.service = service + } + + /// Posts `body` under `payloadId`, and once more after a second if that failed in a way that may pass. The + /// completion runs once, with the outcome of the last attempt. + func send(_ body: Data, payloadId: String, eventCount: Int, completion: @escaping (Outcome) -> Void) { + service.publishEventData(body, payloadId) { response in + let outcome = self.outcome(sentEvents: eventCount, response: response.urlResponse as? HTTPURLResponse, error: response.error) + guard case .retryable = outcome + else { + completion(outcome) + return + } + + os_log("%s Retrying event post after delay.", log: self.service.config.logger, type: .debug, self.typeName(and: #function)) + DispatchQueue.global().asyncAfter(deadline: DispatchTime.now() + 1.0) { + self.service.publishEventData(body, payloadId) { retryResponse in + completion(self.outcome(sentEvents: eventCount, response: retryResponse.urlResponse as? HTTPURLResponse, error: retryResponse.error)) + } + } + } + } + + private func outcome(sentEvents: Int, response: HTTPURLResponse?, error: Error?) -> Outcome { + if error == nil && (200..<300).contains(response?.statusCode ?? 0) { + os_log("%s Completed sending %d event(s)", log: service.config.logger, type: .debug, typeName(and: #function), sentEvents) + return .accepted(serverTime: response?.headerDate) + } + + if let statusCode = response?.statusCode, HTTPURLResponse.StatusCodes.isTerminalStatusCode(statusCode) { + os_log("%s dropping events due to non-retriable response: %s", log: service.config.logger, type: .debug, typeName(and: #function), String(describing: response)) + return .refused(response) + } + + os_log("%s Sending events failed with error: %s response: %s", log: service.config.logger, type: .debug, typeName(and: #function), String(describing: error), String(describing: response)) + if let error = error { + return .retryable(.request(error)) + } + return .retryable(.response(response)) + } +} + +extension AnalyticsEventSender: TypeIdentifying { } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 0faa6c1c5..88f92ce2a 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -47,6 +47,8 @@ class DirectEventProcessor: EventProcessor { private let onSyncComplete: EventSyncCompleteClosure? + private let sender: AnalyticsEventSender + /// Its summarizer is guarded by `recordLock`, and its encoders are reached only under `commitLock`. let eventBuffer: OutboundEventBuffer @@ -175,6 +177,7 @@ class DirectEventProcessor: EventProcessor { commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.DirectEventProcessor.commitQueue", qos: .userInitiated)) { self.service = service self.onSyncComplete = onSyncComplete + self.sender = AnalyticsEventSender(service: service) self.responseDate = Date() self.eventCapacity = service.config.eventCapacity self.commitQueue = commitQueue @@ -692,77 +695,32 @@ class DirectEventProcessor: EventProcessor { } } - private func publish(_ batch: EventBatch, _ body: Data, _ completion: @escaping (DeliveryOutcome) -> Void) { - service.publishEventData(body, batch.payloadId) { response in - let outcome = self.outcome(sentEvents: batch.eventCount, response: response.urlResponse as? HTTPURLResponse, error: response.error, isRetry: false) + /// Sends one batch and acts on what LaunchDarkly made of it, before handing the outcome on. + private func publish(_ batch: EventBatch, _ body: Data, _ completion: @escaping (AnalyticsEventSender.Outcome) -> Void) { + sender.send(body, payloadId: batch.payloadId, eventCount: batch.eventCount) { outcome in switch outcome { - case .accepted, .refused: - self.store.remove(batch) - completion(outcome) - case .retryable: - os_log("%s Retrying event post after delay.", log: self.service.config.logger, type: .debug, self.typeName(and: #function)) - DispatchQueue.global().asyncAfter(deadline: DispatchTime.now() + 1.0) { - self.service.publishEventData(body, batch.payloadId) { retryResponse in - let retried = self.outcome(sentEvents: batch.eventCount, response: retryResponse.urlResponse as? HTTPURLResponse, error: retryResponse.error, isRetry: true) - switch retried { - case .accepted, .refused: - self.store.remove(batch) - case .retryable: - // The batch stays on disk. A later delivery, in this run of the application or the next - // one, sends it under the same payload ID, so LaunchDarkly can tell it is not new. - os_log("%s Keeping %d event(s) on disk to retry later", log: self.service.config.logger, type: .debug, self.typeName(and: #function), batch.eventCount) - } - completion(retried) + case .accepted(let serverTime): + self.stateLock.withLock { + if let serverTime = serverTime, serverTime > self.responseDate { + self.responseDate = serverTime } } + self.reportSyncComplete(nil) + self.store.remove(batch) + case .refused(let response): + self.isOnline = false + self.reportSyncComplete(.response(response)) + self.store.remove(batch) + case .retryable(let error): + self.reportSyncComplete(error) + // The batch stays on disk. A later delivery, in this run of the application or the next one, sends it + // under the same payload ID, so LaunchDarkly can tell it is not new. + os_log("%s Keeping %d event(s) on disk to retry later", log: self.service.config.logger, type: .debug, self.typeName(and: #function), batch.eventCount) } + completion(outcome) } } - private enum DeliveryOutcome { - /// LaunchDarkly took the events. - case accepted - /// LaunchDarkly will never take these events, so keeping them only wastes space. - case refused - /// The events may still be deliverable. - case retryable - } - - private func outcome(sentEvents: Int, response: HTTPURLResponse?, error: Error?, isRetry: Bool) -> DeliveryOutcome { - if error == nil && (200..<300).contains(response?.statusCode ?? 0) { - let serverTime = response?.headerDate - stateLock.withLock { - if let serverTime = serverTime, serverTime > responseDate { - responseDate = serverTime - } - } - - os_log("%s Completed sending %d event(s)", log: service.config.logger, type: .debug, typeName(and: #function), sentEvents) - self.reportSyncComplete(nil) - return .accepted - } - - if let statusCode = response?.statusCode, HTTPURLResponse.StatusCodes.isTerminalStatusCode(statusCode) { - os_log("%s dropping events due to non-retriable response: %s", log: service.config.logger, type: .debug, typeName(and: #function), String(describing: response)) - isOnline = false - self.reportSyncComplete(.response(response)) - return .refused - } - - os_log("%s Sending events failed with error: %s response: %s", log: service.config.logger, type: .debug, typeName(and: #function), String(describing: error), String(describing: response)) - - if isRetry { - if let error = error { - reportSyncComplete(.request(error)) - } else { - reportSyncComplete(.response(response)) - } - return .retryable - } - - return .retryable - } - private func reportSyncComplete(_ result: SynchronizingError?) { // The eventProcessor is created when the LDClient singleton is created, and kept for the app's lifetime. So while the use of self in the async block does setup a retain cycle, it's not going to cause a memory leak guard let onSyncComplete = onSyncComplete From ab53b778386739a0aeeae8b96943dbccfa76f8f7 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Sun, 11 Oct 2026 23:40:24 -0700 Subject: [PATCH 033/158] Count a commit's events while they are being written, so a slow write cannot open room under capacity Co-authored-by: Cursor --- .../ServiceObjects/EventStore.swift | 15 +++++++-- .../ServiceObjects/EventStoreSpec.swift | 32 +++++++++++++++++++ 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index adf388644..b3f0644c8 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -110,6 +110,9 @@ final class EventStore: EventStoring { private var bufferData = Data() private var bufferedEventCount = 0 private var reservedEvents = 0 + /// Taken out of the buffer by a commit and not yet in the log. Counted so a slow write cannot open room under + /// capacity for events the store is still holding. + private var writingEvents = 0 private var committedEvents = 0 private var closedEvents = 0 /// Whether events are being written to disk, which is what the application asked for until a write fails and the @@ -200,7 +203,7 @@ final class EventStore: EventStoring { // MARK: Recording var pendingEventCount: Int { - bufferLock.withLock { reservedEvents + bufferedEventCount + committedEvents + closedEvents } + bufferLock.withLock { reservedEvents + bufferedEventCount + writingEvents + committedEvents + closedEvents } } var isPersisting: Bool { @@ -235,7 +238,8 @@ final class EventStore: EventStoring { // Whether the staged bytes are due a write, or nil where the event was refused for capacity. let needsCommit = bufferLock.withLock { () -> Bool? in - guard bypassingCapacity || reservedEvents + bufferedEventCount + committedEvents + closedEvents < capacity + guard bypassingCapacity + || reservedEvents + bufferedEventCount + writingEvents + committedEvents + closedEvents < capacity else { return nil } // In the same critical section as the append, so the event is never counted twice or not at all. @@ -530,17 +534,22 @@ final class EventStore: EventStoring { let staged = (bytes: bufferData, events: bufferedEventCount) bufferData = Data() bufferedEventCount = 0 + writingEvents = staged.events return staged } guard let taken else { return } if append(taken.bytes) { - bufferLock.withLock { committedEvents += taken.events } + bufferLock.withLock { + writingEvents = 0 + committedEvents += taken.events + } } else { // Put back, to be delivered from memory rather than lost. Ahead of anything staged while the write was in // flight, so the events keep the order they were recorded in. bufferLock.withLock { + writingEvents = 0 bufferData = taken.bytes + bufferData bufferedEventCount += taken.events } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 1cd5891de..3508c1407 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -666,6 +666,38 @@ final class EventStoreSpec: QuickSpec { expect(store.stage(EventStoreSpec.payload("second"))) == false } + it("counts the events a commit is still writing, so a slow write does not make room") { + let writing = DispatchSemaphore(value: 0) + let resume = DispatchSemaphore(value: 0) + var blocked = false + store = EventStore.temporary(capacity: 2, writeLog: { file, bytes in + if !blocked { + blocked = true + writing.signal() + resume.wait() + } + return file.writeAll(bytes) + }) + expect(store.stage(EventStoreSpec.payload("first"))) == true + expect(store.stage(EventStoreSpec.payload("second"))) == true + + let committed = DispatchSemaphore(value: 0) + DispatchQueue.global().async { + store.commit() + committed.signal() + } + writing.wait() + + expect(store.pendingEventCount) == 2 + expect(store.stage(EventStoreSpec.payload("third"))) == false + + resume.signal() + committed.wait() + + expect(store.pendingEventCount) == 2 + expect(store.stage(EventStoreSpec.payload("fourth"))) == false + } + it("takes a summary even when it is full, since those evaluations were already counted") { store = EventStore.temporary(capacity: 1) From 1db8cc7453e51baf8a703482cba299fff3f66e1f Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Sun, 11 Oct 2026 23:40:24 -0700 Subject: [PATCH 034/158] Lay out long tier-3 signatures one parameter per line Co-authored-by: Cursor --- .../ServiceObjects/ClientServiceFactory.swift | 8 ++++++-- .../ServiceObjects/DirectEventProcessor.swift | 4 +++- .../LaunchDarkly/ServiceObjects/DiskEventLog.swift | 6 +++++- .../LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift | 6 +++++- .../Mocks/ClientServiceMockFactory.swift | 4 +++- 5 files changed, 22 insertions(+), 6 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/ClientServiceFactory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/ClientServiceFactory.swift index 1027d9ab5..874702c00 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/ClientServiceFactory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/ClientServiceFactory.swift @@ -16,7 +16,9 @@ protocol ClientServiceCreating { onSyncComplete: FlagSyncCompleteClosure?) -> LDFlagSynchronizing func makeFlagChangeNotifier() -> FlagChangeNotifying func makeEventProcessor(config: LDConfig, service: DarklyServiceProvider) -> EventProcessor - func makeEventProcessor(config: LDConfig, service: DarklyServiceProvider, onSyncComplete: EventSyncCompleteClosure?) -> EventProcessor + func makeEventProcessor(config: LDConfig, + service: DarklyServiceProvider, + onSyncComplete: EventSyncCompleteClosure?) -> EventProcessor func makeStreamingProvider(url: URL, httpHeaders: [String: String], connectMethod: String, connectBody: Data?, handler: EventHandler, delegate: RequestHeaderTransform?, errorHandler: ConnectionErrorHandler?) -> DarklyStreamingProvider func makeEnvironmentReporter(config: LDConfig) -> EnvironmentReporting func makeThrottler(environmentReporter: EnvironmentReporting) -> Throttling @@ -70,7 +72,9 @@ final class ClientServiceFactory: ClientServiceCreating { makeEventProcessor(config: config, service: service, onSyncComplete: nil) } - func makeEventProcessor(config: LDConfig, service: DarklyServiceProvider, onSyncComplete: EventSyncCompleteClosure? = nil) -> EventProcessor { + func makeEventProcessor(config: LDConfig, + service: DarklyServiceProvider, + onSyncComplete: EventSyncCompleteClosure? = nil) -> EventProcessor { if config.sendEvents { return DirectEventProcessor(service: service, onSyncComplete: onSyncComplete) } else { diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 88f92ce2a..246aa413f 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -696,7 +696,9 @@ class DirectEventProcessor: EventProcessor { } /// Sends one batch and acts on what LaunchDarkly made of it, before handing the outcome on. - private func publish(_ batch: EventBatch, _ body: Data, _ completion: @escaping (AnalyticsEventSender.Outcome) -> Void) { + private func publish(_ batch: EventBatch, + _ body: Data, + _ completion: @escaping (AnalyticsEventSender.Outcome) -> Void) { sender.send(body, payloadId: batch.payloadId, eventCount: batch.eventCount) { outcome in switch outcome { case .accepted(let serverTime): diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift index 60124b2f1..7d53b5626 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift @@ -47,7 +47,11 @@ final class DiskEventLog { var isLocked = true #endif - init(directory: URL, processName: String, ioLock: UnfairLock, logger: OSLog, writeLog: @escaping (EventLogFile, Data) -> Int32?) { + init(directory: URL, + processName: String, + ioLock: UnfairLock, + logger: OSLog, + writeLog: @escaping (EventLogFile, Data) -> Int32?) { self.directory = directory self.ioLock = ioLock self.logger = logger diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift index 5f5996bb9..0fe9691b3 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift @@ -44,7 +44,11 @@ final class OutboundEventBuffer { /// /// - Returns: false if the evaluation was not counted, because its context is not counted yet and as many contexts /// as the summarizer allows already are. - func summarize(flagKey: LDFlagKey, reportedValue: LDValue, featureFlag: FeatureFlag?, defaultValue: LDValue, context: LDContext) -> Bool { + func summarize(flagKey: LDFlagKey, + reportedValue: LDValue, + featureFlag: FeatureFlag?, + defaultValue: LDValue, + context: LDContext) -> Bool { contextSummarizer.trackRequest(flagKey: flagKey, reportedValue: reportedValue, featureFlag: featureFlag, defaultValue: defaultValue, context: context) } diff --git a/LaunchDarkly/LaunchDarklyTests/Mocks/ClientServiceMockFactory.swift b/LaunchDarkly/LaunchDarklyTests/Mocks/ClientServiceMockFactory.swift index b4c859ebb..3426240b5 100644 --- a/LaunchDarkly/LaunchDarklyTests/Mocks/ClientServiceMockFactory.swift +++ b/LaunchDarkly/LaunchDarklyTests/Mocks/ClientServiceMockFactory.swift @@ -76,7 +76,9 @@ final class ClientServiceMockFactory: ClientServiceCreating { var makeEventProcessorReceivedConfig: LDConfig? = nil var makeEventProcessorReceivedService: DarklyServiceProvider? = nil var onEventSyncComplete: EventSyncCompleteClosure? = nil - func makeEventProcessor(config: LDConfig, service: DarklyServiceProvider, onSyncComplete: EventSyncCompleteClosure?) -> EventProcessor { + func makeEventProcessor(config: LDConfig, + service: DarklyServiceProvider, + onSyncComplete: EventSyncCompleteClosure?) -> EventProcessor { makeEventProcessorCallCount += 1 makeEventProcessorReceivedConfig = config makeEventProcessorReceivedService = service From 366889058f98fe408851cf7c46ae801de0189e9c Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Sun, 11 Oct 2026 23:45:14 -0700 Subject: [PATCH 035/158] Fix lint warnings in files the event durability tiers touched Moves the reporting and writing methods into same-file extensions to keep the type bodies under the limit, moves the LDValue non-finite check to LDCommon so LDContext stays under the file limit, and clears redundant nil coalescing, explicit nil initialization and void ternaries. No behavior change. Co-authored-by: Cursor --- LaunchDarkly/LaunchDarkly/LDClient.swift | 2 +- .../LaunchDarkly/LDClientVariation.swift | 2 +- LaunchDarkly/LaunchDarkly/LDCommon.swift | 12 +++++++ .../Models/Context/LDContext.swift | 12 ------- .../ServiceObjects/DirectEventProcessor.swift | 4 ++- .../ServiceObjects/EventStore.swift | 20 ++++++----- .../ServiceObjects/NullEventProcessor.swift | 1 + .../LDClientEvaluationHookSpec.swift | 2 +- .../LaunchDarklyTests/LDClientSpec.swift | 36 +++++++++++++++---- .../DirectEventProcessorSpec.swift | 2 +- .../ServiceObjects/EventStoreSpec.swift | 6 ++-- 11 files changed, 64 insertions(+), 35 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/LDClient.swift b/LaunchDarkly/LaunchDarkly/LDClient.swift index 516e54002..cdcffe366 100644 --- a/LaunchDarkly/LaunchDarkly/LDClient.swift +++ b/LaunchDarkly/LaunchDarkly/LDClient.swift @@ -1041,7 +1041,7 @@ public class LDClient { context = AutoEnvContextModifier(environmentReporter: environmentReporter, logger: config.logger).modifyContext(context) } - var hookState: IdentifyHookState? = nil + var hookState: IdentifyHookState? if !hooks.isEmpty { let seriesContext = IdentifySeriesContext(context: context, methodName: "init") let seriesData = hooks.map { hook in hook.beforeIdentify(seriesContext: seriesContext, seriesData: EvaluationSeriesData()) } diff --git a/LaunchDarkly/LaunchDarkly/LDClientVariation.swift b/LaunchDarkly/LaunchDarkly/LDClientVariation.swift index a3dd71352..76c1f4836 100644 --- a/LaunchDarkly/LaunchDarkly/LDClientVariation.swift +++ b/LaunchDarkly/LaunchDarkly/LDClientVariation.swift @@ -170,7 +170,7 @@ extension LDClient { } private func variationDetailInternal(_ flagKey: LDFlagKey, _ defaultValue: T, needsReason: Bool, methodName: String) -> LDEvaluationDetail where T: Decodable, T: LDValueConvertible { - var visited: Set? = nil + var visited: Set? return variationDetailInternal(flagKey, defaultValue, needsReason: needsReason, methodName: methodName, visited: &visited) } diff --git a/LaunchDarkly/LaunchDarkly/LDCommon.swift b/LaunchDarkly/LaunchDarkly/LDCommon.swift index af2628422..5d82e1895 100644 --- a/LaunchDarkly/LaunchDarkly/LDCommon.swift +++ b/LaunchDarkly/LaunchDarkly/LDCommon.swift @@ -184,3 +184,15 @@ public enum LDValue: Codable, } } } + +extension LDValue { + /// Whether this holds NaN or an infinity anywhere, which JSON cannot represent. + var containsNonFiniteNumber: Bool { + switch self { + case .number(let number): return !number.isFinite + case .array(let values): return values.contains { $0.containsNonFiniteNumber } + case .object(let values): return values.values.contains { $0.containsNonFiniteNumber } + case .null, .bool, .string: return false + } + } +} diff --git a/LaunchDarkly/LaunchDarkly/Models/Context/LDContext.swift b/LaunchDarkly/LaunchDarkly/Models/Context/LDContext.swift index 36095c3ef..b2e43468f 100644 --- a/LaunchDarkly/LaunchDarkly/Models/Context/LDContext.swift +++ b/LaunchDarkly/LaunchDarkly/Models/Context/LDContext.swift @@ -652,18 +652,6 @@ extension LDContext { attributes.values.contains { $0.containsNonFiniteNumber } || contexts.contains { $0.containsNonFiniteNumber() } } } - -private extension LDValue { - var containsNonFiniteNumber: Bool { - switch self { - case .number(let number): return !number.isFinite - case .array(let values): return values.contains { $0.containsNonFiniteNumber } - case .object(let values): return values.values.contains { $0.containsNonFiniteNumber } - case .null, .bool, .string: return false - } - } -} - enum LDContextBuilderKey { case generateKey case key(String) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 246aa413f..680afe8d8 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -441,9 +441,11 @@ class DirectEventProcessor: EventProcessor { } } } +} - // MARK: Reporting +// MARK: Reporting +extension DirectEventProcessor { private func startReporting() { guard eventReportTimer == nil else { return } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index b3f0644c8..8b12567f5 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -521,8 +521,18 @@ final class EventStore: EventStoring { os_log("%s recovered %d event(s) from a previous run", log: logger, type: .debug, typeName(and: #function), events) } - // MARK: Writing + deinit { + // One acquisition: the lock is not recursive, and `commit()` would take it again. + ioLock.withLock { + writeStagedBytes() + diskLog.closeDescriptor() + } + } +} + +// MARK: Writing +extension EventStore { /// Requires `ioLock`. private func writeStagedBytes() { ioLock.assertOwned() @@ -573,14 +583,6 @@ final class EventStore: EventStoring { } return true } - - deinit { - // One acquisition: the lock is not recursive, and `commit()` would take it again. - ioLock.withLock { - writeStagedBytes() - diskLog.closeDescriptor() - } - } } /// A batch the filesystem would not take, kept where the SDK used to keep all of them. diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventProcessor.swift index fa3658500..cde064a5c 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventProcessor.swift @@ -7,6 +7,7 @@ class NullEventProcessor: EventProcessor { func record(_ event: Event) { } + // swiftlint:disable:next function_parameter_count func recordFlagEvaluationEvents(flagKey: LDFlagKey, value: LDValue, defaultValue: LDValue, diff --git a/LaunchDarkly/LaunchDarklyTests/LDClientEvaluationHookSpec.swift b/LaunchDarkly/LaunchDarklyTests/LDClientEvaluationHookSpec.swift index c54528d5a..1decc2aae 100644 --- a/LaunchDarkly/LaunchDarklyTests/LDClientEvaluationHookSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/LDClientEvaluationHookSpec.swift @@ -147,7 +147,7 @@ final class LDClientEvaluationHookSpec: XCTestCase { // The evaluation returns the default value, and the event it records reports that value under no variation and // no version, so that is the result the hook is told about too. - guard let key = keys.first ?? nil + guard let key = keys.first.flatMap({ $0 }) else { fail("The hook was not told about the evaluation.") return diff --git a/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift b/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift index efea986d4..418d0afa7 100644 --- a/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift @@ -86,7 +86,11 @@ final class LDClientSpec: QuickSpec { context("when configured to start online") { beforeEach { testContext = TestContext(startOnline: true) - withTimeout ? testContext.start(timeOut: 10.0) : testContext.start() + if withTimeout { + testContext.start(timeOut: 10.0) + } else { + testContext.start() + } } it("takes the client and service objects online") { expect(testContext.subject.isOnline) == true @@ -133,7 +137,11 @@ final class LDClientSpec: QuickSpec { context("when configured to start offline") { beforeEach { testContext = TestContext() - withTimeout ? testContext.start(timeOut: 10.0) : testContext.start() + if withTimeout { + testContext.start(timeOut: 10.0) + } else { + testContext.start() + } } it("leaves the client and service objects offline") { expect(testContext.subject.isOnline) == false @@ -177,7 +185,11 @@ final class LDClientSpec: QuickSpec { context("after setting context") { beforeEach { testContext = TestContext(startOnline: true).withContext(nil) - withTimeout ? testContext.start(timeOut: 10.0) : testContext.start() + if withTimeout { + testContext.start(timeOut: 10.0) + } else { + testContext.start() + } testContext.context = LDContext.stub() testContext.subject.internalIdentify(newContext: testContext.context, useCache: .yes) @@ -215,7 +227,11 @@ final class LDClientSpec: QuickSpec { context("without setting context") { beforeEach { testContext = TestContext(startOnline: true).withContext(nil) - withTimeout ? testContext.start(timeOut: 10.0) : testContext.start() + if withTimeout { + testContext.start(timeOut: 10.0) + } else { + testContext.start() + } } it("saves the config") { expect(testContext.subject.config) == testContext.config @@ -248,7 +264,11 @@ final class LDClientSpec: QuickSpec { it("when called with cached flags for the context and environment") { let cachedFlags = ["test-flag": StorageItem.item(FeatureFlag(flagKey: "test-flag"))] let testContext = TestContext().withCached(flags: cachedFlags.featureFlags) - withTimeout ? testContext.start(timeOut: 10.0) : testContext.start() + if withTimeout { + testContext.start(timeOut: 10.0) + } else { + testContext.start() + } expect(testContext.featureFlagCachingMock.getCachedDataCallCount) == 1 expect(testContext.featureFlagCachingMock.getCachedDataReceivedArguments?.cacheKey) == testContext.context.fullyQualifiedHashedKey() @@ -261,7 +281,11 @@ final class LDClientSpec: QuickSpec { } it("when called without cached flags for the context") { let testContext = TestContext() - withTimeout ? testContext.start(timeOut: 10.0) : testContext.start() + if withTimeout { + testContext.start(timeOut: 10.0) + } else { + testContext.start() + } expect(testContext.featureFlagCachingMock.getCachedDataCallCount) == 1 expect(testContext.featureFlagCachingMock.getCachedDataReceivedArguments?.cacheKey) == testContext.context.fullyQualifiedHashedKey() diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 10e2bc551..7152ffd2f 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -2144,7 +2144,7 @@ private final class ClosingHookStore: EventStoring { beforeClosing = nil before?() let batch = inner.closeBatch() - if let batch, let body = try? inner.body(of: batch) ?? nil { + if let batch, let body = try? inner.body(of: batch) { closedKeys.append(customKeys(body)) } let after = afterClosing diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 3508c1407..cb3f50c3a 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -242,7 +242,7 @@ final class EventStoreSpec: QuickSpec { let persistingAgain = EventStoreSpec.reader(sharing: persisting) persistingAgain.recoverInterruptedLog() - let bodies = persistingAgain.pendingBatches().map { EventStoreSpec.keys(ofBody: try? persistingAgain.body(of: $0) ?? nil) } + let bodies = persistingAgain.pendingBatches().map { EventStoreSpec.keys(ofBody: try? persistingAgain.body(of: $0)) } expect(bodies) == [["closed"], ["open"]] } } @@ -429,7 +429,7 @@ final class EventStoreSpec: QuickSpec { next.commit() _ = next.closeBatch() - let bodies = next.pendingBatches().map { EventStoreSpec.keys(ofBody: try? next.body(of: $0) ?? nil) } + let bodies = next.pendingBatches().map { EventStoreSpec.keys(ofBody: try? next.body(of: $0)) } expect(bodies) == [["previous-0"], ["this-run", "later"]] expect(next.pendingEventCount) == 3 } @@ -783,7 +783,7 @@ final class EventStoreSpec: QuickSpec { // A failed write leaves events on both sides, and closing only the log would leave the rest to a // delivery that reported success without them. - let bodies = interrupted.pendingBatches().map { EventStoreSpec.keys(ofBody: try? interrupted.body(of: $0) ?? nil) } + let bodies = interrupted.pendingBatches().map { EventStoreSpec.keys(ofBody: try? interrupted.body(of: $0)) } expect(bodies) == [["written"], ["held"]] expect(interrupted.pendingEventCount) == 2 } From d6b1e95c95709ba25769141e304098bfdfd07c7f Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Mon, 12 Oct 2026 00:00:20 -0700 Subject: [PATCH 036/158] Run the process assertion specs only where the platform has assertions On macOS processAssertion runs the work inline and never asks the system, so the specs that wait for the system to end the assertion timed out there and failed test-swiftpm. macOS gets a spec for its own behavior instead. Co-authored-by: Cursor --- .../BackgroundActivitySpec.swift | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundActivitySpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundActivitySpec.swift index 3975aa459..f3de67f69 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundActivitySpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundActivitySpec.swift @@ -77,6 +77,7 @@ final class BackgroundActivitySpec: QuickSpec { } } + #if os(iOS) || os(tvOS) || os(watchOS) describe("an assertion from the process") { /// Stands in for `performExpiringActivity`, which answers on a system queue rather than the caller's. /// @@ -140,5 +141,24 @@ final class BackgroundActivitySpec: QuickSpec { expect(Date().timeIntervalSince(started)) > 0.2 } } + #else + describe("an assertion from the process") { + it("runs the work on the calling thread without asking the system, which has no assertions here") { + var asked = false + let assertion = BackgroundActivity.processAssertion(logger: .disabled, + perform: { _, _ in asked = true }, + maximumDuration: 30) + + var ran = false + assertion("test") { finished in + ran = true + finished() + } + + expect(ran) == true + expect(asked) == false + } + } + #endif } } From f2a8bd6fa4bd5f1d0213610cc58b88d06cdc732c Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Mon, 12 Oct 2026 06:20:23 -0700 Subject: [PATCH 037/158] Take a delivered batch out of the store before telling observers about the delivery publish announced an accepted or refused batch before removing it, and the announcement runs on the main thread, so an observer could find the delivered events still pending. v11 emptied its buffer before posting, so observers never saw them. Seen in 4 of 400 test runs; the new spec reproduces it every time by holding the store in a slow write. Co-authored-by: Cursor --- .../ServiceObjects/DirectEventProcessor.swift | 5 +- .../DirectEventProcessorSpec.swift | 51 +++++++++++++++++++ 2 files changed, 54 insertions(+), 2 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 680afe8d8..cfd2a2215 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -701,6 +701,7 @@ extension DirectEventProcessor { private func publish(_ batch: EventBatch, _ body: Data, _ completion: @escaping (AnalyticsEventSender.Outcome) -> Void) { + // The store is settled before observers are told, so that one looking at it on hearing the outcome sees it. sender.send(body, payloadId: batch.payloadId, eventCount: batch.eventCount) { outcome in switch outcome { case .accepted(let serverTime): @@ -709,12 +710,12 @@ extension DirectEventProcessor { self.responseDate = serverTime } } - self.reportSyncComplete(nil) self.store.remove(batch) + self.reportSyncComplete(nil) case .refused(let response): self.isOnline = false - self.reportSyncComplete(.response(response)) self.store.remove(batch) + self.reportSyncComplete(.response(response)) case .retryable(let error): self.reportSyncComplete(error) // The batch stays on disk. A later delivery, in this run of the application or the next one, sends it diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 7152ffd2f..721575c1f 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -578,6 +578,57 @@ final class DirectEventProcessorSpec: QuickSpec { expect(testContext.syncResult).to(beNil()) } } + context("while a commit is stuck writing") { + it("takes the delivered batch out of the store before telling observers") { + let writeStarted = DispatchSemaphore(value: 0) + let finishWrite = DispatchSemaphore(value: 0) + let blockLock = NSLock() + var blockNextWrite = false + // A slow disk, holding the store's I/O, which removing a delivered batch also needs. + let store = EventStore.temporary(writeLog: { file, bytes in + blockLock.lock() + let blocks = blockNextWrite + blockNextWrite = false + blockLock.unlock() + if blocks { + writeStarted.signal() + finishWrite.wait() + } + return file.writeAll(bytes) + }) + var pendingWhenTold: Int? + testContext = TestContext(onSyncComplete: { _ in + pendingWhenTold = store.pendingEventCount + }, store: store) + let reporter = testContext.eventProcessor! + let context = testContext.context! + reporter.isOnline = true + testContext.serviceMock.holdsEventCompletions = true + reporter.record(CustomEvent(key: "delivered", context: context)) + reporter.flush(completion: nil) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) + + blockLock.lock() + blockNextWrite = true + blockLock.unlock() + DispatchQueue.global().async { + reporter.record(CustomEvent(key: "written", context: context)) + } + expect(writeStarted.wait(timeout: .now() + 2)) == .success + DispatchQueue.global().async { + testContext.serviceMock.releaseHeldEventCompletions() + } + // Long enough for an announcement made ahead of the removal to reach the main thread. + waitUntil { done in + DispatchQueue.main.asyncAfter(deadline: .now() + 0.3, execute: done) + } + finishWrite.signal() + + expect(pendingWhenTold).toEventuallyNot(beNil()) + // Only the event still being written. The delivered one was gone before anyone heard. + expect(pendingWhenTold) == 1 + } + } } context("failure") { context("server error") { From 6a6afc8f1b6627da737664b2c3f003b6f5fc24cf Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Mon, 12 Oct 2026 06:20:23 -0700 Subject: [PATCH 038/158] Make the unit tests build and pass on watchOS The watchOS test target had not compiled since 9.0: CFNetworkErrors is unavailable there, and WatchOSEnvironmentReporterSpec used removed API. Specs that force-unwrapped the lifecycle notifications, which watchOS does not post, are now pending there; expectations that assumed streaming follow what the platform allows; and the DarklyService specs answered through OHHTTPStubs are skipped, since on the watchOS simulator those requests reach the network. Co-authored-by: Cursor --- .../LaunchDarklyTests/LDClientSpec.swift | 73 +++++++++++++------ .../Mocks/DarklyServiceMock.swift | 2 +- .../Networking/DarklyServiceSpec.swift | 10 ++- .../WatchOSEnvironmentReporterSpec.swift | 15 ++-- 4 files changed, 65 insertions(+), 35 deletions(-) diff --git a/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift b/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift index 418d0afa7..211f9a36d 100644 --- a/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift @@ -104,7 +104,7 @@ final class LDClientSpec: QuickSpec { it("saves the config") { expect(testContext.subject.config) == testContext.config expect(testContext.subject.service.config) == testContext.config - expect(testContext.makeFlagSynchronizerStreamingMode) == testContext.config.streamingMode + expect(testContext.makeFlagSynchronizerStreamingMode) == testContext.config.effectiveStreamingMode expect(testContext.makeFlagSynchronizerPollingInterval) == testContext.config.flagPollingInterval(runMode: testContext.subject.runMode) expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.config) == testContext.config } @@ -151,7 +151,7 @@ final class LDClientSpec: QuickSpec { it("saves the config") { expect(testContext.subject.config) == testContext.config expect(testContext.subject.service.config) == testContext.config - expect(testContext.makeFlagSynchronizerStreamingMode) == testContext.config.streamingMode + expect(testContext.makeFlagSynchronizerStreamingMode) == testContext.config.effectiveStreamingMode expect(testContext.makeFlagSynchronizerPollingInterval) == testContext.config.flagPollingInterval(runMode: testContext.subject.runMode) expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.config) == testContext.config } @@ -197,7 +197,7 @@ final class LDClientSpec: QuickSpec { it("saves the config") { expect(testContext.subject.config) == testContext.config expect(testContext.subject.service.config) == testContext.config - expect(testContext.makeFlagSynchronizerStreamingMode) == testContext.config.streamingMode + expect(testContext.makeFlagSynchronizerStreamingMode) == testContext.config.effectiveStreamingMode expect(testContext.makeFlagSynchronizerPollingInterval) == testContext.config.flagPollingInterval(runMode: testContext.subject.runMode) expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.config) == testContext.config } @@ -236,7 +236,7 @@ final class LDClientSpec: QuickSpec { it("saves the config") { expect(testContext.subject.config) == testContext.config expect(testContext.subject.service.config) == testContext.config - expect(testContext.makeFlagSynchronizerStreamingMode) == testContext.config.streamingMode + expect(testContext.makeFlagSynchronizerStreamingMode) == testContext.config.effectiveStreamingMode expect(testContext.makeFlagSynchronizerPollingInterval) == testContext.config.flagPollingInterval(runMode: testContext.subject.runMode) expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.config) == testContext.config } @@ -656,7 +656,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.subject.isInitialized) == false expect(completed).toEventually(beTrue()) } - it("takes the client back online at the foreground notification") { + it("takes the client back online at the foreground notification", onlyWhere: platformPostsLifecycleNotifications) { let testContext = backgroundedOnlineClient() testContext.subject.internalIdentify(newContext: LDContext.stub(), useCache: .no) @@ -665,7 +665,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.subject.isOnline).toEventually(beTrue()) expect(testContext.flagSynchronizerMock.isOnline).toEventually(beTrue()) } - it("recovers after repeated identifies while backgrounded") { + it("recovers after repeated identifies while backgrounded", onlyWhere: platformPostsLifecycleNotifications) { let testContext = backgroundedOnlineClient() testContext.subject.internalIdentify(newContext: LDContext.stub(), useCache: .no) testContext.subject.internalIdentify(newContext: LDContext.stub(), useCache: .no) @@ -728,7 +728,7 @@ final class LDClientSpec: QuickSpec { expect(isOnlineDuringSwap) == true expect(eventProcessorOnlineDuringSwap) == true } - it("recovers when the app backgrounds midway through an identify") { + it("recovers when the app backgrounds midway through an identify", onlyWhere: platformPostsLifecycleNotifications) { let testContext = TestContext(startOnline: true, enableBackgroundUpdates: false) testContext.start() let original = testContext.featureFlagCachingMock.getCachedDataCallback @@ -750,7 +750,7 @@ final class LDClientSpec: QuickSpec { // The mirror image of the background-then-identify cases above: identify then // background must end in the same state, so the two transitions compose in either // order. - it("stays online with the data source down when backgrounded after an identify") { + it("stays online with the data source down when backgrounded after an identify", onlyWhere: platformPostsLifecycleNotifications) { let testContext = TestContext(startOnline: true, enableBackgroundUpdates: false) testContext.start() testContext.subject.internalIdentify(newContext: LDContext.stub(), useCache: .no) @@ -800,7 +800,7 @@ final class LDClientSpec: QuickSpec { waitUntil { done in DispatchQueue.main.async { done() } } expect(count) == 1 } - it("leaves a client the application set offline offline") { + it("leaves a client the application set offline offline", onlyWhere: platformPostsLifecycleNotifications) { let testContext = TestContext(startOnline: false, enableBackgroundUpdates: false) testContext.start() testContext.subject.setRunMode(.background) @@ -1326,7 +1326,7 @@ final class LDClientSpec: QuickSpec { testContext.onSyncComplete?(.error(.response(forbiddenError))) expect(testContext.flagSynchronizerMock.isOnline).to(beTrue()) - expect(testContext.subject.getConnectionInformation().currentConnectionMode) == .establishingStreamingConnection + expect(testContext.subject.getConnectionInformation().currentConnectionMode) == (platformStreams ? .establishingStreamingConnection : .polling) expect(testContext.subject.isInitialized).to(beFalse()) expect(testContext.subject.isOnline).to(beTrue()) expect(testContext.eventProcessorMock.isOnline).to(beTrue()) @@ -1354,7 +1354,7 @@ final class LDClientSpec: QuickSpec { expect(after.lastConnectionFailureReason.description) == before.lastConnectionFailureReason.description expect(after.lastFailedConnection) == before.lastFailedConnection } - it("recovers the connection mode when a streaming sync succeeds after a terminal error") { + it("recovers the connection mode when a streaming sync succeeds after a terminal error", onlyWhere: platformStreams) { let testContext = TestContext(startOnline: true) testContext.start() testContext.subject.flagChangeNotifier = ClientServiceMockFactory(config: testContext.config).makeFlagChangeNotifier() @@ -1385,7 +1385,7 @@ final class LDClientSpec: QuickSpec { context("when online") { OperatingSystem.allOperatingSystems.forEach { os in context("on \(os)") { - it("background updates disabled") { + it("background updates disabled", onlyWhere: platformPostsLifecycleNotifications) { let testContext = TestContext(startOnline: true, enableBackgroundUpdates: false) testContext.start() NotificationCenter.default.post(name: SystemCapabilities.backgroundNotification!, object: self) @@ -1396,7 +1396,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.eventProcessorMock.isOnline) == true expect(testContext.flagSynchronizerMock.isOnline) == false } - it("tries to deliver what it has before the process is suspended, where it is suspended") { + it("tries to deliver what it has before the process is suspended, where it is suspended", onlyWhere: platformPostsLifecycleNotifications) { let testContext = TestContext(startOnline: true, enableBackgroundUpdates: false) testContext.start() NotificationCenter.default.post(name: SystemCapabilities.backgroundNotification!, object: self) @@ -1409,7 +1409,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.eventProcessorMock.flushReportingOutcomeCallCount) == 0 #endif } - it("background updates enabled") { + it("background updates enabled", onlyWhere: platformPostsLifecycleNotifications) { let testContext = TestContext(startOnline: true) testContext.start() @@ -1431,7 +1431,7 @@ final class LDClientSpec: QuickSpec { } } } - it("when offline") { + it("when offline", onlyWhere: platformPostsLifecycleNotifications) { let testContext = TestContext() testContext.start() NotificationCenter.default.post(name: SystemCapabilities.backgroundNotification!, object: self) @@ -1448,7 +1448,7 @@ final class LDClientSpec: QuickSpec { context("after starting client") { OperatingSystem.allOperatingSystems.forEach { os in context("on \(os)") { - it("when online at foreground notification") { + it("when online at foreground notification", onlyWhere: platformPostsLifecycleNotifications) { let testContext = TestContext(startOnline: true) testContext.start(runMode: .background) NotificationCenter.default.post(name: SystemCapabilities.foregroundNotification!, object: self) @@ -1458,7 +1458,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.eventProcessorMock.isOnline) == true expect(testContext.flagSynchronizerMock.isOnline) == true } - it("when offline at foreground notification") { + it("when offline at foreground notification", onlyWhere: platformPostsLifecycleNotifications) { let testContext = TestContext() testContext.start(runMode: .background) NotificationCenter.default.post(name: SystemCapabilities.foregroundNotification!, object: self) @@ -1657,8 +1657,10 @@ final class LDClientSpec: QuickSpec { describe("flag synchronizer streaming mode") { OperatingSystem.allOperatingSystems.forEach { os in it("on \(os) sets the flag synchronizer streaming mode") { - // TODO(os-tests): We need to expand this to the other OSs - if os == .watchOS { + // TODO(os-tests): We need to expand this to the other OSs. The client runs as the platform the + // tests do, whatever `os` says. + guard os == SystemCapabilities.operatingSystem + else { return } @@ -1892,7 +1894,7 @@ final class LDClientSpec: QuickSpec { let testContext = TestContext(startOnline: true) testContext.start() expect(testContext.subject.isOnline) == true - expect(testContext.subject.connectionInformation.currentConnectionMode).to(equal(.establishingStreamingConnection)) + expect(testContext.subject.connectionInformation.currentConnectionMode) == (platformStreams ? .establishingStreamingConnection : .polling) } it("when client was started in background") { let testContext = TestContext(startOnline: true, enableBackgroundUpdates: false) @@ -2000,7 +2002,7 @@ final class LDClientSpec: QuickSpec { testContext.subject.internalIdentify(newContext: LDContext.stub(), useCache: .yes) expect(testContext.subject.isInitialized) == true } - it("becomes true when setOnline(true) is refused in the background") { + it("becomes true when setOnline(true) is refused in the background", onlyWhere: platformPostsLifecycleNotifications) { let testContext = TestContext(startOnline: true, enableBackgroundUpdates: false) testContext.start() testContext.subject.setRunMode(.background) @@ -2012,7 +2014,7 @@ final class LDClientSpec: QuickSpec { NotificationCenter.default.post(name: SystemCapabilities.foregroundNotification!, object: self) expect(testContext.subject.isInitialized) == true } - it("remains false across a background and foreground cycle without flags") { + it("remains false across a background and foreground cycle without flags", onlyWhere: platformPostsLifecycleNotifications) { let testContext = TestContext(startOnline: true, enableBackgroundUpdates: false) testContext.start() expect(testContext.subject.isInitialized) == false @@ -2063,4 +2065,31 @@ extension OperatingSystem { } } +private extension LDConfig { + /// The mode a foreground client connects in. Platforms that cannot stream, watchOS among them, poll whatever the + /// config asks for. + var effectiveStreamingMode: LDStreamingMode { + allowStreamingMode ? streamingMode : .polling + } +} + +/// watchOS posts no lifecycle notifications, and does not stream. +private let platformPostsLifecycleNotifications = SystemCapabilities.foregroundNotification != nil + && SystemCapabilities.backgroundNotification != nil +private let platformStreams = SystemCapabilities.operatingSystem.isStreamingEnabled + +/// `it`, for a spec that only applies where `condition` holds. Elsewhere the spec is reported pending rather than run. +private func it(_ description: String, + onlyWhere condition: Bool, + file: FileString = #file, + line: UInt = #line, + closure: @escaping () throws -> Void) { + guard condition + else { + pending(description) { } + return + } + it(description, file: file, line: line, closure: closure) +} + private class ErrorMock: Error { } diff --git a/LaunchDarkly/LaunchDarklyTests/Mocks/DarklyServiceMock.swift b/LaunchDarkly/LaunchDarklyTests/Mocks/DarklyServiceMock.swift index 01987f368..5e82aac17 100644 --- a/LaunchDarkly/LaunchDarklyTests/Mocks/DarklyServiceMock.swift +++ b/LaunchDarkly/LaunchDarklyTests/Mocks/DarklyServiceMock.swift @@ -46,7 +46,7 @@ final class DarklyServiceMock: DarklyServiceProvider { } struct Constants { - static let error = NSError(domain: NSURLErrorDomain, code: Int(CFNetworkErrors.cfurlErrorResourceUnavailable.rawValue), userInfo: nil) + static let error = NSError(domain: NSURLErrorDomain, code: NSURLErrorResourceUnavailable, userInfo: nil) static let jsonErrorString = "Bad json data" static let errorData = jsonErrorString.data(using: .utf8)! diff --git a/LaunchDarkly/LaunchDarklyTests/Networking/DarklyServiceSpec.swift b/LaunchDarkly/LaunchDarklyTests/Networking/DarklyServiceSpec.swift index d7d6d227b..32497b036 100644 --- a/LaunchDarkly/LaunchDarklyTests/Networking/DarklyServiceSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/Networking/DarklyServiceSpec.swift @@ -49,13 +49,17 @@ final class DarklyServiceSpec: QuickSpec { } override func spec() { - getFeatureFlagsSpec() - flagRequestEtagSpec() clearFlagRequestCacheSpec() createEventSourceSpec() - publishEventDataSpec() diagnosticCacheSpec() + // On the watchOS simulator these requests reach the network even with the OHHTTPStubs protocol installed in + // the session, so nothing answers them. + #if !os(watchOS) + getFeatureFlagsSpec() + flagRequestEtagSpec() + publishEventDataSpec() publishDiagnosticSpec() + #endif afterEach { HTTPStubs.removeAllStubs() diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EnvironmentReporting/WatchOSEnvironmentReporterSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EnvironmentReporting/WatchOSEnvironmentReporterSpec.swift index 9026d8c98..8373efd2f 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EnvironmentReporting/WatchOSEnvironmentReporterSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EnvironmentReporting/WatchOSEnvironmentReporterSpec.swift @@ -1,29 +1,26 @@ #if os(watchOS) import Foundation +import WatchKit import XCTest @testable import LaunchDarkly -final class WatchOSEnvironmentReporterSpec: XCTest { +final class WatchOSEnvironmentReporterSpec: XCTestCase { func testDefaultReporterBehavior() { let chain = EnvironmentReporterChainBase() chain.setNext(WatchOSEnvironmentReporter()) - XCTAssert(!chain.applicationInfo.isEmpty()) + XCTAssertNotNil(chain.applicationInfo.applicationId) XCTAssertNotEqual(chain.deviceModel, "UNKNOWN") XCTAssertNotEqual(chain.systemVersion, "UNKNOWN") - XCTAssertNotEqual(chain.systemName, "UNKNOWN") - - XCTAssertEqual(chain.operatingSystem, .watchOS) XCTAssertNil(chain.vendorUUID) } func testBuilderDoesNotIncludeWatchInfoWithoutExplicitOptIn() { - let builder = EnvironmentReporterBuilder() - let reporting = builder.build() + let reporting = EnvironmentReporterBuilder().build() - XCTAssertEqual(reporting.operatingSystem, .watchOS) + XCTAssertEqual(reporting.deviceModel, EnvironmentReporterChainBase().deviceModel) } func testEnsureBuilderUsesCorrectReporter() { @@ -32,7 +29,7 @@ final class WatchOSEnvironmentReporterSpec: XCTest { let reporting = builder.build() - XCTAssertEqual(reporting.operatingSystem, .watchOS) + XCTAssertEqual(reporting.deviceModel, WKInterfaceDevice.current().model) } } #endif From 50ebcd001036b90edc97100a962686cdda3c93fb Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Mon, 12 Oct 2026 07:20:38 -0700 Subject: [PATCH 039/158] Keep the closed-batch count equal to the batches it counts A listing worked out the total under its locks and wrote it to closedEvents after releasing them, so a batch closed in between had its increment overwritten, and with persistence off one held in between could be counted twice. And remove() subtracted a batch's events whether or not it removed anything: two deliveries reaching the same batch subtracted it twice, as did removing a batch another process had delivered, which the listing before had already stopped counting. closedEvents is now the sum of eventCounts and inMemoryBatches, changed only in the critical section that changes one of them. The listing recounts before releasing its locks, holding a batch counts it in the same section as the append, remove() subtracts only the count it took away, and recovery counts the log it closes, which it had been leaving to the first listing. The same fix as on Android. Co-authored-by: Cursor --- .../ServiceObjects/EventStore.swift | 49 ++++++++------ .../ServiceObjects/EventStoreSpec.swift | 65 +++++++++++++++++++ 2 files changed, 94 insertions(+), 20 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index 8b12567f5..b51537b89 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -114,6 +114,9 @@ final class EventStore: EventStoring { /// capacity for events the store is still holding. private var writingEvents = 0 private var committedEvents = 0 + /// Events in closed batches: the sum of `eventCounts` and `inMemoryBatches`. Changed only in the critical section + /// that changes one of those, so that it cannot drift from them; a total worked out from them and written + /// afterwards would overwrite a close or a removal in between. private var closedEvents = 0 /// Whether events are being written to disk, which is what the application asked for until a write fails and the /// store gives up on persistence for the rest of the session. @@ -146,7 +149,7 @@ final class EventStore: EventStoring { /// /// Its own lock rather than `ioLock`, so that a store the application told to keep off the disk -- where held /// batches are the only ones there are -- never waits on the lock disk I/O is serialized under. Taken inside - /// `ioLock` where there is a disk, never the other way round. + /// `ioLock` where there is a disk, never the other way round, and taken before `bufferLock`, never after it. private let heldLock = UnfairLock() /// How many events are in each batch this process closed or recovered, so that listing them does not have to read @@ -356,18 +359,19 @@ final class EventStore: EventStoring { func pendingBatches() -> [EventBatch] { // Listed in one critical section where there is a disk, so that the files and what is held cannot be counted // either side of a close that moves events between them. - let batches = readsDisk ? ioLock.withLock { batchesOnDisk() + heldBatches() } : heldBatches() - - bufferLock.withLock { closedEvents = batches.reduce(0) { $0 + $1.eventCount } } - - return batches + readsDisk ? ioLock.withLock { heldBatchesAndRecount(after: batchesOnDisk()) } : heldBatchesAndRecount(after: []) } - /// The batches the filesystem would not take, after the files: anything held was closed by this run and so is - /// newer than whatever a previous run left on the disk. - private func heldBatches() -> [EventBatch] { + /// Adds the batches the filesystem would not take, after the files: anything held was closed by this run and so is + /// newer than whatever a previous run left on the disk. Then counts everything listed as closed, which is how + /// batches another process closed or delivered come to be counted or not. + /// + /// Requires `ioLock` where there is a disk, so that the count is of the listing as it stands. + private func heldBatchesAndRecount(after onDisk: [EventBatch]) -> [EventBatch] { heldLock.withLock { - inMemoryBatches.map { EventBatch(payloadId: $0.payloadId, eventCount: $0.eventCount) } + let batches = onDisk + inMemoryBatches.map { EventBatch(payloadId: $0.payloadId, eventCount: $0.eventCount) } + bufferLock.withLock { closedEvents = batches.reduce(0) { $0 + $1.eventCount } } + return batches } } @@ -448,21 +452,24 @@ final class EventStore: EventStoring { } func remove(_ batch: EventBatch) { + // Only a count this call takes away stops counting. Two deliveries can both get to the same batch, and a listing + // drops a batch another process delivered before this one gets to remove it. let wasHeld = heldLock.withLock { () -> Bool in guard let index = inMemoryBatches.firstIndex(where: { $0.payloadId == batch.payloadId }) else { return false } - inMemoryBatches.remove(at: index) + let held = inMemoryBatches.remove(at: index) + bufferLock.withLock { closedEvents -= held.eventCount } return true } - if !wasHeld, readsDisk { - ioLock.withLock { - try? FileManager.default.removeItem(at: url(of: batch.payloadId)) - eventCounts[batch.payloadId] = nil + guard !wasHeld, readsDisk + else { return } + ioLock.withLock { + try? FileManager.default.removeItem(at: url(of: batch.payloadId)) + if let counted = eventCounts.removeValue(forKey: batch.payloadId) { + bufferLock.withLock { closedEvents -= counted } } } - - bufferLock.withLock { closedEvents = max(0, closedEvents - batch.eventCount) } } /// Closes any log left open by a previous run of this process, so its events join the batches to deliver. @@ -518,6 +525,7 @@ final class EventStore: EventStoring { else { return } eventCounts[payloadId] = events + bufferLock.withLock { closedEvents += events } os_log("%s recovered %d event(s) from a previous run", log: logger, type: .debug, typeName(and: #function), events) } @@ -627,9 +635,10 @@ private extension EventStore { else { return nil } let payloadId = UUID().uuidString - heldLock.withLock { inMemoryBatches.append(HeldBatch(payloadId: payloadId, body: body, eventCount: events)) } - - bufferLock.withLock { closedEvents += events } + heldLock.withLock { + inMemoryBatches.append(HeldBatch(payloadId: payloadId, body: body, eventCount: events)) + bufferLock.withLock { closedEvents += events } + } return EventBatch(payloadId: payloadId, eventCount: events) } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index cb3f50c3a..33a76fa41 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -705,6 +705,71 @@ final class EventStoreSpec: QuickSpec { expect(store.stage(EventStoreSpec.payload("summary"), bypassingCapacity: true)) == true expect(store.pendingEventCount) == 2 } + + for persistEvents in [true, false] { + it("stops counting a batch once when two deliveries both remove it (persisting: \(persistEvents))") { + store = EventStore.temporary(persistEvents: persistEvents) + _ = store.stage(EventStoreSpec.payload("first")) + let first = store.closeBatch()! + _ = store.stage(EventStoreSpec.payload("second")) + _ = store.closeBatch() + _ = store.pendingBatches() + + store.remove(first) + store.remove(first) + + expect(store.pendingEventCount) == 1 + } + } + + it("stops counting a batch once when another process delivered it and a listing came between") { + store = EventStore.temporary() + _ = store.stage(EventStoreSpec.payload("first")) + let first = store.closeBatch()! + _ = store.stage(EventStoreSpec.payload("second")) + _ = store.closeBatch() + + EventStoreSpec.reader(sharing: store).remove(first) + _ = store.pendingBatches() + store.remove(first) + + expect(store.pendingEventCount) == 1 + } + + for persistEvents in [true, false] { + it("neither loses nor doubles a batch closed while a listing runs (persisting: \(persistEvents))") { + store = EventStore.temporary(capacity: 100_000, persistEvents: persistEvents) + let listing = store! + let doneLock = UnfairLock() + var done = false + let listerFinished = DispatchSemaphore(value: 0) + DispatchQueue.global().async { + while !doneLock.withLock({ done }) { + _ = listing.pendingBatches() + // The locks are not fair, and without this the listings can hold them so continuously that + // the closes starve. + sched_yield() + } + listerFinished.signal() + } + + // Checked after every close, because the next listing would correct a miscount before the end of + // the loop could see it. + var miscounts: [String] = [] + for closed in 1...150 { + _ = store.stage(EventStoreSpec.payload("event-\(closed)")) + _ = store.closeBatch() + let counted = store.pendingEventCount + if counted != closed { + miscounts.append("\(counted) after \(closed) closes") + } + } + doneLock.withLock { done = true } + listerFinished.wait() + + expect(miscounts).to(beEmpty()) + } + } } } From 8ef1c2aa356d71e5b50b2f1e3c5b6b1103a3ad1b Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Mon, 12 Oct 2026 07:30:43 -0700 Subject: [PATCH 040/158] Document that the event capacity counts an earlier launch's events only once they are read The reporter reads what an earlier launch left in the background after starting, so events recorded before then are checked without that backlog. Counting it up front would put file I/O on the client-start path, and the overshoot only keeps extra events, so it is documented on eventCapacity and pinned by a store spec. Co-authored-by: Cursor --- .../LaunchDarkly/Models/LDConfig.swift | 4 ++++ .../ServiceObjects/EventStoreSpec.swift | 18 ++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift index 65c52f595..7bc385a41 100644 --- a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift @@ -342,6 +342,10 @@ public struct LDConfig { public var sendEvents: Bool = Defaults.sendEvents /// The maximum number of analytics events the LDClient can store. When the LDClient event store reaches the eventCapacity, the SDK discards events until it successfully reports them to LaunchDarkly. (Default: 1000) + /// + /// Where `eventPersistence` keeps events across launches, this includes the events an earlier launch left undelivered, from the + /// moment the SDK has read them, which it does in the background just after starting. Events recorded before then are + /// counted without them, so for that moment the SDK can hold up to this many on top of what the earlier launch left. public var eventCapacity: Int = Defaults.eventCapacity /// How far the SDK goes to make a recorded event outlive the process that recorded it. diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 33a76fa41..7d0b971b8 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -666,6 +666,24 @@ final class EventStoreSpec: QuickSpec { expect(store.stage(EventStoreSpec.payload("second"))) == false } + // The bound `LDConfig.eventCapacity` documents: the reporter reads the disk in the background after starting, + // so what an earlier run left only counts from that listing on. + it("counts what an earlier run left undelivered from the first listing on") { + let earlier = EventStore.temporary(capacity: 2) + defer { earlier.deleteEverything() } + _ = earlier.stage(EventStoreSpec.payload("earlier-1")) + _ = earlier.stage(EventStoreSpec.payload("earlier-2")) + expect(earlier.closeBatch()).toNot(beNil()) + store = EventStore(directory: earlier.directory, capacity: 2, logger: .disabled) + + expect(store.stage(EventStoreSpec.payload("before-listing"))) == true + + store.recoverInterruptedLog() + expect(store.pendingBatches().count) == 1 + expect(store.pendingEventCount) == 3 + expect(store.stage(EventStoreSpec.payload("after-listing"))) == false + } + it("counts the events a commit is still writing, so a slow write does not make room") { let writing = DispatchSemaphore(value: 0) let resume = DispatchSemaphore(value: 0) From f289900df1480ccb054ce61e10613a9d637f3056 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Mon, 12 Oct 2026 08:27:58 -0700 Subject: [PATCH 041/158] Give the processor specs' waits room for a loaded CI simulator Each flush and toEventually in DirectEventProcessorSpec waited Nimble's default second, and on a busy CI simulator a flush that takes milliseconds locally took over a second, failing tests whose behaviour was right. They now wait up to ten seconds; a wait ends as soon as what it waits for happens, so passing tests take no longer. The main-thread write test also polls every 200ms instead of 10ms, because each poll decodes every event on disk on the main thread, which kept the run loop too busy to see the count arrive. Co-authored-by: Cursor --- .../DirectEventProcessorSpec.swift | 113 +++++++++--------- 1 file changed, 59 insertions(+), 54 deletions(-) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 721575c1f..a53e9f187 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -7,6 +7,9 @@ final class DirectEventProcessorSpec: QuickSpec { struct Constants { static let eventFlushInterval: TimeInterval = 10.0 static let eventFlushIntervalHalfSecond: TimeInterval = 0.5 + /// How long a flush or delivery may take before a test gives up on it. A wait ends as soon as what it waits + /// for happens, so this costs a passing test nothing, and a loaded CI simulator can take over a second. + static let deliveryTimeout: DispatchTimeInterval = .seconds(10) } struct TestContext { @@ -151,7 +154,7 @@ final class DirectEventProcessorSpec: QuickSpec { reporter.flush { record(true) } reporter.flushReportingOutcome(completion: record) - expect(lock.withLock { answers.count }).toEventually(equal(2)) + expect(lock.withLock { answers.count }).toEventually(equal(2), timeout: Constants.deliveryTimeout) lock.withLock { expect(answers.map { $0.delivered }) == [true, true] expect(answers.contains { $0.onCaller }) == false @@ -322,14 +325,14 @@ final class DirectEventProcessorSpec: QuickSpec { reporter.isOnline = true } it("drops the poisoned batch and delivers a later event") { - waitUntil { done in + waitUntil(timeout: Constants.deliveryTimeout) { done in reporter.record(CustomEvent(key: "poison", context: ldContext, metricValue: .nan)) reporter.flush(completion: done) } expect(serviceMock.publishEventDataCallCount) == 0 expect(store.pendingEventCount) == 0 - waitUntil { done in + waitUntil(timeout: Constants.deliveryTimeout) { done in reporter.record(CustomEvent(key: "after-poison", context: ldContext, metricValue: 1.0)) reporter.flush(completion: done) } @@ -350,7 +353,7 @@ final class DirectEventProcessorSpec: QuickSpec { reporter.isOnline = true } it("drops only the poisoned event") { - waitUntil { done in + waitUntil(timeout: Constants.deliveryTimeout) { done in reporter.record(CustomEvent(key: "poison", context: ldContext, metricValue: .nan)) reporter.record(CustomEvent(key: "kept", context: ldContext, metricValue: 1.0)) reporter.flush(completion: done) @@ -372,7 +375,7 @@ final class DirectEventProcessorSpec: QuickSpec { reporter.isOnline = true } it("drops only the poisoned summary") { - waitUntil { done in + waitUntil(timeout: Constants.deliveryTimeout) { done in reporter.recordFlagEvaluationEvents(flagKey: "poison-flag", value: .number(.nan), defaultValue: .number(0), @@ -399,7 +402,7 @@ final class DirectEventProcessorSpec: QuickSpec { reporter.isOnline = true } it("drops the poisoned summary and delivers a later one") { - waitUntil { done in + waitUntil(timeout: Constants.deliveryTimeout) { done in reporter.recordFlagEvaluationEvents(flagKey: "poison-flag", value: .number(.nan), defaultValue: .number(0), @@ -412,7 +415,7 @@ final class DirectEventProcessorSpec: QuickSpec { expect(store.pendingEventCount) == 0 expect(reporter.eventBuffer.contextSummarizer.hasLoggedRequests) == false - waitUntil { done in + waitUntil(timeout: Constants.deliveryTimeout) { done in reporter.recordFlagEvaluationEvents(flagKey: "after-poison", value: .bool(true), defaultValue: .bool(false), @@ -456,7 +459,7 @@ final class DirectEventProcessorSpec: QuickSpec { context("success") { context("with events and tracked requests") { beforeEach { - waitUntil { syncComplete in + waitUntil(timeout: Constants.deliveryTimeout) { syncComplete in testContext = TestContext(eventStubResponseDate: eventStubResponseDate, onSyncComplete: { result in testContext.syncResult = result syncComplete() @@ -496,7 +499,7 @@ final class DirectEventProcessorSpec: QuickSpec { } context("with events only") { beforeEach { - waitUntil { syncComplete in + waitUntil(timeout: Constants.deliveryTimeout) { syncComplete in testContext = TestContext(eventStubResponseDate: eventStubResponseDate, onSyncComplete: { result in testContext.syncResult = result syncComplete() @@ -521,7 +524,7 @@ final class DirectEventProcessorSpec: QuickSpec { } context("with tracked requests only") { beforeEach { - waitUntil { syncComplete in + waitUntil(timeout: Constants.deliveryTimeout) { syncComplete in testContext = TestContext(eventStubResponseDate: eventStubResponseDate, onSyncComplete: { result in testContext.syncResult = result syncComplete() @@ -559,7 +562,7 @@ final class DirectEventProcessorSpec: QuickSpec { } context("without events or tracked requests") { beforeEach { - waitUntil { syncComplete in + waitUntil(timeout: Constants.deliveryTimeout) { syncComplete in testContext = TestContext(eventStubResponseDate: eventStubResponseDate, onSyncComplete: { result in testContext.syncResult = result syncComplete() @@ -606,7 +609,7 @@ final class DirectEventProcessorSpec: QuickSpec { testContext.serviceMock.holdsEventCompletions = true reporter.record(CustomEvent(key: "delivered", context: context)) reporter.flush(completion: nil) - expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) blockLock.lock() blockNextWrite = true @@ -619,12 +622,12 @@ final class DirectEventProcessorSpec: QuickSpec { testContext.serviceMock.releaseHeldEventCompletions() } // Long enough for an announcement made ahead of the removal to reach the main thread. - waitUntil { done in + waitUntil(timeout: Constants.deliveryTimeout) { done in DispatchQueue.main.asyncAfter(deadline: .now() + 0.3, execute: done) } finishWrite.signal() - expect(pendingWhenTold).toEventuallyNot(beNil()) + expect(pendingWhenTold).toEventuallyNot(beNil(), timeout: Constants.deliveryTimeout) // Only the event still being written. The delivered one was gone before anyone heard. expect(pendingWhenTold) == 1 } @@ -791,7 +794,7 @@ final class DirectEventProcessorSpec: QuickSpec { } it("drops events and stops reporting") { expect(testContext.serviceMock.publishEventDataCallCount) == 1 - expect(testContext.store.pendingEventCount).toEventually(equal(0)) + expect(testContext.store.pendingEventCount).toEventually(equal(0), timeout: Constants.deliveryTimeout) expect(testContext.eventProcessor.isOnline).to(beFalse()) expect(testContext.eventProcessor.isReportingActive).to(beFalse()) if case let .response(error) = testContext.syncResult { @@ -810,7 +813,7 @@ final class DirectEventProcessorSpec: QuickSpec { } context("offline") { beforeEach { - waitUntil { syncComplete in + waitUntil(timeout: Constants.deliveryTimeout) { syncComplete in testContext = TestContext(eventStubResponseDate: eventStubResponseDate, onSyncComplete: { result in testContext.syncResult = result syncComplete() @@ -1044,10 +1047,10 @@ final class DirectEventProcessorSpec: QuickSpec { testContext.recordEvents(Event.Kind.allKinds.count) } it("reports events") { - expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) - expect(testContext.store.pendingEventCount).toEventually(equal(0)) - expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount).toEventually(equal(1)) - expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch).toEventually(equal(testContext.events.count)) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + expect(testContext.store.pendingEventCount).toEventually(equal(0), timeout: Constants.deliveryTimeout) + expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch).toEventually(equal(testContext.events.count), timeout: Constants.deliveryTimeout) let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) valueIsArray(published) { valueArray in expect(valueArray.count) == testContext.events.count @@ -1062,7 +1065,7 @@ final class DirectEventProcessorSpec: QuickSpec { testContext.recordEvents(Event.Kind.nonSummaryKinds.count) } it("publishes events recorded after the first fire") { - expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) testContext.recordEvents(Event.Kind.nonSummaryKinds.count) expect(testContext.serviceMock.publishEventDataCallCount) .toEventually(equal(2), timeout: .seconds(2)) @@ -1072,7 +1075,7 @@ final class DirectEventProcessorSpec: QuickSpec { testContext = TestContext(eventFlushInterval: Constants.eventFlushIntervalHalfSecond) testContext.eventProcessor.isOnline = true - waitUntil { done in + waitUntil(timeout: Constants.deliveryTimeout) { done in DispatchQueue.main.asyncAfter(deadline: DispatchTime.now() + Constants.eventFlushIntervalHalfSecond) { expect(testContext.serviceMock.publishEventDataCallCount) == 0 done() @@ -1119,7 +1122,7 @@ extension DirectEventProcessorSpec { // The promise is weaker by exactly one scheduling hop: the event reaches the disk, just not before // recording it returned. - expect(eventsLeftOnDisk().compactMap { $0.kindField }).toEventually(equal(["custom"])) + expect(eventsLeftOnDisk().compactMap { $0.kindField }).toEventually(equal(["custom"]), timeout: Constants.deliveryTimeout) } it("has written the evaluations that came before a tracked event") { @@ -1171,8 +1174,10 @@ extension DirectEventProcessorSpec { commits.resume() + // Each poll reads and decodes every event on this thread, so polling often keeps the run loop too busy + // to notice the count is already there. expect(eventsLeftOnDisk().compactMap { $0.kindField }.filter { $0 == "feature" }.count) - .toEventually(equal(400)) + .toEventually(equal(400), timeout: Constants.deliveryTimeout, pollInterval: .milliseconds(200)) } it("reports the events a previous run left behind") { @@ -1185,7 +1190,7 @@ extension DirectEventProcessorSpec { let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) nextRun.isOnline = true - waitUntil { done in + waitUntil(timeout: Constants.deliveryTimeout) { done in nextRun.flush(completion: done) } @@ -1210,8 +1215,8 @@ extension DirectEventProcessorSpec { // only thing that can deliver these is the reporter coming online. nextRun.isOnline = true - expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) - expect(recovered.pendingEventCount).toEventually(equal(0)) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + expect(recovered.pendingEventCount).toEventually(equal(0), timeout: Constants.deliveryTimeout) nextRun.isOnline = false } @@ -1257,7 +1262,7 @@ extension DirectEventProcessorSpec { // Answered from the delivery's own threads and polled from this one. let first = Answer() testContext.eventProcessor.flush { first.record(true) } - expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) let second = Answer() testContext.eventProcessor.flush { second.record(true) } @@ -1270,8 +1275,8 @@ extension DirectEventProcessorSpec { // The second caller finds the only batch claimed by the first delivery, so it waits on that claim // rather than sending anything. The batch goes out once in total. - expect(first.value()).toEventually(beTrue()) - expect(second.value()).toEventually(beTrue()) + expect(first.value()).toEventually(beTrue(), timeout: Constants.deliveryTimeout) + expect(second.value()).toEventually(beTrue(), timeout: Constants.deliveryTimeout) expect(testContext.serviceMock.publishEventDataCallCount) == 1 expect(testContext.store.pendingEventCount) == 0 } @@ -1299,7 +1304,7 @@ extension DirectEventProcessorSpec { expect(answered.wait(timeout: .now() + 2)) == .success store.release.signal() - expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) } it("writes a deferred track while a delivery holds the delivery queue") { @@ -1312,7 +1317,7 @@ extension DirectEventProcessorSpec { testContext.eventProcessor.record(CustomEvent(key: "tracked", context: testContext.context)) - expect(eventsLeftOnDisk().compactMap { $0.kindField }).toEventually(equal(["custom"])) + expect(eventsLeftOnDisk().compactMap { $0.kindField }).toEventually(equal(["custom"]), timeout: Constants.deliveryTimeout) } it("has written everything by the time a flush returns, even while a delivery holds the delivery queue") { @@ -1410,10 +1415,10 @@ extension DirectEventProcessorSpec { nextRun.isOnline = true defer { nextRun.isOnline = false } - expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(3)) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(3), timeout: Constants.deliveryTimeout) expect(customKeys(testContext.serviceMock.publishedEventPayloads.last!)) == ["kept"] expect(testContext.serviceMock.publishedPayloadIds.last) == payloadId - expect(recovered.pendingEventCount).toEventually(equal(0)) + expect(recovered.pendingEventCount).toEventually(equal(0), timeout: Constants.deliveryTimeout) } } } @@ -1447,7 +1452,7 @@ extension DirectEventProcessorSpec { expect(testContext.store.pendingEventCount) == 0 testContext.eventProcessor.isOnline = true - waitUntil { done in + waitUntil(timeout: Constants.deliveryTimeout) { done in testContext.eventProcessor.flush(completion: done) } let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) @@ -1639,7 +1644,7 @@ extension DirectEventProcessorSpec { testContext = TestContext() testContext.eventProcessor.isOnline = true var delivered: Bool? - waitUntil { done in + waitUntil(timeout: Constants.deliveryTimeout) { done in testContext.eventProcessor.flushReportingOutcome { result in delivered = result done() @@ -1652,7 +1657,7 @@ extension DirectEventProcessorSpec { testContext = TestContext() testContext.recordEvents(1) var delivered: Bool? - waitUntil { done in + waitUntil(timeout: Constants.deliveryTimeout) { done in testContext.eventProcessor.flushReportingOutcome { result in delivered = result done() @@ -1667,7 +1672,7 @@ extension DirectEventProcessorSpec { testContext.recordEvents(1) testContext.eventProcessor.isOnline = true var delivered: Bool? - waitUntil { done in + waitUntil(timeout: Constants.deliveryTimeout) { done in testContext.eventProcessor.flushReportingOutcome { result in delivered = result done() @@ -1692,7 +1697,7 @@ extension DirectEventProcessorSpec { failing.record(true) var delivered: Bool? - waitUntil { done in + waitUntil(timeout: Constants.deliveryTimeout) { done in testContext.eventProcessor.flushReportingOutcome { result in delivered = result done() @@ -1703,7 +1708,7 @@ extension DirectEventProcessorSpec { expect(store.pendingBatches().count) == 1 failing.record(false) - waitUntil { done in + waitUntil(timeout: Constants.deliveryTimeout) { done in testContext.eventProcessor.flushReportingOutcome { result in delivered = result done() @@ -1729,7 +1734,7 @@ extension DirectEventProcessorSpec { testContext.eventProcessor.isOnline = true var delivered: Bool? - waitUntil { done in + waitUntil(timeout: Constants.deliveryTimeout) { done in testContext.eventProcessor.flushReportingOutcome { result in delivered = result done() @@ -1747,7 +1752,7 @@ extension DirectEventProcessorSpec { testContext.recordEvents(1) testContext.eventProcessor.isOnline = true var delivered: Bool? - waitUntil { done in + waitUntil(timeout: Constants.deliveryTimeout) { done in testContext.eventProcessor.flushReportingOutcome { result in delivered = result done() @@ -1757,7 +1762,7 @@ extension DirectEventProcessorSpec { expect(testContext.serviceMock.publishEventDataCallCount) == 1 // Told once, to the caller who was there to hear it. - waitUntil { done in + waitUntil(timeout: Constants.deliveryTimeout) { done in testContext.eventProcessor.flushReportingOutcome { result in delivered = result done() @@ -1774,7 +1779,7 @@ extension DirectEventProcessorSpec { testContext.eventProcessor.record(CustomEvent(key: "fine", context: LDContext.stub())) testContext.eventProcessor.isOnline = true var delivered: Bool? - waitUntil { done in + waitUntil(timeout: Constants.deliveryTimeout) { done in testContext.eventProcessor.flushReportingOutcome { result in delivered = result done() @@ -1810,7 +1815,7 @@ extension DirectEventProcessorSpec { testContext.recordEvents(1) testContext.eventProcessor.isOnline = true var delivered: Bool? - waitUntil { done in + waitUntil(timeout: Constants.deliveryTimeout) { done in testContext.eventProcessor.flushReportingOutcome { result in delivered = result done() @@ -1829,13 +1834,13 @@ extension DirectEventProcessorSpec { testContext.serviceMock.holdsEventCompletions = true testContext.eventProcessor.flushReportingOutcome { _ in } - expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) // Holding this back until the first delivery answers would let the store fill through a retry, which // can run for twice the connection timeout, and events recorded past the capacity are lost. testContext.recordEvents(1) testContext.eventProcessor.flushReportingOutcome { _ in } - expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(2)) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(2), timeout: Constants.deliveryTimeout) // The first batch is still awaiting its response, so the second delivery left it alone rather than // listing it again and sending it a second time under the same payload ID. @@ -1852,7 +1857,7 @@ extension DirectEventProcessorSpec { testContext.serviceMock.holdsEventCompletions = true testContext.eventProcessor.flushReportingOutcome { _ in } - expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) // Everything on disk belongs to the delivery already running, and that may include this caller's // events: a commit only stages them, and whichever delivery runs next is what closes them into a @@ -1878,7 +1883,7 @@ extension DirectEventProcessorSpec { // A closes a batch of its own and waits on it. Nothing has been lost at the point it closed. let first = Answer() testContext.eventProcessor.flushReportingOutcome(completion: first.record) - expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) // B closes a batch of its own, dropping an event the store will never take as it does, so the loss // falls between the two closes. B has a post of its own outstanding, which is what leaves A to be @@ -1887,15 +1892,15 @@ extension DirectEventProcessorSpec { testContext.recordEvents(1) let second = Answer() testContext.eventProcessor.flushReportingOutcome(completion: second.record) - expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(2)) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(2), timeout: Constants.deliveryTimeout) // Responses land in the order they were made, so A is answered while B's post is still out. With // one shared flag A would take B's loss here, and B would report success for the event it dropped. testContext.serviceMock.holdsEventCompletions = false testContext.serviceMock.releaseHeldEventCompletions() - expect(first.value()).toEventually(beTrue()) - expect(second.value()).toEventually(beFalse()) + expect(first.value()).toEventually(beTrue(), timeout: Constants.deliveryTimeout) + expect(second.value()).toEventually(beFalse(), timeout: Constants.deliveryTimeout) } it("answers away from the queue a delivery runs on, after a delivery") { @@ -1951,7 +1956,7 @@ extension DirectEventProcessorSpec { let answers = Answers() testContext.eventProcessor.flushReportingOutcome(completion: answers.record) - expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) // Held on the first post, each of these finds the only batch claimed by it. None of them is told // its events arrived on the strength of having nothing left of its own to send. @@ -2079,7 +2084,7 @@ extension DirectEventProcessorSpec { testContext.eventProcessor.record(CustomEvent(key: "kept", context: testContext.context)) testContext.eventProcessor.isOnline = true testContext.eventProcessor.flush(completion: nil) - expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) // Well inside the one-second retry delay: a delivery queue sleeping through it would not get to this. testContext.eventProcessor.flush(completion: nil) From d95893a35a66a9ac1e44d6021ffa8b883736555e Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Mon, 12 Oct 2026 08:27:58 -0700 Subject: [PATCH 042/158] Fix compiler warnings in tests the event durability tiers added Integer literals used as LDValue are deprecated, so the tier-added ones are written as .number(...), the same values. 9007199254740993 is not representable as a Double and was already rounded to 9007199254740992, which is now written out. Warnings in code that predates the tiers are left alone. Co-authored-by: Cursor --- .../LaunchDarklyTests/EventJSONWriterTests.swift | 16 ++++++++-------- .../ServiceObjects/ContextSummarizerSpec.swift | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/LaunchDarkly/LaunchDarklyTests/EventJSONWriterTests.swift b/LaunchDarkly/LaunchDarklyTests/EventJSONWriterTests.swift index 7d268b147..171675f19 100644 --- a/LaunchDarkly/LaunchDarklyTests/EventJSONWriterTests.swift +++ b/LaunchDarkly/LaunchDarklyTests/EventJSONWriterTests.swift @@ -28,13 +28,13 @@ final class EventJSONWriterTests: XCTestCase { _ = builder.trySetValue("firstName", "Ada") _ = builder.trySetValue("lastName", "Lovelace") _ = builder.trySetValue("email", "ada@example.com") - _ = builder.trySetValue("age", 36) + _ = builder.trySetValue("age", .number(36)) _ = builder.trySetValue("score", 12.5) - _ = builder.trySetValue("negative", -17) + _ = builder.trySetValue("negative", .number(-17)) _ = builder.trySetValue("verified", true) _ = builder.trySetValue("nothing", .null) _ = builder.trySetValue("tags", ["a", "b", "c"]) - _ = builder.trySetValue("address", ["street": "1 Main St", "city": "Springfield", "zip": 12345]) + _ = builder.trySetValue("address", ["street": "1 Main St", "city": "Springfield", "zip": .number(12345)]) _ = builder.trySetValue("deep", ["a": ["b": ["c": "d"]]]) return try! builder.build().get() } @@ -97,7 +97,7 @@ final class EventJSONWriterTests: XCTestCase { var device = LDContextBuilder(key: "device-key") device.kind("device") device.anonymous(true) - _ = device.trySetValue("os", ["name": "iOS", "version": 18]) + _ = device.trySetValue("os", ["name": "iOS", "version": .number(18)]) var builder = LDMultiContextBuilder() builder.addContext(try! user.build().get()) @@ -132,7 +132,7 @@ final class EventJSONWriterTests: XCTestCase { variation: 2, version: 11, trackEvents: true, - reason: ["kind": "RULE_MATCH", "ruleIndex": 1, "ruleId": "rule-id"], + reason: ["kind": "RULE_MATCH", "ruleIndex": .number(1), "ruleId": "rule-id"], trackReason: true) let unknownFlag = FeatureFlag(flagKey: "flag-key", value: true) @@ -144,11 +144,11 @@ final class EventJSONWriterTests: XCTestCase { return [ ("feature", FeatureEvent(key: "flag-key", context: context, value: true, defaultValue: false, featureFlag: plainFlag, includeReason: false, isDebug: false)), ("feature, reason", FeatureEvent(key: "flag-key", context: context, value: "v", defaultValue: "d", featureFlag: reasonFlag, includeReason: true, isDebug: false)), - ("feature, no flag", FeatureEvent(key: "flag-key", context: context, value: .null, defaultValue: [1, 2], featureFlag: nil, includeReason: false, isDebug: false)), + ("feature, no flag", FeatureEvent(key: "flag-key", context: context, value: .null, defaultValue: [.number(1), .number(2)], featureFlag: nil, includeReason: false, isDebug: false)), ("debug", FeatureEvent(key: "flag-key", context: context, value: true, defaultValue: false, featureFlag: plainFlag, includeReason: false, isDebug: true)), ("identify", IdentifyEvent(context: context)), ("custom, bare", CustomEvent(key: "custom-key", context: context)), - ("custom, data", CustomEvent(key: "custom \"key\"", context: context, data: ["a": 1, "b": [true, .null, "x"]])), + ("custom, data", CustomEvent(key: "custom \"key\"", context: context, data: ["a": .number(1), "b": [true, .null, "x"]])), ("custom, metric", CustomEvent(key: "custom-key", context: context, data: "payload", metricValue: 3.25)), ("summary", SummaryEvent(flagRequestTracker: tracker, context: context)) ] @@ -465,7 +465,7 @@ final class EventJSONWriterTests: XCTestCase { func testNumberFormatting() throws { let numbers: [Double] = [0, -0, 1, -1, 17, -17, 0.5, -0.5, 1.25, 3.141592653589793, - 1e10, 1e-10, 1e20, -1e20, Double(Int32.max), Double(Int64.max), 9007199254740993, + 1e10, 1e-10, 1e20, -1e20, Double(Int32.max), Double(Int64.max), 9007199254740992, 1e16, 1e17, -1e17, 1.5e17, 1e18, 9.2e18, -9.2e18, 9223372036854774784] for number in numbers { diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/ContextSummarizerSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/ContextSummarizerSpec.swift index 94dd06683..eea0da02d 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/ContextSummarizerSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/ContextSummarizerSpec.swift @@ -176,10 +176,10 @@ final class ContextSummarizerSpec: QuickSpec { privateAttributes.forEach { builder.addPrivateAttribute(Reference($0)) } return try! builder.build().get() } - let nested: LDValue = ["x": 1, "y": ["z", true]] - let forward = context([("email", "a@example.com"), ("plan", nested), ("score", 2)], + let nested: LDValue = ["x": .number(1), "y": ["z", true]] + let forward = context([("email", "a@example.com"), ("plan", nested), ("score", .number(2))], privateAttributes: ["email", "plan"]) - let backward = context([("score", 2), ("plan", nested), ("email", "a@example.com")], + let backward = context([("score", .number(2)), ("plan", nested), ("email", "a@example.com")], privateAttributes: ["/plan", "email"]) expect(forward) == backward From 2f180e4768d441c43bfde87f6211d3893611563a Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Mon, 12 Oct 2026 08:27:58 -0700 Subject: [PATCH 043/158] Say which lock guards the outbound buffer's encoder The comment said the encoder was shared by the threads recording events and so must not be mutated, but JSONEncoder is a class and the encoder is only ever used during a commit, under the processor's commitLock. The comment now says that. Co-authored-by: Cursor --- .../LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift index 0fe9691b3..aa007270d 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift @@ -22,7 +22,7 @@ final class OutboundEventBuffer { /// Only to be used while holding the processor's `recordLock`. private(set) var contextSummarizer: ContextSummarizer - /// Shared by the threads recording events, so it must not be mutated after `init`. + /// Only to be used while holding the processor's `commitLock`. private let encoder: JSONEncoder /// Outlives a commit rather than being built per commit, so that its buffer and context cache span them. From 409b07646c31ce5581a8a36399c7a53dc6db0280 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Mon, 12 Oct 2026 08:27:58 -0700 Subject: [PATCH 044/158] List the no-directory store's batches oldest first NullEventStore, the fallback where there is no directory to write to, kept its closed batches in a dictionary and listed them in hash order, so a delivery could send later events ahead of earlier ones. It keeps them in close order now, as the store with a disk lists its files. Co-authored-by: Cursor --- .../ServiceObjects/NullEventStore.swift | 13 +++++----- .../ServiceObjects/EventStoreSpec.swift | 24 +++++++++++++++++++ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventStore.swift index 5f74cfc61..f1b7f5206 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventStore.swift @@ -6,10 +6,11 @@ final class NullEventStore: EventStoring { private let lock = UnfairLock() private var staged: [Data] = [] private var reserved = 0 - private var closed: [String: [Data]] = [:] + /// Oldest first, as the store with a disk lists its batches, so that delivery goes in the order events were closed. + private var closed: [(payloadId: String, events: [Data])] = [] var pendingEventCount: Int { - lock.withLock { reserved + staged.count + closed.values.reduce(0) { $0 + $1.count } } + lock.withLock { reserved + staged.count + closed.reduce(0) { $0 + $1.events.count } } } var isPersisting: Bool { false } @@ -46,17 +47,17 @@ final class NullEventStore: EventStoring { let payloadId = UUID().uuidString let events = staged staged = [] - closed[payloadId] = events + closed.append((payloadId, events)) return EventBatch(payloadId: payloadId, eventCount: events.count) } } func pendingBatches() -> [EventBatch] { - lock.withLock { closed.map { EventBatch(payloadId: $0.key, eventCount: $0.value.count) } } + lock.withLock { closed.map { EventBatch(payloadId: $0.payloadId, eventCount: $0.events.count) } } } func body(of batch: EventBatch) -> Data? { - guard let events = lock.withLock({ closed[batch.payloadId] }) + guard let events = lock.withLock({ closed.first { $0.payloadId == batch.payloadId }?.events }) else { return nil } var body = Data("[".utf8) @@ -71,7 +72,7 @@ final class NullEventStore: EventStoring { } func remove(_ batch: EventBatch) { - lock.withLock { closed[batch.payloadId] = nil } + lock.withLock { closed.removeAll { $0.payloadId == batch.payloadId } } } func recoverInterruptedLog() { diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 7d0b971b8..d07ef810a 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -13,6 +13,7 @@ final class EventStoreSpec: QuickSpec { fallbackSpec() optOutSpec() multiProcessSpec() + nullStoreSpec() #if os(macOS) || targetEnvironment(macCatalyst) sameNameInstancesSpec() #endif @@ -641,6 +642,29 @@ final class EventStoreSpec: QuickSpec { } } + private func nullStoreSpec() { + describe("the store used where there is nowhere to write") { + it("lists its batches oldest first, as the store with a disk does") { + let store = NullEventStore() + var closed: [String] = [] + // Enough that hash order coming out right by chance is not a worry. + for index in 0..<20 { + _ = store.stage(EventStoreSpec.payload("event-\(index)"), bypassingCapacity: false) + if let batch = store.closeBatch() { + closed.append(batch.payloadId) + } + } + + expect(store.pendingBatches().map(\.payloadId)) == closed + + store.remove(EventBatch(payloadId: closed[3], eventCount: 1)) + closed.remove(at: 3) + expect(store.pendingBatches().map(\.payloadId)) == closed + expect(EventStoreSpec.keys(ofBody: store.body(of: EventBatch(payloadId: closed[3], eventCount: 1)))) == ["event-4"] + } + } + } + private func capacitySpec() { describe("capacity") { var store: EventStore! From b85328d38b383bca49d371ae9151b33fa545d424 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Mon, 12 Oct 2026 09:34:17 -0700 Subject: [PATCH 045/158] Stop counting a batch against capacity while its request is out A delivered batch counted against capacity until the SDK had processed the response, so an event recorded between LaunchDarkly receiving a full store's batch and the SDK hearing back was refused. The contract tests' "buffer is reset after flush" records exactly there, and failed whenever the response was slow enough. Before events were persisted, a flush emptied the buffer as it sent, so this could not happen. A batch now stops counting just before its request goes out. Accepted or permanently refused, it is removed as before; a send to be retried makes it count again. Removing it stops it being left out in the same critical section that takes its events away, so it is never subtracted twice, and a listing that no longer finds it (another process delivered it) stops leaving it out too. The trade: after a failed send a full store briefly holds its capacity plus the events that request carried, which eventCapacity now documents. The same change as on Android. Co-authored-by: Cursor --- .../LaunchDarkly/Models/LDConfig.swift | 3 + .../ServiceObjects/DirectEventProcessor.swift | 2 + .../ServiceObjects/EventStore.swift | 79 ++++++++++++++++--- .../ServiceObjects/NullEventStore.swift | 23 +++++- .../DirectEventProcessorSpec.swift | 28 +++++++ .../ServiceObjects/EventStoreSpec.swift | 43 ++++++++++ 6 files changed, 166 insertions(+), 12 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift index 7bc385a41..2dd68e9ee 100644 --- a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift @@ -346,6 +346,9 @@ public struct LDConfig { /// Where `eventPersistence` keeps events across launches, this includes the events an earlier launch left undelivered, from the /// moment the SDK has read them, which it does in the background just after starting. Events recorded before then are /// counted without them, so for that moment the SDK can hold up to this many on top of what the earlier launch left. + /// + /// Events being sent stop counting once the request is out, so a flush makes room straight away. If the request fails they are kept + /// to send again and count again, so after a failed send the SDK can hold up to this many plus the events that request carried. public var eventCapacity: Int = Defaults.eventCapacity /// How far the SDK goes to make a recorded event outlive the process that recorded it. diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index cfd2a2215..718a4da81 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -701,6 +701,7 @@ extension DirectEventProcessor { private func publish(_ batch: EventBatch, _ body: Data, _ completion: @escaping (AnalyticsEventSender.Outcome) -> Void) { + store.startedSending(batch) // The store is settled before observers are told, so that one looking at it on hearing the outcome sees it. sender.send(body, payloadId: batch.payloadId, eventCount: batch.eventCount) { outcome in switch outcome { @@ -717,6 +718,7 @@ extension DirectEventProcessor { self.store.remove(batch) self.reportSyncComplete(.response(response)) case .retryable(let error): + self.store.stoppedSending(batch) self.reportSyncComplete(error) // The batch stays on disk. A later delivery, in this run of the application or the next one, sends it // under the same payload ID, so LaunchDarkly can tell it is not new. diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index b51537b89..3ec0f8bb6 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -19,8 +19,13 @@ struct EventBatch: Equatable { } protocol EventStoring { - /// Events that have been recorded but not yet accepted by LaunchDarkly, whether they are reserved while being - /// encoded, staged in memory, written to the open log, or sitting in a closed batch. + /// Events counted against capacity: those recorded and not yet accepted by LaunchDarkly, whether they are reserved + /// while being encoded, staged in memory, written to the open log, or sitting in a closed batch -- except a batch + /// whose request is out. + /// + /// A batch being sent is left out so that a flush makes room as soon as it sends, as it did when events were only + /// held in memory; otherwise an event recorded while the request is out is refused for a batch LaunchDarkly may + /// already have. A send that fails puts the batch back, so a full store can briefly hold capacity plus that batch. var pendingEventCount: Int { get } /// Whether a commit writes anything, which is false where persistence is off and once a write has failed and the @@ -62,6 +67,12 @@ protocol EventStoring { /// Forgets a batch, which is only correct once LaunchDarkly has accepted it or permanently refused it. func remove(_ batch: EventBatch) + /// Stops counting a batch against capacity while its request is out. Ended by `remove(_:)` or `stoppedSending(_:)`. + func startedSending(_ batch: EventBatch) + + /// Counts a batch against capacity again after a send that will be retried later. + func stoppedSending(_ batch: EventBatch) + /// Closes any log left open by a previous run of this process, so its events join the batches to deliver. func recoverInterruptedLog() } @@ -118,6 +129,10 @@ final class EventStore: EventStoring { /// that changes one of those, so that it cannot drift from them; a total worked out from them and written /// afterwards would overwrite a close or a removal in between. private var closedEvents = 0 + /// Closed batches whose request is out, by payload ID, and the events in them. Every one is also in `closedEvents`, + /// and leaves both in the same critical section. + private var sendingBatches: [String: Int] = [:] + private var sendingEvents = 0 /// Whether events are being written to disk, which is what the application asked for until a write fails and the /// store gives up on persistence for the rest of the session. private var persistEvents = true @@ -206,7 +221,13 @@ final class EventStore: EventStoring { // MARK: Recording var pendingEventCount: Int { - bufferLock.withLock { reservedEvents + bufferedEventCount + writingEvents + committedEvents + closedEvents } + bufferLock.withLock { countedEvents } + } + + /// Every event counted against capacity. Only to be used while holding `bufferLock`. + private var countedEvents: Int { + // Clamped because a batch another process delivered can be marked as sending after a listing stopped counting it. + reservedEvents + bufferedEventCount + writingEvents + committedEvents + max(0, closedEvents - sendingEvents) } var isPersisting: Bool { @@ -241,8 +262,7 @@ final class EventStore: EventStoring { // Whether the staged bytes are due a write, or nil where the event was refused for capacity. let needsCommit = bufferLock.withLock { () -> Bool? in - guard bypassingCapacity - || reservedEvents + bufferedEventCount + writingEvents + committedEvents + closedEvents < capacity + guard bypassingCapacity || countedEvents < capacity else { return nil } // In the same critical section as the append, so the event is never counted twice or not at all. @@ -370,7 +390,12 @@ final class EventStore: EventStoring { private func heldBatchesAndRecount(after onDisk: [EventBatch]) -> [EventBatch] { heldLock.withLock { let batches = onDisk + inMemoryBatches.map { EventBatch(payloadId: $0.payloadId, eventCount: $0.eventCount) } - bufferLock.withLock { closedEvents = batches.reduce(0) { $0 + $1.eventCount } } + bufferLock.withLock { + closedEvents = batches.reduce(0) { $0 + $1.eventCount } + // One no longer listed is no longer counted, so it cannot be left out of the count either. + let listed = Set(batches.map { $0.payloadId }) + sendingBatches.keys.filter { !listed.contains($0) }.forEach(forgetSending) + } return batches } } @@ -458,16 +483,26 @@ final class EventStore: EventStoring { guard let index = inMemoryBatches.firstIndex(where: { $0.payloadId == batch.payloadId }) else { return false } let held = inMemoryBatches.remove(at: index) - bufferLock.withLock { closedEvents -= held.eventCount } + bufferLock.withLock { + closedEvents -= held.eventCount + forgetSending(batch.payloadId) + } return true } - guard !wasHeld, readsDisk + guard !wasHeld else { return } + guard readsDisk + else { + bufferLock.withLock { forgetSending(batch.payloadId) } + return + } ioLock.withLock { try? FileManager.default.removeItem(at: url(of: batch.payloadId)) - if let counted = eventCounts.removeValue(forKey: batch.payloadId) { - bufferLock.withLock { closedEvents -= counted } + let counted = eventCounts.removeValue(forKey: batch.payloadId) ?? 0 + bufferLock.withLock { + closedEvents -= counted + forgetSending(batch.payloadId) } } } @@ -538,6 +573,30 @@ final class EventStore: EventStoring { } } +// MARK: Sending + +extension EventStore { + func startedSending(_ batch: EventBatch) { + bufferLock.withLock { + guard sendingBatches[batch.payloadId] == nil + else { return } + sendingBatches[batch.payloadId] = batch.eventCount + sendingEvents += batch.eventCount + } + } + + func stoppedSending(_ batch: EventBatch) { + bufferLock.withLock { forgetSending(batch.payloadId) } + } + + /// Requires `bufferLock`. + fileprivate func forgetSending(_ payloadId: String) { + if let events = sendingBatches.removeValue(forKey: payloadId) { + sendingEvents -= events + } + } +} + // MARK: Writing extension EventStore { diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventStore.swift index f1b7f5206..9d48f0bf0 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventStore.swift @@ -8,9 +8,13 @@ final class NullEventStore: EventStoring { private var reserved = 0 /// Oldest first, as the store with a disk lists its batches, so that delivery goes in the order events were closed. private var closed: [(payloadId: String, events: [Data])] = [] + /// Closed batches whose request is out, which do not count against capacity. + private var sending: Set = [] var pendingEventCount: Int { - lock.withLock { reserved + staged.count + closed.reduce(0) { $0 + $1.events.count } } + lock.withLock { + reserved + staged.count + closed.reduce(0) { $0 + (sending.contains($1.payloadId) ? 0 : $1.events.count) } + } } var isPersisting: Bool { false } @@ -72,7 +76,22 @@ final class NullEventStore: EventStoring { } func remove(_ batch: EventBatch) { - lock.withLock { closed.removeAll { $0.payloadId == batch.payloadId } } + lock.withLock { + closed.removeAll { $0.payloadId == batch.payloadId } + sending.remove(batch.payloadId) + } + } + + func startedSending(_ batch: EventBatch) { + lock.withLock { + if closed.contains(where: { $0.payloadId == batch.payloadId }) { + sending.insert(batch.payloadId) + } + } + } + + func stoppedSending(_ batch: EventBatch) { + _ = lock.withLock { sending.remove(batch.payloadId) } } func recoverInterruptedLog() { diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index a53e9f187..123aecb5f 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -1281,6 +1281,30 @@ extension DirectEventProcessorSpec { expect(testContext.store.pendingEventCount) == 0 } + // What the contract tests' "buffer is reset after flush" does: they have the payload as soon as it is sent, + // and record the next event before the SDK has heard back. + it("makes room in a full store as soon as its batch is sent") { + testContext = TestContext(eventCapacity: 2) + testContext.eventProcessor.isOnline = true + testContext.serviceMock.holdsEventCompletions = true + testContext.eventProcessor.record(CustomEvent(key: "first", context: testContext.context)) + testContext.eventProcessor.record(CustomEvent(key: "second", context: testContext.context)) + + let first = Answer() + testContext.eventProcessor.flush { first.record(true) } + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + + testContext.eventProcessor.record(CustomEvent(key: "one-more", context: testContext.context)) + testContext.serviceMock.holdsEventCompletions = false + testContext.serviceMock.releaseHeldEventCompletions() + expect(first.value()).toEventually(beTrue(), timeout: Constants.deliveryTimeout) + + waitUntil(timeout: Constants.deliveryTimeout) { done in + testContext.eventProcessor.flush(completion: done) + } + expect(testContext.serviceMock.publishedEventPayloads.map(customKeys)) == [["first", "second"], ["one-more"]] + } + it("answers an evaluating thread while a delivery is listing the store") { testContext = TestContext() // Standing in for the listing a delivery makes on a device whose disk is busy, which on the first one @@ -2193,6 +2217,8 @@ private final class ClosingHookStore: EventStoring { func pendingBatches() -> [EventBatch] { inner.pendingBatches() } func body(of batch: EventBatch) throws -> Data? { try inner.body(of: batch) } func remove(_ batch: EventBatch) { inner.remove(batch) } + func startedSending(_ batch: EventBatch) { inner.startedSending(batch) } + func stoppedSending(_ batch: EventBatch) { inner.stoppedSending(batch) } func recoverInterruptedLog() { inner.recoverInterruptedLog() } func closeBatch() -> EventBatch? { @@ -2267,6 +2293,8 @@ private final class SlowListingStore: EventStoring { func closeBatch() -> EventBatch? { inner.closeBatch() } func body(of batch: EventBatch) throws -> Data? { try inner.body(of: batch) } func remove(_ batch: EventBatch) { inner.remove(batch) } + func startedSending(_ batch: EventBatch) { inner.startedSending(batch) } + func stoppedSending(_ batch: EventBatch) { inner.stoppedSending(batch) } func recoverInterruptedLog() { inner.recoverInterruptedLog() } /// Every listing but the first is held, the first being the one the reporter makes as it starts up. diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index d07ef810a..37bccdd10 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -690,6 +690,49 @@ final class EventStoreSpec: QuickSpec { expect(store.stage(EventStoreSpec.payload("second"))) == false } + it("does not count a batch while its request is out, and counts it again if the send is to be retried") { + store = EventStore.temporary(capacity: 1) + expect(store.stage(EventStoreSpec.payload("first"))) == true + let batch = store.closeBatch()! + + store.startedSending(batch) + expect(store.pendingEventCount) == 0 + expect(store.stage(EventStoreSpec.payload("while-sending"))) == true + + store.stoppedSending(batch) + expect(store.pendingEventCount) == 2 + expect(store.stage(EventStoreSpec.payload("after-retryable"))) == false + } + + it("takes a sent batch out of the count only once when it is delivered") { + store = EventStore.temporary(capacity: 2) + expect(store.stage(EventStoreSpec.payload("first"))) == true + let batch = store.closeBatch()! + expect(store.stage(EventStoreSpec.payload("second"))) == true + + store.startedSending(batch) + store.remove(batch) + expect(store.pendingEventCount) == 1 + store.stoppedSending(batch) + expect(store.pendingEventCount) == 1 + _ = store.pendingBatches() + expect(store.pendingEventCount) == 1 + } + + it("does the same for the store used where there is nowhere to write") { + let nullStore = NullEventStore() + expect(nullStore.stage(EventStoreSpec.payload("first"), bypassingCapacity: false)) == true + let batch = nullStore.closeBatch()! + + nullStore.startedSending(batch) + expect(nullStore.pendingEventCount) == 0 + nullStore.stoppedSending(batch) + expect(nullStore.pendingEventCount) == 1 + nullStore.startedSending(batch) + nullStore.remove(batch) + expect(nullStore.pendingEventCount) == 0 + } + // The bound `LDConfig.eventCapacity` documents: the reporter reads the disk in the background after starting, // so what an earlier run left only counts from that listing on. it("counts what an earlier run left undelivered from the first listing on") { From 35453a819a4c6de50f742a391d1c31f9ecc1d1fb Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Mon, 12 Oct 2026 10:56:06 -0700 Subject: [PATCH 046/158] disabled unit testd --- LaunchDarkly.xcodeproj/project.pbxproj | 10 - .../ServiceObjects/DirectEventProcessor.swift | 7 +- .../ServiceObjects/EventStore.swift | 57 ++++- .../ServiceObjects/NullEventStore.swift | 99 -------- .../Mocks/DarklyServiceMock.swift | 24 +- .../DirectEventProcessorSpec.swift | 215 ++++++++++++++++-- .../ServiceObjects/EventStoreSpec.swift | 36 +-- 7 files changed, 289 insertions(+), 159 deletions(-) delete mode 100644 LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventStore.swift diff --git a/LaunchDarkly.xcodeproj/project.pbxproj b/LaunchDarkly.xcodeproj/project.pbxproj index 0c0cd2618..563782241 100644 --- a/LaunchDarkly.xcodeproj/project.pbxproj +++ b/LaunchDarkly.xcodeproj/project.pbxproj @@ -18,7 +18,6 @@ 29FE129B280413D4008CC918 /* Util.swift in Sources */ = {isa = PBXBuildFile; fileRef = 29FE1297280413D4008CC918 /* Util.swift */; }; 2CA9D36B3566D78850D6F0AF /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; B8F228923BE11E444D1ECB8E /* DiskEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */; }; - 513F2EA8711D6DA3DA7AE41B /* NullEventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = BA3ED412A285F54822108659 /* NullEventStore.swift */; }; 3C55384C9E3485B493E1F86A /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; C3F7E502BD916C40AE238F64 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; AB4827D11084D58661E24B05 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; @@ -55,7 +54,6 @@ 50EE85C72EA0749C007CC662 /* TimeoutExecutorSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50EE85C62EA0749C007CC662 /* TimeoutExecutorSpec.swift */; }; 5AEA1F832DCA974314E07D21 /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; D4225F1F05F2C55E9518941B /* DiskEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */; }; - BCB7C8017479FD22963B3BB2 /* NullEventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = BA3ED412A285F54822108659 /* NullEventStore.swift */; }; DDA788E3056F8AC3CA203933 /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; D4080613CEA27D51BF349075 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; ECBE45806B470ACC5187B7D6 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; @@ -405,7 +403,6 @@ AFCE752CB1A75DEBABC23034 /* LDContextJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88DAB8D1864C14D9C7E06F11 /* LDContextJSONWriter.swift */; }; B0B1AD7CADE8DBB596C2DE9B /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; DAA2C9B9C144F129217A7C55 /* DiskEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */; }; - 99050D62DBD6A90685F6E28C /* NullEventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = BA3ED412A285F54822108659 /* NullEventStore.swift */; }; 44E0BE449714952A68628CEB /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; E5191724DFB38E62C0451086 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; CA2ADB3D20A229D70CA34F73 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; @@ -461,7 +458,6 @@ C4AF311E4DA8C3AB7040637D /* EventJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = B430884DBF2E541B919C08FA /* EventJSONWriter.swift */; }; CF9A1EB116D8AA3B7B5535B2 /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; E2F8080AB2B3CD6E9A9C1FCC /* DiskEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */; }; - 9DAA86F524C6D802D69B8553 /* NullEventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = BA3ED412A285F54822108659 /* NullEventStore.swift */; }; F0F504E5EC32223905356098 /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; B2E6D4F1AC805B3F9D127E53 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; 625ADAC0CCFE3BE324C45752 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; @@ -695,7 +691,6 @@ B4F689132497B2FC004D3CE0 /* DiagnosticEventSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DiagnosticEventSpec.swift; sourceTree = ""; }; C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventStore.swift; sourceTree = ""; }; E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = DiskEventLog.swift; sourceTree = ""; }; - BA3ED412A285F54822108659 /* NullEventStore.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = NullEventStore.swift; sourceTree = ""; }; 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventLogFile.swift; sourceTree = ""; }; A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = DeliveryClaims.swift; sourceTree = ""; }; B58551BB165F28784C785720 /* EventLogFormat.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventLogFormat.swift; sourceTree = ""; }; @@ -1089,7 +1084,6 @@ 9A771B316E19ED598CD4FB0B /* BackgroundActivity.swift */, C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */, E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */, - BA3ED412A285F54822108659 /* NullEventStore.swift */, 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */, A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */, B58551BB165F28784C785720 /* EventLogFormat.swift */, @@ -1625,7 +1619,6 @@ A32FCDB22D53263851DDB727 /* BackgroundActivity.swift in Sources */, CF9A1EB116D8AA3B7B5535B2 /* EventStore.swift in Sources */, E2F8080AB2B3CD6E9A9C1FCC /* DiskEventLog.swift in Sources */, - 9DAA86F524C6D802D69B8553 /* NullEventStore.swift in Sources */, F0F504E5EC32223905356098 /* EventLogFile.swift in Sources */, B2E6D4F1AC805B3F9D127E53 /* DeliveryClaims.swift in Sources */, 625ADAC0CCFE3BE324C45752 /* EventLogFormat.swift in Sources */, @@ -1729,7 +1722,6 @@ 7185A3C30E7FFC4F725769F1 /* BackgroundActivity.swift in Sources */, 2CA9D36B3566D78850D6F0AF /* EventStore.swift in Sources */, B8F228923BE11E444D1ECB8E /* DiskEventLog.swift in Sources */, - 513F2EA8711D6DA3DA7AE41B /* NullEventStore.swift in Sources */, 3C55384C9E3485B493E1F86A /* EventLogFile.swift in Sources */, C3F7E502BD916C40AE238F64 /* DeliveryClaims.swift in Sources */, AB4827D11084D58661E24B05 /* EventLogFormat.swift in Sources */, @@ -1832,7 +1824,6 @@ A20146E7AF098560627C614E /* BackgroundActivity.swift in Sources */, 5AEA1F832DCA974314E07D21 /* EventStore.swift in Sources */, D4225F1F05F2C55E9518941B /* DiskEventLog.swift in Sources */, - BCB7C8017479FD22963B3BB2 /* NullEventStore.swift in Sources */, DDA788E3056F8AC3CA203933 /* EventLogFile.swift in Sources */, D4080613CEA27D51BF349075 /* DeliveryClaims.swift in Sources */, ECBE45806B470ACC5187B7D6 /* EventLogFormat.swift in Sources */, @@ -2002,7 +1993,6 @@ 3D34EEEF37158657D49335BB /* BackgroundActivity.swift in Sources */, B0B1AD7CADE8DBB596C2DE9B /* EventStore.swift in Sources */, DAA2C9B9C144F129217A7C55 /* DiskEventLog.swift in Sources */, - 99050D62DBD6A90685F6E28C /* NullEventStore.swift in Sources */, 44E0BE449714952A68628CEB /* EventLogFile.swift in Sources */, E5191724DFB38E62C0451086 /* DeliveryClaims.swift in Sources */, CA2ADB3D20A229D70CA34F73 /* EventLogFormat.swift in Sources */, diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 718a4da81..af73695e6 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -194,12 +194,7 @@ class DirectEventProcessor: EventProcessor { } private static func makeStore(config: LDConfig) -> EventStoring { - guard let directory = EventStore.defaultDirectory(mobileKey: config.mobileKey) - else { - os_log("Events cannot be persisted: no writable directory was available", log: config.logger, type: .debug) - return NullEventStore() - } - return EventStore(directory: directory, + EventStore(directory: EventStore.defaultDirectory(mobileKey: config.mobileKey), capacity: config.eventCapacity, persistEvents: config.eventPersistence != .disabled, logger: config.logger) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index 3ec0f8bb6..3dc9f8a6f 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -99,7 +99,10 @@ final class EventStore: EventStoring { /// application that only evaluates flags and so never reaches a commit point of its own. private static let stagingThreshold = 16 * 1024 - let directory: URL + /// Where the batches are kept. + var directory: URL { + ioLock.withLock { disk.directory } + } private let capacity: Int private let logger: OSLog /// How a stored log is read, injectable so a test can make a read fail while the file is intact. @@ -146,8 +149,28 @@ final class EventStore: EventStoring { /// Whether a commit is already on its way, so that a burst of recordings queues one write rather than one each. private var isCommitScheduled = false + private let locateDirectory: () -> URL + private let processName: String + private let writeLog: (EventLogFile, Data) -> Int32? /// Only to be used while holding `ioLock`. - private let diskLog: DiskEventLog + private var located: (directory: URL, log: DiskEventLog)? + + /// The directory and the log in it, worked out the first time the disk is reached for, so that a store told to keep + /// off the disk makes no filesystem call at all, not even to find out where it would have written. Requires `ioLock`. + private var disk: (directory: URL, log: DiskEventLog) { + ioLock.assertOwned() + if let located { + return located + } + let directory = locateDirectory() + let disk = (directory: directory, + log: DiskEventLog(directory: directory, processName: processName, ioLock: ioLock, logger: logger, writeLog: writeLog)) + located = disk + return disk + } + + /// Requires `ioLock`. + private var diskLog: DiskEventLog { disk.log } /// Whether a log left by a previous run has been dealt with. Only to be used while holding `ioLock`. private var hasRecoveredInterruptedLog = false @@ -711,12 +734,29 @@ extension EventStore { func pendingEventPayloads() -> [Data] { commit() - guard readsDisk - else { return [] } - ioLock.lock() defer { ioLock.unlock() } + var payloads = readsDisk ? payloadsOnDisk() : [] + // After the disk, which is older: a store that gave up on persistence holds what came later in memory. + let held = heldLock.withLock { inMemoryBatches.map { $0.body } } + for body in held { + let events = (try? JSONDecoder().decode([LDValue].self, from: body)) ?? [] + payloads.append(contentsOf: events.compactMap { try? JSONEncoder().encode($0) }) + } + var staged = EventLogFormat.fileHeader + staged.append(bufferLock.withLock { bufferData }) + _ = EventLogFormat.forEachFrame(in: staged) { type, payload in + if type == EventLogFormat.eventFrame { + payloads.append(payload) + } + } + return payloads + } + + /// Requires `ioLock`. + private func payloadsOnDisk() -> [Data] { + ioLock.assertOwned() let contents = (try? FileManager.default.contentsOfDirectory(at: directory, includingPropertiesForKeys: nil, options: [.skipsHiddenFiles])) ?? [] @@ -740,12 +780,13 @@ extension EventStore { } extension EventStore { - /// The directory the store for a mobile key belongs in, or nil where the platform gave us nowhere to write. + /// The directory the store for a mobile key belongs in. /// /// tvOS is the reason this is not simply Application Support: a tvOS application is only allowed to persist to /// purgeable caches, so there the events are kept somewhere the system may reclaim, which is the most durability - /// the platform offers. - static func defaultDirectory(mobileKey: String) -> URL? { + /// the platform offers. A directory that turns out not to be writable is no different from a full device: the store + /// gives up on the disk and holds events in memory. + static func defaultDirectory(mobileKey: String) -> URL { #if os(tvOS) let searchPath = FileManager.SearchPathDirectory.cachesDirectory #else diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventStore.swift deleted file mode 100644 index 9d48f0bf0..000000000 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventStore.swift +++ /dev/null @@ -1,99 +0,0 @@ -import Foundation - -/// Stands in where the platform gave the SDK nowhere to write. Events are recorded and delivered from memory for the -/// life of the process, which is the behavior the SDK had before it kept a log. -final class NullEventStore: EventStoring { - private let lock = UnfairLock() - private var staged: [Data] = [] - private var reserved = 0 - /// Oldest first, as the store with a disk lists its batches, so that delivery goes in the order events were closed. - private var closed: [(payloadId: String, events: [Data])] = [] - /// Closed batches whose request is out, which do not count against capacity. - private var sending: Set = [] - - var pendingEventCount: Int { - lock.withLock { - reserved + staged.count + closed.reduce(0) { $0 + (sending.contains($1.payloadId) ? 0 : $1.events.count) } - } - } - - var isPersisting: Bool { false } - - func stage(_ encodedEvent: Data, bypassingCapacity: Bool) -> Bool { - lock.withLock { staged.append(encodedEvent) } - return true - } - - func reserve(_ events: Int) { - lock.withLock { reserved += events } - } - - func stageReserved(_ encodedEvent: Data) -> Bool { - lock.withLock { - reserved = max(0, reserved - 1) - staged.append(encodedEvent) - } - return true - } - - func releaseReservations() { - lock.withLock { reserved = 0 } - } - - func commit() { - } - - func closeBatch() -> EventBatch? { - lock.withLock { - guard !staged.isEmpty - else { return nil } - - let payloadId = UUID().uuidString - let events = staged - staged = [] - closed.append((payloadId, events)) - return EventBatch(payloadId: payloadId, eventCount: events.count) - } - } - - func pendingBatches() -> [EventBatch] { - lock.withLock { closed.map { EventBatch(payloadId: $0.payloadId, eventCount: $0.events.count) } } - } - - func body(of batch: EventBatch) -> Data? { - guard let events = lock.withLock({ closed.first { $0.payloadId == batch.payloadId }?.events }) - else { return nil } - - var body = Data("[".utf8) - for (index, event) in events.enumerated() { - if index > 0 { - body.append(UInt8(ascii: ",")) - } - body.append(event) - } - body.append(UInt8(ascii: "]")) - return body - } - - func remove(_ batch: EventBatch) { - lock.withLock { - closed.removeAll { $0.payloadId == batch.payloadId } - sending.remove(batch.payloadId) - } - } - - func startedSending(_ batch: EventBatch) { - lock.withLock { - if closed.contains(where: { $0.payloadId == batch.payloadId }) { - sending.insert(batch.payloadId) - } - } - } - - func stoppedSending(_ batch: EventBatch) { - _ = lock.withLock { sending.remove(batch.payloadId) } - } - - func recoverInterruptedLog() { - } -} diff --git a/LaunchDarkly/LaunchDarklyTests/Mocks/DarklyServiceMock.swift b/LaunchDarkly/LaunchDarklyTests/Mocks/DarklyServiceMock.swift index 5e82aac17..c71a58326 100644 --- a/LaunchDarkly/LaunchDarklyTests/Mocks/DarklyServiceMock.swift +++ b/LaunchDarkly/LaunchDarklyTests/Mocks/DarklyServiceMock.swift @@ -143,6 +143,18 @@ final class DarklyServiceMock: DarklyServiceProvider { private var _publishedPayloadIds: [String] = [] private var _holdsEventCompletions = false private var heldEventCompletions: [ServiceCompletionHandler] = [] + private var _eventStatusForPost: ((Int) -> Int)? + private var _publishedEventStatuses: [Int?] = [] + + /// The status to answer the nth post with, counting from 1, in place of `stubbedEventResponse`. + var eventStatusForPost: ((Int) -> Int)? { + get { withEventLock { _eventStatusForPost } } + set { withEventLock { _eventStatusForPost = newValue } } + } + /// The status each post was answered with, in the order of `publishedEventPayloads`; nil where it had none. + var publishedEventStatuses: [Int?] { + withEventLock { _publishedEventStatuses } + } var stubbedEventResponse: ServiceResponse? { get { withEventLock { _stubbedEventResponse } } @@ -179,12 +191,22 @@ final class DarklyServiceMock: DarklyServiceProvider { _publishedPayloadIds.append(payloadId) guard !_holdsEventCompletions else { + _publishedEventStatuses.append(nil) if let completion { heldEventCompletions.append(completion) } return nil } - return _stubbedEventResponse ?? (nil, nil, nil, nil) + var response = _stubbedEventResponse ?? (nil, nil, nil, nil) + if let statusForPost = _eventStatusForPost { + response = (nil, + HTTPURLResponse(url: config.eventsUrl, statusCode: statusForPost(_publishEventDataCallCount), + httpVersion: Constants.httpVersion, headerFields: nil), + nil, + nil) + } + _publishedEventStatuses.append((response.urlResponse as? HTTPURLResponse)?.statusCode) + return response } if let response { completion?(response) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 123aecb5f..c8bb87497 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -3,7 +3,18 @@ import Quick import Nimble @testable import LaunchDarkly -final class DirectEventProcessorSpec: QuickSpec { +class DirectEventProcessorSpec: QuickSpec { + /// The persistence a `TestContext` gets where it names neither one nor a store. Immediate here, as the durable + /// behaviour is what most of these are about; `DirectEventProcessorWithoutPersistenceSpec` runs them all again with + /// it off, which is what an application gets. + class var persistence: EventPersistence { .immediate } + + /// `persistence` of the spec whose examples are running, for the `TestContext` default, which cannot see the class. + fileprivate static var runningPersistence: EventPersistence = .immediate + + /// Whether the spec runs with a disk, for examples whose subject is the disk. + var persists: Bool { type(of: self).persistence != .disabled } + struct Constants { static let eventFlushInterval: TimeInterval = 10.0 static let eventFlushIntervalHalfSecond: TimeInterval = 0.5 @@ -34,14 +45,16 @@ final class DirectEventProcessorSpec: QuickSpec { onSyncComplete: EventSyncCompleteClosure? = nil, store: EventStore? = nil, eventCapacity: Int = Event.Kind.allKinds.count, - eventPersistence: EventPersistence = .immediate, + eventPersistence: EventPersistence? = nil, commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.tests.DirectEventProcessor.commitQueue")) { config = LDConfig.stub config.eventCapacity = eventCapacity - // The durable behaviour is what most of these are about, so it is the default here even though an - // application has to ask for it. - config.eventPersistence = eventPersistence + // A store a test brings decides, so the processor is never configured for a disk its store does not have. + let persistence = eventPersistence + ?? store.map { $0.isPersisting ? .immediate : .disabled } + ?? DirectEventProcessorSpec.runningPersistence + config.eventPersistence = persistence config.eventFlushInterval = eventFlushInterval ?? Constants.eventFlushInterval context = LDContext.stub() @@ -55,7 +68,7 @@ final class DirectEventProcessorSpec: QuickSpec { serviceMock.diagnosticCache = diagnosticCache self.lastEventResponseDate = lastEventResponseDate.adjustedForHttpUrlHeaderUse - self.store = store ?? EventStore.temporary(capacity: config.eventCapacity) + self.store = store ?? EventStore.temporary(capacity: config.eventCapacity, persistEvents: persistence != .disabled) eventProcessor = DirectEventProcessor(service: serviceMock, onSyncComplete: onSyncComplete, store: self.store, commitQueue: commitQueue) (0.. [String] { + let mock = testContext.serviceMock! + var payloadIds = Set() + var keys: [String] = [] + for (index, status) in mock.publishedEventStatuses.enumerated() where status == 202 { + if payloadIds.insert(mock.publishedPayloadIds[index]).inserted { + keys += customKeys(mock.publishedEventPayloads[index]) + } + } + return keys + } + + func record(_ key: String) { + testContext.eventProcessor.record(CustomEvent(key: key, context: testContext.context)) + } + + it("never touches its directory through refusals, a retry, deliveries and a refusal that takes it offline") { + testContext = TestContext() + // Refused and refused again on the retry; then the batch held from that and the next one are accepted; + // then the service says the key is no good. + testContext.serviceMock.eventStatusForPost = { post in post <= 2 ? 503 : post <= 4 ? 202 : 401 } + testContext.eventProcessor.isOnline = true + + record("refused") + expect(testContext.flushOutcome()) == false + record("accepted") + expect(testContext.flushOutcome()) == true + record("refused-for-good") + _ = testContext.flushOutcome() + expect(testContext.eventProcessor.isOnline) == false + record("while-offline") + testContext.eventProcessor.flush(completion: nil) + // Longer than the retry delay, so anything still on its way would have been posted by now. + waitLettingTimersFire(1.5) + + expect(testContext.serviceMock.publishEventDataCallCount) == 5 + expect(acceptedKeys()) == ["refused", "accepted"] + expect(FileManager.default.fileExists(atPath: testContext.store.directory.path)) == false + } + + it("keeps the oldest events up to capacity through an outage, and delivers each once in order") { + // A batch refused twice used to be dropped. Now it waits in memory, still counted against capacity, so + // an outage costs the newest events rather than the oldest and memory stays bounded however long it lasts. + let capacity = 20 + testContext = TestContext(eventCapacity: capacity) + testContext.serviceMock.respondToEvents(with: 503) + testContext.eventProcessor.isOnline = true + + (0..<10).forEach { record("e-\($0)") } + expect(testContext.flushOutcome()) == false + (10..<100).forEach { record("e-\($0)") } + expect(testContext.flushOutcome()) == false + + testContext.serviceMock.stubEventResponse(success: true) + testContext.serviceMock.eventStatusForPost = { _ in 202 } + expect(testContext.flushOutcome()) == true + + expect(acceptedKeys()) == (0.. [LDValue] in + guard case .array(let events)? = try? JSONDecoder().decode(LDValue.self, from: body) + else { return [] } + return events.filter { $0.kindField == "summary" } + } + expect(summaries.count).to(equal(capacity), description: "round \(round)") + } + } + } + } + + /// An example whose subject is the disk, left out where the spec runs without one. + private func itPersisting(_ description: String, file: FileString = #file, line: UInt = #line, closure: @escaping () throws -> Void) { + guard persists + else { return } + it(description, file: file, line: line, closure: closure) } private func nullEventProcessorSpec() { @@ -1104,7 +1276,7 @@ extension DirectEventProcessorSpec { return reader.pendingEventPayloads().compactMap { try? JSONDecoder().decode(LDValue.self, from: $0) } } - it("has written a tracked event by the time recording it returns") { + itPersisting("has written a tracked event by the time recording it returns") { testContext = TestContext() // The sequence that loses events today: something is tracked and the process ends immediately after. @@ -1115,7 +1287,7 @@ extension DirectEventProcessorSpec { expect(onDisk.first?.kindField) == "custom" } - it("still writes a tracked event when the write is not on the caller's thread") { + itPersisting("still writes a tracked event when the write is not on the caller's thread") { testContext = TestContext(eventPersistence: .deferred) testContext.eventProcessor.record(CustomEvent(key: "fatal-error", context: testContext.context, data: ["message": "boom"])) @@ -1125,7 +1297,7 @@ extension DirectEventProcessorSpec { expect(eventsLeftOnDisk().compactMap { $0.kindField }).toEventually(equal(["custom"]), timeout: Constants.deliveryTimeout) } - it("has written the evaluations that came before a tracked event") { + itPersisting("has written the evaluations that came before a tracked event") { // A response date of its own, because the distant past that a fresh context defaults to also satisfies // the debug window comparison and would add a debug event to what is asserted below. testContext = TestContext(lastEventResponseDate: Date()) @@ -1139,7 +1311,7 @@ extension DirectEventProcessorSpec { expect(eventsLeftOnDisk().compactMap { $0.kindField }) == ["custom", "summary"] } - it("persists feature and summary events synchronously on flush") { + itPersisting("persists feature and summary events synchronously on flush") { testContext = TestContext(lastEventResponseDate: Date()) let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: true) @@ -1180,7 +1352,7 @@ extension DirectEventProcessorSpec { .toEventually(equal(400), timeout: Constants.deliveryTimeout, pollInterval: .milliseconds(200)) } - it("reports the events a previous run left behind") { + itPersisting("reports the events a previous run left behind") { testContext = TestContext() testContext.recordEvents(2) testContext.eventProcessor.commitRecordedEvents() @@ -1203,7 +1375,7 @@ extension DirectEventProcessorSpec { nextRun.isOnline = false } - it("delivers what a previous run left behind as soon as it is online, without waiting for a report interval") { + itPersisting("delivers what a previous run left behind as soon as it is online, without waiting for a report interval") { testContext = TestContext() testContext.recordEvents(2) testContext.eventProcessor.commitRecordedEvents() @@ -1232,7 +1404,7 @@ extension DirectEventProcessorSpec { expect(testContext.serviceMock.publishEventDataCallCount) == 0 } - it("delivers a batch a failed run left behind under the same payload id") { + itPersisting("delivers a batch a failed run left behind under the same payload id") { testContext = TestContext(stubResponseSuccess: false) testContext.recordEvents(1) @@ -1331,7 +1503,7 @@ extension DirectEventProcessorSpec { expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) } - it("writes a deferred track while a delivery holds the delivery queue") { + itPersisting("writes a deferred track while a delivery holds the delivery queue") { testContext = TestContext(eventPersistence: .deferred) // Stands in for a post to a network that does not answer, which holds the queue for as long as its // timeouts allow. @@ -1344,7 +1516,7 @@ extension DirectEventProcessorSpec { expect(eventsLeftOnDisk().compactMap { $0.kindField }).toEventually(equal(["custom"]), timeout: Constants.deliveryTimeout) } - it("has written everything by the time a flush returns, even while a delivery holds the delivery queue") { + itPersisting("has written everything by the time a flush returns, even while a delivery holds the delivery queue") { testContext = TestContext(lastEventResponseDate: Date()) let released = DispatchSemaphore(value: 0) defer { released.signal() } @@ -1359,7 +1531,7 @@ extension DirectEventProcessorSpec { expect(eventsLeftOnDisk().compactMap { $0.kindField }) == ["feature", "summary"] } - it("shares writes between tracks recorded at once, and has each on disk when recording it returns") { + itPersisting("shares writes between tracks recorded at once, and has each on disk when recording it returns") { let trackers = 8 let writesLock = NSLock() var writes = 0 @@ -1406,7 +1578,7 @@ extension DirectEventProcessorSpec { expect(totalWrites) < trackers + 1 } - it("tries a batch a previous run left once at start, with its retry, and then waits while it is still refused") { + itPersisting("tries a batch a previous run left once at start, with its retry, and then waits while it is still refused") { testContext = TestContext() testContext.serviceMock.respondToEvents(with: 503) testContext.eventProcessor.record(CustomEvent(key: "kept", context: testContext.context)) @@ -1423,7 +1595,7 @@ extension DirectEventProcessorSpec { expect(testContext.serviceMock.publishEventDataCallCount) == 2 } - it("leaves a batch still refused when the run ended for the next run to deliver, under the same payload id") { + itPersisting("leaves a batch still refused when the run ended for the next run to deliver, under the same payload id") { testContext = TestContext() testContext.serviceMock.respondToEvents(with: 503) testContext.eventProcessor.record(CustomEvent(key: "kept", context: testContext.context)) @@ -1639,7 +1811,7 @@ extension DirectEventProcessorSpec { expect(store.pendingEventCount) == 3 } - it("makes room in the summarizer at every commit point") { + itPersisting("makes room in the summarizer at every commit point") { // Counters become summary events and are reset at every commit point, so the summarizer only holds the // contexts seen since the last one. Without the reset, an application identifying its way through more // contexts than the capacity would start losing counts partway through. @@ -2376,3 +2548,8 @@ extension Event.Kind { [feature, debug, identify, custom] } } + +/// `DirectEventProcessorSpec` with persistence off, which is what an application gets by default. +final class DirectEventProcessorWithoutPersistenceSpec: DirectEventProcessorSpec { + override class var persistence: EventPersistence { .disabled } +} diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 37bccdd10..9c3afde4d 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -13,7 +13,7 @@ final class EventStoreSpec: QuickSpec { fallbackSpec() optOutSpec() multiProcessSpec() - nullStoreSpec() + withoutPersistenceSpec() #if os(macOS) || targetEnvironment(macCatalyst) sameNameInstancesSpec() #endif @@ -642,10 +642,10 @@ final class EventStoreSpec: QuickSpec { } } - private func nullStoreSpec() { - describe("the store used where there is nowhere to write") { + private func withoutPersistenceSpec() { + describe("the store without persistence") { it("lists its batches oldest first, as the store with a disk does") { - let store = NullEventStore() + let store = EventStore.temporary(persistEvents: false) var closed: [String] = [] // Enough that hash order coming out right by chance is not a worry. for index in 0..<20 { @@ -660,7 +660,7 @@ final class EventStoreSpec: QuickSpec { store.remove(EventBatch(payloadId: closed[3], eventCount: 1)) closed.remove(at: 3) expect(store.pendingBatches().map(\.payloadId)) == closed - expect(EventStoreSpec.keys(ofBody: store.body(of: EventBatch(payloadId: closed[3], eventCount: 1)))) == ["event-4"] + expect(EventStoreSpec.keys(ofBody: try store.body(of: EventBatch(payloadId: closed[3], eventCount: 1)))) == ["event-4"] } } } @@ -719,18 +719,22 @@ final class EventStoreSpec: QuickSpec { expect(store.pendingEventCount) == 1 } - it("does the same for the store used where there is nowhere to write") { - let nullStore = NullEventStore() - expect(nullStore.stage(EventStoreSpec.payload("first"), bypassingCapacity: false)) == true - let batch = nullStore.closeBatch()! + it("does the same without persistence") { + store = EventStore.temporary(capacity: 1, persistEvents: false) + expect(store.stage(EventStoreSpec.payload("first"))) == true + let batch = store.closeBatch()! - nullStore.startedSending(batch) - expect(nullStore.pendingEventCount) == 0 - nullStore.stoppedSending(batch) - expect(nullStore.pendingEventCount) == 1 - nullStore.startedSending(batch) - nullStore.remove(batch) - expect(nullStore.pendingEventCount) == 0 + store.startedSending(batch) + expect(store.pendingEventCount) == 0 + expect(store.stage(EventStoreSpec.payload("while-sending"))) == true + store.stoppedSending(batch) + expect(store.pendingEventCount) == 2 + expect(store.stage(EventStoreSpec.payload("after-retryable"))) == false + store.startedSending(batch) + store.remove(batch) + expect(store.pendingEventCount) == 1 + store.stoppedSending(batch) + expect(store.pendingEventCount) == 1 } // The bound `LDConfig.eventCapacity` documents: the reporter reads the disk in the background after starting, From 8bf63c6e151d9677176ff99b976839a18cf6db74 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Mon, 12 Oct 2026 11:05:19 -0700 Subject: [PATCH 047/158] don't touch disk in disabled --- .../ServiceObjects/DirectEventProcessor.swift | 3 +- .../ServiceObjects/EventStore.swift | 31 +++++++++++---- .../DirectEventProcessorSpec.swift | 18 +++++++-- .../ServiceObjects/EventStoreSpec.swift | 39 +++++++++++++++++++ 4 files changed, 79 insertions(+), 12 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index af73695e6..6fc38ce0b 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -194,7 +194,8 @@ class DirectEventProcessor: EventProcessor { } private static func makeStore(config: LDConfig) -> EventStoring { - EventStore(directory: EventStore.defaultDirectory(mobileKey: config.mobileKey), + let mobileKey = config.mobileKey + return EventStore(locatingDirectory: { EventStore.defaultDirectory(mobileKey: mobileKey) }, capacity: config.eventCapacity, persistEvents: config.eventPersistence != .disabled, logger: config.logger) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index 3dc9f8a6f..0f78169eb 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -215,11 +215,26 @@ final class EventStore: EventStoring { /// A batch is identified by its payload ID rather than by a path, so that a batch listed from the directory and the /// same batch as it was closed are one thing. + /// Requires `ioLock`. private func url(of payloadId: String) -> URL { - directory.appendingPathComponent("\(EventStore.batchPrefix)\(payloadId)") + disk.directory.appendingPathComponent("\(EventStore.batchPrefix)\(payloadId)") + } + + convenience init(directory: URL, + capacity: Int, + persistEvents: Bool = true, + processName: String = ProcessInfo.processInfo.processName, + logger: OSLog, + commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.EventStore.commitQueue", qos: .userInitiated), + readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }, + writeLog: @escaping (EventLogFile, Data) -> Int32? = { $0.writeAll($1) }) { + self.init(locatingDirectory: { directory }, capacity: capacity, persistEvents: persistEvents, processName: processName, + logger: logger, commitQueue: commitQueue, readFile: readFile, writeLog: writeLog) } - init(directory: URL, + /// - Parameter locatingDirectory: answers where the batches go. Asked once, and only if the store reaches for the + /// disk, which one told not to persist events never does. + init(locatingDirectory: @escaping () -> URL, capacity: Int, persistEvents: Bool = true, processName: String = ProcessInfo.processInfo.processName, @@ -227,10 +242,10 @@ final class EventStore: EventStoring { commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.EventStore.commitQueue", qos: .userInitiated), readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }, writeLog: @escaping (EventLogFile, Data) -> Int32? = { $0.writeAll($1) }) { - self.directory = directory - let ioLock = UnfairLock() - self.ioLock = ioLock - self.diskLog = DiskEventLog(directory: directory, processName: processName, ioLock: ioLock, logger: logger, writeLog: writeLog) + self.locateDirectory = locatingDirectory + self.processName = processName + self.writeLog = writeLog + self.ioLock = UnfairLock() self.capacity = capacity self.logger = logger self.commitQueue = commitQueue @@ -428,7 +443,7 @@ final class EventStore: EventStoring { /// Requires `ioLock`. private func batchesOnDisk() -> [EventBatch] { ioLock.assertOwned() - let contents = (try? FileManager.default.contentsOfDirectory(at: directory, + let contents = (try? FileManager.default.contentsOfDirectory(at: disk.directory, includingPropertiesForKeys: [.contentModificationDateKey], options: [.skipsHiddenFiles])) ?? [] @@ -757,7 +772,7 @@ extension EventStore { /// Requires `ioLock`. private func payloadsOnDisk() -> [Data] { ioLock.assertOwned() - let contents = (try? FileManager.default.contentsOfDirectory(at: directory, + let contents = (try? FileManager.default.contentsOfDirectory(at: disk.directory, includingPropertiesForKeys: nil, options: [.skipsHiddenFiles])) ?? [] var logs = contents.filter { $0.lastPathComponent.hasPrefix(EventStore.batchPrefix) }.sorted { $0.path < $1.path } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index c8bb87497..a1ecae2c5 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -184,8 +184,19 @@ class DirectEventProcessorSpec: QuickSpec { testContext.eventProcessor.record(CustomEvent(key: key, context: testContext.context)) } - it("never touches its directory through refusals, a retry, deliveries and a refusal that takes it offline") { - testContext = TestContext() + it("never works out where its directory is through refusals, a retry, deliveries and a refusal that takes it offline") { + let lock = UnfairLock() + var located = 0 + let store = EventStore(locatingDirectory: { + lock.withLock { located += 1 } + return FileManager.default.temporaryDirectory + .appendingPathComponent("com.launchdarkly.tests.events", isDirectory: true) + .appendingPathComponent(UUID().uuidString, isDirectory: true) + }, + capacity: Event.Kind.allKinds.count, + persistEvents: false, + logger: .disabled) + testContext = TestContext(store: store) // Refused and refused again on the retry; then the batch held from that and the next one are accepted; // then the service says the key is no good. testContext.serviceMock.eventStatusForPost = { post in post <= 2 ? 503 : post <= 4 ? 202 : 401 } @@ -205,7 +216,8 @@ class DirectEventProcessorSpec: QuickSpec { expect(testContext.serviceMock.publishEventDataCallCount) == 5 expect(acceptedKeys()) == ["refused", "accepted"] - expect(FileManager.default.fileExists(atPath: testContext.store.directory.path)) == false + testContext.eventProcessor.isOnline = false + expect(lock.withLock { located }) == 0 } it("keeps the oldest events up to capacity through an outage, and delivers each once in order") { diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 9c3afde4d..5d0d3faac 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -643,6 +643,45 @@ final class EventStoreSpec: QuickSpec { } private func withoutPersistenceSpec() { + describe("finding its directory") { + for persistEvents in [false, true] { + it(persistEvents ? "works out where its directory is once, when it first reaches for the disk" + : "never works out where its directory is") { + let lock = UnfairLock() + var located = 0 + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("com.launchdarkly.tests.events", isDirectory: true) + .appendingPathComponent(UUID().uuidString, isDirectory: true) + let store = EventStore(locatingDirectory: { lock.withLock { located += 1 }; return directory }, + capacity: 100, + persistEvents: persistEvents, + logger: .disabled) + defer { try? FileManager.default.removeItem(at: directory) } + expect(lock.withLock { located }) == 0 + #if os(macOS) || targetEnvironment(macCatalyst) + if persistEvents { + store.locksOpenLog = false + } + #endif + + store.recoverInterruptedLog() + _ = store.stage(EventStoreSpec.payload("first")) + store.commit() + let batch = store.closeBatch()! + _ = store.stage(EventStoreSpec.payload("second")) + expect(store.pendingBatches().map(\.payloadId)) == [batch.payloadId] + expect(EventStoreSpec.keys(ofBody: try store.body(of: batch))) == ["first"] + store.startedSending(batch) + store.stoppedSending(batch) + store.remove(batch) + expect(store.pendingEventPayloads().count) == 1 + expect(store.pendingEventCount) == 1 + + expect(lock.withLock { located }) == (persistEvents ? 1 : 0) + } + } + } + describe("the store without persistence") { it("lists its batches oldest first, as the store with a disk does") { let store = EventStore.temporary(persistEvents: false) From 7dbc8eaa6fbbabfbc6df1c5de18af51cbbed450e Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Mon, 12 Oct 2026 11:34:29 -0700 Subject: [PATCH 048/158] EventDirectory --- LaunchDarkly.xcodeproj/project.pbxproj | 10 + .../ServiceObjects/DirectEventProcessor.swift | 2 +- .../ServiceObjects/EventDirectory.swift | 373 ++++++++++++++++++ .../ServiceObjects/EventStore.swift | 366 ++++------------- .../DirectEventProcessorSpec.swift | 10 +- .../ServiceObjects/EventStoreSpec.swift | 65 +-- LaunchDarkly/LaunchDarklyTests/TestUtil.swift | 19 +- 7 files changed, 512 insertions(+), 333 deletions(-) create mode 100644 LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift diff --git a/LaunchDarkly.xcodeproj/project.pbxproj b/LaunchDarkly.xcodeproj/project.pbxproj index 563782241..e42847e36 100644 --- a/LaunchDarkly.xcodeproj/project.pbxproj +++ b/LaunchDarkly.xcodeproj/project.pbxproj @@ -18,6 +18,7 @@ 29FE129B280413D4008CC918 /* Util.swift in Sources */ = {isa = PBXBuildFile; fileRef = 29FE1297280413D4008CC918 /* Util.swift */; }; 2CA9D36B3566D78850D6F0AF /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; B8F228923BE11E444D1ECB8E /* DiskEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */; }; + A1B2C3D4E5F60718293A4B01 /* EventDirectory.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D4E5F60718293A4B5C /* EventDirectory.swift */; }; 3C55384C9E3485B493E1F86A /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; C3F7E502BD916C40AE238F64 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; AB4827D11084D58661E24B05 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; @@ -54,6 +55,7 @@ 50EE85C72EA0749C007CC662 /* TimeoutExecutorSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50EE85C62EA0749C007CC662 /* TimeoutExecutorSpec.swift */; }; 5AEA1F832DCA974314E07D21 /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; D4225F1F05F2C55E9518941B /* DiskEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */; }; + A1B2C3D4E5F60718293A4B02 /* EventDirectory.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D4E5F60718293A4B5C /* EventDirectory.swift */; }; DDA788E3056F8AC3CA203933 /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; D4080613CEA27D51BF349075 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; ECBE45806B470ACC5187B7D6 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; @@ -403,6 +405,7 @@ AFCE752CB1A75DEBABC23034 /* LDContextJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88DAB8D1864C14D9C7E06F11 /* LDContextJSONWriter.swift */; }; B0B1AD7CADE8DBB596C2DE9B /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; DAA2C9B9C144F129217A7C55 /* DiskEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */; }; + A1B2C3D4E5F60718293A4B03 /* EventDirectory.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D4E5F60718293A4B5C /* EventDirectory.swift */; }; 44E0BE449714952A68628CEB /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; E5191724DFB38E62C0451086 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; CA2ADB3D20A229D70CA34F73 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; @@ -458,6 +461,7 @@ C4AF311E4DA8C3AB7040637D /* EventJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = B430884DBF2E541B919C08FA /* EventJSONWriter.swift */; }; CF9A1EB116D8AA3B7B5535B2 /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; E2F8080AB2B3CD6E9A9C1FCC /* DiskEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */; }; + A1B2C3D4E5F60718293A4B04 /* EventDirectory.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D4E5F60718293A4B5C /* EventDirectory.swift */; }; F0F504E5EC32223905356098 /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; B2E6D4F1AC805B3F9D127E53 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; 625ADAC0CCFE3BE324C45752 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; @@ -691,6 +695,7 @@ B4F689132497B2FC004D3CE0 /* DiagnosticEventSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DiagnosticEventSpec.swift; sourceTree = ""; }; C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventStore.swift; sourceTree = ""; }; E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = DiskEventLog.swift; sourceTree = ""; }; + A1B2C3D4E5F60718293A4B5C /* EventDirectory.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventDirectory.swift; sourceTree = ""; }; 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventLogFile.swift; sourceTree = ""; }; A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = DeliveryClaims.swift; sourceTree = ""; }; B58551BB165F28784C785720 /* EventLogFormat.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventLogFormat.swift; sourceTree = ""; }; @@ -1084,6 +1089,7 @@ 9A771B316E19ED598CD4FB0B /* BackgroundActivity.swift */, C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */, E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */, + A1B2C3D4E5F60718293A4B5C /* EventDirectory.swift */, 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */, A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */, B58551BB165F28784C785720 /* EventLogFormat.swift */, @@ -1619,6 +1625,7 @@ A32FCDB22D53263851DDB727 /* BackgroundActivity.swift in Sources */, CF9A1EB116D8AA3B7B5535B2 /* EventStore.swift in Sources */, E2F8080AB2B3CD6E9A9C1FCC /* DiskEventLog.swift in Sources */, + A1B2C3D4E5F60718293A4B04 /* EventDirectory.swift in Sources */, F0F504E5EC32223905356098 /* EventLogFile.swift in Sources */, B2E6D4F1AC805B3F9D127E53 /* DeliveryClaims.swift in Sources */, 625ADAC0CCFE3BE324C45752 /* EventLogFormat.swift in Sources */, @@ -1722,6 +1729,7 @@ 7185A3C30E7FFC4F725769F1 /* BackgroundActivity.swift in Sources */, 2CA9D36B3566D78850D6F0AF /* EventStore.swift in Sources */, B8F228923BE11E444D1ECB8E /* DiskEventLog.swift in Sources */, + A1B2C3D4E5F60718293A4B01 /* EventDirectory.swift in Sources */, 3C55384C9E3485B493E1F86A /* EventLogFile.swift in Sources */, C3F7E502BD916C40AE238F64 /* DeliveryClaims.swift in Sources */, AB4827D11084D58661E24B05 /* EventLogFormat.swift in Sources */, @@ -1824,6 +1832,7 @@ A20146E7AF098560627C614E /* BackgroundActivity.swift in Sources */, 5AEA1F832DCA974314E07D21 /* EventStore.swift in Sources */, D4225F1F05F2C55E9518941B /* DiskEventLog.swift in Sources */, + A1B2C3D4E5F60718293A4B02 /* EventDirectory.swift in Sources */, DDA788E3056F8AC3CA203933 /* EventLogFile.swift in Sources */, D4080613CEA27D51BF349075 /* DeliveryClaims.swift in Sources */, ECBE45806B470ACC5187B7D6 /* EventLogFormat.swift in Sources */, @@ -1993,6 +2002,7 @@ 3D34EEEF37158657D49335BB /* BackgroundActivity.swift in Sources */, B0B1AD7CADE8DBB596C2DE9B /* EventStore.swift in Sources */, DAA2C9B9C144F129217A7C55 /* DiskEventLog.swift in Sources */, + A1B2C3D4E5F60718293A4B03 /* EventDirectory.swift in Sources */, 44E0BE449714952A68628CEB /* EventLogFile.swift in Sources */, E5191724DFB38E62C0451086 /* DeliveryClaims.swift in Sources */, CA2ADB3D20A229D70CA34F73 /* EventLogFormat.swift in Sources */, diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 6fc38ce0b..b840945e7 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -195,7 +195,7 @@ class DirectEventProcessor: EventProcessor { private static func makeStore(config: LDConfig) -> EventStoring { let mobileKey = config.mobileKey - return EventStore(locatingDirectory: { EventStore.defaultDirectory(mobileKey: mobileKey) }, + return EventStore(locatingDirectory: { EventDirectory.defaultUrl(mobileKey: mobileKey) }, capacity: config.eventCapacity, persistEvents: config.eventPersistence != .disabled, logger: config.logger) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift new file mode 100644 index 000000000..0607cbc44 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -0,0 +1,373 @@ +import Foundation +import OSLog + +/// The directory one store keeps its events in: the log this process appends to, and beside it the batches that have +/// been closed off for delivery. +/// +/// Everything about the filesystem lives here -- where the directory is, what the files in it are called, and how they +/// are listed, read, moved and deleted -- so that `EventStore` is left with the buffer, the capacity arithmetic, and +/// the batches it holds in memory, none of which need a disk at all. A store the application told to keep events off +/// the disk has no `EventDirectory`, rather than one it is careful never to use. +/// +/// Where the directory is, is worked out on first use rather than at construction, so that a store which never reaches +/// for the disk makes no filesystem call at all, not even to find out where it would have written. +/// +/// Belongs to an `EventStore`, which makes every call holding its `ioLock`, exactly as `DiskEventLog` belongs to this. +/// The lock is handed down so that each call can check it is held, rather than each level taking a lock of its own +/// that would have to be ordered against the others. +final class EventDirectory { + /// Names a log that has been closed off for delivery; the rest of the name is the batch's payload ID. + private static let batchPrefix = "ready-" + + private let locateDirectory: () -> URL + private let processName: String + private let ioLock: UnfairLock + private let logger: OSLog + /// How a stored log is read, injectable so a test can make a read fail while the file is intact. + private let readFile: (URL) throws -> Data + /// How bytes reach the open log, injectable so a test can make a write fail partway through a session. + private let writeLog: (EventLogFile, Data) -> Int32? + + /// Only to be used while holding `ioLock`. + private var located: (directory: URL, log: DiskEventLog)? + /// Whether a log left by a previous run has been dealt with. Only to be used while holding `ioLock`. + private var hasRecoveredInterruptedLog = false + + /// How many events are in each batch this process closed or recovered, so that listing them does not have to read + /// them back. + /// + /// A batch is never appended to once it is closed, so a count taken at the close holds until the batch is + /// delivered. Without this, listing reads every batch file in full while holding `ioLock`, and a commit at a + /// commit point can end up waiting behind those reads. + /// + /// Only to be used while holding `ioLock`. + private var eventCounts: [String: Int] = [:] + + #if os(macOS) || targetEnvironment(macCatalyst) + /// Whether the open log is locked while it is open. Only to be changed before the store first touches the disk. + /// + /// Kept here rather than asked of the log, so that setting it does not build the log and with it work out where + /// the directory is. Only to be used while holding `ioLock`. + var locksOpenLog = true { + didSet { located?.log.isLocked = locksOpenLog } + } + #endif + + /// - Parameter locatingDirectory: answers where the events go. Asked once, and only on first use. + init(locatingDirectory: @escaping () -> URL, + processName: String, + ioLock: UnfairLock, + logger: OSLog, + readFile: @escaping (URL) throws -> Data, + writeLog: @escaping (EventLogFile, Data) -> Int32?) { + self.locateDirectory = locatingDirectory + self.processName = processName + self.ioLock = ioLock + self.logger = logger + self.readFile = readFile + self.writeLog = writeLog + } + + /// The directory and the log in it, worked out the first time either is reached for. Requires `ioLock`. + private var disk: (directory: URL, log: DiskEventLog) { + ioLock.assertOwned() + if let located { + return located + } + let directory = locateDirectory() + let log = DiskEventLog(directory: directory, + processName: processName, + ioLock: ioLock, + logger: logger, + writeLog: writeLog) + #if os(macOS) || targetEnvironment(macCatalyst) + log.isLocked = locksOpenLog + #endif + let disk = (directory: directory, log: log) + located = disk + return disk + } + + /// Where the batches are kept. Requires `ioLock`. + var url: URL { disk.directory } + + /// Where the log this process appends to is, for a test or a diagnosis to look at. Requires `ioLock`. + var openLogUrl: URL { disk.log.url } + + /// A batch is identified by its payload ID rather than by a path, so that a batch listed from the directory and + /// the same batch as it was closed are one thing. Requires `ioLock`. + private func url(of payloadId: String) -> URL { + disk.directory.appendingPathComponent("\(EventDirectory.batchPrefix)\(payloadId)") + } + + // MARK: Writing + + /// Appends to the open log, first closing off one a previous run left behind. + /// + /// Requires `ioLock`. + /// + /// - Returns: how many events that recovery found, and whether every byte landed. False means none of `bytes` is + /// in the log, and that the caller should give up on the disk. + func append(_ bytes: Data) -> (recovered: Int, written: Bool) { + ioLock.assertOwned() + // Recovered before the log is opened, because it may still be the one a previous run died writing. Appended + // to, this run's events would sit behind that run's torn last frame, where a reader can no longer find where + // they start. + let recovered = disk.log.isOpen ? 0 : recoverInterruptedLog() + return (recovered, disk.log.append(bytes)) + } + + /// What closing the open log produced. + enum LogClose { + case closed(EventBatch) + /// The log is still there to try again with, unless `logSurvives` is false -- a purged cache directory on + /// tvOS, say. Then its events are gone, and they have to stop counting against capacity or the store would + /// refuse events for the rest of the session. + case failed(logSurvives: Bool) + } + + /// Closes the open log into a batch, so the next append starts a new one. Requires `ioLock`. + /// + /// - Parameter events: how many events the log holds, which the caller has counted as it staged them. + func closeLog(holding events: Int) -> LogClose { + ioLock.assertOwned() + let payloadId = UUID().uuidString + do { + try disk.log.close(movingTo: url(of: payloadId)) + } catch { + os_log("%s could not close the event log: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) + return .failed(logSurvives: disk.log.exists) + } + + eventCounts[payloadId] = events + return .closed(EventBatch(payloadId: payloadId, eventCount: events)) + } + + /// Closes the descriptor, leaving everything where it is. Requires `ioLock`. + func closeOpenLog() { + ioLock.assertOwned() + // Not `disk`: a store that never wrote has never worked out where it lives, and shutting down or giving up on + // the disk is no reason to start. + located?.log.closeDescriptor() + } + + // MARK: Listing and reading + + /// The batches in the directory, oldest first. Requires `ioLock`. + func listBatches() -> [EventBatch] { + ioLock.assertOwned() + let contents = (try? FileManager.default.contentsOfDirectory(at: disk.directory, + includingPropertiesForKeys: [.contentModificationDateKey], + options: [.skipsHiddenFiles])) ?? [] + + let batches = contents + .filter { $0.lastPathComponent.hasPrefix(EventDirectory.batchPrefix) } + .compactMap { file -> (EventBatch, Date)? in + let payloadId = String(file.lastPathComponent.dropFirst(EventDirectory.batchPrefix.count)) + // Reading is only for a batch this process has not counted: one a previous run left behind. + guard let events = eventCounts[payloadId] ?? countOfEvents(in: file) + else { return nil } + eventCounts[payloadId] = events + let modified = (try? file.resourceValues(forKeys: [.contentModificationDateKey]))?.contentModificationDate + return (EventBatch(payloadId: payloadId, eventCount: events), modified ?? Date.distantPast) + } + .sorted { $0.1 < $1.1 } + .map { $0.0 } + + // A batch that left the directory without going through `remove` -- a purged cache directory on tvOS, say -- + // would otherwise keep its count for the rest of the session. + let listed = Set(batches.map { $0.payloadId }) + eventCounts = eventCounts.filter { listed.contains($0.key) } + + return batches + } + + /// The `[...]` request body for a batch, or nil if it is gone or holds nothing this version can send, neither of + /// which a later attempt changes. + /// + /// Requires `ioLock`. + /// + /// - throws: if the batch is there but could not be read this time, which a later attempt may manage. + func body(of batch: EventBatch) throws -> Data? { + ioLock.assertOwned() + switch read(url(of: batch.payloadId)) { + case .missing: return nil + case .failed(let error): throw error + case .bytes(let file): return EventLogFormat.assembleBody(from: file) + } + } + + /// Forgets a batch. Requires `ioLock`. + /// + /// - Returns: the events it stopped counting, which is zero for a batch that was already gone -- one another + /// process delivered, say, which a listing has stopped counting already. + func remove(_ payloadId: String) -> Int { + ioLock.assertOwned() + try? FileManager.default.removeItem(at: url(of: payloadId)) + return eventCounts.removeValue(forKey: payloadId) ?? 0 + } + + /// Every event in the directory, encoded exactly as it will be sent. Requires `ioLock`. + /// + /// For tests and for diagnosing a store rather than for the recording path: it reads every file. + func storedPayloads() -> [Data] { + ioLock.assertOwned() + let contents = (try? FileManager.default.contentsOfDirectory(at: disk.directory, + includingPropertiesForKeys: nil, + options: [.skipsHiddenFiles])) ?? [] + var logs = contents.filter { $0.lastPathComponent.hasPrefix(EventDirectory.batchPrefix) }.sorted { $0.path < $1.path } + if disk.log.exists { + logs.append(disk.log.url) + } + + var payloads: [Data] = [] + for log in logs { + guard let file = try? Data(contentsOf: log) + else { continue } + _ = EventLogFormat.forEachFrame(in: file) { type, payload in + if type == EventLogFormat.eventFrame { + payloads.append(payload) + } + } + } + return payloads + } + + /// What reading a stored log produced. + /// + /// A read that failed is kept apart from a file that is not there, because the two arrive the same way: a process + /// out of file descriptors fails to open a file that is perfectly intact with the error a missing one gets. Only + /// what is missing, or what this version cannot make sense of, is gone for good. + private enum LogRead { + case bytes(Data) + case missing + case failed(Error) + } + + private func read(_ url: URL) -> LogRead { + do { + return .bytes(try readFile(url)) + } catch { + return FileManager.default.fileExists(atPath: url.path) ? .failed(error) : .missing + } + } + + /// How many events a batch file holds, or nil when it is not one to list this time. + /// + /// Requires `ioLock`. + private func countOfEvents(in file: URL) -> Int? { + ioLock.assertOwned() + switch read(file) { + case .missing: + // Gone since the directory was listed. + return nil + case .failed(let error): + // Left for a later listing rather than deleted: the read may pass then, and the events in it may have been + // intact all along. Without a count it cannot be delivered now. + os_log("%s could not read stored events, will try again later: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) + return nil + case .bytes(let log): + guard let events = EventLogFormat.eventCount(in: log) + else { + // Written by a version of the SDK whose format this one does not read, or damaged beyond what the + // torn tail recovery tolerates. Either way it can never be delivered, so it is not kept. + try? FileManager.default.removeItem(at: file) + return nil + } + return events + } + } + + // MARK: Recovery + + /// Closes off any log left open by a previous run of this process, so its events join the batches to deliver. + /// + /// The events in it were recorded by a process that is gone, so there is no one left to add to it. Only this + /// process's own log is touched: another process's may still be open in a process that is alive. Where logs are + /// locked, any log no live process holds is recovered, whatever it is named. + /// + /// Requires `ioLock`. Does its work once: after that, the open log is one this run opened. + /// + /// - Returns: how many events it closed off, which is zero on every call after the first. + func recoverInterruptedLog() -> Int { + ioLock.assertOwned() + guard !hasRecoveredInterruptedLog + else { return 0 } + hasRecoveredInterruptedLog = true + + var recovered = 0 + disk.log.forEachLogLeftOpen { recovered += events(recoveredFrom: $0) } + return recovered + } + + /// Closes off a log nobody is writing to any more. Requires `ioLock`. + /// + /// - Returns: how many events it closed off, or zero where there was nothing to close off. + private func events(recoveredFrom leftOpen: URL) -> Int { + ioLock.assertOwned() + let events: Int + switch read(leftOpen) { + case .missing: + return 0 + case .failed(let error): + // It cannot stay under this name, which this run is about to append its own events to, and deleting it + // would lose events that may be intact. Closed off uncounted instead, it is read and counted by whichever + // listing first manages to. + os_log("%s could not read events from a previous run, will try again later: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) + try? FileManager.default.moveItem(at: leftOpen, to: url(of: UUID().uuidString)) + return 0 + case .bytes(let log): + guard let counted = EventLogFormat.eventCount(in: log) + else { + // Unreadable, and a log that cannot be read cannot be appended to either. + try? FileManager.default.removeItem(at: leftOpen) + return 0 + } + events = counted + } + + guard events > 0 + else { + try? FileManager.default.removeItem(at: leftOpen) + return 0 + } + + let payloadId = UUID().uuidString + guard (try? FileManager.default.moveItem(at: leftOpen, to: url(of: payloadId))) != nil + else { return 0 } + + eventCounts[payloadId] = events + os_log("%s recovered %d event(s) from a previous run", log: logger, type: .debug, typeName(and: #function), events) + return events + } +} + +extension EventDirectory { + /// The directory the store for a mobile key belongs in. + /// + /// tvOS is the reason this is not simply Application Support: a tvOS application is only allowed to persist to + /// purgeable caches, so there the events are kept somewhere the system may reclaim, which is the most durability + /// the platform offers. A directory that turns out not to be writable is no different from a full device: the store + /// gives up on the disk and holds events in memory. + static func defaultUrl(mobileKey: String) -> URL { + #if os(tvOS) + let searchPath = FileManager.SearchPathDirectory.cachesDirectory + #else + let searchPath = FileManager.SearchPathDirectory.applicationSupportDirectory + #endif + + let root = FileManager.default.urls(for: searchPath, in: .userDomainMask).first + ?? FileManager.default.temporaryDirectory + + var keyComponent = Util.sha256(mobileKey).base64UrlEncodedString + if let bundleId = Bundle.main.bundleIdentifier { + keyComponent = "\(Util.sha256(bundleId).base64UrlEncodedString).\(keyComponent)" + } + + return root + .appendingPathComponent("com.launchdarkly.events", isDirectory: true) + .appendingPathComponent(keyComponent, isDirectory: true) + } +} + +extension EventDirectory: TypeIdentifying { } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index 0f78169eb..25a8a5c51 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -1,14 +1,6 @@ import Foundation import OSLog -#if canImport(Darwin) -import Darwin -#elseif canImport(Glibc) -import Glibc -#elseif canImport(Musl) -import Musl -#endif - /// A group of encoded events that has been closed off for delivery. /// /// The identifier doubles as the payload ID sent to LaunchDarkly, so a batch that is retried after the process died @@ -90,23 +82,26 @@ protocol EventStoring { /// event, which costs milliseconds where this costs microseconds. The trade that buys is stated where the SDK records /// events: a commit point is a point where staged bytes are committed, and the events an application cares most /// about are recorded at one. +/// +/// What a log looks like on disk is `EventLogFormat`; the directory of them, and the log this process appends to, are +/// `EventDirectory` and `DiskEventLog`. This is what decides when events move between memory, that log, and the +/// batches closed off for delivery, and what counts them against capacity on the way. final class EventStore: EventStoring { - /// Names a log that has been closed off for delivery; the rest of the name is the batch's payload ID. - private static let batchPrefix = "ready-" /// How many staged bytes are allowed to accumulate before one of them pays for a write. /// /// This is what keeps a syscall off most recordings while bounding what a crash can take with it, for an /// application that only evaluates flags and so never reaches a commit point of its own. private static let stagingThreshold = 16 * 1024 - /// Where the batches are kept. - var directory: URL { - ioLock.withLock { disk.directory } - } + /// Where the events are kept, or nil where the application asked for them to be kept off the disk. + /// + /// Nil rather than a directory this store is careful never to use, so that there is nowhere for it to write and + /// nothing for it to read: what a run with persistence turned on left behind waits for the next run that has it + /// turned on. A store that gave up on the disk partway through a session keeps its directory, since the batches + /// there are its own. + private let disk: EventDirectory? private let capacity: Int private let logger: OSLog - /// How a stored log is read, injectable so a test can make a read fail while the file is intact. - private let readFile: (URL) throws -> Data /// Where a commit runs when it was not asked for by a caller who needs it to have happened. /// /// Evaluating a flag must not put a write on whichever thread evaluated it, and that thread is usually the main one. @@ -128,9 +123,9 @@ final class EventStore: EventStoring { /// capacity for events the store is still holding. private var writingEvents = 0 private var committedEvents = 0 - /// Events in closed batches: the sum of `eventCounts` and `inMemoryBatches`. Changed only in the critical section - /// that changes one of those, so that it cannot drift from them; a total worked out from them and written - /// afterwards would overwrite a close or a removal in between. + /// Events in closed batches: the sum of what the directory holds and `inMemoryBatches`. Changed only in the + /// critical section that changes one of those, so that it cannot drift from them; a total worked out from them and + /// written afterwards would overwrite a close or a removal in between. private var closedEvents = 0 /// Closed batches whose request is out, by payload ID, and the events in them. Every one is also in `closedEvents`, /// and leaves both in the same critical section. @@ -139,42 +134,10 @@ final class EventStore: EventStoring { /// Whether events are being written to disk, which is what the application asked for until a write fails and the /// store gives up on persistence for the rest of the session. private var persistEvents = true - /// Whether what is on the disk is this store's to recover, list, and deliver. - /// - /// False only where the application did not ask for persistence. What a run with persistence turned on left behind - /// waits for the next run that has it turned on, rather than being delivered by a run that was told to keep - /// events off the disk. A store that gave up on the disk partway through a session still reads it, since the - /// batches there are its own. - private let readsDisk: Bool /// Whether a commit is already on its way, so that a burst of recordings queues one write rather than one each. private var isCommitScheduled = false - private let locateDirectory: () -> URL - private let processName: String - private let writeLog: (EventLogFile, Data) -> Int32? - /// Only to be used while holding `ioLock`. - private var located: (directory: URL, log: DiskEventLog)? - - /// The directory and the log in it, worked out the first time the disk is reached for, so that a store told to keep - /// off the disk makes no filesystem call at all, not even to find out where it would have written. Requires `ioLock`. - private var disk: (directory: URL, log: DiskEventLog) { - ioLock.assertOwned() - if let located { - return located - } - let directory = locateDirectory() - let disk = (directory: directory, - log: DiskEventLog(directory: directory, processName: processName, ioLock: ioLock, logger: logger, writeLog: writeLog)) - located = disk - return disk - } - - /// Requires `ioLock`. - private var diskLog: DiskEventLog { disk.log } - /// Whether a log left by a previous run has been dealt with. Only to be used while holding `ioLock`. - private var hasRecoveredInterruptedLog = false - - /// Batches held in memory because the filesystem would not take them. + /// Batches held in memory because the filesystem would not take them, or because it was never asked. /// /// Persistence failing should cost durability and nothing else, so the store falls back to what the SDK did before /// it kept a log: hold the events, deliver them, lose them only if the process dies. Kept in the order they were @@ -190,36 +153,24 @@ final class EventStore: EventStoring { /// `ioLock` where there is a disk, never the other way round, and taken before `bufferLock`, never after it. private let heldLock = UnfairLock() - /// How many events are in each batch this process closed or recovered, so that listing them does not have to read - /// them back. - /// - /// A batch is never appended to once it is closed, so a count taken at the close holds until the batch is - /// delivered. Without this, listing reads every batch file in full while holding `ioLock`, and a commit at a - /// commit point can end up waiting behind those reads. - /// - /// Only to be used while holding `ioLock`. - private var eventCounts: [String: Int] = [:] + /// Where the batches are kept, or nil for a store told to keep events off the disk. + var directory: URL? { + ioLock.withLock { disk?.url } + } /// Where the log this process appends to is, for a test or a diagnosis to look at. - var openLogUrl: URL { - ioLock.withLock { diskLog.url } + var openLogUrl: URL? { + ioLock.withLock { disk?.openLogUrl } } #if os(macOS) || targetEnvironment(macCatalyst) /// Whether the open log is locked while it is open. Only to be changed before the store first touches the disk. var locksOpenLog: Bool { - get { ioLock.withLock { diskLog.isLocked } } - set { ioLock.withLock { diskLog.isLocked = newValue } } + get { ioLock.withLock { disk?.locksOpenLog ?? false } } + set { ioLock.withLock { disk?.locksOpenLog = newValue } } } #endif - /// A batch is identified by its payload ID rather than by a path, so that a batch listed from the directory and the - /// same batch as it was closed are one thing. - /// Requires `ioLock`. - private func url(of payloadId: String) -> URL { - disk.directory.appendingPathComponent("\(EventStore.batchPrefix)\(payloadId)") - } - convenience init(directory: URL, capacity: Int, persistEvents: Bool = true, @@ -242,18 +193,22 @@ final class EventStore: EventStoring { commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.EventStore.commitQueue", qos: .userInitiated), readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }, writeLog: @escaping (EventLogFile, Data) -> Int32? = { $0.writeAll($1) }) { - self.locateDirectory = locatingDirectory - self.processName = processName - self.writeLog = writeLog - self.ioLock = UnfairLock() + let ioLock = UnfairLock() + self.ioLock = ioLock self.capacity = capacity self.logger = logger self.commitQueue = commitQueue - self.readFile = readFile // An application that has not asked for persistence gets the same store running the same way it runs once a - // write has failed: events are held, delivered, and lost only if the process dies. + // write has failed: events are held, delivered, and lost only if the process dies. It gets no directory at + // all, so there is nothing for it to write to and nothing of a previous run's for it to read. self.persistEvents = persistEvents - self.readsDisk = persistEvents + self.disk = persistEvents ? EventDirectory(locatingDirectory: locatingDirectory, + processName: processName, + ioLock: ioLock, + logger: logger, + readFile: readFile, + writeLog: writeLog) + : nil } // MARK: Recording @@ -341,13 +296,13 @@ final class EventStore: EventStoring { func commit() { // Where the application asked for no disk there is nothing for a commit to reach, and so nothing to serialize. - guard readsDisk + guard disk != nil else { return } ioLock.withLock { writeStagedBytes() } } func closeBatch() -> EventBatch? { - guard readsDisk + guard let disk else { return closeInMemoryBatch() } ioLock.lock() @@ -357,7 +312,7 @@ final class EventStore: EventStoring { let (events, stillBuffered) = bufferLock.withLock { (committedEvents, bufferedEventCount) } - let fromLog = events > 0 ? closeOpenLog(events) : nil + let fromLog = events > 0 ? closeOpenLog(of: disk, holding: events) : nil // Closed after the log, so the older events are delivered first. Both can hold events at once: a write that // fails after earlier ones landed gives up on persistence with those earlier events in the log and the failed // ones back in memory, and closing only the log would leave the rest to a delivery that reports success @@ -367,57 +322,33 @@ final class EventStore: EventStoring { } /// Requires `ioLock`. - private func closeOpenLog(_ events: Int) -> EventBatch? { + private func closeOpenLog(of disk: EventDirectory, holding events: Int) -> EventBatch? { ioLock.assertOwned() - let payloadId = UUID().uuidString - do { - try diskLog.close(movingTo: url(of: payloadId)) - } catch { - os_log("%s could not close the event log: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) - // The log is still there to try again with, unless it is the log that went missing -- a purged cache - // directory on tvOS, say. Then its events are gone, and they have to stop counting against capacity or the - // store would refuse events for the rest of the session. - if !diskLog.exists { + switch disk.closeLog(holding: events) { + case .failed(let logSurvives): + // Where the log went with its events, they have to stop counting against capacity or the store would + // refuse events for the rest of the session. + if !logSurvives { bufferLock.withLock { committedEvents = 0 } } return nil + case .closed(let batch): + bufferLock.withLock { + committedEvents = 0 + closedEvents += events + } + return batch } - - eventCounts[payloadId] = events - - bufferLock.withLock { - committedEvents = 0 - closedEvents += events - } - - return EventBatch(payloadId: payloadId, eventCount: events) } // MARK: Delivery - /// What reading a stored log produced. - /// - /// A read that failed is kept apart from a file that is not there, because the two arrive the same way: a process - /// out of file descriptors fails to open a file that is perfectly intact with the error a missing one gets. Only - /// what is missing, or what this version cannot make sense of, is gone for good. - private enum LogRead { - case bytes(Data) - case missing - case failed(Error) - } - - private func read(_ url: URL) -> LogRead { - do { - return .bytes(try readFile(url)) - } catch { - return FileManager.default.fileExists(atPath: url.path) ? .failed(error) : .missing - } - } - func pendingBatches() -> [EventBatch] { // Listed in one critical section where there is a disk, so that the files and what is held cannot be counted // either side of a close that moves events between them. - readsDisk ? ioLock.withLock { heldBatchesAndRecount(after: batchesOnDisk()) } : heldBatchesAndRecount(after: []) + guard let disk + else { return heldBatchesAndRecount(after: []) } + return ioLock.withLock { heldBatchesAndRecount(after: disk.listBatches()) } } /// Adds the batches the filesystem would not take, after the files: anything held was closed by this run and so is @@ -438,63 +369,6 @@ final class EventStore: EventStoring { } } - /// The batch files in the directory, oldest first. - /// - /// Requires `ioLock`. - private func batchesOnDisk() -> [EventBatch] { - ioLock.assertOwned() - let contents = (try? FileManager.default.contentsOfDirectory(at: disk.directory, - includingPropertiesForKeys: [.contentModificationDateKey], - options: [.skipsHiddenFiles])) ?? [] - - let batches = contents - .filter { $0.lastPathComponent.hasPrefix(EventStore.batchPrefix) } - .compactMap { file -> (EventBatch, Date)? in - let payloadId = String(file.lastPathComponent.dropFirst(EventStore.batchPrefix.count)) - // Reading is only for a batch this process has not counted: one a previous run left behind. - guard let events = eventCounts[payloadId] ?? countOfEvents(in: file) - else { return nil } - eventCounts[payloadId] = events - let modified = (try? file.resourceValues(forKeys: [.contentModificationDateKey]))?.contentModificationDate - return (EventBatch(payloadId: payloadId, eventCount: events), modified ?? Date.distantPast) - } - .sorted { $0.1 < $1.1 } - .map { $0.0 } - - // A batch that left the directory without going through `remove` -- a purged cache directory on tvOS, say -- - // would otherwise keep its count for the rest of the session. - let listed = Set(batches.map { $0.payloadId }) - eventCounts = eventCounts.filter { listed.contains($0.key) } - - return batches - } - - /// How many events a batch file holds, or nil when it is not one to list this time. - /// - /// Requires `ioLock`. - private func countOfEvents(in file: URL) -> Int? { - ioLock.assertOwned() - switch read(file) { - case .missing: - // Gone since the directory was listed. - return nil - case .failed(let error): - // Left for a later listing rather than deleted: the read may pass then, and the events in it may have been - // intact all along. Without a count it cannot be delivered now. - os_log("%s could not read stored events, will try again later: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) - return nil - case .bytes(let log): - guard let events = EventLogFormat.eventCount(in: log) - else { - // Written by a version of the SDK whose format this one does not read, or damaged beyond what the - // torn tail recovery tolerates. Either way it can never be delivered, so it is not kept. - try? FileManager.default.removeItem(at: file) - return nil - } - return events - } - } - func body(of batch: EventBatch) throws -> Data? { if let held = heldLock.withLock({ inMemoryBatches.first { $0.payloadId == batch.payloadId } }) { return held.body @@ -502,16 +376,10 @@ final class EventStore: EventStoring { // A batch is either held or on the disk, never both and never moved between them, so a batch that is not held // in a store with no disk does not exist. - guard readsDisk + guard let disk else { return nil } - return try ioLock.withLock { - switch read(url(of: batch.payloadId)) { - case .missing: return nil - case .failed(let error): throw error - case .bytes(let file): return EventLogFormat.assembleBody(from: file) - } - } + return try ioLock.withLock { try disk.body(of: batch) } } func remove(_ batch: EventBatch) { @@ -530,14 +398,13 @@ final class EventStore: EventStoring { guard !wasHeld else { return } - guard readsDisk + guard let disk else { bufferLock.withLock { forgetSending(batch.payloadId) } return } ioLock.withLock { - try? FileManager.default.removeItem(at: url(of: batch.payloadId)) - let counted = eventCounts.removeValue(forKey: batch.payloadId) ?? 0 + let counted = disk.remove(batch.payloadId) bufferLock.withLock { closedEvents -= counted forgetSending(batch.payloadId) @@ -545,68 +412,25 @@ final class EventStore: EventStoring { } } - /// Closes any log left open by a previous run of this process, so its events join the batches to deliver. - /// - /// The events in it were recorded by a process that is gone, so there is no one left to add to it. Only this - /// process's own log is touched: another process's may still be open in a process that is alive. Where logs are - /// locked, any log no live process holds is recovered, whatever it is named. func recoverInterruptedLog() { - ioLock.withLock { recoverInterruptedLogOnce() } - } - - /// Requires `ioLock`. Does its work once per store: after that, the open log is one this run opened. - private func recoverInterruptedLogOnce() { - ioLock.assertOwned() - guard readsDisk, !hasRecoveredInterruptedLog + guard let disk else { return } - hasRecoveredInterruptedLog = true - diskLog.forEachLogLeftOpen { recover(logAt: $0) } + ioLock.withLock { countRecovered(disk.recoverInterruptedLog()) } } - /// Closes off a log nobody is writing to any more. Requires `ioLock`. - private func recover(logAt leftOpen: URL) { + /// Counts events a recovery closed off into batches of their own. Requires `ioLock`. + private func countRecovered(_ events: Int) { ioLock.assertOwned() - let events: Int - switch read(leftOpen) { - case .missing: - return - case .failed(let error): - // It cannot stay under this name, which this run is about to append its own events to, and deleting it - // would lose events that may be intact. Closed off uncounted instead, it is read and counted by whichever - // listing first manages to. - os_log("%s could not read events from a previous run, will try again later: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) - try? FileManager.default.moveItem(at: leftOpen, to: url(of: UUID().uuidString)) - return - case .bytes(let log): - guard let counted = EventLogFormat.eventCount(in: log) - else { - // Unreadable, and a log that cannot be read cannot be appended to either. - try? FileManager.default.removeItem(at: leftOpen) - return - } - events = counted - } - guard events > 0 - else { - try? FileManager.default.removeItem(at: leftOpen) - return - } - - let payloadId = UUID().uuidString - guard (try? FileManager.default.moveItem(at: leftOpen, to: url(of: payloadId))) != nil else { return } - - eventCounts[payloadId] = events bufferLock.withLock { closedEvents += events } - os_log("%s recovered %d event(s) from a previous run", log: logger, type: .debug, typeName(and: #function), events) } deinit { // One acquisition: the lock is not recursive, and `commit()` would take it again. ioLock.withLock { writeStagedBytes() - diskLog.closeDescriptor() + disk?.closeOpenLog() } } } @@ -674,12 +498,12 @@ extension EventStore { /// Requires `ioLock`. private func append(_ bytes: Data) -> Bool { ioLock.assertOwned() - // Before the log is opened, because it may still be the one a previous run died writing. Appended to, this - // run's events would sit behind that run's torn last frame, where a reader can no longer find where they start. - if !diskLog.isOpen { - recoverInterruptedLogOnce() - } - guard diskLog.append(bytes) + guard let disk + else { return false } + + let (recovered, written) = disk.append(bytes) + countRecovered(recovered) + guard written else { // Never trap on a full disk. The SDK gives up on persistence for the rest of the session rather than taking // the application down with it, which is how other SDKs have crashed their hosts. @@ -708,7 +532,7 @@ private extension EventStore { /// Requires `ioLock`. func disablePersistence() { ioLock.assertOwned() - diskLog.closeDescriptor() + disk?.closeOpenLog() bufferLock.withLock { persistEvents = false } } @@ -752,7 +576,7 @@ extension EventStore { ioLock.lock() defer { ioLock.unlock() } - var payloads = readsDisk ? payloadsOnDisk() : [] + var payloads = disk?.storedPayloads() ?? [] // After the disk, which is older: a store that gave up on persistence holds what came later in memory. let held = heldLock.withLock { inMemoryBatches.map { $0.body } } for body in held { @@ -768,58 +592,6 @@ extension EventStore { } return payloads } - - /// Requires `ioLock`. - private func payloadsOnDisk() -> [Data] { - ioLock.assertOwned() - let contents = (try? FileManager.default.contentsOfDirectory(at: disk.directory, - includingPropertiesForKeys: nil, - options: [.skipsHiddenFiles])) ?? [] - var logs = contents.filter { $0.lastPathComponent.hasPrefix(EventStore.batchPrefix) }.sorted { $0.path < $1.path } - if diskLog.exists { - logs.append(diskLog.url) - } - - var payloads: [Data] = [] - for log in logs { - guard let file = try? Data(contentsOf: log) - else { continue } - _ = EventLogFormat.forEachFrame(in: file) { type, payload in - if type == EventLogFormat.eventFrame { - payloads.append(payload) - } - } - } - return payloads - } -} - -extension EventStore { - /// The directory the store for a mobile key belongs in. - /// - /// tvOS is the reason this is not simply Application Support: a tvOS application is only allowed to persist to - /// purgeable caches, so there the events are kept somewhere the system may reclaim, which is the most durability - /// the platform offers. A directory that turns out not to be writable is no different from a full device: the store - /// gives up on the disk and holds events in memory. - static func defaultDirectory(mobileKey: String) -> URL { - #if os(tvOS) - let searchPath = FileManager.SearchPathDirectory.cachesDirectory - #else - let searchPath = FileManager.SearchPathDirectory.applicationSupportDirectory - #endif - - let root = FileManager.default.urls(for: searchPath, in: .userDomainMask).first - ?? FileManager.default.temporaryDirectory - - var keyComponent = Util.sha256(mobileKey).base64UrlEncodedString - if let bundleId = Bundle.main.bundleIdentifier { - keyComponent = "\(Util.sha256(bundleId).base64UrlEncodedString).\(keyComponent)" - } - - return root - .appendingPathComponent("com.launchdarkly.events", isDirectory: true) - .appendingPathComponent(keyComponent, isDirectory: true) - } } extension EventStore: TypeIdentifying { } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index a1ecae2c5..53c3f1237 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -1284,7 +1284,7 @@ extension DirectEventProcessorSpec { /// The events the next run of the application would find. func eventsLeftOnDisk() -> [LDValue] { - let reader = EventStore.sharing(testContext.store.directory) + let reader = EventStore.sharing(testContext.store.directory!) return reader.pendingEventPayloads().compactMap { try? JSONDecoder().decode(LDValue.self, from: $0) } } @@ -1370,7 +1370,7 @@ extension DirectEventProcessorSpec { testContext.eventProcessor.commitRecordedEvents() // Nothing delivered them and nothing closed the log, as would be the case had the process died here. - let recovered = EventStore.sharing(testContext.store.directory) + let recovered = EventStore.sharing(testContext.store.directory!) let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) nextRun.isOnline = true @@ -1392,7 +1392,7 @@ extension DirectEventProcessorSpec { testContext.recordEvents(2) testContext.eventProcessor.commitRecordedEvents() - let recovered = EventStore.sharing(testContext.store.directory) + let recovered = EventStore.sharing(testContext.store.directory!) let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) // Nothing asks for a flush, and the report interval is several times longer than this waits, so the @@ -1595,7 +1595,7 @@ extension DirectEventProcessorSpec { testContext.serviceMock.respondToEvents(with: 503) testContext.eventProcessor.record(CustomEvent(key: "kept", context: testContext.context)) - let recovered = EventStore.sharing(testContext.store.directory) + let recovered = EventStore.sharing(testContext.store.directory!) let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) nextRun.isOnline = true defer { nextRun.isOnline = false } @@ -1618,7 +1618,7 @@ extension DirectEventProcessorSpec { // The same directory, as the application's next launch would find it. testContext.serviceMock.stubEventResponse(success: true) - let recovered = EventStore.sharing(testContext.store.directory) + let recovered = EventStore.sharing(testContext.store.directory!) let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) nextRun.isOnline = true defer { nextRun.isOnline = false } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 5d0d3faac..3b3b9fef3 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -29,11 +29,11 @@ final class EventStoreSpec: QuickSpec { } func instance() -> EventStore { - EventStore(directory: first.directory, capacity: 100, processName: "app", logger: .disabled) + EventStore(directory: first.directory!, capacity: 100, processName: "app", logger: .disabled) } beforeEach { - first = EventStore(directory: EventStore.temporary().directory, + first = EventStore(directory: EventStore.temporary().directory!, capacity: 100, processName: "app", logger: .disabled) @@ -49,8 +49,8 @@ final class EventStoreSpec: QuickSpec { _ = second.stage(EventStoreSpec.payload("second")) second.commit() - expect(second.openLogUrl) != first.openLogUrl - expect(second.openLogUrl.lastPathComponent).to(contain("-\(getpid())-")) + expect(second.openLogUrl!) != first.openLogUrl! + expect(second.openLogUrl!.lastPathComponent).to(contain("-\(getpid())-")) _ = first.stage(EventStoreSpec.payload("first-again")) first.commit() @@ -80,7 +80,7 @@ final class EventStoreSpec: QuickSpec { it("recovers the log a previous run under the name left open, once nothing holds it") { _ = first.stage(EventStoreSpec.payload("left-open")) first.commit() - let directory = first.directory + let directory = first.directory! first = nil first = EventStore(directory: directory, capacity: 100, processName: "app", logger: .disabled) @@ -103,11 +103,11 @@ final class EventStoreSpec: QuickSpec { /// Another process of the same application, using the same directory under its own name. Unlocked, as on an /// iPhone or iPad, where a process name is one live process and the name alone keeps the logs apart. func otherProcess(_ name: String = "widget") -> EventStore { - EventStore.sharing(app.directory, processName: name) + EventStore.sharing(app.directory!, processName: name) } beforeEach { - app = EventStore.sharing(EventStore.temporary().directory, processName: "app") + app = EventStore.sharing(EventStore.temporary().directory!, processName: "app") } it("leaves a log another process still has open where it is, so that process's later events reach it") { @@ -133,7 +133,7 @@ final class EventStoreSpec: QuickSpec { _ = widget.stage(EventStoreSpec.payload("from-widget")) widget.commit() - expect(app.openLogUrl) != widget.openLogUrl + expect(app.openLogUrl!) != widget.openLogUrl! let fromApp = app.closeBatch() let fromWidget = widget.closeBatch() expect(EventStoreSpec.keys(ofBody: fromApp.flatMap { try? app.body(of: $0) })) == ["from-app"] @@ -183,11 +183,18 @@ final class EventStoreSpec: QuickSpec { } it("writes nothing to the disk, so the directory is never even created") { + // Given the directory rather than asked for it, because this store has none to give: it is told to + // keep events off the disk, and so is built without anywhere to put them. + let directory = EventStore.temporaryDirectory() + let store = EventStore(directory: directory, capacity: 100, persistEvents: false, logger: .disabled) + defer { try? FileManager.default.removeItem(at: directory) } + _ = store.stage(EventStoreSpec.payload("held")) store.commit() - expect(FileManager.default.fileExists(atPath: store.directory.path)) == false - expect(EventStoreSpec.keys(of: EventStoreSpec.reader(sharing: store))).to(beEmpty()) + expect(store.directory).to(beNil()) + expect(FileManager.default.fileExists(atPath: directory.path)) == false + expect(EventStoreSpec.keys(of: EventStore.sharing(directory))).to(beEmpty()) } it("delivers the events from memory, so opting out costs durability and nothing else") { @@ -201,11 +208,16 @@ final class EventStoreSpec: QuickSpec { it("asks the filesystem nothing, even about a batch it does not hold") { var reads = 0 - let store = EventStore.temporary(persistEvents: false, readFile: { _ in - reads += 1 - return Data() - }) - defer { store.deleteEverything() } + let directory = EventStore.temporaryDirectory() + let store = EventStore(directory: directory, + capacity: 100, + persistEvents: false, + logger: .disabled, + readFile: { _ in + reads += 1 + return Data() + }) + defer { try? FileManager.default.removeItem(at: directory) } _ = store.stage(EventStoreSpec.payload("held")) let batch = store.closeBatch() @@ -220,7 +232,8 @@ final class EventStoreSpec: QuickSpec { store.remove(stranger) expect(reads) == 0 - expect(FileManager.default.fileExists(atPath: store.directory.path)) == false + expect(store.directory).to(beNil()) + expect(FileManager.default.fileExists(atPath: directory.path)) == false } it("leaves alone what a run that did persist left behind, for the next run that persists") { @@ -232,7 +245,7 @@ final class EventStoreSpec: QuickSpec { _ = persisting.stage(EventStoreSpec.payload("open")) persisting.commit() - let withoutPersistence = EventStore.sharing(persisting.directory, persistEvents: false) + let withoutPersistence = EventStore.sharing(persisting.directory!, persistEvents: false) withoutPersistence.recoverInterruptedLog() expect(withoutPersistence.pendingBatches()).to(beEmpty()) expect(withoutPersistence.pendingEventCount) == 0 @@ -268,7 +281,7 @@ final class EventStoreSpec: QuickSpec { /// what actually reached the disk, which is the whole question a crash asks. private static func reader(sharing store: EventStore, readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }) -> EventStore { - EventStore.sharing(store.directory, readFile: readFile) + EventStore.sharing(store.directory!, readFile: readFile) } /// What a process out of file descriptors gets for a file that is perfectly intact. @@ -419,7 +432,7 @@ final class EventStoreSpec: QuickSpec { } store.commit() // The previous run died partway through writing its last frame. - let log = store.openLogUrl + let log = store.openLogUrl! try Data(contentsOf: log).dropLast(3).write(to: log) let next = EventStoreSpec.reader(sharing: store) @@ -462,7 +475,7 @@ final class EventStoreSpec: QuickSpec { store.commit() // A process killed partway through a write leaves the last frame short of its declared length. - let log = store.openLogUrl + let log = store.openLogUrl! let whole = try Data(contentsOf: log) try whole.dropLast(8).write(to: log) @@ -513,7 +526,7 @@ final class EventStoreSpec: QuickSpec { readFile: EventStoreSpec.readFailing(while: { failing })) next.recoverInterruptedLog() // Moved out from under the name this run appends to, and not deleted. - expect(FileManager.default.fileExists(atPath: store.openLogUrl.path)) == false + expect(FileManager.default.fileExists(atPath: store.openLogUrl!.path)) == false // So this run has the name to itself, and the events that are intact are only waiting for a read that // works to be counted and delivered. @@ -533,7 +546,7 @@ final class EventStoreSpec: QuickSpec { _ = store.closeBatch() let batchUrl = try FileManager.default - .contentsOfDirectory(at: store.directory, includingPropertiesForKeys: nil) + .contentsOfDirectory(at: store.directory!, includingPropertiesForKeys: nil) .first { $0.lastPathComponent.hasPrefix("ready-") } var corrupted = try Data(contentsOf: batchUrl!) corrupted[4] = 0xFF @@ -595,7 +608,7 @@ final class EventStoreSpec: QuickSpec { // Nothing appends to a batch once it is closed, so the count taken at the close still holds and // listing it has no reason to go to the disk. Damaging the file is how the test tells the two apart: // a listing that read would refuse this batch, which is exactly what the store below does. - let batchUrl = store.directory.appendingPathComponent("ready-\(batch.payloadId)") + let batchUrl = store.directory!.appendingPathComponent("ready-\(batch.payloadId)") try Data("not an event log".utf8).write(to: batchUrl) expect(store.pendingBatches()) == [batch] @@ -784,7 +797,7 @@ final class EventStoreSpec: QuickSpec { _ = earlier.stage(EventStoreSpec.payload("earlier-1")) _ = earlier.stage(EventStoreSpec.payload("earlier-2")) expect(earlier.closeBatch()).toNot(beNil()) - store = EventStore(directory: earlier.directory, capacity: 2, logger: .disabled) + store = EventStore(directory: earlier.directory!, capacity: 2, logger: .disabled) expect(store.stage(EventStoreSpec.payload("before-listing"))) == true @@ -911,7 +924,7 @@ final class EventStoreSpec: QuickSpec { } afterEach { // The blocking file, which is the parent of the directory the store never got to create. - try? FileManager.default.removeItem(at: store.directory.deletingLastPathComponent()) + try? FileManager.default.removeItem(at: store.directory!.deletingLastPathComponent()) } it("delivers the events from memory rather than dropping them") { @@ -1047,7 +1060,7 @@ final class EventStoreSpec: QuickSpec { it("still refuses events once capacity is reached, so memory stays bounded") { let bounded = EventStoreSpec.unwritableStore(capacity: 2) - defer { try? FileManager.default.removeItem(at: bounded.directory.deletingLastPathComponent()) } + defer { try? FileManager.default.removeItem(at: bounded.directory!.deletingLastPathComponent()) } expect(bounded.stage(EventStoreSpec.payload("first"))) == true bounded.commit() diff --git a/LaunchDarkly/LaunchDarklyTests/TestUtil.swift b/LaunchDarkly/LaunchDarklyTests/TestUtil.swift index 51703dd6a..a5a570bc5 100644 --- a/LaunchDarkly/LaunchDarklyTests/TestUtil.swift +++ b/LaunchDarkly/LaunchDarklyTests/TestUtil.swift @@ -61,9 +61,7 @@ extension EventStore { readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }, writeLog: @escaping (EventLogFile, Data) -> Int32? = { $0.writeAll($1) } ) -> EventStore { - let directory = FileManager.default.temporaryDirectory - .appendingPathComponent("com.launchdarkly.tests.events", isDirectory: true) - .appendingPathComponent(UUID().uuidString, isDirectory: true) + let directory = temporaryDirectory() return unlocked(EventStore(directory: directory, capacity: capacity, persistEvents: persistEvents, @@ -73,6 +71,16 @@ extension EventStore { writeLog: writeLog)) } + /// A directory of a test's own under the temporary directory. + /// + /// Handed in where the test has to look at the place a store would have written, which a store told to keep events + /// off the disk will not tell it: that store has no directory at all. + static func temporaryDirectory() -> URL { + FileManager.default.temporaryDirectory + .appendingPathComponent("com.launchdarkly.tests.events", isDirectory: true) + .appendingPathComponent(UUID().uuidString, isDirectory: true) + } + /// Another store on `directory`, standing in for the next run of the application, or for another process. /// /// Unlocked on a Mac too. Tests open it while the first store is still alive, and a locked log would rightly be @@ -96,8 +104,11 @@ extension EventStore { return store } - /// Removes the store's directory, including any batch still waiting to be delivered. + /// Removes the store's directory, including any batch still waiting to be delivered. A store told to keep events + /// off the disk has no directory, and so nothing to remove. func deleteEverything() { + guard let directory + else { return } try? FileManager.default.removeItem(at: directory) } } From 1ae965d676ec347d80631657c221551fe45555d0 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Mon, 12 Oct 2026 11:48:49 -0700 Subject: [PATCH 049/158] Keep events counted while a held batch is being assembled Closing a batch from memory cleared the buffer, assembled the body outside the lock, and only then counted the events as closed. In between they were counted nowhere, so a capacity check in that window admitted that many events more than capacity allows. Counted through writingEvents, which the commit path already uses for the same window around a write, and which Android has always used for both. Co-authored-by: Cursor --- .../ServiceObjects/EventStore.swift | 33 ++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index 25a8a5c51..bbdc41edb 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -119,8 +119,9 @@ final class EventStore: EventStoring { private var bufferData = Data() private var bufferedEventCount = 0 private var reservedEvents = 0 - /// Taken out of the buffer by a commit and not yet in the log. Counted so a slow write cannot open room under - /// capacity for events the store is still holding. + /// Taken out of the buffer to be written to the log, or assembled into a batch held in memory, and not yet + /// counted where they land. Both happen outside `bufferLock`, and for as long as they take the events are in + /// neither place; without this they would be counted nowhere, and capacity would admit that many more. private var writingEvents = 0 private var committedEvents = 0 /// Events in closed batches: the sum of what the directory holds and `inMemoryBatches`. Changed only in the @@ -473,7 +474,7 @@ extension EventStore { let staged = (bytes: bufferData, events: bufferedEventCount) bufferData = Data() bufferedEventCount = 0 - writingEvents = staged.events + writingEvents += staged.events return staged } guard let taken @@ -481,14 +482,14 @@ extension EventStore { if append(taken.bytes) { bufferLock.withLock { - writingEvents = 0 + writingEvents -= taken.events committedEvents += taken.events } } else { // Put back, to be delivered from memory rather than lost. Ahead of anything staged while the write was in // flight, so the events keep the order they were recorded in. bufferLock.withLock { - writingEvents = 0 + writingEvents -= taken.events bufferData = taken.bytes + bufferData bufferedEventCount += taken.events } @@ -541,24 +542,34 @@ private extension EventStore { /// Called under `ioLock` where there is a disk, so that it and the log are closed as one step, and on its own /// where there is not. func closeInMemoryBatch() -> EventBatch? { - let (frames, events) = bufferLock.withLock { () -> (Data, Int) in - let staged = (bufferData, bufferedEventCount) + let (frames, events) = bufferLock.withLock { () -> (frames: Data, events: Int) in + let staged = (frames: bufferData, events: bufferedEventCount) bufferData = Data() bufferedEventCount = 0 + writingEvents += staged.events return staged } - // Assembled through the same reader a batch on disk goes through, so a held batch and a written one produce - // byte-identical request bodies. + // Assembled outside `bufferLock`, because it copies every staged frame; `writingEvents` is what keeps them + // counted meanwhile. Through the same reader a batch on disk goes through, so a held batch and a written one + // produce byte-identical request bodies. var log = EventLogFormat.fileHeader log.append(frames) guard let body = EventLogFormat.assembleBody(from: log) - else { return nil } + else { + bufferLock.withLock { writingEvents -= events } + return nil + } let payloadId = UUID().uuidString heldLock.withLock { inMemoryBatches.append(HeldBatch(payloadId: payloadId, body: body, eventCount: events)) - bufferLock.withLock { closedEvents += events } + // Counted where they landed in the same critical section that stops counting them on the way, so a + // capacity check never sees them in both places or in neither. + bufferLock.withLock { + writingEvents -= events + closedEvents += events + } } return EventBatch(payloadId: payloadId, eventCount: events) From 60c19654b4dcf6fe0241f3bc94121a03ba636977 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Mon, 12 Oct 2026 11:59:54 -0700 Subject: [PATCH 050/158] Stop re-sending a delivered batch whose file will not delete A failed delete was swallowed: the counts came off, the file stayed, and the next listing found it and sent it again, and the one after that. The counts still come off, because LaunchDarkly has the events and holding capacity for a file this process may never be allowed to delete would cost the session events it could still have recorded. The batch goes into a per-session set that every listing skips, and each listing retries the delete, so a directory that stops being read-only clears itself. Co-authored-by: Cursor --- .../ServiceObjects/EventDirectory.swift | 36 ++++++++++++++++++- .../ServiceObjects/EventStoreSpec.swift | 29 +++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index 0607cbc44..3002a3103 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -43,6 +43,19 @@ final class EventDirectory { /// Only to be used while holding `ioLock`. private var eventCounts: [String: Int] = [:] + /// Batches LaunchDarkly has accepted whose file the filesystem would not let go of. + /// + /// Left out of every listing, because a batch that is delivered and merely still on the disk would otherwise be + /// found again by the next delivery, and the one after that, and sent each time. The delete is tried once more on + /// each listing, so a directory that stops being read-only clears itself. + /// + /// Only for this session: a later run, which may well be able to delete the file, treats it as it would any other + /// batch a previous run left behind. LaunchDarkly discards a repeated payload ID, so the cost of that is a wasted + /// request rather than events counted twice. + /// + /// Only to be used while holding `ioLock`. + private var deliveredButUndeleted: Set = [] + #if os(macOS) || targetEnvironment(macCatalyst) /// Whether the open log is locked while it is open. Only to be changed before the store first touches the disk. /// @@ -164,6 +177,8 @@ final class EventDirectory { .filter { $0.lastPathComponent.hasPrefix(EventDirectory.batchPrefix) } .compactMap { file -> (EventBatch, Date)? in let payloadId = String(file.lastPathComponent.dropFirst(EventDirectory.batchPrefix.count)) + guard !isDeliveredButUndeleted(payloadId, at: file) + else { return nil } // Reading is only for a batch this process has not counted: one a previous run left behind. guard let events = eventCounts[payloadId] ?? countOfEvents(in: file) else { return nil } @@ -182,6 +197,18 @@ final class EventDirectory { return batches } + /// Whether a batch has already been delivered and only its file is left, trying the delete that failed once more + /// on the way. Requires `ioLock`. + private func isDeliveredButUndeleted(_ payloadId: String, at file: URL) -> Bool { + guard deliveredButUndeleted.contains(payloadId) + else { return false } + try? FileManager.default.removeItem(at: file) + if !FileManager.default.fileExists(atPath: file.path) { + deliveredButUndeleted.remove(payloadId) + } + return true + } + /// The `[...]` request body for a batch, or nil if it is gone or holds nothing this version can send, neither of /// which a later attempt changes. /// @@ -203,7 +230,14 @@ final class EventDirectory { /// process delivered, say, which a listing has stopped counting already. func remove(_ payloadId: String) -> Int { ioLock.assertOwned() - try? FileManager.default.removeItem(at: url(of: payloadId)) + let file = url(of: payloadId) + try? FileManager.default.removeItem(at: file) + if FileManager.default.fileExists(atPath: file.path) { + // Its events stop counting all the same: LaunchDarkly has them, and holding capacity for a file this + // process may never be allowed to delete would cost the session events it could still have recorded. + os_log("%s could not delete a delivered batch; it will not be sent again this session", log: logger, type: .debug, typeName(and: #function)) + deliveredButUndeleted.insert(payloadId) + } return eventCounts.removeValue(forKey: payloadId) ?? 0 } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 3b3b9fef3..210f95af0 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -863,6 +863,35 @@ final class EventStoreSpec: QuickSpec { } } + it("does not list a delivered batch again when its file will not delete, and deletes it when it can") { + store = EventStore.temporary() + _ = store.stage(EventStoreSpec.payload("first")) + let first = store.closeBatch()! + _ = store.stage(EventStoreSpec.payload("second")) + let second = store.closeBatch()! + + let directory = store.directory! + // A directory with no write permission is one its entries cannot be unlinked from, which is what the + // filesystem refusing a delete looks like from here. + let readOnly: [FileAttributeKey: Any] = [.posixPermissions: 0o555] + let writable: [FileAttributeKey: Any] = [.posixPermissions: 0o755] + try? FileManager.default.setAttributes(readOnly, ofItemAtPath: directory.path) + defer { try? FileManager.default.setAttributes(writable, ofItemAtPath: directory.path) } + + store.remove(first) + + // Delivered, so it neither counts against capacity nor comes back for another delivery, even though + // its file is still sitting there. + expect(store.pendingBatches().map(\.payloadId)) == [second.payloadId] + expect(store.pendingEventCount) == 1 + + try? FileManager.default.setAttributes(writable, ofItemAtPath: directory.path) + + // The listing that can delete it does, rather than leaving it for a run that would send it again. + _ = store.pendingBatches() + expect(FileManager.default.fileExists(atPath: directory.appendingPathComponent("ready-\(first.payloadId)").path)) == false + } + it("stops counting a batch once when another process delivered it and a listing came between") { store = EventStore.temporary() _ = store.stage(EventStoreSpec.payload("first")) From eaa67a456753f49ee3845e7cd56230bec8c3b183 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Mon, 12 Oct 2026 13:58:15 -0700 Subject: [PATCH 051/158] Raise the default event capacity to 2000 Also documents that the same number bounds how many distinct contexts flag evaluations are summarized for between two reports. Co-authored-by: Cursor --- LaunchDarkly/LaunchDarkly/Models/LDConfig.swift | 4 ++-- LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift | 2 +- .../LaunchDarklyTests/Models/DiagnosticEventSpec.swift | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift index 2dd68e9ee..35f54709f 100644 --- a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift @@ -216,7 +216,7 @@ public struct LDConfig { static let sendEvents = true /// The default maximum number of events the LDClient can store - static let eventCapacity = 1000 + static let eventCapacity = 2000 /// The default behavior for the SDK is to keep events in memory only. static let eventPersistence = EventPersistence.disabled @@ -341,7 +341,7 @@ public struct LDConfig { /// only the sending of client-side events, not streaming or polling for events from the server. public var sendEvents: Bool = Defaults.sendEvents - /// The maximum number of analytics events the LDClient can store. When the LDClient event store reaches the eventCapacity, the SDK discards events until it successfully reports them to LaunchDarkly. (Default: 1000) + /// The maximum number of analytics events the LDClient can store. When the LDClient event store reaches the eventCapacity, the SDK discards events until it successfully reports them to LaunchDarkly. The same number bounds how many distinct contexts flag evaluations are summarized for between two reports; evaluations for contexts beyond it are not counted until the next report. (Default: 2000) /// /// Where `eventPersistence` keeps events across launches, this includes the events an earlier launch left undelivered, from the /// moment the SDK has read them, which it does in the background just after starting. Events recorded before then are diff --git a/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift b/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift index 673364af4..efd09f36a 100644 --- a/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift @@ -35,7 +35,7 @@ public final class ObjcLDConfig: NSObject { set { config.streamUrl = newValue } } - /// The maximum number of analytics events the LDClient can store. When the LDClient event store reaches the eventCapacity, the SDK discards events until it successfully reports them to LaunchDarkly. (Default: 1000) + /// The maximum number of analytics events the LDClient can store. When the LDClient event store reaches the eventCapacity, the SDK discards events until it successfully reports them to LaunchDarkly. The same number bounds how many distinct contexts flag evaluations are summarized for between two reports; evaluations for contexts beyond it are not counted until the next report. (Default: 2000) @objc public var eventCapacity: Int { get { config.eventCapacity } set { config.eventCapacity = newValue } diff --git a/LaunchDarkly/LaunchDarklyTests/Models/DiagnosticEventSpec.swift b/LaunchDarkly/LaunchDarklyTests/Models/DiagnosticEventSpec.swift index d693b8dde..3a6089d33 100644 --- a/LaunchDarkly/LaunchDarklyTests/Models/DiagnosticEventSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/Models/DiagnosticEventSpec.swift @@ -174,7 +174,7 @@ final class DiagnosticEventSpec: QuickSpec { customConfig.baseUrl = URL(string: "https://clientstream.launchdarkly.com")! customConfig.eventsUrl = URL(string: "https://app.launchdarkly.com")! customConfig.streamUrl = URL(string: "https://mobile.launchdarkly.com")! - customConfig.eventCapacity = 2_000 + customConfig.eventCapacity = 3_000 customConfig.connectionTimeout = 30.0 customConfig.eventFlushInterval = 60.0 customConfig.streamingMode = .polling @@ -202,7 +202,7 @@ final class DiagnosticEventSpec: QuickSpec { expect(diagnosticConfig.customBaseURI) == false expect(diagnosticConfig.customEventsURI) == false expect(diagnosticConfig.customStreamURI) == false - expect(diagnosticConfig.eventsCapacity) == 1_000 + expect(diagnosticConfig.eventsCapacity) == 2_000 expect(diagnosticConfig.connectTimeoutMillis) == 10_000 expect(diagnosticConfig.eventsFlushIntervalMillis) == 30_000 expect(diagnosticConfig.streamingDisabled) == false @@ -224,7 +224,7 @@ final class DiagnosticEventSpec: QuickSpec { expect(diagnosticConfig.customBaseURI) == true expect(diagnosticConfig.customEventsURI) == true expect(diagnosticConfig.customStreamURI) == true - expect(diagnosticConfig.eventsCapacity) == 2_000 + expect(diagnosticConfig.eventsCapacity) == 3_000 expect(diagnosticConfig.connectTimeoutMillis) == 30_000 expect(diagnosticConfig.eventsFlushIntervalMillis) == 60_000 expect(diagnosticConfig.streamingDisabled) == true From 8d596833a7e1d5606b25fd721b1d643ae576a9c6 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Mon, 12 Oct 2026 14:40:58 -0700 Subject: [PATCH 052/158] Write on flush at .deferred too A flush is what an application calls before ending the process on purpose, so it now writes on the calling thread wherever events persist rather than only queueing the write. The flush and .deferred docs now say what a crash loses and when evaluations are written. Co-authored-by: Cursor --- LaunchDarkly/LaunchDarkly/LDClientFlush.swift | 18 ++++++++-------- .../LaunchDarkly/Models/LDConfig.swift | 9 ++++++-- .../ServiceObjects/DirectEventProcessor.swift | 21 ++++++++++++++----- .../DirectEventProcessorSpec.swift | 15 +++++++++++++ 4 files changed, 47 insertions(+), 16 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/LDClientFlush.swift b/LaunchDarkly/LaunchDarkly/LDClientFlush.swift index be9d10a7b..cfa6e46b8 100644 --- a/LaunchDarkly/LaunchDarkly/LDClientFlush.swift +++ b/LaunchDarkly/LaunchDarkly/LDClientFlush.swift @@ -13,15 +13,15 @@ extension LDClient { Because it does not wait, it cannot report how the delivery went. Use `flush(completion:)` where that matters but blocking does not suit, or `flushAndWait` where blocking is acceptable. - Where `LDConfig.eventPersistence` is on, it also writes before it returns. Recording an event does not on its own - make it outlive the process: events are written in runs, so one recorded shortly before the process ends may - never have been written at all. This call writes everything recorded so far, and those events then survive - whether or not the delivery does. Where it is off, events live in memory only and nothing survives the process, - whether or not this was called. + Where `LDConfig.eventPersistence` is on, it also writes before it returns, on the calling thread, at `.deferred` + as well as at `.immediate`. Recording an event does not on its own make it outlive the process: events are written + in runs, so one recorded shortly before the process ends may never have been written at all. This call writes + everything recorded so far, and those events then survive whether or not the delivery does. Where it is off, + events live in memory only and nothing survives the process, whether or not this was called. That is what makes it worth calling where the process is about to end deliberately. It is not a crash-time - mechanism — Apple gives the SDK no crash hook, and a trap or a `SIGKILL` takes whatever was recorded since the - last write. Those events are not lost so much as late: they reach LaunchDarkly on the next launch. + mechanism — Apple gives the SDK no crash hook, and a trap or a `SIGKILL` loses whatever was recorded since the + last write. What was written before then is delivered on the next launch. */ public func flush() { LDClient.instancesQueue.sync(flags: .barrier) { @@ -146,8 +146,8 @@ extension LDClient { } } - // Started, then given up on. The deliveries are what makes a zero timeout worth calling at all: at - // `.immediate` the events are on disk by the time each `flushReportingOutcome` returns. Waiting is what is + // Started, then given up on. The deliveries are what makes a zero timeout worth calling at all: wherever events + // persist, they are on disk by the time each `flushReportingOutcome` returns. Waiting is what is // skipped, and a reporter that answered during the loop above must not turn that into a `true`. guard timeout > 0 else { return false } diff --git a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift index 35f54709f..5911cbf9b 100644 --- a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift @@ -32,8 +32,13 @@ public enum LDStreamingMode { case disabled /// Events are written to disk, off the thread that recorded them. /// - /// Recording stays as cheap as it is without persistence, and an event becomes durable a moment after it was - /// recorded rather than immediately. A process that dies inside that window still loses it. + /// Recording stays as cheap as it is without persistence. An event from `track` or `identify` becomes durable a + /// moment after it was recorded rather than immediately, and a process that dies inside that window still loses + /// it. Evaluating a flag does not schedule a write of its own: its events wait in memory until enough of them have + /// built up, or until the next `track`, `identify`, flush or delivery writes them. + /// + /// A flush is the exception. It writes on the calling thread, as `.immediate` does, so that an application ending + /// the process on purpose can rely on it. case deferred /// Events are written to disk before `track` and `identify` return. /// diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index b840945e7..85978872d 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -475,15 +475,26 @@ extension DirectEventProcessor { /// Queues a delivery the way a caller asking for one expects, which is as a commit point. /// - /// At `.immediate` everything accepted before the call is on disk by the time it returns, even where the delivery - /// itself cannot run because the client is offline. What it does not do is close those events into a batch: only - /// a delivery does that, so a delivery already under way may be the one holding them. That is why the answer - /// waits on the claims it skips rather than only on what it sent itself. + /// Wherever events persist, everything accepted before the call is on disk by the time it returns, even where the + /// delivery itself cannot run because the client is offline. What it does not do is close those events into a + /// batch: only a delivery does that, so a delivery already under way may be the one holding them. That is why the + /// answer waits on the claims it skips rather than only on what it sent itself. private func commitThenQueueDelivery(_ completion: FlushOutcomeClosure?) { - commitAtCommitPoint() + commitOnFlush() queueDelivery(completion) } + /// Writes on the caller's thread at `.deferred` as well as at `.immediate`, unlike `track` and `identify`. + /// + /// A flush is what an application calls when it is about to end the process on purpose, and a write it only queued + /// is one the process may end before. Deferring is for the recordings an application makes all the time, which a + /// flush is not. Where nothing persists, a commit makes nothing durable, and is left to the delivery. + private func commitOnFlush() { + if service.config.eventPersistence == .immediate || store.isPersisting { + commitRecordedEvents() + } + } + private func queueScheduledDelivery() { queueDelivery(nil) } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 53c3f1237..858fdc0ad 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -1336,6 +1336,21 @@ extension DirectEventProcessorSpec { expect(eventsLeftOnDisk().compactMap { $0.kindField }) == ["feature", "summary"] } + itPersisting("writes on flush at .deferred too, without waiting for the queue a tracked event's write goes to") { + // Suspended, so that the tracked event's own write cannot run and the only write there can be is one the + // flush makes on this thread. + let commits = DispatchQueue(label: "com.launchdarkly.tests.suspended") + commits.suspend() + defer { commits.resume() } + testContext = TestContext(eventPersistence: .deferred, commitQueue: commits) + + testContext.eventProcessor.record(CustomEvent(key: "before-exit", context: testContext.context, data: nil)) + expect(eventsLeftOnDisk()).to(beEmpty()) + + testContext.eventProcessor.flush(completion: nil) + expect(eventsLeftOnDisk().compactMap { $0.kindField }) == ["custom"] + } + it("does not write on the main thread when a flag is evaluated there") { // Suspended for the length of the evaluations, so that a write this thread was going to cause is a // write that cannot happen: an evaluation is expected to be a memory operation, and the thread From d4f6e06b379663b0b11322e3a4a7b30c3ea01934 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Mon, 12 Oct 2026 16:22:30 -0700 Subject: [PATCH 053/158] Leave scheduling writes to the event processor The store scheduled its own write once 16 KB were staged, on a queue of its own. The processor is the only caller of stage, and every path that stages ends in a commit or a close, so that write only ever split or repeated one already under way. The store now writes only when told to, and pendingCommitThreshold is the one background trigger. Co-authored-by: Cursor --- .../ServiceObjects/DirectEventProcessor.swift | 3 ++ .../ServiceObjects/EventStore.swift | 54 +++---------------- .../ServiceObjects/EventStoreSpec.swift | 48 +++-------------- LaunchDarkly/LaunchDarklyTests/TestUtil.swift | 2 - 4 files changed, 17 insertions(+), 90 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 85978872d..751689924 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -32,6 +32,9 @@ class DirectEventProcessor: EventProcessor { /// The second is the worst a commit point can cost, since it encodes whatever is held before returning. The common /// case is far below the bound, because the commit queue keeps the run drained; raising this trades that tail /// against the number of writes. + /// + /// It is the only thing that writes events no caller asked to have written. The store writes only when told to, so + /// every staged byte reaches the disk through a commit that started here, at a commit point, or at a delivery. private static let pendingCommitThreshold = 32 // MARK: Set in init and never changed diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index bbdc41edb..8f43823e4 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -26,7 +26,8 @@ protocol EventStoring { /// Stages an already encoded event, returning false if it was dropped because `capacity` is reached. /// - /// Staging is a copy into a buffer; `commit()` is what makes the event outlive the process. + /// Staging is a copy into a buffer, and never writes however much is staged; `commit()` is what makes the event + /// outlive the process, and deciding when to call it is the caller's. func stage(_ encodedEvent: Data, bypassingCapacity: Bool) -> Bool /// Counts events that have been accepted but are still being encoded, so capacity sees them until they are staged. @@ -87,12 +88,6 @@ protocol EventStoring { /// `EventDirectory` and `DiskEventLog`. This is what decides when events move between memory, that log, and the /// batches closed off for delivery, and what counts them against capacity on the way. final class EventStore: EventStoring { - /// How many staged bytes are allowed to accumulate before one of them pays for a write. - /// - /// This is what keeps a syscall off most recordings while bounding what a crash can take with it, for an - /// application that only evaluates flags and so never reaches a commit point of its own. - private static let stagingThreshold = 16 * 1024 - /// Where the events are kept, or nil where the application asked for them to be kept off the disk. /// /// Nil rather than a directory this store is careful never to use, so that there is nowhere for it to write and @@ -102,11 +97,6 @@ final class EventStore: EventStoring { private let disk: EventDirectory? private let capacity: Int private let logger: OSLog - /// Where a commit runs when it was not asked for by a caller who needs it to have happened. - /// - /// Evaluating a flag must not put a write on whichever thread evaluated it, and that thread is usually the main one. - /// A caller at a commit point still commits synchronously, because for them having returned is the guarantee. - private let commitQueue: DispatchQueue /// Guards the buffer and the counters. It is held for a copy and never across a syscall, so an evaluation /// recording an event waits on another thread's memcpy at worst, never on the disk. @@ -135,8 +125,6 @@ final class EventStore: EventStoring { /// Whether events are being written to disk, which is what the application asked for until a write fails and the /// store gives up on persistence for the rest of the session. private var persistEvents = true - /// Whether a commit is already on its way, so that a burst of recordings queues one write rather than one each. - private var isCommitScheduled = false /// Batches held in memory because the filesystem would not take them, or because it was never asked. /// @@ -177,11 +165,10 @@ final class EventStore: EventStoring { persistEvents: Bool = true, processName: String = ProcessInfo.processInfo.processName, logger: OSLog, - commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.EventStore.commitQueue", qos: .userInitiated), readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }, writeLog: @escaping (EventLogFile, Data) -> Int32? = { $0.writeAll($1) }) { self.init(locatingDirectory: { directory }, capacity: capacity, persistEvents: persistEvents, processName: processName, - logger: logger, commitQueue: commitQueue, readFile: readFile, writeLog: writeLog) + logger: logger, readFile: readFile, writeLog: writeLog) } /// - Parameter locatingDirectory: answers where the batches go. Asked once, and only if the store reaches for the @@ -191,14 +178,12 @@ final class EventStore: EventStoring { persistEvents: Bool = true, processName: String = ProcessInfo.processInfo.processName, logger: OSLog, - commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.EventStore.commitQueue", qos: .userInitiated), readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }, writeLog: @escaping (EventLogFile, Data) -> Int32? = { $0.writeAll($1) }) { let ioLock = UnfairLock() self.ioLock = ioLock self.capacity = capacity self.logger = logger - self.commitQueue = commitQueue // An application that has not asked for persistence gets the same store running the same way it runs once a // write has failed: events are held, delivered, and lost only if the process dies. It gets no directory at // all, so there is nothing for it to write to and nothing of a previous run's for it to read. @@ -254,10 +239,9 @@ final class EventStore: EventStoring { return false } - // Whether the staged bytes are due a write, or nil where the event was refused for capacity. - let needsCommit = bufferLock.withLock { () -> Bool? in + return bufferLock.withLock { guard bypassingCapacity || countedEvents < capacity - else { return nil } + else { return false } // In the same critical section as the append, so the event is never counted twice or not at all. if usingReservation { @@ -265,34 +249,8 @@ final class EventStore: EventStoring { } bufferData.append(EventLogFormat.frame(for: encodedEvent)) bufferedEventCount += 1 - // Nothing to schedule once persistence has been given up on: there is nowhere for a commit to put these - // bytes, so they stay in memory until a delivery closes them into a batch. - let due = persistEvents && bufferData.count >= EventStore.stagingThreshold && !isCommitScheduled - if due { - isCommitScheduled = true - } - return due - } - guard let needsCommit - else { return false } - - // Handed to another thread rather than done here. Recording an event is something a flag evaluation does, and an - // evaluation is expected to be a memory operation: the caller is usually the main thread, where a write that - // happens to find a busy filesystem is a stall an application cannot do anything about. - if needsCommit { - commitQueue.async { [weak self] in - self?.commitScheduled() - } + return true } - return true - } - - private func commitScheduled() { - // Cleared before the commit rather than after it, so that an event staged while this write is in flight can ask - // for another one instead of finding a commit apparently already on its way and waiting for a commit point. - bufferLock.withLock { isCommitScheduled = false } - - commit() } func commit() { diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 210f95af0..ae986bca3 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -330,52 +330,20 @@ final class EventStoreSpec: QuickSpec { expect(EventStoreSpec.keys(of: EventStoreSpec.reader(sharing: store))) == ["committed"] } - it("commits without being asked once enough has been staged") { - // No commit point is reached and nothing is flushed, so only the staging threshold can have written these. - let event = EventStoreSpec.payload(String(repeating: "p", count: 1_024)) - for _ in 0..<32 { - _ = store.stage(event) - } - - expect(EventStoreSpec.reader(sharing: store).pendingEventPayloads()).toEventuallyNot(beEmpty()) - } - - it("does not write on the thread that staged the event") { - // Suspended, so any write this thread was going to cause is a write that cannot happen. - let commits = DispatchQueue(label: "com.launchdarkly.tests.suspended") - commits.suspend() - let deferring = EventStore.temporary(commitQueue: commits) - // Resumed in the body rather than here: a queue resumed twice traps. - defer { deferring.deleteEverything() } - + it("never writes on its own, however much is staged") { + // When to write is the caller's decision. 64 KB is well past what a store that wrote on its own would + // let build up. let event = EventStoreSpec.payload(String(repeating: "p", count: 1_024)) for _ in 0..<64 { - _ = deferring.stage(event) + _ = store.stage(event) } - expect(EventStoreSpec.reader(sharing: deferring).pendingEventPayloads()).to(beEmpty()) - expect(deferring.pendingEventCount) == 64 - - commits.resume() - - expect(EventStoreSpec.reader(sharing: deferring).pendingEventPayloads().count).toEventually(equal(64)) - } + expect(EventStoreSpec.reader(sharing: store).pendingEventPayloads()).to(beEmpty()) + expect(store.pendingEventCount) == 64 - it("still writes on the caller's thread when the caller is at a commit point") { - // The same suspended queue, to show a commit point does not depend on it: `commit()` is the caller's - // guarantee that the event outlived them, so it cannot be handed to someone else. - let commits = DispatchQueue(label: "com.launchdarkly.tests.suspended") - commits.suspend() - let committing = EventStore.temporary(commitQueue: commits) - defer { - commits.resume() - committing.deleteEverything() - } - - _ = committing.stage(EventStoreSpec.payload("tracked")) - committing.commit() + store.commit() - expect(EventStoreSpec.keys(of: EventStoreSpec.reader(sharing: committing))) == ["tracked"] + expect(EventStoreSpec.reader(sharing: store).pendingEventPayloads().count) == 64 } it("keeps events in the order they were recorded") { diff --git a/LaunchDarkly/LaunchDarklyTests/TestUtil.swift b/LaunchDarkly/LaunchDarklyTests/TestUtil.swift index a5a570bc5..bc6d7cebc 100644 --- a/LaunchDarkly/LaunchDarklyTests/TestUtil.swift +++ b/LaunchDarkly/LaunchDarklyTests/TestUtil.swift @@ -57,7 +57,6 @@ extension EventStore { static func temporary( capacity: Int = 100, persistEvents: Bool = true, - commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.tests.commitQueue"), readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }, writeLog: @escaping (EventLogFile, Data) -> Int32? = { $0.writeAll($1) } ) -> EventStore { @@ -66,7 +65,6 @@ extension EventStore { capacity: capacity, persistEvents: persistEvents, logger: .disabled, - commitQueue: commitQueue, readFile: readFile, writeLog: writeLog)) } From 109372db98d43c9cc7a13fe4cdd705b75207604b Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Mon, 12 Oct 2026 16:55:13 -0700 Subject: [PATCH 054/158] Count a batch's events in its file name Batches are now named ready--, so listing one this process did not close - every batch at startup, and those of another process sharing the environment - costs a directory entry rather than a read of the whole file under ioLock. Batches named without a count by an earlier version are still read, once, and delivered under the name they have. Co-authored-by: Cursor --- .../ServiceObjects/EventDirectory.swift | 90 ++++++++++++++----- .../ServiceObjects/EventStore.swift | 2 +- .../ServiceObjects/EventStoreSpec.swift | 85 ++++++++++++++---- 3 files changed, 138 insertions(+), 39 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index 3002a3103..bf401fbcb 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -16,7 +16,15 @@ import OSLog /// The lock is handed down so that each call can check it is held, rather than each level taking a lock of its own /// that would have to be ordered against the others. final class EventDirectory { - /// Names a log that has been closed off for delivery; the rest of the name is the batch's payload ID. + /// Names a log that has been closed off for delivery. The rest of the name is the batch's payload ID and how many + /// events are in it, `ready--`. + /// + /// The count is in the name so that listing a batch this process did not close costs a directory entry rather than + /// a read of the whole file. That is what every batch is at startup: the ones a previous run left behind, and the + /// ones belonging to another process sharing the environment. Reading them all is done holding `ioLock`, which a + /// commit at a commit point then waits behind. + /// + /// A batch is never appended to once it is closed, so the count is as good as the file is. private static let batchPrefix = "ready-" private let locateDirectory: () -> URL @@ -33,15 +41,21 @@ final class EventDirectory { /// Whether a log left by a previous run has been dealt with. Only to be used while holding `ioLock`. private var hasRecoveredInterruptedLog = false - /// How many events are in each batch this process closed or recovered, so that listing them does not have to read - /// them back. + /// A batch in the directory: the file it is, and how many events it holds. + private struct Stored { + let url: URL + let events: Int + } + + /// What this process knows about each batch in the directory, by payload ID, which is how everything outside this + /// class names one. /// - /// A batch is never appended to once it is closed, so a count taken at the close holds until the batch is - /// delivered. Without this, listing reads every batch file in full while holding `ioLock`, and a commit at a - /// commit point can end up waiting behind those reads. + /// The file is remembered rather than composed from the payload ID, because the count is part of the name and a + /// batch closed by a version that did not write one has no count to compose. The count is remembered so that such + /// a batch is read at most once. /// /// Only to be used while holding `ioLock`. - private var eventCounts: [String: Int] = [:] + private var stored: [String: Stored] = [:] /// Batches LaunchDarkly has accepted whose file the filesystem would not let go of. /// @@ -109,8 +123,32 @@ final class EventDirectory { /// A batch is identified by its payload ID rather than by a path, so that a batch listed from the directory and /// the same batch as it was closed are one thing. Requires `ioLock`. - private func url(of payloadId: String) -> URL { - disk.directory.appendingPathComponent("\(EventDirectory.batchPrefix)\(payloadId)") + /// + /// - Parameter events: what the caller holding the batch knows it to contain, which is what names it where this + /// process has neither closed nor listed it -- as when it delivers a batch another process closed. + private func url(of payloadId: String, holding events: Int) -> URL { + stored[payloadId]?.url ?? newUrl(of: payloadId, holding: events) + } + + /// Requires `ioLock`. + private func newUrl(of payloadId: String, holding events: Int) -> URL { + disk.directory.appendingPathComponent("\(EventDirectory.batchPrefix)\(payloadId)-\(events)") + } + + /// A batch as its filename gives it: everything between the prefix and the count is the payload ID. + /// + /// - Returns: the payload ID, and the count the name ends with, which is nil where it has none -- how a version + /// that did not write one named its batches. A payload ID cannot be mistaken for a count: it is a UUID, whose + /// last group is twelve characters, and twelve digits are more than an `Int32` holds. + private static func parse(batchName name: String) -> (payloadId: String, events: Int?) { + let rest = String(name.dropFirst(batchPrefix.count)) + guard let split = rest.lastIndex(of: "-") + else { return (rest, nil) } + let suffix = rest[rest.index(after: split)...] + guard !suffix.isEmpty, suffix.allSatisfy({ $0.isASCII && $0.isNumber }), + let events = Int32(suffix) + else { return (rest, nil) } + return (String(rest[.. LogClose { ioLock.assertOwned() let payloadId = UUID().uuidString + let file = newUrl(of: payloadId, holding: events) do { - try disk.log.close(movingTo: url(of: payloadId)) + try disk.log.close(movingTo: file) } catch { os_log("%s could not close the event log: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) return .failed(logSurvives: disk.log.exists) } - eventCounts[payloadId] = events + stored[payloadId] = Stored(url: file, events: events) return .closed(EventBatch(payloadId: payloadId, eventCount: events)) } @@ -176,13 +215,14 @@ final class EventDirectory { let batches = contents .filter { $0.lastPathComponent.hasPrefix(EventDirectory.batchPrefix) } .compactMap { file -> (EventBatch, Date)? in - let payloadId = String(file.lastPathComponent.dropFirst(EventDirectory.batchPrefix.count)) + let (payloadId, counted) = EventDirectory.parse(batchName: file.lastPathComponent) guard !isDeliveredButUndeleted(payloadId, at: file) else { return nil } - // Reading is only for a batch this process has not counted: one a previous run left behind. - guard let events = eventCounts[payloadId] ?? countOfEvents(in: file) + // Reading is only for a batch whose name does not count it, which a version that named them without a + // count may have left behind, and then only the once. + guard let events = counted ?? stored[payloadId]?.events ?? countOfEvents(in: file) else { return nil } - eventCounts[payloadId] = events + stored[payloadId] = Stored(url: file, events: events) let modified = (try? file.resourceValues(forKeys: [.contentModificationDateKey]))?.contentModificationDate return (EventBatch(payloadId: payloadId, eventCount: events), modified ?? Date.distantPast) } @@ -192,7 +232,7 @@ final class EventDirectory { // A batch that left the directory without going through `remove` -- a purged cache directory on tvOS, say -- // would otherwise keep its count for the rest of the session. let listed = Set(batches.map { $0.payloadId }) - eventCounts = eventCounts.filter { listed.contains($0.key) } + stored = stored.filter { listed.contains($0.key) } return batches } @@ -217,7 +257,7 @@ final class EventDirectory { /// - throws: if the batch is there but could not be read this time, which a later attempt may manage. func body(of batch: EventBatch) throws -> Data? { ioLock.assertOwned() - switch read(url(of: batch.payloadId)) { + switch read(url(of: batch.payloadId, holding: batch.eventCount)) { case .missing: return nil case .failed(let error): throw error case .bytes(let file): return EventLogFormat.assembleBody(from: file) @@ -228,9 +268,9 @@ final class EventDirectory { /// /// - Returns: the events it stopped counting, which is zero for a batch that was already gone -- one another /// process delivered, say, which a listing has stopped counting already. - func remove(_ payloadId: String) -> Int { + func remove(_ payloadId: String, holding events: Int) -> Int { ioLock.assertOwned() - let file = url(of: payloadId) + let file = url(of: payloadId, holding: events) try? FileManager.default.removeItem(at: file) if FileManager.default.fileExists(atPath: file.path) { // Its events stop counting all the same: LaunchDarkly has them, and holding capacity for a file this @@ -238,7 +278,7 @@ final class EventDirectory { os_log("%s could not delete a delivered batch; it will not be sent again this session", log: logger, type: .debug, typeName(and: #function)) deliveredButUndeleted.insert(payloadId) } - return eventCounts.removeValue(forKey: payloadId) ?? 0 + return stored.removeValue(forKey: payloadId)?.events ?? 0 } /// Every event in the directory, encoded exactly as it will be sent. Requires `ioLock`. @@ -348,7 +388,10 @@ final class EventDirectory { // would lose events that may be intact. Closed off uncounted instead, it is read and counted by whichever // listing first manages to. os_log("%s could not read events from a previous run, will try again later: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) - try? FileManager.default.moveItem(at: leftOpen, to: url(of: UUID().uuidString)) + // Named without a count, which is what a batch whose events have never been counted is: the listing that + // first manages to read it is what counts it. + let uncounted = disk.directory.appendingPathComponent("\(EventDirectory.batchPrefix)\(UUID().uuidString)") + try? FileManager.default.moveItem(at: leftOpen, to: uncounted) return 0 case .bytes(let log): guard let counted = EventLogFormat.eventCount(in: log) @@ -367,10 +410,11 @@ final class EventDirectory { } let payloadId = UUID().uuidString - guard (try? FileManager.default.moveItem(at: leftOpen, to: url(of: payloadId))) != nil + let file = newUrl(of: payloadId, holding: events) + guard (try? FileManager.default.moveItem(at: leftOpen, to: file)) != nil else { return 0 } - eventCounts[payloadId] = events + stored[payloadId] = Stored(url: file, events: events) os_log("%s recovered %d event(s) from a previous run", log: logger, type: .debug, typeName(and: #function), events) return events } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index 8f43823e4..1461a4c40 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -363,7 +363,7 @@ final class EventStore: EventStoring { return } ioLock.withLock { - let counted = disk.remove(batch.payloadId) + let counted = disk.remove(batch.payloadId, holding: batch.eventCount) bufferLock.withLock { closedEvents -= counted forgetSending(batch.payloadId) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index ae986bca3..716ad5feb 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -284,6 +284,24 @@ final class EventStoreSpec: QuickSpec { EventStore.sharing(store.directory!, readFile: readFile) } + /// The one `ready-` file in a store's directory. + private static func onlyBatchFile(in store: EventStore) throws -> URL { + try FileManager.default + .contentsOfDirectory(at: store.directory!, includingPropertiesForKeys: nil) + .first { $0.lastPathComponent.hasPrefix("ready-") }! + } + + /// Renames a batch to the name a version that did not count them in the name gave it, which is what such a + /// version leaves behind for this one to deliver. + @discardableResult + private static func asUncountedName(_ batch: URL) throws -> URL { + let name = batch.lastPathComponent + let renamed = batch.deletingLastPathComponent() + .appendingPathComponent(String(name[.. Bool) -> (URL) throws -> Data { { url in @@ -450,11 +468,12 @@ final class EventStoreSpec: QuickSpec { expect(EventStoreSpec.keys(of: EventStoreSpec.reader(sharing: store))) == ["event-0", "event-1"] } - it("is kept for a later listing when the read failed rather than the log") { + it("is kept for a later listing when a batch named without a count cannot be read") { store = EventStore.temporary() _ = store.stage(EventStoreSpec.payload("kept")) store.commit() _ = store.closeBatch() + try EventStoreSpec.asUncountedName(EventStoreSpec.onlyBatchFile(in: store)) var failing = true let next = EventStoreSpec.reader(sharing: store, @@ -469,6 +488,22 @@ final class EventStoreSpec: QuickSpec { expect(EventStoreSpec.keys(ofBody: try next.body(of: batches[0]))) == ["kept"] } + it("is listed even while reads are failing when its name counts it") { + store = EventStore.temporary() + _ = store.stage(EventStoreSpec.payload("kept")) + _ = store.closeBatch() + + // The read a listing used to need is the one a process out of file descriptors cannot make. With the + // count in the name there is nothing to read until the batch is sent, and a body that cannot be read + // is a delivery to retry rather than a batch that is invisible until it can be. + let next = EventStoreSpec.reader(sharing: store, + readFile: EventStoreSpec.readFailing(while: { true })) + let batches = next.pendingBatches() + expect(batches.count) == 1 + expect(batches[0].eventCount) == 1 + expect { try next.body(of: batches[0]) }.to(throwError()) + } + it("is a failure to retry, not nothing to send, when a body cannot be read") { var failing = false store = EventStore.temporary(readFile: EventStoreSpec.readFailing(while: { failing })) @@ -507,24 +542,22 @@ final class EventStoreSpec: QuickSpec { expect(EventStoreSpec.keys(of: EventStoreSpec.reader(sharing: store))) == ["from-the-previous-run", "this-run"] } - it("is discarded when it was written in a format this version does not read") { + it("is discarded when a batch named without a count is in a format this version does not read") { store = EventStore.temporary() _ = store.stage(EventStoreSpec.payload("unreadable")) store.commit() _ = store.closeBatch() - let batchUrl = try FileManager.default - .contentsOfDirectory(at: store.directory!, includingPropertiesForKeys: nil) - .first { $0.lastPathComponent.hasPrefix("ready-") } - var corrupted = try Data(contentsOf: batchUrl!) + let batchUrl = try EventStoreSpec.asUncountedName(EventStoreSpec.onlyBatchFile(in: store)) + var corrupted = try Data(contentsOf: batchUrl) corrupted[4] = 0xFF corrupted[5] = 0xFF - try corrupted.write(to: batchUrl!) + try corrupted.write(to: batchUrl) let next = EventStoreSpec.reader(sharing: store) expect(next.pendingBatches()).to(beEmpty()) // A batch that can never be delivered is not left to occupy the device either. - expect(FileManager.default.fileExists(atPath: batchUrl!.path)) == false + expect(FileManager.default.fileExists(atPath: batchUrl.path)) == false } } } @@ -568,21 +601,42 @@ final class EventStoreSpec: QuickSpec { expect(store.pendingEventCount) == 0 } - it("counts a batch it closed without reading the file back") { + it("counts a closed batch from its name without reading the file back") { _ = store.stage(EventStoreSpec.payload("first")) _ = store.stage(EventStoreSpec.payload("second")) let batch = store.closeBatch()! - // Nothing appends to a batch once it is closed, so the count taken at the close still holds and + // Nothing appends to a batch once it is closed, so the count its name carries still holds and // listing it has no reason to go to the disk. Damaging the file is how the test tells the two apart: - // a listing that read would refuse this batch, which is exactly what the store below does. - let batchUrl = store.directory!.appendingPathComponent("ready-\(batch.payloadId)") + // a listing that read would refuse this batch. + let batchUrl = store.directory!.appendingPathComponent("ready-\(batch.payloadId)-2") try Data("not an event log".utf8).write(to: batchUrl) + // The store that closed it and one that did not both count it from the name alone. expect(store.pendingBatches()) == [batch] + expect(EventStoreSpec.reader(sharing: store).pendingBatches()) == [batch] + + // The damage is found where the batch is read to be sent, and reads as nothing to send rather than + // as a failure to retry, so the delivery removes it instead of trying it forever. + expect(try store.body(of: batch)).to(beNil()) + } + + it("reads a batch named without a count once and then remembers it") { + _ = store.stage(EventStoreSpec.payload("left-by-a-previous-version")) + let batch = store.closeBatch()! + + // The name a version that did not count them in the name gave it, which is what such a version + // leaves behind for this one to deliver. + let counted = store.directory!.appendingPathComponent("ready-\(batch.payloadId)-1") + let uncounted = store.directory!.appendingPathComponent("ready-\(batch.payloadId)") + try FileManager.default.moveItem(at: counted, to: uncounted) - // A store that did not close it has no count to go on, so it reads, and finds the damage. - expect(EventStoreSpec.reader(sharing: store).pendingBatches()).to(beEmpty()) + let reader = EventStoreSpec.reader(sharing: store) + expect(reader.pendingBatches()) == [batch] + // Still deliverable under the name it has, which is not the one its count would give it. + expect(try reader.body(of: batch)).toNot(beNil()) + reader.remove(batch) + expect(FileManager.default.fileExists(atPath: uncounted.path)) == false } it("gives a batch its own payload id, so a retry is not read as a new delivery") { @@ -857,7 +911,8 @@ final class EventStoreSpec: QuickSpec { // The listing that can delete it does, rather than leaving it for a run that would send it again. _ = store.pendingBatches() - expect(FileManager.default.fileExists(atPath: directory.appendingPathComponent("ready-\(first.payloadId)").path)) == false + let left = try FileManager.default.contentsOfDirectory(at: directory, includingPropertiesForKeys: nil) + expect(left.filter { $0.lastPathComponent.hasPrefix("ready-\(first.payloadId)") }).to(beEmpty()) } it("stops counting a batch once when another process delivered it and a listing came between") { From 3aaa3b80ac5f3ddec852ae882859202744fdfefb Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Mon, 12 Oct 2026 17:27:31 -0700 Subject: [PATCH 055/158] unreadable batch --- .../ServiceObjects/DirectEventProcessor.swift | 10 +++++++- .../ServiceObjects/EventDirectory.swift | 15 ++++++++--- .../ServiceObjects/EventStore.swift | 13 +++++++--- .../DirectEventProcessorSpec.swift | 25 +++++++++++++++++++ .../ServiceObjects/EventStoreSpec.swift | 6 ++--- 5 files changed, 58 insertions(+), 11 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 751689924..ec094cc56 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -677,6 +677,13 @@ extension DirectEventProcessor { let read: Data? do { read = try store.body(of: batch) + } catch is UnreadableBatch { + // Lost like a refused batch: nothing will ever send it, so it is removed, the rest are still attempted, and + // the caller hears that this pass lost events. + os_log("%s dropping %d stored event(s) this version cannot read", log: service.config.logger, type: .error, typeName(and: #function), batch.eventCount) + store.remove(batch) + deliver(remaining, lostAnything: true, completion) + return } catch { // Kept: the batch is still there, and a read that failed may not fail next time. os_log("%s could not read stored events: %s", log: service.config.logger, type: .debug, typeName(and: #function), String(describing: error)) @@ -686,7 +693,8 @@ extension DirectEventProcessor { guard let body = read else { - // Gone, or holding nothing this version can send. Either way there is nothing to send and nothing to keep. + // Gone, which is usually another process having delivered it, or holding no events. Either way nothing is + // lost and there is nothing to keep. store.remove(batch) deliver(remaining, lostAnything: lostAnything, completion) return diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index bf401fbcb..6958d3b0d 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -249,18 +249,25 @@ final class EventDirectory { return true } - /// The `[...]` request body for a batch, or nil if it is gone or holds nothing this version can send, neither of - /// which a later attempt changes. + /// The `[...]` request body for a batch, or nil if it is gone or holds no events. /// /// Requires `ioLock`. /// - /// - throws: if the batch is there but could not be read this time, which a later attempt may manage. + /// - throws: `UnreadableBatch` if the batch is there but holds nothing this version can send; anything else if it + /// could not be read this time, which a later attempt may manage. func body(of batch: EventBatch) throws -> Data? { ioLock.assertOwned() switch read(url(of: batch.payloadId, holding: batch.eventCount)) { case .missing: return nil case .failed(let error): throw error - case .bytes(let file): return EventLogFormat.assembleBody(from: file) + case .bytes(let file): + if let body = EventLogFormat.assembleBody(from: file) { + return body + } + // A log this version reads that holds no events has nothing to lose. + guard EventLogFormat.eventCount(in: file) == 0 + else { throw UnreadableBatch(payloadId: batch.payloadId) } + return nil } } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index 1461a4c40..1e470e94d 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -51,10 +51,11 @@ protocol EventStoring { /// Batches awaiting delivery, oldest first, including any recovered from a previous run of the application. func pendingBatches() -> [EventBatch] - /// The `[...]` request body for a batch, or nil if it is gone or holds nothing this version can send, neither of - /// which a later attempt changes. + /// The `[...]` request body for a batch, or nil if it is gone or holds no events, neither of which loses anything. /// - /// - throws: if the batch is there but could not be read this time, which a later attempt may manage. + /// - throws: `UnreadableBatch` if the batch is there but holds nothing this version can send, which no later + /// attempt changes and which loses its events; anything else if it could not be read this time, which a later + /// attempt may manage. func body(of batch: EventBatch) throws -> Data? /// Forgets a batch, which is only correct once LaunchDarkly has accepted it or permanently refused it. @@ -70,6 +71,12 @@ protocol EventStoring { func recoverInterruptedLog() } +/// A batch that is there to read and holds nothing this version can send: damaged, or written in a format it does not +/// read. Unlike a batch that is gone, its events are lost. +struct UnreadableBatch: Error { + let payloadId: String +} + /// A crash-durable append log of encoded analytics events. /// /// Events are appended to an open log, which is closed into a batch when the reporter is ready to deliver one. A batch diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 858fdc0ad..94c9d875a 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -1942,6 +1942,31 @@ extension DirectEventProcessorSpec { expect(store.pendingEventCount) == 0 } + it("reports false and drops a stored batch this version cannot read, and still delivers the rest") { + let store = EventStore.temporary() + testContext = TestContext(store: store) + testContext.recordEvents(1) + testContext.eventProcessor.commitRecordedEvents() + let damaged = store.closeBatch()! + // Its name still counts it, so it is listed without being read; the damage is only found on sending. + let damagedUrl = store.directory!.appendingPathComponent("ready-\(damaged.payloadId)-\(damaged.eventCount)") + try Data("not an event log".utf8).write(to: damagedUrl) + testContext.recordEvents(1) + testContext.eventProcessor.isOnline = true + + var delivered: Bool? + waitUntil(timeout: Constants.deliveryTimeout) { done in + testContext.eventProcessor.flushReportingOutcome { result in + delivered = result + done() + } + } + expect(delivered) == false + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + expect(FileManager.default.fileExists(atPath: damagedUrl.path)) == false + expect(store.pendingEventCount) == 0 + } + it("delivers what memory was left holding beside what the disk took, and reports true") { var writes = 0 let store = EventStore.temporary(writeLog: { file, bytes in diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 716ad5feb..eea8ca311 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -616,9 +616,9 @@ final class EventStoreSpec: QuickSpec { expect(store.pendingBatches()) == [batch] expect(EventStoreSpec.reader(sharing: store).pendingBatches()) == [batch] - // The damage is found where the batch is read to be sent, and reads as nothing to send rather than - // as a failure to retry, so the delivery removes it instead of trying it forever. - expect(try store.body(of: batch)).to(beNil()) + // The damage is found where the batch is read to be sent, and reads as a loss rather than as a failure + // to retry, so the delivery removes it instead of trying it forever, and says it lost events. + expect { try store.body(of: batch) }.to(throwError(errorType: UnreadableBatch.self)) } it("reads a batch named without a count once and then remembers it") { From 9d0b78922a1d866f58aa69f22c052f9f26c3be82 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Mon, 12 Oct 2026 22:55:01 -0700 Subject: [PATCH 056/158] Write on entering the background at .deferred too didEnterBackground only queued the write at .deferred, so on macOS and watchOS, where nothing is flushed afterwards, a process ended in the background could lose what was recorded. It now commits before returning wherever events persist, by the same rule a flush uses, exposed as EventProcessor.commitWherePersisting(). commitAtCommitPoint() had no other caller outside the processor and is now private to it. Co-authored-by: Cursor --- LaunchDarkly/GeneratedCode/mocks.generated.swift | 10 +++++----- LaunchDarkly/LaunchDarkly/LDClient.swift | 7 ++++--- LaunchDarkly/LaunchDarkly/Models/LDConfig.swift | 5 +++-- .../ServiceObjects/DirectEventProcessor.swift | 13 +++++++------ .../ServiceObjects/EventProcessor.swift | 9 ++++++--- .../ServiceObjects/NullEventProcessor.swift | 2 +- LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift | 9 +++++++++ 7 files changed, 35 insertions(+), 20 deletions(-) diff --git a/LaunchDarkly/GeneratedCode/mocks.generated.swift b/LaunchDarkly/GeneratedCode/mocks.generated.swift index aca360bd5..dc9b68aa9 100644 --- a/LaunchDarkly/GeneratedCode/mocks.generated.swift +++ b/LaunchDarkly/GeneratedCode/mocks.generated.swift @@ -238,11 +238,11 @@ final class EventProcessorMock: EventProcessor { try! commitRecordedEventsCallback?() } - var commitAtCommitPointCallCount = 0 - var commitAtCommitPointCallback: (() throws -> Void)? - func commitAtCommitPoint() { - commitAtCommitPointCallCount += 1 - try! commitAtCommitPointCallback?() + var commitWherePersistingCallCount = 0 + var commitWherePersistingCallback: (() throws -> Void)? + func commitWherePersisting() { + commitWherePersistingCallCount += 1 + try! commitWherePersistingCallback?() } } diff --git a/LaunchDarkly/LaunchDarkly/LDClient.swift b/LaunchDarkly/LaunchDarkly/LDClient.swift index cdcffe366..24135bb0b 100644 --- a/LaunchDarkly/LaunchDarkly/LDClient.swift +++ b/LaunchDarkly/LaunchDarkly/LDClient.swift @@ -263,9 +263,10 @@ public class LDClient { @objc private func didEnterBackground() { os_log("%s", log: config.logger, type: .debug, typeName(and: #function)) - // The OS may kill a backgrounded application without warning, so make its events durable now. On every - // platform: macOS does not suspend an application that loses focus, but it can still be quit or killed. - eventProcessor.commitAtCommitPoint() + // The OS may kill a backgrounded application without warning, so make its events durable now, before returning, + // at `.deferred` too. On every platform: macOS does not suspend an application that loses focus, but it can + // still be quit or killed. + eventProcessor.commitWherePersisting() // A backgrounded process is suspended once idle, so deliver inside an activity assertion. Only where suspension // is real: macOS posts this notification whenever the application loses focus, and never suspends it for that. // What does not get out is on disk for the next launch. diff --git a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift index 5911cbf9b..ccfe7180b 100644 --- a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift @@ -37,8 +37,9 @@ public enum LDStreamingMode { /// it. Evaluating a flag does not schedule a write of its own: its events wait in memory until enough of them have /// built up, or until the next `track`, `identify`, flush or delivery writes them. /// - /// A flush is the exception. It writes on the calling thread, as `.immediate` does, so that an application ending - /// the process on purpose can rely on it. + /// A flush and entering the background are the exceptions. Both write on the calling thread, as `.immediate` does: + /// a flush so that an application ending the process on purpose can rely on it, and entering the background because + /// that is when the system is most likely to end it. case deferred /// Events are written to disk before `track` and `identify` return. /// diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index ec094cc56..14aaf77c3 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -305,7 +305,7 @@ class DirectEventProcessor: EventProcessor { /// Committing on the caller's thread is what lets `track` promise its event is on disk by the time it returns. /// Scheduling it instead keeps the encode and the write off that thread, and the event is durable a moment later /// rather than immediately. - func commitAtCommitPoint() { + private func commitAtCommitPoint() { if service.config.eventPersistence == .immediate { commitRecordedEvents() } else { @@ -483,16 +483,17 @@ extension DirectEventProcessor { /// batch: only a delivery does that, so a delivery already under way may be the one holding them. That is why the /// answer waits on the claims it skips rather than only on what it sent itself. private func commitThenQueueDelivery(_ completion: FlushOutcomeClosure?) { - commitOnFlush() + commitWherePersisting() queueDelivery(completion) } /// Writes on the caller's thread at `.deferred` as well as at `.immediate`, unlike `track` and `identify`. /// - /// A flush is what an application calls when it is about to end the process on purpose, and a write it only queued - /// is one the process may end before. Deferring is for the recordings an application makes all the time, which a - /// flush is not. Where nothing persists, a commit makes nothing durable, and is left to the delivery. - private func commitOnFlush() { + /// A flush is what an application calls when it is about to end the process on purpose, and entering the background + /// is when the system is most likely to end it; a write only queued is one the process may end before. Deferring is + /// for the recordings an application makes all the time, which neither is. Where nothing persists, a commit makes + /// nothing durable, and is left to the delivery. + func commitWherePersisting() { if service.config.eventPersistence == .immediate || store.isPersisting { commitRecordedEvents() } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift index 2f2d4ede5..52c13c191 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift @@ -39,7 +39,10 @@ protocol EventProcessor { /// calling directly before deliberately terminating the process. func commitRecordedEvents() - /// Commits as a commit point does: before returning at `.immediate`, on a background queue at `.deferred`, and not - /// at all where nothing is persisted, where a commit would only move the encode earlier. - func commitAtCommitPoint() + /// Commits before returning wherever events persist, at `.deferred` as well as at `.immediate`, and not at all where + /// nothing is persisted, where a commit would only move the encode earlier. + /// + /// What a flush does, and what entering the background does: both are points the process may end soon after, where + /// a write that was only queued may never happen. + func commitWherePersisting() } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventProcessor.swift index cde064a5c..81cc0d82d 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventProcessor.swift @@ -33,6 +33,6 @@ class NullEventProcessor: EventProcessor { func commitRecordedEvents() { } - func commitAtCommitPoint() { + func commitWherePersisting() { } } diff --git a/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift b/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift index 211f9a36d..cc0118f4c 100644 --- a/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift @@ -1409,6 +1409,15 @@ final class LDClientSpec: QuickSpec { expect(testContext.eventProcessorMock.flushReportingOutcomeCallCount) == 0 #endif } + it("writes what it has before the notification returns, on every platform", onlyWhere: platformPostsLifecycleNotifications) { + let testContext = TestContext(startOnline: true, enableBackgroundUpdates: false) + testContext.start() + NotificationCenter.default.post(name: SystemCapabilities.backgroundNotification!, object: self) + + // Checked without waiting: a write only queued is one a process killed in the + // background may never get to, macOS included, where nothing is delivered here. + expect(testContext.eventProcessorMock.commitWherePersistingCallCount) == 1 + } it("background updates enabled", onlyWhere: platformPostsLifecycleNotifications) { let testContext = TestContext(startOnline: true) testContext.start() From ea0b0694267c93d80c7404e08bbf1e1ffe2a17d4 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Mon, 12 Oct 2026 23:00:23 -0700 Subject: [PATCH 057/158] Read the summarizer under its lock in tests DirectEventProcessorSpec read eventBuffer.contextSummarizer without recordLock, which a scheduled commit holds while it takes the counters. The reads now go through withContextSummarizer, a DEBUG accessor that takes the lock. Co-authored-by: Cursor --- .../ServiceObjects/DirectEventProcessor.swift | 5 ++ .../DirectEventProcessorSpec.swift | 52 +++++++++---------- 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 14aaf77c3..983748328 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -788,5 +788,10 @@ extension DirectEventProcessor: TypeIdentifying { } defer { deliveryLock.unlock() } return deliveryRunCount } + + /// Reads the summarizer under `recordLock`, which a scheduled commit may be holding to take its counters. + func withContextSummarizer(_ read: (ContextSummarizer) -> T) -> T { + recordLock.withLock { read(eventBuffer.contextSummarizer) } + } } #endif diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 94c9d875a..6d37a4eca 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -597,7 +597,7 @@ class DirectEventProcessorSpec: QuickSpec { } expect(serviceMock.publishEventDataCallCount) == 0 expect(store.pendingEventCount) == 0 - expect(reporter.eventBuffer.contextSummarizer.hasLoggedRequests) == false + expect(reporter.withContextSummarizer { $0.hasLoggedRequests }) == false waitUntil(timeout: Constants.deliveryTimeout) { done in reporter.recordFlagEvaluationEvents(flagKey: "after-poison", @@ -677,7 +677,7 @@ class DirectEventProcessorSpec: QuickSpec { expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 expect(testContext.store.pendingEventCount) == 0 expect(testContext.eventProcessor.lastEventResponseDate) == testContext.eventStubResponseDate - expect(testContext.eventProcessor.eventBuffer.contextSummarizer.hasLoggedRequests) == false + expect(testContext.eventProcessor.withContextSummarizer { $0.hasLoggedRequests }) == false expect(testContext.syncResult).to(beNil()) } } @@ -702,7 +702,7 @@ class DirectEventProcessorSpec: QuickSpec { expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == testContext.events.count expect(testContext.store.pendingEventCount) == 0 expect(testContext.eventProcessor.lastEventResponseDate) == testContext.eventStubResponseDate - expect(testContext.eventProcessor.eventBuffer.contextSummarizer.hasLoggedRequests) == false + expect(testContext.eventProcessor.withContextSummarizer { $0.hasLoggedRequests }) == false expect(testContext.syncResult).to(beNil()) } } @@ -740,7 +740,7 @@ class DirectEventProcessorSpec: QuickSpec { expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == 1 expect(testContext.store.pendingEventCount) == 0 expect(testContext.eventProcessor.lastEventResponseDate) == testContext.eventStubResponseDate - expect(testContext.eventProcessor.eventBuffer.contextSummarizer.hasLoggedRequests) == false + expect(testContext.eventProcessor.withContextSummarizer { $0.hasLoggedRequests }) == false expect(testContext.syncResult).to(beNil()) } } @@ -761,7 +761,7 @@ class DirectEventProcessorSpec: QuickSpec { expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 0 expect(testContext.store.pendingEventCount) == 0 expect(testContext.eventProcessor.lastEventResponseDate) == Date.distantPast - expect(testContext.eventProcessor.eventBuffer.contextSummarizer.hasLoggedRequests) == false + expect(testContext.eventProcessor.withContextSummarizer { $0.hasLoggedRequests }) == false expect(testContext.syncResult).to(beNil()) } } @@ -855,7 +855,7 @@ class DirectEventProcessorSpec: QuickSpec { expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 expect(testContext.eventProcessor.lastEventResponseDate) == Date.distantPast - expect(testContext.eventProcessor.eventBuffer.contextSummarizer.hasLoggedRequests) == false + expect(testContext.eventProcessor.withContextSummarizer { $0.hasLoggedRequests }) == false guard case let .request(error) = testContext.syncResult else { fail("Expected error result for event send") @@ -901,7 +901,7 @@ class DirectEventProcessorSpec: QuickSpec { expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 expect(testContext.eventProcessor.lastEventResponseDate) == Date.distantPast - expect(testContext.eventProcessor.eventBuffer.contextSummarizer.hasLoggedRequests) == false + expect(testContext.eventProcessor.withContextSummarizer { $0.hasLoggedRequests }) == false let expectedError = testContext.serviceMock.errorEventHTTPURLResponse guard case let .response(error) = testContext.syncResult else { @@ -950,7 +950,7 @@ class DirectEventProcessorSpec: QuickSpec { expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 expect(testContext.eventProcessor.lastEventResponseDate) == Date.distantPast - expect(testContext.eventProcessor.eventBuffer.contextSummarizer.hasLoggedRequests) == false + expect(testContext.eventProcessor.withContextSummarizer { $0.hasLoggedRequests }) == false guard case let .request(error) = testContext.syncResult else { fail("Expected error result for event send") @@ -1027,7 +1027,7 @@ class DirectEventProcessorSpec: QuickSpec { let pending = testContext.committedEvents() expect(pending.dropLast().map { $0 }) == testContext.recordedEventsAsJSON() expect(pending.last?.kindField) == "summary" - expect(testContext.eventProcessor.eventBuffer.contextSummarizer.hasLoggedRequests) == false + expect(testContext.eventProcessor.withContextSummarizer { $0.hasLoggedRequests }) == false guard case .isOffline = testContext.syncResult else { fail("Expected error .isOffline result for event send") @@ -1071,7 +1071,7 @@ class DirectEventProcessorSpec: QuickSpec { reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: tracked, context: LDContext.stub(key: "c"), includeReason: false) expect(diagnosticCache.incrementDroppedEventCountCallCount) == 1 - expect(reporter.eventBuffer.contextSummarizer.getSummaries().count) == 2 + expect(reporter.withContextSummarizer { $0.getSummaries() }.count) == 2 reporter.commitRecordedEvents() expect(recordedEvents(store).compactMap { $0.kindField }) == ["feature", "summary", "summary"] } @@ -1080,8 +1080,8 @@ class DirectEventProcessorSpec: QuickSpec { defer { store.deleteEverything() } reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: nil, context: context, includeReason: true) expect(recordedEvents(store)).to(beEmpty()) - expect(reporter.eventBuffer.contextSummarizer.hasLoggedRequests) == true - let summaries = reporter.eventBuffer.contextSummarizer.getSummaries() + expect(reporter.withContextSummarizer { $0.hasLoggedRequests }) == true + let summaries = reporter.withContextSummarizer { $0.getSummaries() } expect(summaries.count) == 1 let tracker = summaries[0].tracker expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" @@ -1095,8 +1095,8 @@ class DirectEventProcessorSpec: QuickSpec { let flag = FeatureFlag(flagKey: "unused", value: nil, variation: 1, flagVersion: 2, trackEvents: false) reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: true) expect(recordedEvents(store)).to(beEmpty()) - expect(reporter.eventBuffer.contextSummarizer.hasLoggedRequests) == true - let summaries = reporter.eventBuffer.contextSummarizer.getSummaries() + expect(reporter.withContextSummarizer { $0.hasLoggedRequests }) == true + let summaries = reporter.withContextSummarizer { $0.getSummaries() } expect(summaries.count) == 1 let tracker = summaries[0].tracker expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" @@ -1112,8 +1112,8 @@ class DirectEventProcessorSpec: QuickSpec { let expected = FeatureEvent(key: "flag-key", context: context, value: "a", defaultValue: "b", featureFlag: flag, includeReason: true, isDebug: false) expect(expectedEvents(reporter.pendingEventsForTesting)) == expectedEvents([expected]) expect(recordedEvents(store)).to(beEmpty()) - expect(reporter.eventBuffer.contextSummarizer.hasLoggedRequests) == true - let summaries = reporter.eventBuffer.contextSummarizer.getSummaries() + expect(reporter.withContextSummarizer { $0.hasLoggedRequests }) == true + let summaries = reporter.withContextSummarizer { $0.getSummaries() } expect(summaries.count) == 1 let tracker = summaries[0].tracker expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" @@ -1127,8 +1127,8 @@ class DirectEventProcessorSpec: QuickSpec { let flag = FeatureFlag(flagKey: "unused", value: nil, variation: 1, flagVersion: 2, trackEvents: false, debugEventsUntilDate: Date().addingTimeInterval(-1.0)) reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: true) expect(recordedEvents(store)).to(beEmpty()) - expect(reporter.eventBuffer.contextSummarizer.hasLoggedRequests) == true - let summaries = reporter.eventBuffer.contextSummarizer.getSummaries() + expect(reporter.withContextSummarizer { $0.hasLoggedRequests }) == true + let summaries = reporter.withContextSummarizer { $0.getSummaries() } expect(summaries.count) == 1 let tracker = summaries[0].tracker expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" @@ -1144,8 +1144,8 @@ class DirectEventProcessorSpec: QuickSpec { let expected = FeatureEvent(key: "flag-key", context: context, value: "a", defaultValue: "b", featureFlag: flag, includeReason: false, isDebug: true) expect(expectedEvents(reporter.pendingEventsForTesting)) == expectedEvents([expected]) expect(recordedEvents(store)).to(beEmpty()) - expect(reporter.eventBuffer.contextSummarizer.hasLoggedRequests) == true - let summaries = reporter.eventBuffer.contextSummarizer.getSummaries() + expect(reporter.withContextSummarizer { $0.hasLoggedRequests }) == true + let summaries = reporter.withContextSummarizer { $0.getSummaries() } expect(summaries.count) == 1 let tracker = summaries[0].tracker expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" @@ -1160,8 +1160,8 @@ class DirectEventProcessorSpec: QuickSpec { let flag = FeatureFlag(flagKey: "unused", value: nil, variation: 1, flagVersion: 2, trackEvents: false, debugEventsUntilDate: Date().addingTimeInterval(3.0)) reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: true) expect(recordedEvents(store)).to(beEmpty()) - expect(reporter.eventBuffer.contextSummarizer.hasLoggedRequests) == true - let summaries = reporter.eventBuffer.contextSummarizer.getSummaries() + expect(reporter.withContextSummarizer { $0.hasLoggedRequests }) == true + let summaries = reporter.withContextSummarizer { $0.getSummaries() } expect(summaries.count) == 1 let tracker = summaries[0].tracker expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" @@ -1179,8 +1179,8 @@ class DirectEventProcessorSpec: QuickSpec { let expectedDebug = FeatureEvent(key: "flag-key", context: context, value: "a", defaultValue: "b", featureFlag: flag, includeReason: false, isDebug: true) expect(expectedEvents(reporter.pendingEventsForTesting)) == expectedEvents([expectedFeature, expectedDebug]) expect(recordedEvents(store)).to(beEmpty()) - expect(reporter.eventBuffer.contextSummarizer.hasLoggedRequests) == true - let summaries = reporter.eventBuffer.contextSummarizer.getSummaries() + expect(reporter.withContextSummarizer { $0.hasLoggedRequests }) == true + let summaries = reporter.withContextSummarizer { $0.getSummaries() } expect(summaries.count) == 1 let tracker = summaries[0].tracker expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" @@ -1206,8 +1206,8 @@ class DirectEventProcessorSpec: QuickSpec { expect(held.filter { $0.kindField == "feature" }.count) == 10 expect(held.filter { $0.kindField == "debug" }.count) == 10 expect(recordedEvents(store)).to(beEmpty()) - expect(reporter.eventBuffer.contextSummarizer.hasLoggedRequests) == true - let summaries = reporter.eventBuffer.contextSummarizer.getSummaries() + expect(reporter.withContextSummarizer { $0.hasLoggedRequests }) == true + let summaries = reporter.withContextSummarizer { $0.getSummaries() } expect(summaries.count) == 1 let tracker = summaries[0].tracker expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" From 49a4c5693c74bf19c54c433ff9364d5ec26ba4d8 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Mon, 12 Oct 2026 23:38:01 -0700 Subject: [PATCH 058/158] Do not take a payload id ending in zero-led digits for a count A batch named without a count by an earlier version ends in a UUID group, which Int32 accepts once its leading zeros are dropped. A count is never written with a leading zero, so a suffix that has one is part of the id. Co-authored-by: Cursor --- .../ServiceObjects/EventDirectory.swift | 4 +++- .../ServiceObjects/EventStoreSpec.swift | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index 6958d3b0d..11c8cbae9 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -139,13 +139,15 @@ final class EventDirectory { /// /// - Returns: the payload ID, and the count the name ends with, which is nil where it has none -- how a version /// that did not write one named its batches. A payload ID cannot be mistaken for a count: it is a UUID, whose - /// last group is twelve characters, and twelve digits are more than an `Int32` holds. + /// last group is twelve characters. Twelve digits are more than an `Int32` holds unless they start with a zero, + /// which a count never does. private static func parse(batchName name: String) -> (payloadId: String, events: Int?) { let rest = String(name.dropFirst(batchPrefix.count)) guard let split = rest.lastIndex(of: "-") else { return (rest, nil) } let suffix = rest[rest.index(after: split)...] guard !suffix.isEmpty, suffix.allSatisfy({ $0.isASCII && $0.isNumber }), + suffix == "0" || suffix.first != "0", let events = Int32(suffix) else { return (rest, nil) } return (String(rest[.. Date: Mon, 12 Oct 2026 23:38:01 -0700 Subject: [PATCH 059/158] Say what a dropped event-publish completion costs now Co-authored-by: Cursor --- LaunchDarkly/LaunchDarkly/Networking/DarklyService.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LaunchDarkly/LaunchDarkly/Networking/DarklyService.swift b/LaunchDarkly/LaunchDarkly/Networking/DarklyService.swift index e49b7ca27..50a09dc18 100644 --- a/LaunchDarkly/LaunchDarkly/Networking/DarklyService.swift +++ b/LaunchDarkly/LaunchDarkly/Networking/DarklyService.swift @@ -191,7 +191,7 @@ final class DarklyService: DarklyServiceProvider { func publishEventData(_ eventData: Data, _ payloadId: String, completion: ServiceCompletionHandler?) { guard hasMobileKey(#function) else { - // Answered rather than dropped: the reporter holds every later delivery until this one answers. + // Answered rather than dropped: until this answers, its batch stays claimed and its flush never reports. completion?((nil, nil, URLError(.userAuthenticationRequired), nil)) return } From 35163da3353ee3e3ee696b8db1e260f5bbf7b34f Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Mon, 12 Oct 2026 23:52:12 -0700 Subject: [PATCH 060/158] Wait for the after-identify hooks in the hook forwarding specs An identify's completion runs before its after-identify hooks, so a test that only waits on the completion could check the stages before the hook was told. RecordingHook is also written from the identify's thread, so it now guards its stages with a lock. Co-authored-by: Cursor --- .../LDClientEvaluationHookSpec.swift | 58 ++++++++++++------- 1 file changed, 38 insertions(+), 20 deletions(-) diff --git a/LaunchDarkly/LaunchDarklyTests/LDClientEvaluationHookSpec.swift b/LaunchDarkly/LaunchDarklyTests/LDClientEvaluationHookSpec.swift index 1decc2aae..adf43e011 100644 --- a/LaunchDarkly/LaunchDarklyTests/LDClientEvaluationHookSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/LDClientEvaluationHookSpec.swift @@ -110,6 +110,21 @@ final class LDClientEvaluationHookSpec: XCTestCase { return testContext } + /// Identifies and then tracks, with `hook` holding only the stages those two produce. + /// + /// An identify's completion runs before its after-identify hooks, so waiting on the completion is not enough: + /// for the identify the client makes as it starts, and for this one. + private func identifyThenTrack(_ testContext: TestContext, _ hook: RecordingHook) { + expect(hook.recordedStages).toEventually(contain("afterIdentify")) + _ = hook.takeStages() + + waitUntil { done in + testContext.subject.identify(context: LDContext.stub()) { _ in done() } + } + expect(hook.recordedStages).toEventually(contain("afterIdentify")) + testContext.subject.track(key: "event-key") + } + // An evaluation reads the flag once, so every hook it reaches and the result it returns all describe that one read, // whatever the store does in the meantime. func testHooksAndTheResultDescribeOneReadOfTheFlag() { @@ -400,12 +415,7 @@ final class LDClientEvaluationHookSpec: XCTestCase { func testADeduperForwardsTheStagesItDoesNotDeduplicate() { let hook = RecordingHook() let testContext = dedupeTestContext(hooks: [DedupingHook(hook, window: 60)]) - _ = hook.takeStages() - - waitUntil { done in - testContext.subject.identify(context: LDContext.stub()) { _ in done() } - } - testContext.subject.track(key: "event-key") + identifyThenTrack(testContext, hook) // Only evaluations are deduplicated; the identify and track stages are always forwarded. XCTAssertEqual(hook.takeStages(), ["beforeIdentify", "afterIdentify", "afterTrack"]) @@ -488,12 +498,7 @@ final class LDClientEvaluationHookSpec: XCTestCase { func testADecoratorForwardsTheStagesItDoesNotOverride() { let hook = RecordingHook() let testContext = dedupeTestContext(hooks: [CountingDecorator(hook)]) - _ = hook.takeStages() - - waitUntil { done in - testContext.subject.identify(context: LDContext.stub()) { _ in done() } - } - testContext.subject.track(key: "event-key") + identifyThenTrack(testContext, hook) // The decorator mentions neither identify nor track, and the hook it wraps is still told about both: a stage a // decorator leaves alone is forwarded rather than dropped. @@ -545,14 +550,27 @@ final class LDClientEvaluationHookSpec: XCTestCase { // Returned rather than built on demand, so that a test can recognize it by identity: `Metadata` exposes // nothing to compare it by. let hookMetadata = Metadata(name: "recording-hook") + /// Guards `stages`: the identify stages run on whichever thread the identify ended on. + private let lock = UnfairLock() private var stages: [String] = [] + /// The stages recorded since `takeStages` was last called, leaving them in place. + var recordedStages: [String] { + lock.withLock { stages } + } + /// The stages recorded since this was last called, so a test can ignore the identify the client makes as it /// starts. func takeStages() -> [String] { - let recorded = stages - stages = [] - return recorded + lock.withLock { + let recorded = stages + stages = [] + return recorded + } + } + + private func record(_ stage: String) { + lock.withLock { stages.append(stage) } } func metadata() -> Metadata { @@ -560,27 +578,27 @@ final class LDClientEvaluationHookSpec: XCTestCase { } func beforeEvaluation(seriesContext: EvaluationSeriesContext, seriesData: EvaluationSeriesData) -> EvaluationSeriesData { - stages.append("beforeEvaluation") + record("beforeEvaluation") return seriesData } func afterEvaluation(seriesContext: EvaluationSeriesContext, seriesData: EvaluationSeriesData, evaluationDetail: LDEvaluationDetail) -> EvaluationSeriesData { - stages.append("afterEvaluation") + record("afterEvaluation") return seriesData } func beforeIdentify(seriesContext: IdentifySeriesContext, seriesData: IdentifySeriesData) -> IdentifySeriesData { - stages.append("beforeIdentify") + record("beforeIdentify") return seriesData } func afterIdentify(seriesContext: IdentifySeriesContext, seriesData: IdentifySeriesData, result: IdentifyResult) -> IdentifySeriesData { - stages.append("afterIdentify") + record("afterIdentify") return seriesData } func afterTrack(seriesContext: TrackSeriesContext) { - stages.append("afterTrack") + record("afterTrack") } } From 19192a39250225fe2846728ec402af0b342b0764 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 13 Oct 2026 00:26:36 -0700 Subject: [PATCH 061/158] Go online before staging the batch the unreadable-batch spec damages The check for what a previous run left can run after the spec closes its batch. It then delivers on going online, and where that delivery starts before the spec's flush, the flush finds nothing left and reports true. Co-authored-by: Cursor --- .../ServiceObjects/DirectEventProcessorSpec.swift | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 6d37a4eca..c0b6c78e1 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -1945,6 +1945,14 @@ extension DirectEventProcessorSpec { it("reports false and drops a stored batch this version cannot read, and still delivers the rest") { let store = EventStore.temporary() testContext = TestContext(store: store) + // Online, and past the check for what a previous run left, before there is a batch for that check to + // find. Found, it would be delivered on going online, ahead of the flush this test asks about. + testContext.eventProcessor.isOnline = true + var delivered: Bool? + waitUntil(timeout: Constants.deliveryTimeout) { done in + testContext.eventProcessor.flushReportingOutcome { _ in done() } + } + testContext.recordEvents(1) testContext.eventProcessor.commitRecordedEvents() let damaged = store.closeBatch()! @@ -1952,9 +1960,7 @@ extension DirectEventProcessorSpec { let damagedUrl = store.directory!.appendingPathComponent("ready-\(damaged.payloadId)-\(damaged.eventCount)") try Data("not an event log".utf8).write(to: damagedUrl) testContext.recordEvents(1) - testContext.eventProcessor.isOnline = true - var delivered: Bool? waitUntil(timeout: Constants.deliveryTimeout) { done in testContext.eventProcessor.flushReportingOutcome { result in delivered = result From 51edd8646973b23646e2cdbb6b1accb0c7263d4f Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 13 Oct 2026 08:44:57 -0700 Subject: [PATCH 062/158] Make the summary-count spec commit during recording however it is scheduled It required more than one summary, which only held where a commit landed while the evaluations were still being recorded. A starved commit queue and main thread left one. Recording now pauses halfway until the first half is committed, so there are always at least two. Co-authored-by: Cursor --- .../ServiceObjects/DirectEventProcessorSpec.swift | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index c0b6c78e1..f00500a22 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -1693,15 +1693,28 @@ extension DirectEventProcessorSpec { let reporter = testContext.eventProcessor! let context = testContext.context! + // Recording pauses halfway until a commit has taken the first half, so that at least two commits land + // while it runs however the threads are scheduled. Otherwise the recording could finish before the + // first commit, and the single summary that leaves would check nothing about concurrent commits. + let halfway = DispatchSemaphore(value: 0) + let resume = DispatchSemaphore(value: 0) let recorded = DispatchSemaphore(value: 0) DispatchQueue.global().async { - for _ in 0..<2_000 { + for index in 0..<2_000 { + if index == 1_000 { + halfway.signal() + resume.wait() + } reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: false) } recorded.signal() } while recorded.wait(timeout: .now()) == .timedOut { reporter.commitRecordedEvents() + if halfway.wait(timeout: .now()) == .success { + reporter.commitRecordedEvents() + resume.signal() + } } reporter.commitRecordedEvents() From 188bb59ddc9b2205528b58ab7809779769be2f0b Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 20 Oct 2026 08:09:58 -0700 Subject: [PATCH 063/158] Bound stored events by bytes and by age Persisted events are now limited to eventCapacityBytes per environment (default 20 MiB), a fifth of which only summaries may use, and are discarded once older than eventMaxAge (default 7 days). Expiry also removes the directories of environments nothing has written to for that long. Neither applies where eventPersistence is .disabled. Co-authored-by: Cursor --- .../LaunchDarkly/Models/LDConfig.swift | 24 ++ .../ObjectiveC/ObjcLDConfig.swift | 12 + .../ServiceObjects/DirectEventProcessor.swift | 30 ++- .../ServiceObjects/EventDirectory.swift | 207 ++++++++++++--- .../ServiceObjects/EventLogFile.swift | 26 ++ .../ServiceObjects/EventStore.swift | 194 +++++++++++--- .../Models/LDConfigSpec.swift | 8 + .../DirectEventProcessorSpec.swift | 59 +++++ .../ServiceObjects/EventStoreSpec.swift | 241 ++++++++++++++++++ LaunchDarkly/LaunchDarklyTests/TestUtil.swift | 4 + 10 files changed, 744 insertions(+), 61 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift index ccfe7180b..e08ae0cfe 100644 --- a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift @@ -224,6 +224,12 @@ public struct LDConfig { /// The default maximum number of events the LDClient can store static let eventCapacity = 2000 + /// The default maximum number of bytes of events the LDClient can persist per environment. (20 MiB) + static let eventCapacityBytes = 20 * 1024 * 1024 + + /// The default maximum age of persisted events. (7 days) + static let eventMaxAge: TimeInterval = 7 * 24 * 60 * 60 + /// The default behavior for the SDK is to keep events in memory only. static let eventPersistence = EventPersistence.disabled @@ -365,6 +371,22 @@ public struct LDConfig { /// in which the event exists only in memory. (Default: `.disabled`) public var eventPersistence: EventPersistence = Defaults.eventPersistence + /// The maximum number of bytes of events the SDK keeps for each environment where `eventPersistence` keeps events across launches, + /// on disk and in memory together. When it is reached, the SDK discards new events until it successfully reports the stored ones, + /// as it does when `eventCapacity` is reached; it never deletes stored events to make room. A fifth of it is kept for the summaries + /// of flag evaluations, so that evaluations are still counted after full events start being discarded. A value of zero or less + /// means the default. (Default: 20 MiB) + /// + /// It has no effect where `eventPersistence` is `.disabled`. + public var eventCapacityBytes: Int = Defaults.eventCapacityBytes + + /// How long the SDK keeps events it has persisted but not yet reported, where `eventPersistence` keeps events across launches. Events + /// older than this are discarded rather than reported, and so is everything the SDK stored for an environment whose mobile key the + /// application stopped using. A value of zero or less means the default. (Default: 7 days) + /// + /// It has no effect where `eventPersistence` is `.disabled`. + public var eventMaxAge: TimeInterval = Defaults.eventMaxAge + /// The timeout interval for flag requests and event reports. (Default: 10 seconds) public var connectionTimeout: TimeInterval = Defaults.connectionTimeout /// The time interval between event reports (Default: 30 seconds) @@ -595,6 +617,8 @@ extension LDConfig: Equatable { && lhs.streamUrl == rhs.streamUrl && lhs.eventCapacity == rhs.eventCapacity && lhs.eventPersistence == rhs.eventPersistence + && lhs.eventCapacityBytes == rhs.eventCapacityBytes + && lhs.eventMaxAge == rhs.eventMaxAge && lhs.sendEvents == rhs.sendEvents && lhs.connectionTimeout == rhs.connectionTimeout && lhs.eventFlushInterval == rhs.eventFlushInterval diff --git a/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift b/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift index efd09f36a..9960e5542 100644 --- a/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift @@ -52,6 +52,18 @@ public final class ObjcLDConfig: NSObject { set { config.eventPersistence = newValue } } + /// The maximum number of bytes of events the SDK keeps for each environment where `eventPersistence` keeps events across launches. When it is reached, the SDK discards new events until it reports the stored ones; a fifth of it is kept for the summaries of flag evaluations. No effect where `eventPersistence` is `.disabled`. (Default: 20 MiB) + @objc public var eventCapacityBytes: Int { + get { config.eventCapacityBytes } + set { config.eventCapacityBytes = newValue } + } + + /// How long the SDK keeps persisted events it has not yet reported before discarding them, which also removes what it stored for a mobile key the application stopped using. No effect where `eventPersistence` is `.disabled`. (Default: 7 days) + @objc public var eventMaxAge: TimeInterval { + get { config.eventMaxAge } + set { config.eventMaxAge = newValue } + } + /// The timeout interval for flag requests and event reports. (Default: 10 seconds) @objc public var connectionTimeout: TimeInterval { get { config.connectionTimeout } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 983748328..e6a6c4710 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -192,6 +192,7 @@ class DirectEventProcessor: EventProcessor { let store = self.store deliveryQueue.async { [weak self] in store.recoverInterruptedLog() + self?.reportDiscarded(store.discardExpired()) self?.hasEventsFromPreviousRun = !store.pendingBatches().isEmpty } } @@ -200,6 +201,8 @@ class DirectEventProcessor: EventProcessor { let mobileKey = config.mobileKey return EventStore(locatingDirectory: { EventDirectory.defaultUrl(mobileKey: mobileKey) }, capacity: config.eventCapacity, + byteCapacity: config.eventCapacityBytes, + maxAge: config.eventMaxAge, persistEvents: config.eventPersistence != .disabled, logger: config.logger) } @@ -265,9 +268,12 @@ class DirectEventProcessor: EventProcessor { /// Capacity is consulted before anything else, so an event that will not be kept is never encoded. That ordering is /// what bounds an application re-evaluating a tracked flag in a render loop: once the limit is reached an /// evaluation costs no more than its summary counter, however fast the loop runs. + /// + /// The byte capacity can only be consulted as far as whether any room is left, since the event has no size until + /// it is encoded; whether it fits is decided when it is staged. private func admitToPending(_ event: Event) -> Bool { recordLock.assertOwned() - guard pending.count + store.pendingEventCount < eventCapacity + guard pending.count + store.pendingEventCount < eventCapacity, store.hasRoomForEventBytes else { return false } pending.append(event) @@ -279,6 +285,19 @@ class DirectEventProcessor: EventProcessor { service.diagnosticCache?.incrementDroppedEventCount() } + /// Records stored events discarded for being older than the maximum age. + /// + /// Counted as dropped rather than lost: a flush is about the caller's events, and the ones discarded are long + /// past anything a caller is waiting on. + private func reportDiscarded(_ events: Int) { + guard events > 0 + else { return } + os_log("%s discarded %d stored event(s) older than the maximum age", log: service.config.logger, type: .debug, typeName(and: #function), events) + for _ in 0.. (deliverable: [EventBatch], waitingOn: [Claim]) { // Listed before the lock is taken, because listing reads the disk. Taking what appeared after it is the next // delivery's business; all this has to be atomic is deciding which of these batches are this delivery's. + // Expired first, so that a batch too old to report is not sent. + reportDiscarded(store.discardExpired()) let pending = store.pendingBatches() return claimLock.withLock { var deliverable: [EventBatch] = [] diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index 11c8cbae9..42ae3795b 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -1,6 +1,17 @@ import Foundation import OSLog +/// How many events, and how many bytes, something stored holds: the two things capacity counts. +struct EventTally: Equatable { + var events = 0 + var bytes = 0 + + static func += (lhs: inout EventTally, rhs: EventTally) { + lhs.events += rhs.events + lhs.bytes += rhs.bytes + } +} + /// The directory one store keeps its events in: the log this process appends to, and beside it the batches that have /// been closed off for delivery. /// @@ -41,10 +52,13 @@ final class EventDirectory { /// Whether a log left by a previous run has been dealt with. Only to be used while holding `ioLock`. private var hasRecoveredInterruptedLog = false - /// A batch in the directory: the file it is, and how many events it holds. + /// A batch in the directory: the file it is, how many events it holds, and how big it is. private struct Stored { let url: URL let events: Int + let bytes: Int + + var tally: EventTally { EventTally(events: events, bytes: bytes) } } /// What this process knows about each batch in the directory, by payload ID, which is how everything outside this @@ -159,14 +173,14 @@ final class EventDirectory { /// /// Requires `ioLock`. /// - /// - Returns: how many events that recovery found, and whether every byte landed. False means none of `bytes` is - /// in the log, and that the caller should give up on the disk. - func append(_ bytes: Data) -> (recovered: Int, written: Bool) { + /// - Returns: what that recovery found, and whether every byte landed. False means none of `bytes` is in the log, + /// and that the caller should give up on the disk. + func append(_ bytes: Data) -> (recovered: EventTally, written: Bool) { ioLock.assertOwned() // Recovered before the log is opened, because it may still be the one a previous run died writing. Appended // to, this run's events would sit behind that run's torn last frame, where a reader can no longer find where // they start. - let recovered = disk.log.isOpen ? 0 : recoverInterruptedLog() + let recovered = disk.log.isOpen ? EventTally() : recoverInterruptedLog() return (recovered, disk.log.append(bytes)) } @@ -181,9 +195,10 @@ final class EventDirectory { /// Closes the open log into a batch, so the next append starts a new one. Requires `ioLock`. /// - /// - Parameter events: how many events the log holds, which the caller has counted as it staged them. - func closeLog(holding events: Int) -> LogClose { + /// - Parameter held: what the log holds, which the caller has counted as it staged and wrote it. + func closeLog(holding held: EventTally) -> LogClose { ioLock.assertOwned() + let events = held.events let payloadId = UUID().uuidString let file = newUrl(of: payloadId, holding: events) do { @@ -193,7 +208,7 @@ final class EventDirectory { return .failed(logSurvives: disk.log.exists) } - stored[payloadId] = Stored(url: file, events: events) + stored[payloadId] = Stored(url: file, events: events, bytes: held.bytes) return .closed(EventBatch(payloadId: payloadId, eventCount: events)) } @@ -208,10 +223,12 @@ final class EventDirectory { // MARK: Listing and reading /// The batches in the directory, oldest first. Requires `ioLock`. + /// + /// Sizes come from the same listing, so that counting the directory's bytes reads no file either. func listBatches() -> [EventBatch] { ioLock.assertOwned() let contents = (try? FileManager.default.contentsOfDirectory(at: disk.directory, - includingPropertiesForKeys: [.contentModificationDateKey], + includingPropertiesForKeys: EventDirectory.listedKeys, options: [.skipsHiddenFiles])) ?? [] let batches = contents @@ -224,9 +241,9 @@ final class EventDirectory { // count may have left behind, and then only the once. guard let events = counted ?? stored[payloadId]?.events ?? countOfEvents(in: file) else { return nil } - stored[payloadId] = Stored(url: file, events: events) - let modified = (try? file.resourceValues(forKeys: [.contentModificationDateKey]))?.contentModificationDate - return (EventBatch(payloadId: payloadId, eventCount: events), modified ?? Date.distantPast) + let listed = EventDirectory.listed(file) + stored[payloadId] = Stored(url: file, events: events, bytes: listed.bytes ?? stored[payloadId]?.bytes ?? 0) + return (EventBatch(payloadId: payloadId, eventCount: events), listed.modified ?? Date.distantPast) } .sorted { $0.1 < $1.1 } .map { $0.0 } @@ -239,6 +256,26 @@ final class EventDirectory { return batches } + private static let listedKeys: [URLResourceKey] = [.contentModificationDateKey, .fileSizeKey] + + /// What a directory listing says about a file, without opening it. + private static func listed(_ file: URL) -> (bytes: Int?, modified: Date?) { + let values = try? file.resourceValues(forKeys: Set(listedKeys)) + return (values?.fileSize, values?.contentModificationDate) + } + + /// The bytes in every batch the last listing or close counted. Requires `ioLock`. + var storedBytes: Int { + ioLock.assertOwned() + return stored.values.reduce(0) { $0 + $1.bytes } + } + + /// The bytes in a batch, as far as this process has counted them. Requires `ioLock`. + func bytes(of payloadId: String) -> Int { + ioLock.assertOwned() + return stored[payloadId]?.bytes ?? 0 + } + /// Whether a batch has already been delivered and only its file is left, trying the delete that failed once more /// on the way. Requires `ioLock`. private func isDeliveredButUndeleted(_ payloadId: String, at file: URL) -> Bool { @@ -275,9 +312,9 @@ final class EventDirectory { /// Forgets a batch. Requires `ioLock`. /// - /// - Returns: the events it stopped counting, which is zero for a batch that was already gone -- one another - /// process delivered, say, which a listing has stopped counting already. - func remove(_ payloadId: String, holding events: Int) -> Int { + /// - Returns: what it stopped counting, which is nothing for a batch that was already gone -- one another process + /// delivered, say, which a listing has stopped counting already. + func remove(_ payloadId: String, holding events: Int) -> EventTally { ioLock.assertOwned() let file = url(of: payloadId, holding: events) try? FileManager.default.removeItem(at: file) @@ -287,7 +324,7 @@ final class EventDirectory { os_log("%s could not delete a delivered batch; it will not be sent again this session", log: logger, type: .debug, typeName(and: #function)) deliveredButUndeleted.insert(payloadId) } - return stored.removeValue(forKey: payloadId)?.events ?? 0 + return stored.removeValue(forKey: payloadId)?.tally ?? EventTally() } /// Every event in the directory, encoded exactly as it will be sent. Requires `ioLock`. @@ -371,27 +408,28 @@ final class EventDirectory { /// /// Requires `ioLock`. Does its work once: after that, the open log is one this run opened. /// - /// - Returns: how many events it closed off, which is zero on every call after the first. - func recoverInterruptedLog() -> Int { + /// - Returns: what it closed off, which is nothing on every call after the first. + func recoverInterruptedLog() -> EventTally { ioLock.assertOwned() guard !hasRecoveredInterruptedLog - else { return 0 } + else { return EventTally() } hasRecoveredInterruptedLog = true - var recovered = 0 - disk.log.forEachLogLeftOpen { recovered += events(recoveredFrom: $0) } + var recovered = EventTally() + disk.log.forEachLogLeftOpen { recovered += tally(recoveredFrom: $0) } return recovered } /// Closes off a log nobody is writing to any more. Requires `ioLock`. /// - /// - Returns: how many events it closed off, or zero where there was nothing to close off. - private func events(recoveredFrom leftOpen: URL) -> Int { + /// - Returns: what it closed off, or nothing where there was nothing to close off. + private func tally(recoveredFrom leftOpen: URL) -> EventTally { ioLock.assertOwned() let events: Int + let bytes: Int switch read(leftOpen) { case .missing: - return 0 + return EventTally() case .failed(let error): // It cannot stay under this name, which this run is about to append its own events to, and deleting it // would lose events that may be intact. Closed off uncounted instead, it is read and counted by whichever @@ -401,32 +439,143 @@ final class EventDirectory { // first manages to read it is what counts it. let uncounted = disk.directory.appendingPathComponent("\(EventDirectory.batchPrefix)\(UUID().uuidString)") try? FileManager.default.moveItem(at: leftOpen, to: uncounted) - return 0 + return EventTally() case .bytes(let log): guard let counted = EventLogFormat.eventCount(in: log) else { // Unreadable, and a log that cannot be read cannot be appended to either. try? FileManager.default.removeItem(at: leftOpen) - return 0 + return EventTally() } events = counted + bytes = log.count } guard events > 0 else { try? FileManager.default.removeItem(at: leftOpen) - return 0 + return EventTally() } let payloadId = UUID().uuidString let file = newUrl(of: payloadId, holding: events) guard (try? FileManager.default.moveItem(at: leftOpen, to: file)) != nil - else { return 0 } + else { return EventTally() } - stored[payloadId] = Stored(url: file, events: events) + let recovered = Stored(url: file, events: events, bytes: bytes) + stored[payloadId] = recovered os_log("%s recovered %d event(s) from a previous run", log: logger, type: .debug, typeName(and: #function), events) + return recovered.tally + } +} + +// MARK: Expiry + +extension EventDirectory { + /// Deletes the batches in this directory last written before `cutoff`, and the logs of other processes that have + /// not been written since then either. Requires `ioLock`. + /// + /// A batch's modification time is when its newest event was written, since a batch is renamed rather than rewritten + /// when it is closed, so a batch is discarded only once everything in it is that old. A log another process has not + /// appended to for as long belongs to a process that is gone: one that is alive appends far more often than that. + /// + /// - Parameter sending: batches whose request is out, which are left alone however old they are. + /// - Returns: how many events were discarded, and how much of that this process had counted, which is what stops + /// counting against capacity. + func discard(lastWrittenBefore cutoff: Date, sparing sending: Set) -> (events: Int, counted: EventTally) { + ioLock.assertOwned() + let contents = (try? FileManager.default.contentsOfDirectory(at: disk.directory, + includingPropertiesForKeys: EventDirectory.listedKeys, + options: [.skipsHiddenFiles])) ?? [] + var events = 0 + var counted = EventTally() + for file in contents where EventDirectory.listed(file).modified.map({ $0 < cutoff }) ?? false { + let name = file.lastPathComponent + if name.hasPrefix(EventDirectory.batchPrefix) { + let (payloadId, named) = EventDirectory.parse(batchName: name) + guard !sending.contains(payloadId), !deliveredButUndeleted.contains(payloadId) + else { continue } + let held = named ?? stored[payloadId]?.events ?? EventDirectory.eventsReading(file) + guard EventDirectory.delete(file) + else { continue } + events += held + counted += stored.removeValue(forKey: payloadId)?.tally ?? EventTally() + } else if name.hasPrefix(DiskEventLog.prefix), file.standardizedFileURL != disk.log.url.standardizedFileURL { + events += EventDirectory.discardAbandonedLog(file) + } + } + return (events, counted) + } + + /// Empties and removes the directories of other environments that nothing has written to since `cutoff`: those + /// of a mobile key the application no longer uses, which no client will look in again. + /// + /// Age decides rather than the keys the application configures, because another process of the same application + /// may be using a key this one does not configure. Nothing is touched here that this store reads or writes, so it + /// needs no lock. A directory is removed only if it is empty, so a log another process starts in it meanwhile + /// keeps the directory and itself. + /// + /// Only directories carrying this application's bundle identifier are considered. An unsandboxed Mac application + /// shares Application Support with every other one, whose events are not this application's to delete. + /// + /// - Returns: how many events were discarded. + static func discardAbandonedEnvironments(besides current: URL, lastWrittenBefore cutoff: Date) -> Int { + let root = current.deletingLastPathComponent() + let ownName = current.lastPathComponent + let isSameApplication: (String) -> Bool + if let dot = ownName.firstIndex(of: ".") { + let prefix = ownName[...dot] + isSameApplication = { $0.hasPrefix(prefix) } + } else { + isSameApplication = { !$0.contains(".") } + } + + let siblings = (try? FileManager.default.contentsOfDirectory(at: root, + includingPropertiesForKeys: [.isDirectoryKey], + options: [.skipsHiddenFiles])) ?? [] + var events = 0 + for directory in siblings where directory.lastPathComponent != ownName && isSameApplication(directory.lastPathComponent) { + guard (try? directory.resourceValues(forKeys: [.isDirectoryKey]))?.isDirectory == true + else { continue } + let contents = (try? FileManager.default.contentsOfDirectory(at: directory, + includingPropertiesForKeys: listedKeys, + options: [])) ?? [] + for file in contents where listed(file).modified.map({ $0 < cutoff }) ?? false { + let name = file.lastPathComponent + if name.hasPrefix(batchPrefix) { + let held = parse(batchName: name).events ?? eventsReading(file) + if delete(file) { + events += held + } + } else if name.hasPrefix(DiskEventLog.prefix) { + events += discardAbandonedLog(file) + } + } + EventLogFile.removeDirectoryIfEmpty(directory) + } return events } + + /// Deletes a log no live process is appending to, returning how many events it held. + private static func discardAbandonedLog(_ log: URL) -> Int { + #if os(macOS) || targetEnvironment(macCatalyst) + // A Mac runs second instances of an application, each holding its log locked for as long as it lives. + if EventLogFile.isLockedByLiveProcess(log) { + return 0 + } + #endif + let held = eventsReading(log) + return delete(log) ? held : 0 + } + + private static func eventsReading(_ file: URL) -> Int { + (try? Data(contentsOf: file)).flatMap { EventLogFormat.eventCount(in: $0) } ?? 0 + } + + private static func delete(_ file: URL) -> Bool { + try? FileManager.default.removeItem(at: file) + return !FileManager.default.fileExists(atPath: file.path) + } } extension EventDirectory { diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFile.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFile.swift index 87e49acfb..491b095b3 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFile.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFile.swift @@ -49,8 +49,26 @@ struct EventLogFile { } return nil } + + /// Whether a live process holds the lock on a log, which is to say is still appending to it. + static func isLockedByLiveProcess(_ url: URL) -> Bool { + guard let opened = openLocked(url, flags: O_RDONLY) + else { return FileManager.default.fileExists(atPath: url.path) } + opened.close() + return false + } #endif + /// Removes a directory only if it is empty, so that nothing put there since it was last looked at goes with it. + @discardableResult + static func removeDirectoryIfEmpty(_ url: URL) -> Bool { + url.withUnsafeFileSystemRepresentation { path -> Bool in + guard let path = path + else { return false } + return removeDirectory(path) == 0 + } + } + static func excludeFromBackup(_ directory: URL) { #if canImport(Darwin) var url = directory @@ -120,6 +138,10 @@ private func truncateFile(_ descriptor: Int32, _ length: off_t) -> Int32 { private func closeFile(_ descriptor: Int32) { _ = Darwin.close(descriptor) } + +private func removeDirectory(_ path: UnsafePointer) -> Int32 { + Darwin.rmdir(path) +} #else private func openFile(_ path: UnsafePointer, _ flags: Int32, _ mode: mode_t) -> Int32 { open(path, flags, mode) @@ -136,6 +158,10 @@ private func truncateFile(_ descriptor: Int32, _ length: off_t) -> Int32 { private func closeFile(_ descriptor: Int32) { _ = close(descriptor) } + +private func removeDirectory(_ path: UnsafePointer) -> Int32 { + rmdir(path) +} #endif #if os(macOS) || targetEnvironment(macCatalyst) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index 1e470e94d..47edc287c 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -20,20 +20,31 @@ protocol EventStoring { /// already have. A send that fails puts the batch back, so a full store can briefly hold capacity plus that batch. var pendingEventCount: Int { get } + /// Whether a full event of any size could still be staged within the byte capacity. + /// + /// Asked before an event is accepted, when it has not been encoded and so has no size yet. An event accepted while + /// there is room can still be refused at `stageReserved(_:)` if it turns out not to fit. + var hasRoomForEventBytes: Bool { get } + /// Whether a commit writes anything, which is false where persistence is off and once a write has failed and the /// store has given up on it. var isPersisting: Bool { get } - /// Stages an already encoded event, returning false if it was dropped because `capacity` is reached. + /// Stages an already encoded event, returning false if it was dropped because `capacity` or the byte capacity is + /// reached. /// /// Staging is a copy into a buffer, and never writes however much is staged; `commit()` is what makes the event /// outlive the process, and deciding when to call it is the caller's. + /// + /// - Parameter bypassingCapacity: true for a summary, which is not refused for `capacity` and may use the part of + /// the byte capacity held back from full events, but is still refused past the byte capacity itself. func stage(_ encodedEvent: Data, bypassingCapacity: Bool) -> Bool /// Counts events that have been accepted but are still being encoded, so capacity sees them until they are staged. func reserve(_ events: Int) - /// Stages an event using up one reservation, rather than checking capacity again. + /// Stages an event using up one reservation, rather than checking `capacity` again. Its bytes are checked, since + /// they were not known when it was accepted. func stageReserved(_ encodedEvent: Data) -> Bool /// Gives back every reservation not used up by `stageReserved(_:)`, such as one for an event that failed to encode. @@ -69,6 +80,14 @@ protocol EventStoring { /// Closes any log left open by a previous run of this process, so its events join the batches to deliver. func recoverInterruptedLog() + + /// Deletes what has been stored for longer than the maximum age, except a batch whose request is out. + /// + /// The first call also empties and removes the directories of other environments of this application that nothing + /// has written to for as long, which is what becomes of a mobile key the application stopped using. + /// + /// - Returns: how many events were discarded. + func discardExpired() -> Int } /// A batch that is there to read and holds nothing this version can send: damaged, or written in a format it does not @@ -103,6 +122,13 @@ final class EventStore: EventStoring { /// there are its own. private let disk: EventDirectory? private let capacity: Int + /// The most bytes summaries may take the store to. + private let byteCapacity: Int + /// The most bytes full events may take the store to: the byte capacity less a fifth of it, held back so that + /// summaries, which bypass `capacity`, still have room once full events are being refused. + private let eventByteCapacity: Int + /// How long a batch may go unwritten before it is discarded rather than delivered. + private let maxAge: TimeInterval private let logger: OSLog /// Guards the buffer and the counters. It is held for a copy and never across a syscall, so an evaluation @@ -125,10 +151,22 @@ final class EventStore: EventStoring { /// critical section that changes one of those, so that it cannot drift from them; a total worked out from them and /// written afterwards would overwrite a close or a removal in between. private var closedEvents = 0 - /// Closed batches whose request is out, by payload ID, and the events in them. Every one is also in `closedEvents`, + /// Closed batches whose request is out, by payload ID, and what is in them. Every one is also in `closedEvents`, /// and leaves both in the same critical section. - private var sendingBatches: [String: Int] = [:] + private var sendingBatches: [String: EventTally] = [:] private var sendingEvents = 0 + + /// The bytes of the events counted above, state for state. Staged bytes are `bufferData` itself. Changed in the + /// same critical sections as the event counts beside them, so that bytes and events are never counted in + /// different places. + private var writingBytes = 0 + private var committedBytes = 0 + private var closedBytes = 0 + private var sendingBytes = 0 + + /// Whether the directories of other environments have been looked at for ones nobody uses any more, which is + /// done once per process. Only to be used while holding `ioLock`. + private var hasDiscardedAbandonedEnvironments = false /// Whether events are being written to disk, which is what the application asked for until a write fails and the /// store gives up on persistence for the rest of the session. private var persistEvents = true @@ -169,19 +207,24 @@ final class EventStore: EventStoring { convenience init(directory: URL, capacity: Int, + byteCapacity: Int = LDConfig.Defaults.eventCapacityBytes, + maxAge: TimeInterval = LDConfig.Defaults.eventMaxAge, persistEvents: Bool = true, processName: String = ProcessInfo.processInfo.processName, logger: OSLog, readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }, writeLog: @escaping (EventLogFile, Data) -> Int32? = { $0.writeAll($1) }) { - self.init(locatingDirectory: { directory }, capacity: capacity, persistEvents: persistEvents, processName: processName, - logger: logger, readFile: readFile, writeLog: writeLog) + self.init(locatingDirectory: { directory }, capacity: capacity, byteCapacity: byteCapacity, maxAge: maxAge, + persistEvents: persistEvents, processName: processName, logger: logger, readFile: readFile, + writeLog: writeLog) } /// - Parameter locatingDirectory: answers where the batches go. Asked once, and only if the store reaches for the /// disk, which one told not to persist events never does. init(locatingDirectory: @escaping () -> URL, capacity: Int, + byteCapacity: Int = LDConfig.Defaults.eventCapacityBytes, + maxAge: TimeInterval = LDConfig.Defaults.eventMaxAge, persistEvents: Bool = true, processName: String = ProcessInfo.processInfo.processName, logger: OSLog, @@ -190,6 +233,9 @@ final class EventStore: EventStoring { let ioLock = UnfairLock() self.ioLock = ioLock self.capacity = capacity + self.byteCapacity = byteCapacity > 0 ? byteCapacity : LDConfig.Defaults.eventCapacityBytes + self.eventByteCapacity = self.byteCapacity - self.byteCapacity / 5 + self.maxAge = maxAge > 0 ? maxAge : LDConfig.Defaults.eventMaxAge self.logger = logger // An application that has not asked for persistence gets the same store running the same way it runs once a // write has failed: events are held, delivered, and lost only if the process dies. It gets no directory at @@ -216,6 +262,18 @@ final class EventStore: EventStoring { reservedEvents + bufferedEventCount + writingEvents + committedEvents + max(0, closedEvents - sendingEvents) } + /// Every byte counted against the byte capacity. Only to be used while holding `bufferLock`. + /// + /// Events reserved while being encoded have no bytes yet; they are counted when they are staged. + private var countedBytes: Int { + bufferData.count + writingBytes + committedBytes + max(0, closedBytes - sendingBytes) + } + + var hasRoomForEventBytes: Bool { + // A store kept off the disk holds events the way the SDK did before persistence, bounded by count alone. + disk == nil || bufferLock.withLock { countedBytes < eventByteCapacity } + } + var isPersisting: Bool { bufferLock.withLock { persistEvents } } @@ -246,18 +304,28 @@ final class EventStore: EventStoring { return false } - return bufferLock.withLock { - guard bypassingCapacity || countedEvents < capacity - else { return false } - + // Framed before the lock is taken, so that the critical section is the append. + let frame = EventLogFormat.frame(for: encodedEvent) + // Only a summary bypasses capacity without a reservation, and only a summary may use the bytes held back. + let isSummary = bypassingCapacity && !usingReservation + let boundedByBytes = disk != nil + let staged = bufferLock.withLock { () -> Bool in // In the same critical section as the append, so the event is never counted twice or not at all. if usingReservation { reservedEvents = max(0, reservedEvents - 1) } - bufferData.append(EventLogFormat.frame(for: encodedEvent)) + guard bypassingCapacity || countedEvents < capacity, + !boundedByBytes || countedBytes + frame.count <= (isSummary ? byteCapacity : eventByteCapacity) + else { return false } + + bufferData.append(frame) bufferedEventCount += 1 return true } + if !staged { + os_log("%s dropping an event the store has no room for", log: logger, type: .debug, typeName(and: #function)) + } + return staged } func commit() { @@ -276,9 +344,11 @@ final class EventStore: EventStoring { writeStagedBytes() - let (events, stillBuffered) = bufferLock.withLock { (committedEvents, bufferedEventCount) } + let (committed, stillBuffered) = bufferLock.withLock { + (EventTally(events: committedEvents, bytes: committedBytes), bufferedEventCount) + } - let fromLog = events > 0 ? closeOpenLog(of: disk, holding: events) : nil + let fromLog = committed.events > 0 ? closeOpenLog(of: disk, holding: committed) : nil // Closed after the log, so the older events are delivered first. Both can hold events at once: a write that // fails after earlier ones landed gives up on persistence with those earlier events in the log and the failed // ones back in memory, and closing only the log would leave the rest to a delivery that reports success @@ -288,20 +358,27 @@ final class EventStore: EventStoring { } /// Requires `ioLock`. - private func closeOpenLog(of disk: EventDirectory, holding events: Int) -> EventBatch? { + private func closeOpenLog(of disk: EventDirectory, holding committed: EventTally) -> EventBatch? { ioLock.assertOwned() - switch disk.closeLog(holding: events) { + // The file is the frames behind its header. A listing counts it from the file itself. + let closed = EventTally(events: committed.events, bytes: committed.bytes + EventLogFormat.fileHeader.count) + switch disk.closeLog(holding: closed) { case .failed(let logSurvives): // Where the log went with its events, they have to stop counting against capacity or the store would // refuse events for the rest of the session. if !logSurvives { - bufferLock.withLock { committedEvents = 0 } + bufferLock.withLock { + committedEvents = 0 + committedBytes = 0 + } } return nil case .closed(let batch): bufferLock.withLock { committedEvents = 0 - closedEvents += events + committedBytes = 0 + closedEvents += closed.events + closedBytes += closed.bytes } return batch } @@ -313,8 +390,11 @@ final class EventStore: EventStoring { // Listed in one critical section where there is a disk, so that the files and what is held cannot be counted // either side of a close that moves events between them. guard let disk - else { return heldBatchesAndRecount(after: []) } - return ioLock.withLock { heldBatchesAndRecount(after: disk.listBatches()) } + else { return heldBatchesAndRecount(after: [], storedBytes: 0) } + return ioLock.withLock { + let listed = disk.listBatches() + return heldBatchesAndRecount(after: listed, storedBytes: disk.storedBytes) + } } /// Adds the batches the filesystem would not take, after the files: anything held was closed by this run and so is @@ -322,11 +402,14 @@ final class EventStore: EventStoring { /// batches another process closed or delivered come to be counted or not. /// /// Requires `ioLock` where there is a disk, so that the count is of the listing as it stands. - private func heldBatchesAndRecount(after onDisk: [EventBatch]) -> [EventBatch] { + /// + /// - Parameter storedBytes: the bytes in the batches listed from the disk. + private func heldBatchesAndRecount(after onDisk: [EventBatch], storedBytes: Int) -> [EventBatch] { heldLock.withLock { let batches = onDisk + inMemoryBatches.map { EventBatch(payloadId: $0.payloadId, eventCount: $0.eventCount) } bufferLock.withLock { closedEvents = batches.reduce(0) { $0 + $1.eventCount } + closedBytes = inMemoryBatches.reduce(storedBytes) { $0 + $1.body.count } // One no longer listed is no longer counted, so it cannot be left out of the count either. let listed = Set(batches.map { $0.payloadId }) sendingBatches.keys.filter { !listed.contains($0) }.forEach(forgetSending) @@ -357,6 +440,7 @@ final class EventStore: EventStoring { let held = inMemoryBatches.remove(at: index) bufferLock.withLock { closedEvents -= held.eventCount + closedBytes -= held.body.count forgetSending(batch.payloadId) } return true @@ -372,7 +456,8 @@ final class EventStore: EventStoring { ioLock.withLock { let counted = disk.remove(batch.payloadId, holding: batch.eventCount) bufferLock.withLock { - closedEvents -= counted + closedEvents -= counted.events + closedBytes -= counted.bytes forgetSending(batch.payloadId) } } @@ -385,11 +470,38 @@ final class EventStore: EventStoring { } /// Counts events a recovery closed off into batches of their own. Requires `ioLock`. - private func countRecovered(_ events: Int) { + private func countRecovered(_ recovered: EventTally) { ioLock.assertOwned() - guard events > 0 + guard recovered.events > 0 else { return } - bufferLock.withLock { closedEvents += events } + bufferLock.withLock { + closedEvents += recovered.events + closedBytes += recovered.bytes + } + } + + func discardExpired() -> Int { + guard let disk + else { return 0 } + let cutoff = Date().addingTimeInterval(-maxAge) + + let (discarded, directory) = ioLock.withLock { () -> (Int, URL?) in + let sending = bufferLock.withLock { Set(sendingBatches.keys) } + let (events, counted) = disk.discard(lastWrittenBefore: cutoff, sparing: sending) + // In the same critical section as the delete, as for any other path that loses events without delivering + // them, so the store cannot go on counting what is gone. + bufferLock.withLock { + closedEvents -= counted.events + closedBytes -= counted.bytes + } + defer { hasDiscardedAbandonedEnvironments = true } + return (events, hasDiscardedAbandonedEnvironments ? nil : disk.url) + } + + // Outside `ioLock`: none of it is this store's directory, and a commit point should not wait on it. + guard let directory + else { return discarded } + return discarded + EventDirectory.discardAbandonedEnvironments(besides: directory, lastWrittenBefore: cutoff) } deinit { @@ -405,11 +517,13 @@ final class EventStore: EventStoring { extension EventStore { func startedSending(_ batch: EventBatch) { + let bytes = bytes(of: batch.payloadId) bufferLock.withLock { guard sendingBatches[batch.payloadId] == nil else { return } - sendingBatches[batch.payloadId] = batch.eventCount + sendingBatches[batch.payloadId] = EventTally(events: batch.eventCount, bytes: bytes) sendingEvents += batch.eventCount + sendingBytes += bytes } } @@ -417,10 +531,21 @@ extension EventStore { bufferLock.withLock { forgetSending(batch.payloadId) } } + /// The bytes a closed batch is counted as, held or on the disk. + private func bytes(of payloadId: String) -> Int { + if let held = heldLock.withLock({ inMemoryBatches.first { $0.payloadId == payloadId } }) { + return held.body.count + } + guard let disk + else { return 0 } + return ioLock.withLock { disk.bytes(of: payloadId) } + } + /// Requires `bufferLock`. fileprivate func forgetSending(_ payloadId: String) { - if let events = sendingBatches.removeValue(forKey: payloadId) { - sendingEvents -= events + if let sending = sendingBatches.removeValue(forKey: payloadId) { + sendingEvents -= sending.events + sendingBytes -= sending.bytes } } } @@ -440,6 +565,7 @@ extension EventStore { bufferData = Data() bufferedEventCount = 0 writingEvents += staged.events + writingBytes += staged.bytes.count return staged } guard let taken @@ -448,13 +574,16 @@ extension EventStore { if append(taken.bytes) { bufferLock.withLock { writingEvents -= taken.events + writingBytes -= taken.bytes.count committedEvents += taken.events + committedBytes += taken.bytes.count } } else { // Put back, to be delivered from memory rather than lost. Ahead of anything staged while the write was in // flight, so the events keep the order they were recorded in. bufferLock.withLock { writingEvents -= taken.events + writingBytes -= taken.bytes.count bufferData = taken.bytes + bufferData bufferedEventCount += taken.events } @@ -512,6 +641,7 @@ private extension EventStore { bufferData = Data() bufferedEventCount = 0 writingEvents += staged.events + writingBytes += staged.frames.count return staged } @@ -522,7 +652,10 @@ private extension EventStore { log.append(frames) guard let body = EventLogFormat.assembleBody(from: log) else { - bufferLock.withLock { writingEvents -= events } + bufferLock.withLock { + writingEvents -= events + writingBytes -= frames.count + } return nil } @@ -530,10 +663,13 @@ private extension EventStore { heldLock.withLock { inMemoryBatches.append(HeldBatch(payloadId: payloadId, body: body, eventCount: events)) // Counted where they landed in the same critical section that stops counting them on the way, so a - // capacity check never sees them in both places or in neither. + // capacity check never sees them in both places or in neither. A held batch is counted as its body, which + // is what it keeps. bufferLock.withLock { writingEvents -= events + writingBytes -= frames.count closedEvents += events + closedBytes += body.count } } diff --git a/LaunchDarkly/LaunchDarklyTests/Models/LDConfigSpec.swift b/LaunchDarkly/LaunchDarklyTests/Models/LDConfigSpec.swift index 7f86d0cc7..0e18e078c 100644 --- a/LaunchDarkly/LaunchDarklyTests/Models/LDConfigSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/Models/LDConfigSpec.swift @@ -8,6 +8,8 @@ final class LDConfigSpec: XCTestCase { fileprivate static let alternateMockUrl = URL(string: "https://dummy.alternate.com")! fileprivate static let eventCapacity = 10 fileprivate static let eventPersistence = EventPersistence.immediate + fileprivate static let eventCapacityBytes = 4096 + fileprivate static let eventMaxAge: TimeInterval = 60 fileprivate static let connectionTimeout: TimeInterval = 0.01 fileprivate static let eventFlushInterval: TimeInterval = 0.01 fileprivate static let flagPollingInterval: TimeInterval = 0.01 @@ -39,6 +41,8 @@ final class LDConfigSpec: XCTestCase { ("stream URL", Constants.alternateMockUrl, { c, v in c.streamUrl = v as! URL }), ("event capacity", Constants.eventCapacity, { c, v in c.eventCapacity = v as! Int }), ("event persistence", Constants.eventPersistence, { c, v in c.eventPersistence = v as! EventPersistence }), + ("event capacity bytes", Constants.eventCapacityBytes, { c, v in c.eventCapacityBytes = v as! Int }), + ("event max age", Constants.eventMaxAge, { c, v in c.eventMaxAge = v as! TimeInterval }), ("connection timeout", Constants.connectionTimeout, { c, v in c.connectionTimeout = v as! TimeInterval }), ("event flush interval", Constants.eventFlushInterval, { c, v in c.eventFlushInterval = v as! TimeInterval }), ("poll interval", Constants.flagPollingInterval, { c, v in c.flagPollingInterval = v as! TimeInterval }), @@ -69,6 +73,8 @@ final class LDConfigSpec: XCTestCase { XCTAssertEqual(config.streamUrl, LDConfig.Defaults.streamUrl) XCTAssertEqual(config.eventCapacity, LDConfig.Defaults.eventCapacity) XCTAssertEqual(config.eventPersistence, LDConfig.Defaults.eventPersistence) + XCTAssertEqual(config.eventCapacityBytes, 20 * 1024 * 1024) + XCTAssertEqual(config.eventMaxAge, 7 * 24 * 60 * 60) XCTAssertEqual(config.connectionTimeout, LDConfig.Defaults.connectionTimeout) XCTAssertEqual(config.eventFlushInterval, LDConfig.Defaults.eventFlushInterval) XCTAssertEqual(config.flagPollingInterval, LDConfig.Defaults.flagPollingInterval) @@ -100,6 +106,8 @@ final class LDConfigSpec: XCTestCase { XCTAssertEqual(config.eventsUrl, Constants.alternateMockUrl, "\(os)") XCTAssertEqual(config.streamUrl, Constants.alternateMockUrl, "\(os)") XCTAssertEqual(config.eventCapacity, Constants.eventCapacity, "\(os)") + XCTAssertEqual(config.eventCapacityBytes, Constants.eventCapacityBytes, "\(os)") + XCTAssertEqual(config.eventMaxAge, Constants.eventMaxAge, "\(os)") XCTAssertEqual(config.connectionTimeout, Constants.connectionTimeout, "\(os)") XCTAssertEqual(config.eventFlushInterval, Constants.eventFlushInterval, "\(os)") XCTAssertEqual(config.flagPollingInterval, Constants.flagPollingInterval, "\(os)") diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index f00500a22..54479e4c5 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -1419,6 +1419,61 @@ extension DirectEventProcessorSpec { nextRun.isOnline = false } + itPersisting("discards what a previous run left longer ago than the maximum age, and still answers a flush with success") { + testContext = TestContext() + testContext.recordEvents(2) + testContext.eventProcessor.commitRecordedEvents() + expect(testContext.store.closeBatch()).toNot(beNil()) + let directory = testContext.store.directory! + let longAgo = Date().addingTimeInterval(-8 * 24 * 60 * 60) + for file in try FileManager.default.contentsOfDirectory(at: directory, includingPropertiesForKeys: nil) { + try FileManager.default.setAttributes([.modificationDate: longAgo], ofItemAtPath: file.path) + } + + let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: EventStore.sharing(directory)) + nextRun.isOnline = true + var delivered: Bool? + waitUntil(timeout: Constants.deliveryTimeout) { done in + nextRun.flushReportingOutcome { result in + delivered = result + done() + } + } + + // The caller's own events all got out; the ones discarded were no one's to wait on. + expect(delivered) == true + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + expect(testContext.diagnosticCache.incrementDroppedEventCountCallCount) == 2 + nextRun.isOnline = false + } + + itPersisting("refuses an event without encoding it once the stored bytes leave no room for full events") { + let store = EventStore.temporary(capacity: 100, byteCapacity: 1000) + // One frame of 800 bytes, which is all the room full events have. + expect(store.stage(Data(repeating: UInt8(ascii: "x"), count: 800 - EventLogFormat.frameHeaderSize))) == true + testContext = TestContext(store: store, eventCapacity: 100) + + let refused = EncodingGate(key: "refused", context: testContext.context) + refused.proceed.signal() + testContext.eventProcessor.record(refused) + + expect(refused.started.wait(timeout: .now() + 0.2)) == .timedOut + expect(testContext.diagnosticCache.incrementDroppedEventCountCallCount) == 1 + } + + itPersisting("reports a flush as failed when an accepted event turns out too large for the bytes left") { + let store = EventStore.temporary(capacity: 100, byteCapacity: 1000) + expect(store.stage(Data(repeating: UInt8(ascii: "x"), count: 700 - EventLogFormat.frameHeaderSize))) == true + testContext = TestContext(store: store, eventCapacity: 100) + testContext.eventProcessor.isOnline = true + + testContext.eventProcessor.record(CustomEvent(key: "large", context: testContext.context, + data: LDValue.string(String(repeating: "y", count: 300)))) + + expect(testContext.flushOutcome()) == false + expect(testContext.serviceMock.publishedEventPayloads.flatMap(customKeys)).toNot(contain("large")) + } + it("does not bring a delivery forward when a previous run left nothing behind") { testContext = TestContext() @@ -2451,6 +2506,7 @@ private final class ClosingHookStore: EventStoring { } var pendingEventCount: Int { inner.pendingEventCount } + var hasRoomForEventBytes: Bool { inner.hasRoomForEventBytes } var isPersisting: Bool { inner.isPersisting } func stage(_ encodedEvent: Data, bypassingCapacity: Bool) -> Bool { inner.stage(encodedEvent, bypassingCapacity: bypassingCapacity) } func reserve(_ events: Int) { inner.reserve(events) } @@ -2463,6 +2519,7 @@ private final class ClosingHookStore: EventStoring { func startedSending(_ batch: EventBatch) { inner.startedSending(batch) } func stoppedSending(_ batch: EventBatch) { inner.stoppedSending(batch) } func recoverInterruptedLog() { inner.recoverInterruptedLog() } + func discardExpired() -> Int { inner.discardExpired() } func closeBatch() -> EventBatch? { let before = beforeClosing @@ -2527,6 +2584,7 @@ private final class SlowListingStore: EventStoring { } var pendingEventCount: Int { inner.pendingEventCount } + var hasRoomForEventBytes: Bool { inner.hasRoomForEventBytes } var isPersisting: Bool { inner.isPersisting } func stage(_ encodedEvent: Data, bypassingCapacity: Bool) -> Bool { inner.stage(encodedEvent, bypassingCapacity: bypassingCapacity) } func reserve(_ events: Int) { inner.reserve(events) } @@ -2539,6 +2597,7 @@ private final class SlowListingStore: EventStoring { func startedSending(_ batch: EventBatch) { inner.startedSending(batch) } func stoppedSending(_ batch: EventBatch) { inner.stoppedSending(batch) } func recoverInterruptedLog() { inner.recoverInterruptedLog() } + func discardExpired() -> Int { inner.discardExpired() } /// Every listing but the first is held, the first being the one the reporter makes as it starts up. func pendingBatches() -> [EventBatch] { diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 9f1c5bb69..995fbc185 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -10,6 +10,8 @@ final class EventStoreSpec: QuickSpec { corruptionSpec() batchSpec() capacitySpec() + byteCapacitySpec() + expirySpec() fallbackSpec() optOutSpec() multiProcessSpec() @@ -315,6 +317,14 @@ final class EventStoreSpec: QuickSpec { Data("{\"kind\":\"custom\",\"key\":\"\(key)\"}".utf8) } + /// An event whose frame is exactly `frameBytes` long, padded out in a `data` field. + private static func payload(framedIn frameBytes: Int, key: String) -> Data { + let bare = "{\"kind\":\"custom\",\"key\":\"\(key)\",\"data\":\"\"}" + let padding = frameBytes - EventLogFormat.frameHeaderSize - bare.utf8.count + precondition(padding >= 0, "\(key) does not fit in \(frameBytes) bytes") + return Data("{\"kind\":\"custom\",\"key\":\"\(key)\",\"data\":\"\(String(repeating: "x", count: padding))\"}".utf8) + } + private static func keys(of store: EventStore) -> [String] { store.pendingEventPayloads().compactMap { guard case .object(let fields) = try? JSONDecoder().decode(LDValue.self, from: $0), @@ -979,6 +989,237 @@ final class EventStoreSpec: QuickSpec { } } + /// A byte capacity of 1000 lets full events take the store to 800 bytes and summaries to 1000. Every event here is + /// framed in exactly 100 bytes, so eight full events fill it and two summaries fill the rest. + private func byteCapacitySpec() { + describe("byte capacity") { + var store: EventStore! + afterEach { + store.deleteEverything() + } + + it("refuses full events past what it holds back for summaries") { + store = EventStore.temporary(capacity: 100, byteCapacity: 1000) + + for index in 1...8 { + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-\(index)"))) == true + } + expect(store.hasRoomForEventBytes) == false + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-9"))) == false + expect(store.pendingEventCount) == 8 + } + + it("takes summaries into what it holds back, and refuses them past the byte capacity") { + store = EventStore.temporary(capacity: 100, byteCapacity: 1000) + for index in 1...8 { + _ = store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-\(index)")) + } + + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "summary-1"), bypassingCapacity: true)) == true + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "summary-2"), bypassingCapacity: true)) == true + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "summary-3"), bypassingCapacity: true)) == false + expect(store.pendingEventCount) == 10 + } + + it("refuses a reserved event that turns out not to fit, and gives up its reservation") { + store = EventStore.temporary(capacity: 100, byteCapacity: 1000) + for index in 1...7 { + _ = store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-\(index)")) + } + store.reserve(1) + + expect(store.stageReserved(EventStoreSpec.payload(framedIn: 200, key: "too-big"))) == false + expect(store.pendingEventCount) == 7 + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "fits"))) == true + } + + it("makes room again once a batch is delivered") { + store = EventStore.temporary(capacity: 100, byteCapacity: 1000) + for index in 1...8 { + _ = store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-\(index)")) + } + let batch = store.closeBatch()! + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "while-held"))) == false + + store.startedSending(batch) + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "while-sending"))) == true + store.remove(batch) + expect(store.hasRoomForEventBytes) == true + } + + it("bounds the batches it holds in memory once it has given up on the disk") { + store = EventStore.temporary(capacity: 100, byteCapacity: 1000, writeLog: { _, _ in ENOSPC }) + for index in 1...8 { + _ = store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-\(index)")) + } + store.commit() + expect(store.isPersisting) == false + expect(store.closeBatch()).toNot(beNil()) + + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "past-capacity"))) == false + } + + it("is not bounded by bytes where it keeps events off the disk") { + store = EventStore.temporary(capacity: 100, byteCapacity: 1000, persistEvents: false) + for index in 1...20 { + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-\(index)"))) == true + } + expect(store.hasRoomForEventBytes) == true + expect(store.pendingEventCount) == 20 + } + + it("counts committed bytes as well as staged ones") { + store = EventStore.temporary(capacity: 100, byteCapacity: 1000) + for index in 1...8 { + _ = store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-\(index)")) + } + store.commit() + + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "after-commit"))) == false + } + + it("counts the bytes of a batch another process closed from the listing, without reading it") { + store = EventStore.temporary() + for index in 1...8 { + _ = store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-\(index)")) + } + expect(store.closeBatch()).toNot(beNil()) + + var reads = 0 + let other = EventStore.sharing(store.directory!, byteCapacity: 1000, processName: "other") { url in + reads += 1 + return try Data(contentsOf: url) + } + expect(other.pendingBatches().count) == 1 + + expect(reads) == 0 + expect(other.hasRoomForEventBytes) == false + expect(other.stage(EventStoreSpec.payload(framedIn: 100, key: "other"))) == false + } + } + } + + /// Each store here is the current environment of an application `app` under a root of the test's own, since the + /// first expiry also looks at every other environment beside it. + private func expirySpec() { + describe("expiry") { + var root: URL! + var store: EventStore! + beforeEach { + root = EventStore.temporaryDirectory() + store = EventStore.sharing(root.appendingPathComponent("app.current", isDirectory: true)) + } + afterEach { + try? FileManager.default.removeItem(at: root) + } + + it("discards a batch last written longer ago than the maximum age, and stops counting it") { + _ = store.stage(EventStoreSpec.payload("old-1")) + _ = store.stage(EventStoreSpec.payload("old-2")) + _ = store.closeBatch() + try EventStoreSpec.age(try EventStoreSpec.onlyBatchFile(in: store), byDays: 8) + + expect(store.discardExpired()) == 2 + + expect(store.pendingBatches()).to(beEmpty()) + expect(store.pendingEventCount) == 0 + expect(store.stage(EventStoreSpec.payload("new"))) == true + } + + it("keeps a batch younger than the maximum age") { + _ = store.stage(EventStoreSpec.payload("recent")) + _ = store.closeBatch() + try EventStoreSpec.age(try EventStoreSpec.onlyBatchFile(in: store), byDays: 6) + + expect(store.discardExpired()) == 0 + expect(store.pendingBatches().count) == 1 + } + + it("keeps a batch whose request is out, however old") { + _ = store.stage(EventStoreSpec.payload("sending")) + let batch = store.closeBatch()! + try EventStoreSpec.age(try EventStoreSpec.onlyBatchFile(in: store), byDays: 30) + store.startedSending(batch) + + expect(store.discardExpired()) == 0 + expect(try store.body(of: batch)).toNot(beNil()) + } + + it("discards another process's log nobody has written to for the maximum age, and leaves a younger one") { + let gone = EventStore.sharing(store.directory!, processName: "gone") + _ = gone.stage(EventStoreSpec.payload("gone")) + gone.commit() + let alive = EventStore.sharing(store.directory!, processName: "alive") + _ = alive.stage(EventStoreSpec.payload("alive")) + alive.commit() + try EventStoreSpec.age(gone.openLogUrl!, byDays: 8) + try EventStoreSpec.age(alive.openLogUrl!, byDays: 1) + + expect(store.discardExpired()) == 1 + + expect(FileManager.default.fileExists(atPath: gone.openLogUrl!.path)) == false + expect(FileManager.default.fileExists(atPath: alive.openLogUrl!.path)) == true + } + + it("leaves its own open log alone, however long since it was written") { + _ = store.stage(EventStoreSpec.payload("own")) + store.commit() + try EventStoreSpec.age(store.openLogUrl!, byDays: 8) + + expect(store.discardExpired()) == 0 + expect(EventStoreSpec.keys(of: store)) == ["own"] + } + + it("removes the directories of this application's other environments once nothing has written to them for the maximum age") { + let abandoned = try EventStoreSpec.environment("app.abandoned", in: root, events: 2) + let inUse = try EventStoreSpec.environment("app.in-use", in: root, events: 1) + let otherApplication = try EventStoreSpec.environment("other.abandoned", in: root, events: 1) + try EventStoreSpec.age(contentsOf: abandoned, byDays: 8) + try EventStoreSpec.age(contentsOf: inUse, byDays: 1) + try EventStoreSpec.age(contentsOf: otherApplication, byDays: 30) + + expect(store.discardExpired()) == 2 + // Once per process: the next call has nothing else to look at. + expect(store.discardExpired()) == 0 + + expect(FileManager.default.fileExists(atPath: abandoned.path)) == false + expect(try FileManager.default.contentsOfDirectory(atPath: inUse.path)).toNot(beEmpty()) + expect(try FileManager.default.contentsOfDirectory(atPath: otherApplication.path)).toNot(beEmpty()) + } + + it("does nothing where the application did not ask for persistence") { + let abandoned = try EventStoreSpec.environment("app.abandoned", in: root, events: 1) + try EventStoreSpec.age(contentsOf: abandoned, byDays: 8) + store = EventStore.sharing(root.appendingPathComponent("app.current", isDirectory: true), persistEvents: false) + + expect(store.discardExpired()) == 0 + expect(try FileManager.default.contentsOfDirectory(atPath: abandoned.path)).toNot(beEmpty()) + } + } + } + + /// An environment directory under `root` holding one closed batch of `events` events. + private static func environment(_ name: String, in root: URL, events: Int) throws -> URL { + let directory = root.appendingPathComponent(name, isDirectory: true) + let writer = EventStore.sharing(directory) + for index in 0.. Data = { try Data(contentsOf: $0) }, writeLog: @escaping (EventLogFile, Data) -> Int32? = { $0.writeAll($1) } @@ -63,6 +64,7 @@ extension EventStore { let directory = temporaryDirectory() return unlocked(EventStore(directory: directory, capacity: capacity, + byteCapacity: byteCapacity, persistEvents: persistEvents, logger: .disabled, readFile: readFile, @@ -84,11 +86,13 @@ extension EventStore { /// Unlocked on a Mac too. Tests open it while the first store is still alive, and a locked log would rightly be /// left to the live instance holding it; the locking itself has tests of its own. static func sharing(_ directory: URL, + byteCapacity: Int = LDConfig.Defaults.eventCapacityBytes, persistEvents: Bool = true, processName: String = ProcessInfo.processInfo.processName, readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }) -> EventStore { unlocked(EventStore(directory: directory, capacity: 100, + byteCapacity: byteCapacity, persistEvents: persistEvents, processName: processName, logger: .disabled, From 4be95a8962ba257b64d3cb4609df47de4fe02d71 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 20 Oct 2026 17:41:24 -0700 Subject: [PATCH 064/158] Checksum every frame of the event log Log format version 2 stores a CRC-32 (IEEE) of each frame's type, size and payload after the size. The reader stops at the first frame that fails it, as it does at a torn tail, and still reads version 1 logs. A batch whose first frame is damaged is lost rather than empty. Co-authored-by: Cursor --- .../Extensions/Data+Compression.swift | 12 ++ .../ServiceObjects/EventDirectory.swift | 4 +- .../ServiceObjects/EventLogFormat.swift | 111 ++++++++++++------ .../ServiceObjects/EventStoreSpec.swift | 94 +++++++++++++++ 4 files changed, 186 insertions(+), 35 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/Extensions/Data+Compression.swift b/LaunchDarkly/LaunchDarkly/Extensions/Data+Compression.swift index 7ab1080ad..7c4f0472a 100644 --- a/LaunchDarkly/LaunchDarkly/Extensions/Data+Compression.swift +++ b/LaunchDarkly/LaunchDarkly/Extensions/Data+Compression.swift @@ -284,6 +284,18 @@ struct Crc32: CustomStringConvertible { } } + /// Advance the current checksum with bytes already in memory, without wrapping them in `Data` first. + /// - parameter bytes: bytes to advance the checksum + mutating func advance(withBytes bytes: UnsafeRawBufferPointer) { + guard let base = bytes.baseAddress?.assumingMemoryBound(to: UInt8.self) + else { return } + if let fastCrc32 = Crc32.zLibCrc32 { + checksum = fastCrc32(checksum, base, UInt32(bytes.count)) + } else { + checksum = slowCrc32(start: checksum, data: Data(bytes)) + } + } + /// Formatted checksum. var description: String { return String(format: "%08x", checksum) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index 42ae3795b..ebb185295 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -303,8 +303,8 @@ final class EventDirectory { if let body = EventLogFormat.assembleBody(from: file) { return body } - // A log this version reads that holds no events has nothing to lose. - guard EventLogFormat.eventCount(in: file) == 0 + // A log this version reads that holds no events has nothing to lose, unless its first frame is damaged. + guard EventLogFormat.eventCount(in: file) == 0, !EventLogFormat.isDamaged(file) else { throw UnreadableBatch(payloadId: batch.payloadId) } return nil } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift index e9e0af536..ca3efdc1e 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift @@ -4,23 +4,28 @@ import Foundation /// /// A log opens with a magic and a format version, and holds a sequence of frames: /// -/// +- 4 bytes -+- 2 bytes -+ +- 2 bytes -+- 4 bytes -+- n bytes -+ -/// | LDEV | version | ... | type | length (n) | payload | -/// +-----------+-----------+ +-----------+-------------+-----------+ +/// +- 4 bytes -+- 2 bytes -+ +- 2 bytes -+- 4 bytes -+- 4 bytes -+- n bytes -+ +/// | LDEV | version | ... | type | size (n) | CRC-32 | payload | +/// +-----------+-----------+ +-----------+-----------+-----------+-----------+ /// -/// The length prefix is what makes a log recoverable: a reader can tell a frame the writer finished from one it did not, -/// without having to parse the event to find where it ends. The version is what makes an upgrade safe -- a log this -/// version does not understand is discarded rather than misread -- and the frame type leaves room for a later version to -/// write something new into a log this one still reads. +/// The size prefix is what makes a log recoverable: a reader can tell a frame the writer finished from one it did not, +/// without having to parse the event to find where it ends. The checksum, over the type, the size and the payload, is +/// what tells a frame the writer finished from one the storage has since damaged; a reader treats either the same way. +/// The version is what makes an upgrade safe -- a log this version does not understand is discarded rather than +/// misread -- and the frame type leaves room for a later version to write something new into a log this one still +/// reads. enum EventLogFormat { /// Bump this whenever the framing or the meaning of a frame changes. - static let version: UInt16 = 1 + static let version: UInt16 = 2 + /// The version before frames carried a checksum, which this one still reads. + static let versionWithoutChecksum: UInt16 = 1 static let magic = Data("LDEV".utf8) static let fileHeaderSize = 6 - static let frameHeaderSize = 6 + static let frameHeaderSize = 10 + static let frameHeaderSizeWithoutChecksum = 6 /// The only frame type written today; a reader skips a frame whose type it does not know. static let eventFrame: UInt16 = 1 - /// A ceiling on a single frame, so a corrupt length cannot make recovery allocate wildly. + /// A ceiling on a single frame, so a corrupt size cannot make recovery allocate wildly. static let maxFrameSize = 8 * 1024 * 1024 static var fileHeader: Data { @@ -31,34 +36,68 @@ enum EventLogFormat { static func frame(for encodedEvent: Data) -> Data { var frame = Data(capacity: frameHeaderSize + encodedEvent.count) - frame.append(bigEndian(eventFrame)) - frame.append(bigEndian(UInt32(encodedEvent.count))) + withUnsafeBytes(of: eventFrame.bigEndian) { frame.append(contentsOf: $0) } + withUnsafeBytes(of: UInt32(encodedEvent.count).bigEndian) { frame.append(contentsOf: $0) } + var checksum = Crc32() + frame.withUnsafeBytes { checksum.advance(withBytes: $0) } + encodedEvent.withUnsafeBytes { checksum.advance(withBytes: $0) } + withUnsafeBytes(of: checksum.checksum.bigEndian) { frame.append(contentsOf: $0) } frame.append(encodedEvent) return frame } - /// Walks the frames of a log, stopping at the first one that is not wholly there. + /// What walking a log's frames found. + struct FrameWalk { + /// False when the walk stopped at a frame that is torn or fails its checksum, rather than at the end of the + /// file. + let reachedEnd: Bool + } + + /// Walks the frames of a log, stopping at the first one that is not wholly there or fails its checksum. /// /// A process that died partway through a write leaves a torn frame at the end. Everything before it is intact, so - /// recovery keeps that and drops only the tail. Returns false when the file is not a log this version reads. - static func forEachFrame(in file: Data, _ visit: (UInt16, Data) -> Void) -> Bool { - guard file.count >= fileHeaderSize, - file.prefix(magic.count) == magic, - readUInt16(file, at: magic.count) == version - else { return false } - - var cursor = fileHeaderSize - while cursor + frameHeaderSize <= file.count { - let type = readUInt16(file, at: cursor) - let length = Int(readUInt32(file, at: cursor + 2)) - let start = cursor + frameHeaderSize - guard length > 0, length <= maxFrameSize, start + length <= file.count - else { break } - - visit(type, file.subdata(in: start..<(start + length))) - cursor = start + length + /// recovery keeps that and drops only the tail. A frame the storage damaged afterwards ends the walk the same way, + /// because its size can no longer be trusted to say where the next frame starts. Returns nil when the file is not a + /// log this version reads. + @discardableResult + static func forEachFrame(in file: Data, _ visit: (UInt16, Data) -> Void) -> FrameWalk? { + guard file.count >= fileHeaderSize, file.prefix(magic.count) == magic + else { return nil } + let isChecksummed: Bool + switch readUInt16(file, at: magic.count) { + case version: isChecksummed = true + case versionWithoutChecksum: isChecksummed = false + default: return nil + } + let headerSize = isChecksummed ? frameHeaderSize : frameHeaderSizeWithoutChecksum + + return file.withUnsafeBytes { bytes in + var cursor = fileHeaderSize + while cursor + headerSize <= bytes.count { + let type = UInt16(bytes[cursor]) << 8 | UInt16(bytes[cursor + 1]) + let length = Int(readUInt32(bytes, at: cursor + 2)) + let start = cursor + headerSize + guard length > 0, length <= maxFrameSize, start + length <= bytes.count + else { break } + + let payload = UnsafeRawBufferPointer(rebasing: bytes[start..<(start + length)]) + if isChecksummed { + var checksum = Crc32() + checksum.advance(withBytes: UnsafeRawBufferPointer(rebasing: bytes[cursor..<(cursor + 6)])) + checksum.advance(withBytes: payload) + guard checksum.checksum == readUInt32(bytes, at: cursor + 6) + else { break } + } + + visit(type, Data(payload)) + cursor = start + length + } + return FrameWalk(reachedEnd: cursor == bytes.count) } - return true + } + + private static func readUInt32(_ bytes: UnsafeRawBufferPointer, at offset: Int) -> UInt32 { + UInt32(bytes[offset]) << 24 | UInt32(bytes[offset + 1]) << 16 | UInt32(bytes[offset + 2]) << 8 | UInt32(bytes[offset + 3]) } /// How many events a log holds, or nil where it is not one this version reads. @@ -68,11 +107,17 @@ enum EventLogFormat { if type == eventFrame { events += 1 } - }) + }) != nil else { return nil } return events } + /// Whether a log holds frames that cannot be read back: torn, or failing their checksum. False for a log this + /// version does not read at all. + static func isDamaged(_ file: Data) -> Bool { + forEachFrame(in: file) { _, _ in }?.reachedEnd == false + } + /// Assembles the JSON array LaunchDarkly expects out of the frames, without parsing the events themselves: they were /// serialized on the way in and are shipped exactly as they were recorded. static func assembleBody(from file: Data) -> Data? { @@ -88,7 +133,7 @@ enum EventLogFormat { isFirst = false } - guard readable, !isFirst + guard readable != nil, !isFirst else { return nil } body.append(UInt8(ascii: "]")) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 995fbc185..1146db088 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -317,6 +317,17 @@ final class EventStoreSpec: QuickSpec { Data("{\"kind\":\"custom\",\"key\":\"\(key)\"}".utf8) } + /// Where each frame of a log written by this version starts. + private static func frameOffsets(in log: Data) -> [Int] { + var offsets: [Int] = [] + var cursor = EventLogFormat.fileHeaderSize + while cursor + EventLogFormat.frameHeaderSize <= log.count { + offsets.append(cursor) + cursor += EventLogFormat.frameHeaderSize + Int(EventLogFormat.readUInt32(log, at: cursor + 2)) + } + return offsets + } + /// An event whose frame is exactly `frameBytes` long, padded out in a `data` field. private static func payload(framedIn frameBytes: Int, key: String) -> Data { let bare = "{\"kind\":\"custom\",\"key\":\"\(key)\",\"data\":\"\"}" @@ -478,6 +489,83 @@ final class EventStoreSpec: QuickSpec { expect(EventStoreSpec.keys(of: EventStoreSpec.reader(sharing: store))) == ["event-0", "event-1"] } + context("where the storage changed a frame after it was written") { + var log: URL! + var second: Int! + beforeEach { + store = EventStore.temporary() + for index in 0..<3 { + _ = store.stage(EventStoreSpec.payload("event-\(index)")) + } + store.commit() + log = store.openLogUrl! + second = EventStoreSpec.frameOffsets(in: try! Data(contentsOf: log))[1] + } + + it("keeps every event before a payload with a flipped bit") { + var damaged = try Data(contentsOf: log) + damaged[second + EventLogFormat.frameHeaderSize + 3] ^= 0x04 + try damaged.write(to: log) + + expect(EventStoreSpec.keys(of: EventStoreSpec.reader(sharing: store))) == ["event-0"] + } + + it("keeps every event before a zero-filled payload") { + var damaged = try Data(contentsOf: log) + let start = second + EventLogFormat.frameHeaderSize + let size = Int(EventLogFormat.readUInt32(damaged, at: second + 2)) + damaged.replaceSubrange(start..<(start + size), with: Data(count: size)) + try damaged.write(to: log) + + expect(EventStoreSpec.keys(of: EventStoreSpec.reader(sharing: store))) == ["event-0"] + } + + it("keeps every event before a size that still fits in the file") { + // A size that is merely wrong rather than impossible passes every bound a torn tail is caught by, + // and would have the reader start the next frame partway through this one's payload. + var damaged = try Data(contentsOf: log) + damaged[second + 5] -= 1 + try damaged.write(to: log) + + expect(EventStoreSpec.keys(of: EventStoreSpec.reader(sharing: store))) == ["event-0"] + } + } + + it("loses a batch whose first frame is damaged, rather than taking it for one with nothing to send") { + store = EventStore.temporary() + _ = store.stage(EventStoreSpec.payload("damaged")) + let batch = store.closeBatch()! + let file = try EventStoreSpec.onlyBatchFile(in: store) + var damaged = try Data(contentsOf: file) + damaged[EventLogFormat.fileHeaderSize + EventLogFormat.frameHeaderSize] ^= 0x01 + try damaged.write(to: file) + + expect { try store.body(of: batch) }.to(throwError(errorType: UnreadableBatch.self)) + } + + it("still reads a log written before frames carried a checksum") { + store = EventStore.temporary() + _ = store.stage(EventStoreSpec.payload("placeholder")) + _ = store.closeBatch() + let file = try EventStoreSpec.onlyBatchFile(in: store) + + var log = EventLogFormat.magic + log.append(EventLogFormat.bigEndian(EventLogFormat.versionWithoutChecksum)) + for key in ["first", "second"] { + let payload = EventStoreSpec.payload(key) + log.append(EventLogFormat.bigEndian(EventLogFormat.eventFrame)) + log.append(EventLogFormat.bigEndian(UInt32(payload.count))) + log.append(payload) + } + try log.write(to: file) + try EventStoreSpec.asUncountedName(file) + + let next = EventStoreSpec.reader(sharing: store) + let batches = next.pendingBatches() + expect(batches.map(\.eventCount)) == [2] + expect(EventStoreSpec.keys(ofBody: try next.body(of: batches[0]))) == ["first", "second"] + } + it("is kept for a later listing when a batch named without a count cannot be read") { store = EventStore.temporary() _ = store.stage(EventStoreSpec.payload("kept")) @@ -593,6 +681,12 @@ final class EventStoreSpec: QuickSpec { expect(body) == Data("[{\"kind\":\"custom\",\"key\":\"first\"},{\"kind\":\"custom\",\"key\":\"second\"}]".utf8) } + it("frames an event as the Android SDK does") { + // The checksum is zlib's CRC-32, so these bytes are what the Android SDK writes for the same event too. + let frame = EventLogFormat.frame(for: Data("{\"kind\":\"custom\",\"key\":\"k\"}".utf8)) + expect(frame.prefix(EventLogFormat.frameHeaderSize)) == Data([0x00, 0x01, 0x00, 0x00, 0x00, 0x1B, 0xCB, 0x2B, 0x5E, 0x6F]) + } + it("has nothing to close when nothing was recorded") { expect(store.closeBatch()).to(beNil()) } From a8362df7cb88985a07a228a1e3b8d73532f9e3c5 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 20 Oct 2026 17:44:36 -0700 Subject: [PATCH 065/158] Move the in-memory CRC advance out of the vendored compression file Crc32.advance(withBytes:) now lives in Data+CRC.swift, so Data+Compression.swift differs from DataCompression 3.9.0 only in the access of the members it needs. Co-authored-by: Cursor --- LaunchDarkly.xcodeproj/project.pbxproj | 10 ++++++++++ .../LaunchDarkly/Extensions/Data+CRC.swift | 15 +++++++++++++++ .../Extensions/Data+Compression.swift | 18 +++--------------- 3 files changed, 28 insertions(+), 15 deletions(-) create mode 100644 LaunchDarkly/LaunchDarkly/Extensions/Data+CRC.swift diff --git a/LaunchDarkly.xcodeproj/project.pbxproj b/LaunchDarkly.xcodeproj/project.pbxproj index e42847e36..82a32c4d2 100644 --- a/LaunchDarkly.xcodeproj/project.pbxproj +++ b/LaunchDarkly.xcodeproj/project.pbxproj @@ -324,6 +324,10 @@ A350387D2F4F5A380032BA9F /* Data+Compression.swift in Sources */ = {isa = PBXBuildFile; fileRef = A350387B2F4F5A380032BA9F /* Data+Compression.swift */; }; A350387E2F4F5A380032BA9F /* Data+Compression.swift in Sources */ = {isa = PBXBuildFile; fileRef = A350387B2F4F5A380032BA9F /* Data+Compression.swift */; }; A350387F2F4F5A380032BA9F /* Data+Compression.swift in Sources */ = {isa = PBXBuildFile; fileRef = A350387B2F4F5A380032BA9F /* Data+Compression.swift */; }; + A35038812F4F5A380032BA9F /* Data+CRC.swift in Sources */ = {isa = PBXBuildFile; fileRef = A35038802F4F5A380032BA9F /* Data+CRC.swift */; }; + A35038822F4F5A380032BA9F /* Data+CRC.swift in Sources */ = {isa = PBXBuildFile; fileRef = A35038802F4F5A380032BA9F /* Data+CRC.swift */; }; + A35038832F4F5A380032BA9F /* Data+CRC.swift in Sources */ = {isa = PBXBuildFile; fileRef = A35038802F4F5A380032BA9F /* Data+CRC.swift */; }; + A35038842F4F5A380032BA9F /* Data+CRC.swift in Sources */ = {isa = PBXBuildFile; fileRef = A35038802F4F5A380032BA9F /* Data+CRC.swift */; }; A35038812F4F965E0032BA9F /* LDClientIdentifyHookSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = A35038802F4F965E0032BA9F /* LDClientIdentifyHookSpec.swift */; }; A35038832F4F96680032BA9F /* LDClientEvaluationHookSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = A35038822F4F96680032BA9F /* LDClientEvaluationHookSpec.swift */; }; A35038852F4F96CA0032BA9F /* WatchOSEnvironmentReporterSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = A35038842F4F96CA0032BA9F /* WatchOSEnvironmentReporterSpec.swift */; }; @@ -657,6 +661,7 @@ A35038762F4F58660032BA9F /* LDClientIdentifyHook.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LDClientIdentifyHook.swift; sourceTree = ""; }; 86CB57159BC81898DE96A352 /* LDClientFlush.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LDClientFlush.swift; sourceTree = ""; }; A350387B2F4F5A380032BA9F /* Data+Compression.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Data+Compression.swift"; sourceTree = ""; }; + A35038802F4F5A380032BA9F /* Data+CRC.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Data+CRC.swift"; sourceTree = ""; }; A35038802F4F965E0032BA9F /* LDClientIdentifyHookSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LDClientIdentifyHookSpec.swift; sourceTree = ""; }; A35038822F4F96680032BA9F /* LDClientEvaluationHookSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LDClientEvaluationHookSpec.swift; sourceTree = ""; }; A35038842F4F96CA0032BA9F /* WatchOSEnvironmentReporterSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WatchOSEnvironmentReporterSpec.swift; sourceTree = ""; }; @@ -989,6 +994,7 @@ isa = PBXGroup; children = ( A350387B2F4F5A380032BA9F /* Data+Compression.swift */, + A35038802F4F5A380032BA9F /* Data+CRC.swift */, 83DDBEF51FA24A7E00E428B6 /* Data.swift */, 83DDBEFD1FA24F9600E428B6 /* Date.swift */, 831D2AAE2061AAA000B4AC3C /* Thread.swift */, @@ -1572,6 +1578,7 @@ A35038772F4F58660032BA9F /* LDClientIdentifyHook.swift in Sources */, 99686398CCB7515A18247662 /* LDClientFlush.swift in Sources */, A350387C2F4F5A380032BA9F /* Data+Compression.swift in Sources */, + A35038812F4F5A380032BA9F /* Data+CRC.swift in Sources */, A3BA7CF62BD05A280000DB28 /* EvaluationSeriesContext.swift in Sources */, A3470C3A2B7C1ACE00951CEE /* LDValueDecoder.swift in Sources */, C443A41223186A4F00145710 /* ConnectionModeChangeObserver.swift in Sources */, @@ -1692,6 +1699,7 @@ 831EF35A20655E730001C643 /* HTTPHeaders.swift in Sources */, 831EF35B20655E730001C643 /* DarklyService.swift in Sources */, A350387F2F4F5A380032BA9F /* Data+Compression.swift in Sources */, + A35038842F4F5A380032BA9F /* Data+CRC.swift in Sources */, 831EF35C20655E730001C643 /* HTTPURLResponse.swift in Sources */, C443A40723145FEE00145710 /* ConnectionInformationStore.swift in Sources */, 29FE129A280413D4008CC918 /* Util.swift in Sources */, @@ -1775,6 +1783,7 @@ A35038782F4F58660032BA9F /* LDClientIdentifyHook.swift in Sources */, B04B36074760CCA9F48E1851 /* LDClientFlush.swift in Sources */, A350387D2F4F5A380032BA9F /* Data+Compression.swift in Sources */, + A35038822F4F5A380032BA9F /* Data+CRC.swift in Sources */, A3BA7CF32BD05A280000DB28 /* EvaluationSeriesContext.swift in Sources */, A3470C372B7C1ACE00951CEE /* LDValueDecoder.swift in Sources */, C43C37E1236BA050003C1624 /* LDEvaluationDetail.swift in Sources */, @@ -1945,6 +1954,7 @@ A350387A2F4F58660032BA9F /* LDClientIdentifyHook.swift in Sources */, 43DC6AA5DB0BE9A7B697C10F /* LDClientFlush.swift in Sources */, A350387E2F4F5A380032BA9F /* Data+Compression.swift in Sources */, + A35038832F4F5A380032BA9F /* Data+CRC.swift in Sources */, A3BA7CF42BD05A280000DB28 /* EvaluationSeriesContext.swift in Sources */, A3470C382B7C1ACE00951CEE /* LDValueDecoder.swift in Sources */, C43C37E6238DF22B003C1624 /* LDEvaluationDetail.swift in Sources */, diff --git a/LaunchDarkly/LaunchDarkly/Extensions/Data+CRC.swift b/LaunchDarkly/LaunchDarkly/Extensions/Data+CRC.swift new file mode 100644 index 000000000..159d9a069 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/Extensions/Data+CRC.swift @@ -0,0 +1,15 @@ +import Foundation + +extension Crc32 { + /// Advance the current checksum with bytes already in memory, without wrapping them in `Data` first. + /// - parameter bytes: bytes to advance the checksum + mutating func advance(withBytes bytes: UnsafeRawBufferPointer) { + guard let base = bytes.baseAddress?.assumingMemoryBound(to: UInt8.self) + else { return } + if let fastCrc32 = Crc32.zLibCrc32 { + checksum = fastCrc32(checksum, base, UInt32(bytes.count)) + } else { + checksum = slowCrc32(start: checksum, data: Data(bytes)) + } + } +} diff --git a/LaunchDarkly/LaunchDarkly/Extensions/Data+Compression.swift b/LaunchDarkly/LaunchDarkly/Extensions/Data+Compression.swift index 7c4f0472a..b746d0f15 100644 --- a/LaunchDarkly/LaunchDarkly/Extensions/Data+Compression.swift +++ b/LaunchDarkly/LaunchDarkly/Extensions/Data+Compression.swift @@ -258,12 +258,12 @@ internal extension Data { /// Struct based type representing a Crc32 checksum. struct Crc32: CustomStringConvertible { - private static let zLibCrc32: ZLibCrc32FuncPtr? = loadCrc32fromZLib() + static let zLibCrc32: ZLibCrc32FuncPtr? = loadCrc32fromZLib() init() {} // C convention function pointer type matching the signature of `libz::crc32` - private typealias ZLibCrc32FuncPtr = @convention(c) ( + typealias ZLibCrc32FuncPtr = @convention(c) ( _ cks: UInt32, _ buf: UnsafePointer, _ len: UInt32 @@ -284,18 +284,6 @@ struct Crc32: CustomStringConvertible { } } - /// Advance the current checksum with bytes already in memory, without wrapping them in `Data` first. - /// - parameter bytes: bytes to advance the checksum - mutating func advance(withBytes bytes: UnsafeRawBufferPointer) { - guard let base = bytes.baseAddress?.assumingMemoryBound(to: UInt8.self) - else { return } - if let fastCrc32 = Crc32.zLibCrc32 { - checksum = fastCrc32(checksum, base, UInt32(bytes.count)) - } else { - checksum = slowCrc32(start: checksum, data: Data(bytes)) - } - } - /// Formatted checksum. var description: String { return String(format: "%08x", checksum) @@ -312,7 +300,7 @@ struct Crc32: CustomStringConvertible { /// Rudimentary fallback implementation of the crc32 checksum. This is only a backup used /// when zlib can't be found under '/usr/lib/libz.dylib'. /// - returns: crc32 checksum (4 byte) - private func slowCrc32(start: UInt32, data: Data) -> UInt32 { + func slowCrc32(start: UInt32, data: Data) -> UInt32 { return ~data.reduce(~start) { (crc: UInt32, next: UInt8) -> UInt32 in let tableOffset = (crc ^ UInt32(next)) & 0xff return lookUpTable[Int(tableOffset)] ^ crc >> 8 From 8a6af04c10bb3fb208c8407ad45292ebe1322ec6 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 20 Oct 2026 18:47:15 -0700 Subject: [PATCH 066/158] experimental --- .../LaunchDarkly/Models/LDConfig.swift | 11 ++++++- .../ObjectiveC/ObjcLDConfig.swift | 3 +- .../ServiceObjects/EventLogFormat.swift | 32 +++++++------------ .../ServiceObjects/EventStoreSpec.swift | 23 ------------- 4 files changed, 23 insertions(+), 46 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift index e08ae0cfe..51a7af690 100644 --- a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift @@ -24,6 +24,10 @@ public enum LDStreamingMode { /// An event lives in memory until it is delivered, so a process that dies before the next flush takes everything /// recorded since the last one — including the crash an application was reporting when it died. Writing events to /// disk is what closes that gap: they are delivered on a later launch instead of being lost. +/// +/// `.deferred` and `.immediate` are not stable in this major version, and not subject to any backwards compatibility +/// guarantees or semantic versioning. They are experimental: their behavior, their cost on the calling thread, and +/// what they leave on disk may change in a minor release. @objc(LDEventPersistence) public enum EventPersistence: Int { /// Events are kept in memory only, and nothing survives the process ending. /// @@ -40,12 +44,16 @@ public enum LDStreamingMode { /// A flush and entering the background are the exceptions. Both write on the calling thread, as `.immediate` does: /// a flush so that an application ending the process on purpose can rely on it, and entering the background because /// that is when the system is most likely to end it. + /// + /// This level is experimental; see `EventPersistence`. case deferred /// Events are written to disk before `track` and `identify` return. /// /// The write happens on the calling thread, costing tens of microseconds depending on how many evaluations are /// waiting to be encoded alongside it. In exchange there is no window: a process that dies the instant after /// `track` returns still reports that event on the next launch. Evaluating a flag stays in memory regardless. + /// + /// This level is experimental; see `EventPersistence`. case immediate } @@ -368,7 +376,8 @@ public struct LDConfig { /// Anything other than `.disabled` appends events to a log under Application Support and delivers them on a later /// run, which is what lets the crash an application was reporting when it died reach LaunchDarkly at all. /// `.immediate` additionally puts the write on the thread that called `track` or `identify`, so there is no window - /// in which the event exists only in memory. (Default: `.disabled`) + /// in which the event exists only in memory. Both levels other than `.disabled` are experimental, and not subject to + /// any backwards compatibility guarantees or semantic versioning. (Default: `.disabled`) public var eventPersistence: EventPersistence = Defaults.eventPersistence /// The maximum number of bytes of events the SDK keeps for each environment where `eventPersistence` keeps events across launches, diff --git a/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift b/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift index 9960e5542..77536aab5 100644 --- a/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift @@ -46,7 +46,8 @@ public final class ObjcLDConfig: NSObject { /// Without this, an event lives in memory until it is delivered, and a process that dies before the next flush /// takes everything recorded since the last one, including the crash an application was reporting when it died. /// `.immediate` additionally puts the write on the thread that called `track` or `identify`, so there is no window - /// in which the event exists only in memory. (Default: `.disabled`) + /// in which the event exists only in memory. Both levels other than `.disabled` are experimental, and not subject to + /// any backwards compatibility guarantees or semantic versioning. (Default: `.disabled`) @objc public var eventPersistence: EventPersistence { get { config.eventPersistence } set { config.eventPersistence = newValue } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift index ca3efdc1e..427e4d394 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift @@ -16,13 +16,10 @@ import Foundation /// reads. enum EventLogFormat { /// Bump this whenever the framing or the meaning of a frame changes. - static let version: UInt16 = 2 - /// The version before frames carried a checksum, which this one still reads. - static let versionWithoutChecksum: UInt16 = 1 + static let version: UInt16 = 1 static let magic = Data("LDEV".utf8) static let fileHeaderSize = 6 static let frameHeaderSize = 10 - static let frameHeaderSizeWithoutChecksum = 6 /// The only frame type written today; a reader skips a frame whose type it does not know. static let eventFrame: UInt16 = 1 /// A ceiling on a single frame, so a corrupt size cannot make recovery allocate wildly. @@ -61,33 +58,26 @@ enum EventLogFormat { /// log this version reads. @discardableResult static func forEachFrame(in file: Data, _ visit: (UInt16, Data) -> Void) -> FrameWalk? { - guard file.count >= fileHeaderSize, file.prefix(magic.count) == magic + guard file.count >= fileHeaderSize, + file.prefix(magic.count) == magic, + readUInt16(file, at: magic.count) == version else { return nil } - let isChecksummed: Bool - switch readUInt16(file, at: magic.count) { - case version: isChecksummed = true - case versionWithoutChecksum: isChecksummed = false - default: return nil - } - let headerSize = isChecksummed ? frameHeaderSize : frameHeaderSizeWithoutChecksum return file.withUnsafeBytes { bytes in var cursor = fileHeaderSize - while cursor + headerSize <= bytes.count { + while cursor + frameHeaderSize <= bytes.count { let type = UInt16(bytes[cursor]) << 8 | UInt16(bytes[cursor + 1]) let length = Int(readUInt32(bytes, at: cursor + 2)) - let start = cursor + headerSize + let start = cursor + frameHeaderSize guard length > 0, length <= maxFrameSize, start + length <= bytes.count else { break } let payload = UnsafeRawBufferPointer(rebasing: bytes[start..<(start + length)]) - if isChecksummed { - var checksum = Crc32() - checksum.advance(withBytes: UnsafeRawBufferPointer(rebasing: bytes[cursor..<(cursor + 6)])) - checksum.advance(withBytes: payload) - guard checksum.checksum == readUInt32(bytes, at: cursor + 6) - else { break } - } + var checksum = Crc32() + checksum.advance(withBytes: UnsafeRawBufferPointer(rebasing: bytes[cursor..<(cursor + 6)])) + checksum.advance(withBytes: payload) + guard checksum.checksum == readUInt32(bytes, at: cursor + 6) + else { break } visit(type, Data(payload)) cursor = start + length diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 1146db088..41ab0d3cd 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -543,29 +543,6 @@ final class EventStoreSpec: QuickSpec { expect { try store.body(of: batch) }.to(throwError(errorType: UnreadableBatch.self)) } - it("still reads a log written before frames carried a checksum") { - store = EventStore.temporary() - _ = store.stage(EventStoreSpec.payload("placeholder")) - _ = store.closeBatch() - let file = try EventStoreSpec.onlyBatchFile(in: store) - - var log = EventLogFormat.magic - log.append(EventLogFormat.bigEndian(EventLogFormat.versionWithoutChecksum)) - for key in ["first", "second"] { - let payload = EventStoreSpec.payload(key) - log.append(EventLogFormat.bigEndian(EventLogFormat.eventFrame)) - log.append(EventLogFormat.bigEndian(UInt32(payload.count))) - log.append(payload) - } - try log.write(to: file) - try EventStoreSpec.asUncountedName(file) - - let next = EventStoreSpec.reader(sharing: store) - let batches = next.pendingBatches() - expect(batches.map(\.eventCount)) == [2] - expect(EventStoreSpec.keys(ofBody: try next.body(of: batches[0]))) == ["first", "second"] - } - it("is kept for a later listing when a batch named without a count cannot be read") { store = EventStore.temporary() _ = store.stage(EventStoreSpec.payload("kept")) From 067e676042d99531d25e2b68d75c220bed6c46b5 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 20 Oct 2026 20:22:42 -0700 Subject: [PATCH 067/158] Name each event log by a UUID and recover every log no live process holds A log was named for its process, so recovery could only take its own and a log left by a process that never ran again was stranded; on a Mac a second instance fell back to a per-pid name for the same reason. Each log is now open-, given its header under new- and only then renamed, and the logs open in this process are remembered (and on a Mac still flock'd). Recovery and expiry take any open- log no live process holds, including ones named by the previous scheme, and appending no longer runs recovery. Co-authored-by: Cursor --- .../ServiceObjects/DiskEventLog.swift | 225 ++++++++++-------- .../ServiceObjects/EventDirectory.swift | 78 +++--- .../ServiceObjects/EventStore.swift | 26 +- .../DirectEventProcessorSpec.swift | 13 +- .../ServiceObjects/EventStoreSpec.swift | 196 +++++++++------ LaunchDarkly/LaunchDarklyTests/TestUtil.swift | 49 +++- 6 files changed, 345 insertions(+), 242 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift index 7d53b5626..c955108cd 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift @@ -9,18 +9,35 @@ import Glibc import Musl #endif -/// The file of events on disk this process appends to, not to be confused with diagnostic logging. +/// The file of events on disk a store appends to, not to be confused with diagnostic logging. /// -/// One per process rather than one per directory, so that only one process ever writes a given log and recovery only -/// ever touches its own. With one shared log, a second process starting up would close off, as if a previous run had -/// left it, a log the first is still appending to through its open descriptor, and the first's later events would land -/// in a batch already sent or deleted. Another process's log is left for that process to recover the next time it runs. +/// One per store rather than one per directory, so that only one writer ever appends to a given log. Each is named for a +/// UUID picked when it is opened, which is what keeps two writers apart: nothing about it comes from the application, and +/// no two processes, or two stores in one, ever pick the same one. +/// +/// A log nobody is appending to belongs to no one alive, whoever wrote it, and recovery takes it. The logs open in this +/// process are remembered, which is all it takes where only this process uses the directory. On a Mac, which runs a +/// second instance of an application over the same directory on request, the owner also holds its log locked for as +/// long as it is open, and the kernel releases the lock when the descriptor closes, including when the process dies. +/// Elsewhere a lock is not compiled in at all: iOS kills a suspended application that holds a lock on a file in a shared +/// container. +/// +/// A log is given its header under a name recovery does not look for, and only then moved to one it does, so that a log +/// being opened is never taken for one whose owner is gone. /// /// Belongs to an `EventStore`, which makes every call holding its `ioLock`. The lock is handed in so that each call can /// check it is held, rather than this taking a lock of its own that would have to be ordered against the store's. final class DiskEventLog { - /// Names a log a single process appends to; the rest of the name is that process. + /// Names a log its owner appends to; the rest of the name is the UUID it was opened under. static let prefix = "open-" + /// Names a log while it is being opened, before it has its header. Recovery never looks at these; one left by a + /// process that died opening it holds no events, and expiry removes it. + static let openingPrefix = "new-" + + /// The paths of the logs open in this process, which recovery and expiry leave alone. Guarded by + /// `openInThisProcessLock`. + private static var openInThisProcess: Set = [] + private static let openInThisProcessLock = UnfairLock() private let directory: URL private let ioLock: UnfairLock @@ -29,26 +46,18 @@ final class DiskEventLog { /// does. Answers with the errno it stopped on, or nil where every byte landed. private let writeLog: (EventLogFile, Data) -> Int32? - /// Named for the process, unless on a Mac `isLocked` found another live instance holding that name, in which case - /// it is named for this instance's process ID as well. + /// Where the log is, whether or not it exists yet. private(set) var url: URL /// Nil until the first append opens it, and again once it is closed. private var file: EventLogFile? #if os(macOS) || targetEnvironment(macCatalyst) - /// The readable, collision-proof form of the process name every log this process opens is named after. - private let processLogName: String - /// Whether the log is locked while it is open, so that another live process can tell it is taken. Only to be - /// changed before the log is first opened or recovered. - /// - /// Only on a Mac, which runs a second instance of an application under the same process name on request, so that - /// a name alone cannot keep two live processes apart. Elsewhere a name is one live process, and a lock is not - /// compiled in at all: iOS kills a suspended application that holds a lock on a file in a shared container. + /// Whether the log is locked while it is open, so that another live instance can tell it is taken. Only to be + /// changed before the log is first opened. var isLocked = true #endif init(directory: URL, - processName: String, ioLock: UnfairLock, logger: OSLog, writeLog: @escaping (EventLogFile, Data) -> Int32?) { @@ -56,31 +65,27 @@ final class DiskEventLog { self.ioLock = ioLock self.logger = logger self.writeLog = writeLog - let processLogName = DiskEventLog.name(for: processName) - self.url = directory.appendingPathComponent(DiskEventLog.prefix + processLogName) - #if os(macOS) || targetEnvironment(macCatalyst) - self.processLogName = processLogName - #endif + self.url = DiskEventLog.newUrl(in: directory) + } + + private static func newUrl(in directory: URL) -> URL { + directory.appendingPathComponent(prefix + UUID().uuidString) } - /// Names the log belonging to one process. + /// Whether some live process is appending to a log, and recovery and expiry must leave it alone. /// - /// The readable part is the process name with anything awkward in a filename replaced, which keeps a directory of - /// these diagnosable. That reduction is not one-to-one, and two processes sharing a log is the one thing this name - /// exists to prevent, so a digest of the original name is appended. - static func name(for processName: String) -> String { - let trimmed = processName.trimmingCharacters(in: .whitespacesAndNewlines) - let name = trimmed.isEmpty ? "unknown" : trimmed - let readable = String(name.unicodeScalars.map { scalar -> Character in - switch scalar { - case "a"..."z", "A"..."Z", "0"..."9", "-", "_": - return Character(scalar) - default: - return "_" - } - }) - let digest = Util.sha256(name).prefix(4).map { String(format: "%02x", $0) }.joined() - return "\(readable)-\(digest)" + /// A log open in this process is answered from memory. On a Mac any other is answered by its lock, and one that + /// cannot be looked at counts as held: leaving a dead process's log for a later run costs nothing, while taking a + /// live one's would lose what it appends next. + static func isHeldByLiveProcess(_ log: URL) -> Bool { + if isOpenInThisProcess(log) { + return true + } + #if os(macOS) || targetEnvironment(macCatalyst) + return EventLogFile.isLockedByLiveProcess(log) + #else + return false + #endif } /// Requires `ioLock`. @@ -124,34 +129,47 @@ final class DiskEventLog { /// Requires `ioLock`. func close(movingTo destination: URL) throws { ioLock.assertOwned() - // Closed after the rename rather than before it, so that a locked log is still locked while it moves. Another - // instance recovering unlocked logs could otherwise take it in between. + // Closed after the move rather than before it, so that the log is still held while it moves. Another store + // recovering logs could otherwise take it in between. defer { closeDescriptor() } try FileManager.default.moveItem(at: url, to: destination) } /// Closes the descriptor, leaving the log where it is. Requires `ioLock`. + /// + /// No longer held, it is from here on a log like one a process that died left behind, which recovery may take at + /// any moment. So this log is never opened again: the next append starts a new one. func closeDescriptor() { ioLock.assertOwned() - file?.close() - file = nil + guard let file + else { return } + file.close() + self.file = nil + DiskEventLog.forget(url) } - /// Hands `recover` each log left open by a process that is gone, which it has to move or remove. - /// - /// Only this process's own log, by name: another process's may still be open in a process that is alive. Where - /// logs are locked, any log no live process holds, whatever it is named. + /// Hands `recover` every log no live process holds, which it has to move or remove: whoever wrote it is gone, + /// whatever it is named, including a log an earlier version named for its process. /// /// Requires `ioLock`. - func forEachLogLeftOpen(_ recover: (URL) -> Void) { + func forEachUnheldLog(_ recover: (URL) -> Void) { ioLock.assertOwned() - #if os(macOS) || targetEnvironment(macCatalyst) - if isLocked { - forEachUnheldLog(recover) - return + let contents = (try? FileManager.default.contentsOfDirectory(at: directory, + includingPropertiesForKeys: nil, + options: [.skipsHiddenFiles])) ?? [] + for log in contents where log.lastPathComponent.hasPrefix(DiskEventLog.prefix) { + guard !DiskEventLog.isOpenInThisProcess(log) + else { continue } + #if os(macOS) || targetEnvironment(macCatalyst) + // Locked for as long as it is recovered, so that another instance recovering at the same time leaves it be. + guard let held = EventLogFile.openLocked(log, flags: O_RDONLY) + else { continue } + recover(log) + held.close() + #else + recover(log) + #endif } - #endif - recover(url) } /// Requires `ioLock`. @@ -169,19 +187,13 @@ final class DiskEventLog { } EventLogFile.excludeFromBackup(directory) - guard let opened = open() - else { - os_log("%s could not open the event log: errno %d", log: logger, type: .debug, typeName(and: #function), errno) - return nil + if FileManager.default.fileExists(atPath: url.path) { + // Left by an earlier open this session and closed since, so no longer this store's to write. + url = DiskEventLog.newUrl(in: directory) } - + guard let opened = open() + else { return nil } file = opened - - if opened.isEmpty { - guard write(EventLogFormat.fileHeader, to: opened) - else { return nil } - } - return opened } @@ -192,58 +204,63 @@ final class DiskEventLog { return false } - /// Opens the log, locked where logs are. Requires `ioLock`. + /// Opens a new log: created under `openingPrefix`, locked where logs are, given its header, and only then moved to + /// the name recovery looks for. Requires `ioLock`. private func open() -> EventLogFile? { ioLock.assertOwned() - // O_APPEND is what makes each write land at the end of the file as one step, so that clients for several - // environments writing their own logs, or a commit racing a reader, cannot produce a spliced frame. - let flags = O_WRONLY | O_APPEND | O_CREAT + let opening = directory.appendingPathComponent( + DiskEventLog.openingPrefix + url.lastPathComponent.dropFirst(DiskEventLog.prefix.count)) + // O_APPEND is what makes each write land at the end of the file as one step, so that a commit racing a reader + // cannot produce a spliced frame. + let flags = O_WRONLY | O_APPEND | O_CREAT | O_EXCL + DiskEventLog.remember(url) + guard let opened = openFile(at: opening, flags: flags) + else { + os_log("%s could not open the event log: errno %d", log: logger, type: .debug, typeName(and: #function), errno) + DiskEventLog.forget(url) + return nil + } + + guard write(EventLogFormat.fileHeader, to: opened) + else { return abandon(opened, at: opening) } + do { + try FileManager.default.moveItem(at: opening, to: url) + } catch { + os_log("%s could not name the event log: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) + return abandon(opened, at: opening) + } + return opened + } + + /// Closes and removes a log that never got the name recovery looks for. Requires `ioLock`. + private func abandon(_ opened: EventLogFile, at opening: URL) -> EventLogFile? { + opened.close() + try? FileManager.default.removeItem(at: opening) + DiskEventLog.forget(url) + return nil + } + + /// Opens a file for a new log, locked where logs are. Requires `ioLock`. + private func openFile(at url: URL, flags: Int32) -> EventLogFile? { #if os(macOS) || targetEnvironment(macCatalyst) if isLocked { - return openLocked(flags: flags) + return EventLogFile.openLocked(url, flags: flags) } #endif return EventLogFile.open(at: url, flags: flags) } - #if os(macOS) || targetEnvironment(macCatalyst) - /// Opens the log for this process name locked, or one of this instance's own where another live instance holds - /// it. Requires `ioLock`. - private func openLocked(flags: Int32) -> EventLogFile? { - ioLock.assertOwned() - if let opened = EventLogFile.openLocked(url, flags: flags) { - return opened - } + private static func isOpenInThisProcess(_ log: URL) -> Bool { + openInThisProcessLock.withLock { openInThisProcess.contains(log.standardizedFileURL.path) } + } - // Another live instance of the application holds the log for this process name, so this one takes a log of its - // own. The process ID keeps it apart from every other live process, and the suffix from another store in this - // one, as a closed client's can be while it is still going away. Recovery finds the log by its lock rather than - // its name, so it is not orphaned when this instance ends. - let ownLog = directory.appendingPathComponent( - "\(DiskEventLog.prefix)\(processLogName)-\(getpid())-\(UUID().uuidString.prefix(8))") - guard let opened = EventLogFile.openLocked(ownLog, flags: flags) - else { return nil } - url = ownLog - return opened + private static func remember(_ log: URL) { + openInThisProcessLock.withLock { _ = openInThisProcess.insert(log.standardizedFileURL.path) } } - /// Hands `recover` every open log no live process holds. Requires `ioLock`. - /// - /// A log that can be locked is one no live process has open, so whoever wrote it is gone. That includes a log a - /// second instance opened under its process ID, which no later run would otherwise come back for by name. - private func forEachUnheldLog(_ recover: (URL) -> Void) { - ioLock.assertOwned() - let contents = (try? FileManager.default.contentsOfDirectory(at: directory, - includingPropertiesForKeys: nil, - options: [.skipsHiddenFiles])) ?? [] - for log in contents where log.lastPathComponent.hasPrefix(DiskEventLog.prefix) { - guard let held = EventLogFile.openLocked(log, flags: O_RDONLY) - else { continue } - recover(log) - held.close() - } + private static func forget(_ log: URL) { + openInThisProcessLock.withLock { _ = openInThisProcess.remove(log.standardizedFileURL.path) } } - #endif } extension DiskEventLog: TypeIdentifying { } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index ebb185295..e02065b38 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -49,8 +49,6 @@ final class EventDirectory { /// Only to be used while holding `ioLock`. private var located: (directory: URL, log: DiskEventLog)? - /// Whether a log left by a previous run has been dealt with. Only to be used while holding `ioLock`. - private var hasRecoveredInterruptedLog = false /// A batch in the directory: the file it is, how many events it holds, and how big it is. private struct Stored { @@ -117,7 +115,6 @@ final class EventDirectory { } let directory = locateDirectory() let log = DiskEventLog(directory: directory, - processName: processName, ioLock: ioLock, logger: logger, writeLog: writeLog) @@ -169,19 +166,13 @@ final class EventDirectory { // MARK: Writing - /// Appends to the open log, first closing off one a previous run left behind. + /// Appends to the open log, opening a new one where none is open. Requires `ioLock`. /// - /// Requires `ioLock`. - /// - /// - Returns: what that recovery found, and whether every byte landed. False means none of `bytes` is in the log, - /// and that the caller should give up on the disk. - func append(_ bytes: Data) -> (recovered: EventTally, written: Bool) { + /// - Returns: whether every byte landed. False means none of `bytes` is in the log, and that the caller should give + /// up on the disk. + func append(_ bytes: Data) -> Bool { ioLock.assertOwned() - // Recovered before the log is opened, because it may still be the one a previous run died writing. Appended - // to, this run's events would sit behind that run's torn last frame, where a reader can no longer find where - // they start. - let recovered = disk.log.isOpen ? EventTally() : recoverInterruptedLog() - return (recovered, disk.log.append(bytes)) + return disk.log.append(bytes) } /// What closing the open log produced. @@ -400,23 +391,19 @@ final class EventDirectory { // MARK: Recovery - /// Closes off any log left open by a previous run of this process, so its events join the batches to deliver. + /// Closes off every log no live process holds, so its events join the batches to deliver. /// - /// The events in it were recorded by a process that is gone, so there is no one left to add to it. Only this - /// process's own log is touched: another process's may still be open in a process that is alive. Where logs are - /// locked, any log no live process holds is recovered, whatever it is named. + /// The events in it were recorded by a store that is gone, so there is no one left to add to it: a previous run of + /// this process, or another process that has since ended, whatever it was called. A log a live process is still + /// appending to is left alone. /// - /// Requires `ioLock`. Does its work once: after that, the open log is one this run opened. + /// Requires `ioLock`. /// - /// - Returns: what it closed off, which is nothing on every call after the first. + /// - Returns: what it closed off. func recoverInterruptedLog() -> EventTally { ioLock.assertOwned() - guard !hasRecoveredInterruptedLog - else { return EventTally() } - hasRecoveredInterruptedLog = true - var recovered = EventTally() - disk.log.forEachLogLeftOpen { recovered += tally(recoveredFrom: $0) } + disk.log.forEachUnheldLog { recovered += tally(recoveredFrom: $0) } return recovered } @@ -431,9 +418,9 @@ final class EventDirectory { case .missing: return EventTally() case .failed(let error): - // It cannot stay under this name, which this run is about to append its own events to, and deleting it - // would lose events that may be intact. Closed off uncounted instead, it is read and counted by whichever - // listing first manages to. + // Deleting it would lose events that may be intact, and left under this name it would wait for the next + // run's recovery. Closed off uncounted instead, it is read and counted by whichever listing first manages + // to. os_log("%s could not read events from a previous run, will try again later: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) // Named without a count, which is what a batch whose events have never been counted is: the listing that // first manages to read it is what counts it. @@ -472,12 +459,13 @@ final class EventDirectory { // MARK: Expiry extension EventDirectory { - /// Deletes the batches in this directory last written before `cutoff`, and the logs of other processes that have + /// Deletes the batches in this directory last written before `cutoff`, and the logs no live process holds that have /// not been written since then either. Requires `ioLock`. /// /// A batch's modification time is when its newest event was written, since a batch is renamed rather than rewritten - /// when it is closed, so a batch is discarded only once everything in it is that old. A log another process has not - /// appended to for as long belongs to a process that is gone: one that is alive appends far more often than that. + /// when it is closed, so a batch is discarded only once everything in it is that old. A log is discarded only once + /// nothing is appending to it, however long ago it was written: a live process may have gone that long without an + /// event. /// /// - Parameter sending: batches whose request is out, which are left alone however old they are. /// - Returns: how many events were discarded, and how much of that this process had counted, which is what stops @@ -500,8 +488,8 @@ extension EventDirectory { else { continue } events += held counted += stored.removeValue(forKey: payloadId)?.tally ?? EventTally() - } else if name.hasPrefix(DiskEventLog.prefix), file.standardizedFileURL != disk.log.url.standardizedFileURL { - events += EventDirectory.discardAbandonedLog(file) + } else { + events += EventDirectory.discardIfAbandonedLog(file) } } return (events, counted) @@ -547,8 +535,8 @@ extension EventDirectory { if delete(file) { events += held } - } else if name.hasPrefix(DiskEventLog.prefix) { - events += discardAbandonedLog(file) + } else { + events += discardIfAbandonedLog(file) } } EventLogFile.removeDirectoryIfEmpty(directory) @@ -556,22 +544,26 @@ extension EventDirectory { return events } - /// Deletes a log no live process is appending to, returning how many events it held. - private static func discardAbandonedLog(_ log: URL) -> Int { - #if os(macOS) || targetEnvironment(macCatalyst) - // A Mac runs second instances of an application, each holding its log locked for as long as it lives. - if EventLogFile.isLockedByLiveProcess(log) { + /// Deletes a log no live process is appending to, or one a process died opening, returning how many events it + /// held. Anything else is left alone. + private static func discardIfAbandonedLog(_ file: URL) -> Int { + let name = file.lastPathComponent + if name.hasPrefix(DiskEventLog.openingPrefix) { + // Opening a log takes moments, so one this old was never finished, and holds no events. + delete(file) return 0 } - #endif - let held = eventsReading(log) - return delete(log) ? held : 0 + guard name.hasPrefix(DiskEventLog.prefix), !DiskEventLog.isHeldByLiveProcess(file) + else { return 0 } + let held = eventsReading(file) + return delete(file) ? held : 0 } private static func eventsReading(_ file: URL) -> Int { (try? Data(contentsOf: file)).flatMap { EventLogFormat.eventCount(in: $0) } ?? 0 } + @discardableResult private static func delete(_ file: URL) -> Bool { try? FileManager.default.removeItem(at: file) return !FileManager.default.fileExists(atPath: file.path) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index 47edc287c..af028c69e 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -78,7 +78,8 @@ protocol EventStoring { /// Counts a batch against capacity again after a send that will be retried later. func stoppedSending(_ batch: EventBatch) - /// Closes any log left open by a previous run of this process, so its events join the batches to deliver. + /// Closes every log no live process holds, so its events join the batches to deliver: one a previous run left open, + /// or another process that has since ended. func recoverInterruptedLog() /// Deletes what has been stored for longer than the maximum age, except a batch whose request is out. @@ -466,17 +467,14 @@ final class EventStore: EventStoring { func recoverInterruptedLog() { guard let disk else { return } - ioLock.withLock { countRecovered(disk.recoverInterruptedLog()) } - } - - /// Counts events a recovery closed off into batches of their own. Requires `ioLock`. - private func countRecovered(_ recovered: EventTally) { - ioLock.assertOwned() - guard recovered.events > 0 - else { return } - bufferLock.withLock { - closedEvents += recovered.events - closedBytes += recovered.bytes + ioLock.withLock { + let recovered = disk.recoverInterruptedLog() + guard recovered.events > 0 + else { return } + bufferLock.withLock { + closedEvents += recovered.events + closedBytes += recovered.bytes + } } } @@ -596,9 +594,7 @@ extension EventStore { guard let disk else { return false } - let (recovered, written) = disk.append(bytes) - countRecovered(recovered) - guard written + guard disk.append(bytes) else { // Never trap on a full disk. The SDK gives up on persistence for the rest of the session rather than taking // the application down with it, which is how other SDKs have crashed their hosts. diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 54479e4c5..63ef54ad9 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -1273,8 +1273,8 @@ class DirectEventProcessorSpec: QuickSpec { extension DirectEventProcessorSpec { /// What the SDK is able to report after the process it was recording in is gone. /// - /// These are written against a second store reading the same directory, which is what the next run of the - /// application amounts to: it sees the bytes that reached the disk and nothing that was still in memory. + /// These are written against what is in the directory, and a store on a copy of it, which is what the next run of + /// the application amounts to: it sees the bytes that reached the disk and nothing that was still in memory. private func durabilitySpec() { describe("surviving the process") { var testContext: TestContext! @@ -1284,8 +1284,7 @@ extension DirectEventProcessorSpec { /// The events the next run of the application would find. func eventsLeftOnDisk() -> [LDValue] { - let reader = EventStore.sharing(testContext.store.directory!) - return reader.pendingEventPayloads().compactMap { try? JSONDecoder().decode(LDValue.self, from: $0) } + testContext.store.payloadsOnDisk.compactMap { try? JSONDecoder().decode(LDValue.self, from: $0) } } itPersisting("has written a tracked event by the time recording it returns") { @@ -1385,7 +1384,7 @@ extension DirectEventProcessorSpec { testContext.eventProcessor.commitRecordedEvents() // Nothing delivered them and nothing closed the log, as would be the case had the process died here. - let recovered = EventStore.sharing(testContext.store.directory!) + let recovered = testContext.store.nextRun() let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) nextRun.isOnline = true @@ -1407,7 +1406,7 @@ extension DirectEventProcessorSpec { testContext.recordEvents(2) testContext.eventProcessor.commitRecordedEvents() - let recovered = EventStore.sharing(testContext.store.directory!) + let recovered = testContext.store.nextRun() let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) // Nothing asks for a flush, and the report interval is several times longer than this waits, so the @@ -1665,7 +1664,7 @@ extension DirectEventProcessorSpec { testContext.serviceMock.respondToEvents(with: 503) testContext.eventProcessor.record(CustomEvent(key: "kept", context: testContext.context)) - let recovered = EventStore.sharing(testContext.store.directory!) + let recovered = testContext.store.nextRun() let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) nextRun.isOnline = true defer { nextRun.isOnline = false } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 41ab0d3cd..16f280d3f 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -17,31 +17,28 @@ final class EventStoreSpec: QuickSpec { multiProcessSpec() withoutPersistenceSpec() #if os(macOS) || targetEnvironment(macCatalyst) - sameNameInstancesSpec() + lockedInstancesSpec() #endif } #if os(macOS) || targetEnvironment(macCatalyst) - private func sameNameInstancesSpec() { - // A second instance of a Mac application runs under the first one's process name. - describe("two live instances under one process name") { + private func lockedInstancesSpec() { + // A Mac runs a second instance of an application over the same directory on request. + describe("live instances holding their logs locked") { var first: EventStore! afterEach { first.deleteEverything() } func instance() -> EventStore { - EventStore(directory: first.directory!, capacity: 100, processName: "app", logger: .disabled) + EventStore(directory: first.directory!, capacity: 100, logger: .disabled) } beforeEach { - first = EventStore(directory: EventStore.temporary().directory!, - capacity: 100, - processName: "app", - logger: .disabled) + first = EventStore(directory: EventStore.temporary().directory!, capacity: 100, logger: .disabled) } - it("gives the second a log of its own, named for its process, and leaves the first's alone") { + it("gives each a log of its own, and leaves the first's alone") { _ = first.stage(EventStoreSpec.payload("first")) first.commit() @@ -52,7 +49,6 @@ final class EventStoreSpec: QuickSpec { _ = second.stage(EventStoreSpec.payload("second")) second.commit() expect(second.openLogUrl!) != first.openLogUrl! - expect(second.openLogUrl!.lastPathComponent).to(contain("-\(getpid())-")) _ = first.stage(EventStoreSpec.payload("first-again")) first.commit() @@ -62,7 +58,24 @@ final class EventStoreSpec: QuickSpec { expect(EventStoreSpec.keys(ofBody: fromSecond.flatMap { try? second.body(of: $0) })) == ["second"] } - it("recovers the log an instance that has ended left open, though no run is named for it") { + it("leaves a log another process holds locked alone until the lock is let go") { + var ended: EventStore? = instance() + _ = ended?.stage(EventStoreSpec.payload("held")) + ended?.commit() + let log = ended!.openLogUrl! + ended = nil + // A descriptor of its own, as another live instance's would be, so nothing in this process knows of it. + let held = EventLogFile.openLocked(log, flags: O_RDONLY)! + + first.recoverInterruptedLog() + expect(first.pendingBatches()).to(beEmpty()) + + held.close() + first.recoverInterruptedLog() + expect(EventStoreSpec.keys(ofBody: first.pendingBatches().first.flatMap { try? first.body(of: $0) })) == ["held"] + } + + it("recovers the log an instance that has ended left open") { _ = first.stage(EventStoreSpec.payload("first")) first.commit() @@ -79,12 +92,12 @@ final class EventStoreSpec: QuickSpec { expect(EventStoreSpec.keys(ofBody: batches.first.flatMap { try? next.body(of: $0) })) == ["from-the-ended-instance"] } - it("recovers the log a previous run under the name left open, once nothing holds it") { + it("recovers the log a previous run left open, once nothing holds it") { _ = first.stage(EventStoreSpec.payload("left-open")) first.commit() let directory = first.directory! first = nil - first = EventStore(directory: directory, capacity: 100, processName: "app", logger: .disabled) + first = EventStore(directory: directory, capacity: 100, logger: .disabled) first.recoverInterruptedLog() let batches = first.pendingBatches() @@ -102,21 +115,21 @@ final class EventStoreSpec: QuickSpec { app.deleteEverything() } - /// Another process of the same application, using the same directory under its own name. Unlocked, as on an - /// iPhone or iPad, where a process name is one live process and the name alone keeps the logs apart. - func otherProcess(_ name: String = "widget") -> EventStore { - EventStore.sharing(app.directory!, processName: name) + /// Another process of the same application, using the same directory. Unlocked, as on an iPhone or iPad, + /// where the logs a live store has open are known without one. + func otherProcess() -> EventStore { + EventStore.sharing(app.directory!) } beforeEach { - app = EventStore.sharing(EventStore.temporary().directory!, processName: "app") + app = EventStore.sharing(EventStore.temporary().directory!) } it("leaves a log another process still has open where it is, so that process's later events reach it") { _ = app.stage(EventStoreSpec.payload("before")) app.commit() - // Starting up, the other process recovers what its own previous run left open, and nothing else. + // Starting up, the other process recovers what no live process holds, and nothing else. let widget = otherProcess() widget.recoverInterruptedLog() expect(widget.pendingBatches()).to(beEmpty()) @@ -152,24 +165,44 @@ final class EventStoreSpec: QuickSpec { expect(widget.pendingBatches().map(\.payloadId)) == [closed?.payloadId].compactMap { $0 } } - it("recovers a process's own log on its next run") { - _ = app.stage(EventStoreSpec.payload("left-open")) - app.commit() + it("recovers the log a process that has ended left open, whichever process it was") { + var widget: EventStore? = otherProcess() + _ = widget?.stage(EventStoreSpec.payload("left-open")) + widget?.commit() + // Going away closes its log, as the process ending would. + widget = nil - let nextRun = otherProcess("app") - nextRun.recoverInterruptedLog() - let batches = nextRun.pendingBatches() + app.recoverInterruptedLog() + let batches = app.pendingBatches() expect(batches.count) == 1 - expect(EventStoreSpec.keys(ofBody: batches.first.flatMap { try? nextRun.body(of: $0) })) == ["left-open"] + expect(EventStoreSpec.keys(ofBody: batches.first.flatMap { try? app.body(of: $0) })) == ["left-open"] } - it("names logs so that process names reducing to the same characters stay apart") { - let colon = DiskEventLog.name(for: "com.app:remote") - let dot = DiskEventLog.name(for: "com.app.remote") - expect(colon) != dot - expect(colon.hasPrefix("com_app_remote-")) == true - expect(dot.hasPrefix("com_app_remote-")) == true - expect(DiskEventLog.name(for: " ").hasPrefix("unknown-")) == true + it("recovers a log an earlier version named for its process") { + var earlierVersion: EventStore? = otherProcess() + _ = earlierVersion?.stage(EventStoreSpec.payload("named-for-its-process")) + earlierVersion?.commit() + let log = earlierVersion!.openLogUrl! + earlierVersion = nil + try FileManager.default.moveItem(at: log, to: log.deletingLastPathComponent() + .appendingPathComponent("\(DiskEventLog.prefix)com_example_app-1a2b3c4d")) + + app.recoverInterruptedLog() + let batches = app.pendingBatches() + expect(batches.count) == 1 + expect(EventStoreSpec.keys(ofBody: batches.first.flatMap { try? app.body(of: $0) })) == ["named-for-its-process"] + } + + it("names a log for a UUID rather than for the process") { + _ = app.stage(EventStoreSpec.payload("named")) + app.commit() + + let logs = try FileManager.default.contentsOfDirectory(atPath: app.directory!.path) + .filter { $0.hasPrefix(DiskEventLog.prefix) } + expect(logs.count) == 1 + expect(logs.first.flatMap { UUID(uuidString: String($0.dropFirst(DiskEventLog.prefix.count))) }).toNot(beNil()) + expect(try FileManager.default.contentsOfDirectory(atPath: app.directory!.path) + .filter { $0.hasPrefix(DiskEventLog.openingPrefix) }).to(beEmpty()) } } } @@ -256,7 +289,7 @@ final class EventStoreSpec: QuickSpec { let own = withoutPersistence.closeBatch() expect(withoutPersistence.pendingBatches().map(\.payloadId)) == [own?.payloadId] - let persistingAgain = EventStoreSpec.reader(sharing: persisting) + let persistingAgain = persisting.nextRun() persistingAgain.recoverInterruptedLog() let bodies = persistingAgain.pendingBatches().map { EventStoreSpec.keys(ofBody: try? persistingAgain.body(of: $0)) } expect(bodies) == [["closed"], ["open"]] @@ -279,8 +312,8 @@ final class EventStoreSpec: QuickSpec { logger: .disabled) } - /// A store reading the same directory as another, standing in for the next run of the application: it sees only - /// what actually reached the disk, which is the whole question a crash asks. + /// A store reading the same directory as another, standing in for another process: it sees the batches, and leaves + /// alone the log the live store appends to. `nextRun()` stands in for the next run of the application. private static func reader(sharing store: EventStore, readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }) -> EventStore { EventStore.sharing(store.directory!, readFile: readFile) @@ -337,7 +370,11 @@ final class EventStoreSpec: QuickSpec { } private static func keys(of store: EventStore) -> [String] { - store.pendingEventPayloads().compactMap { + keys(in: store.pendingEventPayloads()) + } + + private static func keys(in payloads: [Data]) -> [String] { + payloads.compactMap { guard case .object(let fields) = try? JSONDecoder().decode(LDValue.self, from: $0), case .string(let key) = fields["key"] else { return nil } @@ -358,7 +395,7 @@ final class EventStoreSpec: QuickSpec { it("keeps a staged event out of the log until it is committed") { _ = store.stage(EventStoreSpec.payload("staged-only")) - expect(EventStoreSpec.keys(of: EventStoreSpec.reader(sharing: store))).to(beEmpty()) + expect(EventStoreSpec.keys(in: store.payloadsOnDisk)).to(beEmpty()) expect(store.pendingEventCount) == 1 } @@ -366,7 +403,7 @@ final class EventStoreSpec: QuickSpec { _ = store.stage(EventStoreSpec.payload("committed")) store.commit() - expect(EventStoreSpec.keys(of: EventStoreSpec.reader(sharing: store))) == ["committed"] + expect(EventStoreSpec.keys(in: store.payloadsOnDisk)) == ["committed"] } it("never writes on its own, however much is staged") { @@ -377,12 +414,12 @@ final class EventStoreSpec: QuickSpec { _ = store.stage(event) } - expect(EventStoreSpec.reader(sharing: store).pendingEventPayloads()).to(beEmpty()) + expect(store.payloadsOnDisk).to(beEmpty()) expect(store.pendingEventCount) == 64 store.commit() - expect(EventStoreSpec.reader(sharing: store).pendingEventPayloads().count) == 64 + expect(store.payloadsOnDisk.count) == 64 } it("keeps events in the order they were recorded") { @@ -391,7 +428,7 @@ final class EventStoreSpec: QuickSpec { } store.commit() - expect(EventStoreSpec.keys(of: EventStoreSpec.reader(sharing: store))) == ["event-0", "event-1", "event-2", "event-3", "event-4"] + expect(EventStoreSpec.keys(in: store.payloadsOnDisk)) == ["event-0", "event-1", "event-2", "event-3", "event-4"] } } } @@ -409,7 +446,7 @@ final class EventStoreSpec: QuickSpec { store.commit() // Nothing closed the log, as nothing would if the process had died here. - let next = EventStoreSpec.reader(sharing: store) + let next = store.nextRun() next.recoverInterruptedLog() let batches = next.pendingBatches() @@ -425,7 +462,7 @@ final class EventStoreSpec: QuickSpec { let batch = store.closeBatch() store.remove(batch!) - let next = EventStoreSpec.reader(sharing: store) + let next = store.nextRun() next.recoverInterruptedLog() expect(next.pendingBatches()).to(beEmpty()) @@ -442,7 +479,7 @@ final class EventStoreSpec: QuickSpec { let log = store.openLogUrl! try Data(contentsOf: log).dropLast(3).write(to: log) - let next = EventStoreSpec.reader(sharing: store) + let next = store.nextRun() _ = next.stage(EventStoreSpec.payload("this-run")) next.commit() next.recoverInterruptedLog() @@ -486,7 +523,7 @@ final class EventStoreSpec: QuickSpec { let whole = try Data(contentsOf: log) try whole.dropLast(8).write(to: log) - expect(EventStoreSpec.keys(of: EventStoreSpec.reader(sharing: store))) == ["event-0", "event-1"] + expect(EventStoreSpec.keys(in: store.payloadsOnDisk)) == ["event-0", "event-1"] } context("where the storage changed a frame after it was written") { @@ -507,7 +544,7 @@ final class EventStoreSpec: QuickSpec { damaged[second + EventLogFormat.frameHeaderSize + 3] ^= 0x04 try damaged.write(to: log) - expect(EventStoreSpec.keys(of: EventStoreSpec.reader(sharing: store))) == ["event-0"] + expect(EventStoreSpec.keys(in: store.payloadsOnDisk)) == ["event-0"] } it("keeps every event before a zero-filled payload") { @@ -517,7 +554,7 @@ final class EventStoreSpec: QuickSpec { damaged.replaceSubrange(start..<(start + size), with: Data(count: size)) try damaged.write(to: log) - expect(EventStoreSpec.keys(of: EventStoreSpec.reader(sharing: store))) == ["event-0"] + expect(EventStoreSpec.keys(in: store.payloadsOnDisk)) == ["event-0"] } it("keeps every event before a size that still fits in the file") { @@ -527,7 +564,7 @@ final class EventStoreSpec: QuickSpec { damaged[second + 5] -= 1 try damaged.write(to: log) - expect(EventStoreSpec.keys(of: EventStoreSpec.reader(sharing: store))) == ["event-0"] + expect(EventStoreSpec.keys(in: store.payloadsOnDisk)) == ["event-0"] } } @@ -600,21 +637,20 @@ final class EventStoreSpec: QuickSpec { store.commit() var failing = true - let next = EventStoreSpec.reader(sharing: store, - readFile: EventStoreSpec.readFailing(while: { failing })) + let next = store.nextRun(readFile: EventStoreSpec.readFailing(while: { failing })) + let leftOpen = next.directory!.appendingPathComponent(store.openLogUrl!.lastPathComponent) next.recoverInterruptedLog() - // Moved out from under the name this run appends to, and not deleted. - expect(FileManager.default.fileExists(atPath: store.openLogUrl!.path)) == false + // Moved out from under the name recovery looks for, and not deleted. + expect(FileManager.default.fileExists(atPath: leftOpen.path)) == false - // So this run has the name to itself, and the events that are intact are only waiting for a read that - // works to be counted and delivered. + // The events that are intact are only waiting for a read that works to be counted and delivered. _ = next.stage(EventStoreSpec.payload("this-run")) next.commit() failing = false let batches = next.pendingBatches() expect(batches.count) == 1 expect(EventStoreSpec.keys(ofBody: try next.body(of: batches[0]))) == ["from-the-previous-run"] - expect(EventStoreSpec.keys(of: EventStoreSpec.reader(sharing: store))) == ["from-the-previous-run", "this-run"] + expect(EventStoreSpec.keys(in: next.payloadsOnDisk)) == ["from-the-previous-run", "this-run"] } it("is discarded when a batch named without a count is in a format this version does not read") { @@ -1216,22 +1252,46 @@ final class EventStoreSpec: QuickSpec { expect(try store.body(of: batch)).toNot(beNil()) } - it("discards another process's log nobody has written to for the maximum age, and leaves a younger one") { - let gone = EventStore.sharing(store.directory!, processName: "gone") - _ = gone.stage(EventStoreSpec.payload("gone")) - gone.commit() - let alive = EventStore.sharing(store.directory!, processName: "alive") - _ = alive.stage(EventStoreSpec.payload("alive")) - alive.commit() - try EventStoreSpec.age(gone.openLogUrl!, byDays: 8) - try EventStoreSpec.age(alive.openLogUrl!, byDays: 1) + it("discards a log nobody holds once nothing has written to it for the maximum age, and leaves a younger one") { + func logLeftBy(_ key: String) -> URL { + // Going away closes its log, as its process ending would. + let ended = EventStore.sharing(store.directory!) + _ = ended.stage(EventStoreSpec.payload(key)) + ended.commit() + return ended.openLogUrl! + } + let old = logLeftBy("gone") + let young = logLeftBy("recent") + try EventStoreSpec.age(old, byDays: 8) + try EventStoreSpec.age(young, byDays: 1) expect(store.discardExpired()) == 1 - expect(FileManager.default.fileExists(atPath: gone.openLogUrl!.path)) == false + expect(FileManager.default.fileExists(atPath: old.path)) == false + expect(FileManager.default.fileExists(atPath: young.path)) == true + } + + it("leaves the log of another live store alone, however long since it was written") { + // A live process may go longer than the maximum age without an event. + let alive = EventStore.sharing(store.directory!) + _ = alive.stage(EventStoreSpec.payload("alive")) + alive.commit() + try EventStoreSpec.age(alive.openLogUrl!, byDays: 8) + + expect(store.discardExpired()) == 0 expect(FileManager.default.fileExists(atPath: alive.openLogUrl!.path)) == true } + it("discards a log a process died while opening, which holds no events") { + try FileManager.default.createDirectory(at: store.directory!, withIntermediateDirectories: true) + let opening = store.directory!.appendingPathComponent(DiskEventLog.openingPrefix + UUID().uuidString) + try EventLogFormat.fileHeader.write(to: opening) + try EventStoreSpec.age(opening, byDays: 8) + + expect(store.discardExpired()) == 0 + expect(FileManager.default.fileExists(atPath: opening.path)) == false + } + it("leaves its own open log alone, however long since it was written") { _ = store.stage(EventStoreSpec.payload("own")) store.commit() @@ -1417,7 +1477,7 @@ final class EventStoreSpec: QuickSpec { expect(EventStoreSpec.keys(ofBody: fromMemory.flatMap { try? interrupted.body(of: $0) })) == ["landed", "torn"] // Delivered from memory this run, so the next must not find any of it again in the log. - let next = EventStoreSpec.reader(sharing: interrupted) + let next = interrupted.nextRun() next.recoverInterruptedLog() expect(next.pendingBatches()).to(beEmpty()) } diff --git a/LaunchDarkly/LaunchDarklyTests/TestUtil.swift b/LaunchDarkly/LaunchDarklyTests/TestUtil.swift index 65247f21e..cc74b1f7c 100644 --- a/LaunchDarkly/LaunchDarklyTests/TestUtil.swift +++ b/LaunchDarkly/LaunchDarklyTests/TestUtil.swift @@ -81,10 +81,10 @@ extension EventStore { .appendingPathComponent(UUID().uuidString, isDirectory: true) } - /// Another store on `directory`, standing in for the next run of the application, or for another process. + /// Another store on `directory`, standing in for another process: it shares the batches, and leaves alone the log a + /// live store appends to. `nextRun` is the next run of the application. /// - /// Unlocked on a Mac too. Tests open it while the first store is still alive, and a locked log would rightly be - /// left to the live instance holding it; the locking itself has tests of its own. + /// Unlocked on a Mac too, like every store from `temporary`; the locking itself has tests of its own. static func sharing(_ directory: URL, byteCapacity: Int = LDConfig.Defaults.eventCapacityBytes, persistEvents: Bool = true, @@ -106,11 +106,50 @@ extension EventStore { return store } - /// Removes the store's directory, including any batch still waiting to be delivered. A store told to keep events - /// off the disk has no directory, and so nothing to remove. + /// The next run of the application, had the process ended where it stands: a store on a copy of this one's + /// directory, which holds what reached the disk and nothing that was still in memory. No live process holds the + /// logs in the copy, as none would once the process was gone, so recovery takes them. + func nextRun(readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }) -> EventStore { + let original = directory! + let copy = original.deletingLastPathComponent() + .appendingPathComponent("\(original.lastPathComponent)\(EventStore.nextRunSuffix)\(UUID().uuidString)", + isDirectory: true) + try? FileManager.default.copyItem(at: original, to: copy) + return EventStore.sharing(copy, readFile: readFile) + } + + private static let nextRunSuffix = "-next-run-" + + /// Every event in the store's directory, read straight from the files rather than through a store: the batches, + /// then the logs still open. What the next run of the application would find there. + var payloadsOnDisk: [Data] { + let files = ((try? FileManager.default.contentsOfDirectory(atPath: directory!.path)) ?? []).sorted() + let batches = files.filter { $0.hasPrefix("ready-") } + let logs = files.filter { $0.hasPrefix(DiskEventLog.prefix) } + var payloads: [Data] = [] + for name in batches + logs { + guard let file = try? Data(contentsOf: directory!.appendingPathComponent(name)) + else { continue } + _ = EventLogFormat.forEachFrame(in: file) { type, payload in + if type == EventLogFormat.eventFrame { + payloads.append(payload) + } + } + } + return payloads + } + + /// Removes the store's directory, including any batch still waiting to be delivered, and any copy `nextRun` made of + /// it. A store told to keep events off the disk has no directory, and so nothing to remove. func deleteEverything() { guard let directory else { return } + let parent = directory.deletingLastPathComponent() + let copies = ((try? FileManager.default.contentsOfDirectory(atPath: parent.path)) ?? []) + .filter { $0.hasPrefix(directory.lastPathComponent + EventStore.nextRunSuffix) } + for copy in copies { + try? FileManager.default.removeItem(at: parent.appendingPathComponent(copy)) + } try? FileManager.default.removeItem(at: directory) } } From 1cdd573beab56d8f310eea7258f836831210bd24 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 20 Oct 2026 20:26:14 -0700 Subject: [PATCH 068/158] Stop handing the process name to the event store Logs are named for a UUID now, so nothing reads it: drop the processName parameter from EventStore and EventDirectory, and from the test helpers. Co-authored-by: Cursor --- .../LaunchDarkly/ServiceObjects/EventDirectory.swift | 3 --- LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift | 6 +----- .../LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift | 2 +- LaunchDarkly/LaunchDarklyTests/TestUtil.swift | 2 -- 4 files changed, 2 insertions(+), 11 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index e02065b38..3b28dd6c0 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -39,7 +39,6 @@ final class EventDirectory { private static let batchPrefix = "ready-" private let locateDirectory: () -> URL - private let processName: String private let ioLock: UnfairLock private let logger: OSLog /// How a stored log is read, injectable so a test can make a read fail while the file is intact. @@ -94,13 +93,11 @@ final class EventDirectory { /// - Parameter locatingDirectory: answers where the events go. Asked once, and only on first use. init(locatingDirectory: @escaping () -> URL, - processName: String, ioLock: UnfairLock, logger: OSLog, readFile: @escaping (URL) throws -> Data, writeLog: @escaping (EventLogFile, Data) -> Int32?) { self.locateDirectory = locatingDirectory - self.processName = processName self.ioLock = ioLock self.logger = logger self.readFile = readFile diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index af028c69e..4c4e7eacc 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -211,13 +211,11 @@ final class EventStore: EventStoring { byteCapacity: Int = LDConfig.Defaults.eventCapacityBytes, maxAge: TimeInterval = LDConfig.Defaults.eventMaxAge, persistEvents: Bool = true, - processName: String = ProcessInfo.processInfo.processName, logger: OSLog, readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }, writeLog: @escaping (EventLogFile, Data) -> Int32? = { $0.writeAll($1) }) { self.init(locatingDirectory: { directory }, capacity: capacity, byteCapacity: byteCapacity, maxAge: maxAge, - persistEvents: persistEvents, processName: processName, logger: logger, readFile: readFile, - writeLog: writeLog) + persistEvents: persistEvents, logger: logger, readFile: readFile, writeLog: writeLog) } /// - Parameter locatingDirectory: answers where the batches go. Asked once, and only if the store reaches for the @@ -227,7 +225,6 @@ final class EventStore: EventStoring { byteCapacity: Int = LDConfig.Defaults.eventCapacityBytes, maxAge: TimeInterval = LDConfig.Defaults.eventMaxAge, persistEvents: Bool = true, - processName: String = ProcessInfo.processInfo.processName, logger: OSLog, readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }, writeLog: @escaping (EventLogFile, Data) -> Int32? = { $0.writeAll($1) }) { @@ -243,7 +240,6 @@ final class EventStore: EventStoring { // all, so there is nothing for it to write to and nothing of a previous run's for it to read. self.persistEvents = persistEvents self.disk = persistEvents ? EventDirectory(locatingDirectory: locatingDirectory, - processName: processName, ioLock: ioLock, logger: logger, readFile: readFile, diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 16f280d3f..ad74f01c3 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -1193,7 +1193,7 @@ final class EventStoreSpec: QuickSpec { expect(store.closeBatch()).toNot(beNil()) var reads = 0 - let other = EventStore.sharing(store.directory!, byteCapacity: 1000, processName: "other") { url in + let other = EventStore.sharing(store.directory!, byteCapacity: 1000) { url in reads += 1 return try Data(contentsOf: url) } diff --git a/LaunchDarkly/LaunchDarklyTests/TestUtil.swift b/LaunchDarkly/LaunchDarklyTests/TestUtil.swift index cc74b1f7c..7217c7797 100644 --- a/LaunchDarkly/LaunchDarklyTests/TestUtil.swift +++ b/LaunchDarkly/LaunchDarklyTests/TestUtil.swift @@ -88,13 +88,11 @@ extension EventStore { static func sharing(_ directory: URL, byteCapacity: Int = LDConfig.Defaults.eventCapacityBytes, persistEvents: Bool = true, - processName: String = ProcessInfo.processInfo.processName, readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }) -> EventStore { unlocked(EventStore(directory: directory, capacity: 100, byteCapacity: byteCapacity, persistEvents: persistEvents, - processName: processName, logger: .disabled, readFile: readFile)) } From 4d33b2832aaa3140827f60d1289a07e86e4471dd Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 20 Oct 2026 20:36:53 -0700 Subject: [PATCH 069/158] Name an event log pending- rather than open- The file outlives the process that opened it, and recovery closes it off for whichever process runs next, so open- described it wrongly. Co-authored-by: Cursor --- .../ServiceObjects/DiskEventLog.swift | 8 ++++---- .../ServiceObjects/EventStoreSpec.swift | 15 --------------- 2 files changed, 4 insertions(+), 19 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift index c955108cd..2e0405efd 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift @@ -28,8 +28,9 @@ import Musl /// Belongs to an `EventStore`, which makes every call holding its `ioLock`. The lock is handed in so that each call can /// check it is held, rather than this taking a lock of its own that would have to be ordered against the store's. final class DiskEventLog { - /// Names a log its owner appends to; the rest of the name is the UUID it was opened under. - static let prefix = "open-" + /// Names a log whose events are not yet in a batch: one its owner is appending to, or one left by an owner that is + /// gone, for recovery to close off. The rest of the name is the UUID it was opened under. + static let prefix = "pending-" /// Names a log while it is being opened, before it has its header. Recovery never looks at these; one left by a /// process that died opening it holds no events, and expiry removes it. static let openingPrefix = "new-" @@ -148,8 +149,7 @@ final class DiskEventLog { DiskEventLog.forget(url) } - /// Hands `recover` every log no live process holds, which it has to move or remove: whoever wrote it is gone, - /// whatever it is named, including a log an earlier version named for its process. + /// Hands `recover` every log no live process holds, which it has to move or remove: whoever wrote it is gone. /// /// Requires `ioLock`. func forEachUnheldLog(_ recover: (URL) -> Void) { diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index ad74f01c3..46147f502 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -178,21 +178,6 @@ final class EventStoreSpec: QuickSpec { expect(EventStoreSpec.keys(ofBody: batches.first.flatMap { try? app.body(of: $0) })) == ["left-open"] } - it("recovers a log an earlier version named for its process") { - var earlierVersion: EventStore? = otherProcess() - _ = earlierVersion?.stage(EventStoreSpec.payload("named-for-its-process")) - earlierVersion?.commit() - let log = earlierVersion!.openLogUrl! - earlierVersion = nil - try FileManager.default.moveItem(at: log, to: log.deletingLastPathComponent() - .appendingPathComponent("\(DiskEventLog.prefix)com_example_app-1a2b3c4d")) - - app.recoverInterruptedLog() - let batches = app.pendingBatches() - expect(batches.count) == 1 - expect(EventStoreSpec.keys(ofBody: batches.first.flatMap { try? app.body(of: $0) })) == ["named-for-its-process"] - } - it("names a log for a UUID rather than for the process") { _ = app.stage(EventStoreSpec.payload("named")) app.commit() From e29e9e266a1377af747416e9887d1314d5be7a4e Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 20 Oct 2026 20:37:01 -0700 Subject: [PATCH 070/158] Log events from a previous run as prepared, not recovered Delivering what a previous run left is ordinary persistence, not a fault. One line for the total, worded as on Android. Co-authored-by: Cursor --- LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index 3b28dd6c0..9b83c6c27 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -401,6 +401,9 @@ final class EventDirectory { ioLock.assertOwned() var recovered = EventTally() disk.log.forEachUnheldLog { recovered += tally(recoveredFrom: $0) } + if recovered.events > 0 { + os_log("%s prepared %d event(s) from a previous run of this application for delivery", log: logger, type: .debug, typeName(and: #function), recovered.events) + } return recovered } @@ -448,7 +451,6 @@ final class EventDirectory { let recovered = Stored(url: file, events: events, bytes: bytes) stored[payloadId] = recovered - os_log("%s recovered %d event(s) from a previous run", log: logger, type: .debug, typeName(and: #function), events) return recovered.tally } } From 82322ad378212918ce1bcd2d407a58866fe8403e Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 20 Oct 2026 20:51:24 -0700 Subject: [PATCH 071/158] Rename recoverInterruptedLog to preparePayloadsFromInterruptedLogs Closing off a log a previous run left is how its events get delivered, which recover made sound like a fault. Named as on Android. Co-authored-by: Cursor --- .../ServiceObjects/DirectEventProcessor.swift | 2 +- .../ServiceObjects/EventDirectory.swift | 6 ++-- .../ServiceObjects/EventStore.swift | 6 ++-- .../DirectEventProcessorSpec.swift | 4 +-- .../ServiceObjects/EventStoreSpec.swift | 34 +++++++++---------- 5 files changed, 26 insertions(+), 26 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index e6a6c4710..fef5c5672 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -191,7 +191,7 @@ class DirectEventProcessor: EventProcessor { // doing it on the caller's thread would put file I/O in the way of the client starting up. let store = self.store deliveryQueue.async { [weak self] in - store.recoverInterruptedLog() + store.preparePayloadsFromInterruptedLogs() self?.reportDiscarded(store.discardExpired()) self?.hasEventsFromPreviousRun = !store.pendingBatches().isEmpty } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index 9b83c6c27..0222ae4c6 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -397,10 +397,10 @@ final class EventDirectory { /// Requires `ioLock`. /// /// - Returns: what it closed off. - func recoverInterruptedLog() -> EventTally { + func preparePayloadsFromInterruptedLogs() -> EventTally { ioLock.assertOwned() var recovered = EventTally() - disk.log.forEachUnheldLog { recovered += tally(recoveredFrom: $0) } + disk.log.forEachUnheldLog { recovered += preparePayload(fromInterruptedLog: $0) } if recovered.events > 0 { os_log("%s prepared %d event(s) from a previous run of this application for delivery", log: logger, type: .debug, typeName(and: #function), recovered.events) } @@ -410,7 +410,7 @@ final class EventDirectory { /// Closes off a log nobody is writing to any more. Requires `ioLock`. /// /// - Returns: what it closed off, or nothing where there was nothing to close off. - private func tally(recoveredFrom leftOpen: URL) -> EventTally { + private func preparePayload(fromInterruptedLog leftOpen: URL) -> EventTally { ioLock.assertOwned() let events: Int let bytes: Int diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index 4c4e7eacc..8eee854e7 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -80,7 +80,7 @@ protocol EventStoring { /// Closes every log no live process holds, so its events join the batches to deliver: one a previous run left open, /// or another process that has since ended. - func recoverInterruptedLog() + func preparePayloadsFromInterruptedLogs() /// Deletes what has been stored for longer than the maximum age, except a batch whose request is out. /// @@ -460,11 +460,11 @@ final class EventStore: EventStoring { } } - func recoverInterruptedLog() { + func preparePayloadsFromInterruptedLogs() { guard let disk else { return } ioLock.withLock { - let recovered = disk.recoverInterruptedLog() + let recovered = disk.preparePayloadsFromInterruptedLogs() guard recovered.events > 0 else { return } bufferLock.withLock { diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 63ef54ad9..581bf19db 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -2517,7 +2517,7 @@ private final class ClosingHookStore: EventStoring { func remove(_ batch: EventBatch) { inner.remove(batch) } func startedSending(_ batch: EventBatch) { inner.startedSending(batch) } func stoppedSending(_ batch: EventBatch) { inner.stoppedSending(batch) } - func recoverInterruptedLog() { inner.recoverInterruptedLog() } + func preparePayloadsFromInterruptedLogs() { inner.preparePayloadsFromInterruptedLogs() } func discardExpired() -> Int { inner.discardExpired() } func closeBatch() -> EventBatch? { @@ -2595,7 +2595,7 @@ private final class SlowListingStore: EventStoring { func remove(_ batch: EventBatch) { inner.remove(batch) } func startedSending(_ batch: EventBatch) { inner.startedSending(batch) } func stoppedSending(_ batch: EventBatch) { inner.stoppedSending(batch) } - func recoverInterruptedLog() { inner.recoverInterruptedLog() } + func preparePayloadsFromInterruptedLogs() { inner.preparePayloadsFromInterruptedLogs() } func discardExpired() -> Int { inner.discardExpired() } /// Every listing but the first is held, the first being the one the reporter makes as it starts up. diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 46147f502..5e2089dfc 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -43,7 +43,7 @@ final class EventStoreSpec: QuickSpec { first.commit() let second = instance() - second.recoverInterruptedLog() + second.preparePayloadsFromInterruptedLogs() expect(second.pendingBatches()).to(beEmpty()) _ = second.stage(EventStoreSpec.payload("second")) @@ -67,11 +67,11 @@ final class EventStoreSpec: QuickSpec { // A descriptor of its own, as another live instance's would be, so nothing in this process knows of it. let held = EventLogFile.openLocked(log, flags: O_RDONLY)! - first.recoverInterruptedLog() + first.preparePayloadsFromInterruptedLogs() expect(first.pendingBatches()).to(beEmpty()) held.close() - first.recoverInterruptedLog() + first.preparePayloadsFromInterruptedLogs() expect(EventStoreSpec.keys(ofBody: first.pendingBatches().first.flatMap { try? first.body(of: $0) })) == ["held"] } @@ -86,7 +86,7 @@ final class EventStoreSpec: QuickSpec { ended = nil let next = instance() - next.recoverInterruptedLog() + next.preparePayloadsFromInterruptedLogs() let batches = next.pendingBatches() expect(batches.count) == 1 expect(EventStoreSpec.keys(ofBody: batches.first.flatMap { try? next.body(of: $0) })) == ["from-the-ended-instance"] @@ -99,7 +99,7 @@ final class EventStoreSpec: QuickSpec { first = nil first = EventStore(directory: directory, capacity: 100, logger: .disabled) - first.recoverInterruptedLog() + first.preparePayloadsFromInterruptedLogs() let batches = first.pendingBatches() expect(batches.count) == 1 expect(EventStoreSpec.keys(ofBody: batches.first.flatMap { try? first.body(of: $0) })) == ["left-open"] @@ -131,7 +131,7 @@ final class EventStoreSpec: QuickSpec { // Starting up, the other process recovers what no live process holds, and nothing else. let widget = otherProcess() - widget.recoverInterruptedLog() + widget.preparePayloadsFromInterruptedLogs() expect(widget.pendingBatches()).to(beEmpty()) _ = app.stage(EventStoreSpec.payload("after")) @@ -160,7 +160,7 @@ final class EventStoreSpec: QuickSpec { let closed = app.closeBatch() let widget = otherProcess() - widget.recoverInterruptedLog() + widget.preparePayloadsFromInterruptedLogs() expect(closed).toNot(beNil()) expect(widget.pendingBatches().map(\.payloadId)) == [closed?.payloadId].compactMap { $0 } } @@ -172,7 +172,7 @@ final class EventStoreSpec: QuickSpec { // Going away closes its log, as the process ending would. widget = nil - app.recoverInterruptedLog() + app.preparePayloadsFromInterruptedLogs() let batches = app.pendingBatches() expect(batches.count) == 1 expect(EventStoreSpec.keys(ofBody: batches.first.flatMap { try? app.body(of: $0) })) == ["left-open"] @@ -266,7 +266,7 @@ final class EventStoreSpec: QuickSpec { persisting.commit() let withoutPersistence = EventStore.sharing(persisting.directory!, persistEvents: false) - withoutPersistence.recoverInterruptedLog() + withoutPersistence.preparePayloadsFromInterruptedLogs() expect(withoutPersistence.pendingBatches()).to(beEmpty()) expect(withoutPersistence.pendingEventCount) == 0 @@ -275,7 +275,7 @@ final class EventStoreSpec: QuickSpec { expect(withoutPersistence.pendingBatches().map(\.payloadId)) == [own?.payloadId] let persistingAgain = persisting.nextRun() - persistingAgain.recoverInterruptedLog() + persistingAgain.preparePayloadsFromInterruptedLogs() let bodies = persistingAgain.pendingBatches().map { EventStoreSpec.keys(ofBody: try? persistingAgain.body(of: $0)) } expect(bodies) == [["closed"], ["open"]] } @@ -432,7 +432,7 @@ final class EventStoreSpec: QuickSpec { // Nothing closed the log, as nothing would if the process had died here. let next = store.nextRun() - next.recoverInterruptedLog() + next.preparePayloadsFromInterruptedLogs() let batches = next.pendingBatches() expect(batches.count) == 1 @@ -448,7 +448,7 @@ final class EventStoreSpec: QuickSpec { store.remove(batch!) let next = store.nextRun() - next.recoverInterruptedLog() + next.preparePayloadsFromInterruptedLogs() expect(next.pendingBatches()).to(beEmpty()) expect(next.pendingEventCount) == 0 @@ -467,7 +467,7 @@ final class EventStoreSpec: QuickSpec { let next = store.nextRun() _ = next.stage(EventStoreSpec.payload("this-run")) next.commit() - next.recoverInterruptedLog() + next.preparePayloadsFromInterruptedLogs() _ = next.stage(EventStoreSpec.payload("later")) next.commit() _ = next.closeBatch() @@ -624,7 +624,7 @@ final class EventStoreSpec: QuickSpec { var failing = true let next = store.nextRun(readFile: EventStoreSpec.readFailing(while: { failing })) let leftOpen = next.directory!.appendingPathComponent(store.openLogUrl!.lastPathComponent) - next.recoverInterruptedLog() + next.preparePayloadsFromInterruptedLogs() // Moved out from under the name recovery looks for, and not deleted. expect(FileManager.default.fileExists(atPath: leftOpen.path)) == false @@ -814,7 +814,7 @@ final class EventStoreSpec: QuickSpec { } #endif - store.recoverInterruptedLog() + store.preparePayloadsFromInterruptedLogs() _ = store.stage(EventStoreSpec.payload("first")) store.commit() let batch = store.closeBatch()! @@ -938,7 +938,7 @@ final class EventStoreSpec: QuickSpec { expect(store.stage(EventStoreSpec.payload("before-listing"))) == true - store.recoverInterruptedLog() + store.preparePayloadsFromInterruptedLogs() expect(store.pendingBatches().count) == 1 expect(store.pendingEventCount) == 3 expect(store.stage(EventStoreSpec.payload("after-listing"))) == false @@ -1463,7 +1463,7 @@ final class EventStoreSpec: QuickSpec { // Delivered from memory this run, so the next must not find any of it again in the log. let next = interrupted.nextRun() - next.recoverInterruptedLog() + next.preparePayloadsFromInterruptedLogs() expect(next.pendingBatches()).to(beEmpty()) } From 0533f11e2e6838c3bb483808e320600466cbb517 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 20 Oct 2026 21:53:33 -0700 Subject: [PATCH 072/158] Send a batch's events no more than once when a failed write stays on disk A write that fails is cut back off the log and its events are delivered from memory. Where the cut-back fails too, the whole frames it left in the log were sent from the batch file as well. A batch now delivers only the events it was closed holding, and a failed cut-back is logged as an error. Co-authored-by: Cursor --- .../ServiceObjects/DiskEventLog.swift | 6 +++--- .../ServiceObjects/EventDirectory.swift | 4 +++- .../ServiceObjects/EventLogFormat.swift | 8 ++++++-- .../ServiceObjects/EventStoreSpec.swift | 17 +++++++++++++++++ 4 files changed, 29 insertions(+), 6 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift index 2e0405efd..b5e5fff2b 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift @@ -114,10 +114,10 @@ final class DiskEventLog { guard write(bytes, to: file) else { // A write that fails partway can leave whole frames behind. The caller keeps every one of `bytes` in memory - // to deliver from there, so frames left here as well would be sent twice: beside the memory copy when the - // log is closed this session, or by recovery on the next launch. + // to deliver from there, so frames left here as well would be sent twice. Where they cannot be cut back, the + // batch this log is closed into still delivers only the events it was closed holding. if let start, !file.truncate(to: start) { - os_log("%s could not cut back a failed write: errno %d", log: logger, type: .debug, typeName(and: #function), errno) + os_log("%s could not cut back a failed write: errno %d", log: logger, type: .error, typeName(and: #function), errno) } return false } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index 0222ae4c6..f0351dd1c 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -288,7 +288,9 @@ final class EventDirectory { case .missing: return nil case .failed(let error): throw error case .bytes(let file): - if let body = EventLogFormat.assembleBody(from: file) { + // No more than the batch was closed holding. A write that failed and could not be truncated leaves whole + // frames after them, whose events are delivered from memory instead, and would otherwise be sent twice. + if let body = EventLogFormat.assembleBody(from: file, maxEvents: batch.eventCount > 0 ? batch.eventCount : .max) { return body } // A log this version reads that holds no events has nothing to lose, unless its first frame is damaged. diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift index 427e4d394..d126a57a0 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift @@ -110,12 +110,16 @@ enum EventLogFormat { /// Assembles the JSON array LaunchDarkly expects out of the frames, without parsing the events themselves: they were /// serialized on the way in and are shipped exactly as they were recorded. - static func assembleBody(from file: Data) -> Data? { + /// + /// Only the first `maxEvents` events go into it; any after them are left out. + static func assembleBody(from file: Data, maxEvents: Int = .max) -> Data? { var body = Data("[".utf8) var isFirst = true + var events = 0 let readable = forEachFrame(in: file) { type, payload in - guard type == eventFrame + guard type == eventFrame, events < maxEvents else { return } + events += 1 if !isFirst { body.append(UInt8(ascii: ",")) } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 5e2089dfc..510d61c73 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -565,6 +565,23 @@ final class EventStoreSpec: QuickSpec { expect { try store.body(of: batch) }.to(throwError(errorType: UnreadableBatch.self)) } + it("sends no frames a failed write left behind where it could not be cut back") { + store = EventStore.temporary() + _ = store.stage(EventStoreSpec.payload("written")) + let batch = store.closeBatch()! + let file = try EventStoreSpec.onlyBatchFile(in: store) + // Whole frames past the one the batch was closed holding, which memory delivers instead. + var leftBehind = try Data(contentsOf: file) + leftBehind.append(EventLogFormat.frame(for: EventStoreSpec.payload("from-memory"))) + try leftBehind.write(to: file) + + expect(EventStoreSpec.keys(ofBody: try? store.body(of: batch))) == ["written"] + let next = store.nextRun() + let listed = next.pendingBatches() + expect(listed.map { $0.eventCount }) == [1] + expect(EventStoreSpec.keys(ofBody: listed.first.flatMap { try? next.body(of: $0) })) == ["written"] + } + it("is kept for a later listing when a batch named without a count cannot be read") { store = EventStore.temporary() _ = store.stage(EventStoreSpec.payload("kept")) From 5ad00a1292df630185fa9d5c0bd1ec9837aabf6b Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 20 Oct 2026 21:53:33 -0700 Subject: [PATCH 073/158] Log when events from a previous run cannot be moved for delivery Moving an unreadable log aside, and moving a readable one into a batch, could each fail without a word, leaving the events for the next launch. Co-authored-by: Cursor --- .../ServiceObjects/EventDirectory.swift | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index f0351dd1c..0907a0276 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -423,11 +423,15 @@ final class EventDirectory { // Deleting it would lose events that may be intact, and left under this name it would wait for the next // run's recovery. Closed off uncounted instead, it is read and counted by whichever listing first manages // to. - os_log("%s could not read events from a previous run, will try again later: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) // Named without a count, which is what a batch whose events have never been counted is: the listing that // first manages to read it is what counts it. let uncounted = disk.directory.appendingPathComponent("\(EventDirectory.batchPrefix)\(UUID().uuidString)") - try? FileManager.default.moveItem(at: leftOpen, to: uncounted) + if (try? FileManager.default.moveItem(at: leftOpen, to: uncounted)) != nil { + os_log("%s could not read events from a previous run, will try again later: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) + } else { + // Left where it is, unlocked and appended to by no one, for the next launch to recover. + os_log("%s could not read events from a previous run, will try again at the next launch: %s", log: logger, type: .info, typeName(and: #function), String(describing: error)) + } return EventTally() case .bytes(let log): guard let counted = EventLogFormat.eventCount(in: log) @@ -449,7 +453,10 @@ final class EventDirectory { let payloadId = UUID().uuidString let file = newUrl(of: payloadId, holding: events) guard (try? FileManager.default.moveItem(at: leftOpen, to: file)) != nil - else { return EventTally() } + else { + os_log("%s could not prepare %d event(s) from a previous run, will try again at the next launch", log: logger, type: .info, typeName(and: #function), events) + return EventTally() + } let recovered = Stored(url: file, events: events, bytes: bytes) stored[payloadId] = recovered From b21ef40d4fd338d363c26a69ffc911c4c2b20ea1 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 20 Oct 2026 22:39:14 -0700 Subject: [PATCH 074/158] Keep every batch within 2 MiB, so no request is too large to accept While the SDK is offline nothing closes the open log, so one batch could grow to the whole byte capacity, 20 MiB, and go out as one request. The events service refuses a request that large with a 4xx no retry changes, which drops the batch and stops events for the session. A commit now closes the open log before any frame that would take it past 2 MiB and carries on in a new one; events held in memory are split the same way. Splitting the file rather than the request keeps one payload ID per batch, so a part that arrived is never sent again. The largest event is lowered from 8 MiB to what fits in a batch on its own. Co-authored-by: Cursor --- .../ServiceObjects/EventLogFormat.swift | 25 +++- .../ServiceObjects/EventStore.swift | 124 ++++++++++++------ .../ServiceObjects/EventStoreSpec.swift | 82 ++++++++++++ 3 files changed, 192 insertions(+), 39 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift index d126a57a0..6b989f425 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift @@ -22,8 +22,12 @@ enum EventLogFormat { static let frameHeaderSize = 10 /// The only frame type written today; a reader skips a frame whose type it does not know. static let eventFrame: UInt16 = 1 - /// A ceiling on a single frame, so a corrupt size cannot make recovery allocate wildly. - static let maxFrameSize = 8 * 1024 * 1024 + /// The most a batch may hold, file header included, so that no request it becomes is larger than the events + /// service accepts. + static let maxBatchSize = 2 * 1024 * 1024 + /// A ceiling on a single frame's payload: the most that still fits in a batch on its own. It is also what keeps a + /// corrupt size from making recovery allocate wildly. + static let maxFrameSize = maxBatchSize - fileHeaderSize - frameHeaderSize static var fileHeader: Data { var header = magic @@ -145,6 +149,23 @@ enum EventLogFormat { UInt8(truncatingIfNeeded: value)]) } + /// The size, header included, of the frame at `start` in frames this process wrote. + static func frameSize(in frames: Data, at start: Int) -> Int { + frameHeaderSize + Int(readUInt32(frames, at: start + 2)) + } + + /// Where a run of whole frames from `start`, in frames this process wrote, ends if it is to take no more than + /// `room` bytes, and how many frames it takes. It takes the first frame whatever its size. + static func framesFitting(in frames: Data, from start: Int, room: Int) -> (end: Int, count: Int) { + var end = start + frameSize(in: frames, at: start) + var count = 1 + while end < frames.count, end + frameSize(in: frames, at: end) - start <= room { + end += frameSize(in: frames, at: end) + count += 1 + } + return (end, count) + } + static func readUInt16(_ data: Data, at offset: Int) -> UInt16 { let start = data.startIndex + offset return UInt16(data[start]) << 8 | UInt16(data[start + 1]) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index 8eee854e7..9428ee555 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -55,8 +55,10 @@ protocol EventStoring { /// Commits, then closes the open log into a batch to deliver, and whatever is still staged in memory into another. /// - /// Returns the batch closed from the log, or the one closed from memory where the log held nothing, and nil when - /// there is nothing to send. + /// Returns the batch closed from the log, or the first closed from memory where the log held nothing, and nil when + /// there is nothing to send. A log a commit filled was closed into a batch of its own as it filled, and memory is + /// closed into as many batches as keep each within `EventLogFormat.maxBatchSize`; `pendingBatches()` lists every + /// one. func closeBatch() -> EventBatch? /// Batches awaiting delivery, oldest first, including any recovered from a previous run of the application. @@ -334,7 +336,7 @@ final class EventStore: EventStoring { func closeBatch() -> EventBatch? { guard let disk - else { return closeInMemoryBatch() } + else { return closeInMemoryBatches().first } ioLock.lock() defer { ioLock.unlock() } @@ -349,8 +351,8 @@ final class EventStore: EventStoring { // Closed after the log, so the older events are delivered first. Both can hold events at once: a write that // fails after earlier ones landed gives up on persistence with those earlier events in the log and the failed // ones back in memory, and closing only the log would leave the rest to a delivery that reports success - // without them. - let fromMemory = stillBuffered > 0 ? closeInMemoryBatch() : nil + // without them. A full log that could not be closed leaves events staged too. + let fromMemory = stillBuffered > 0 ? closeInMemoryBatches().first : nil return fromLog ?? fromMemory } @@ -548,6 +550,9 @@ extension EventStore { extension EventStore { /// Requires `ioLock`. + /// + /// Closes the open log into a batch wherever the next frame would take it past `EventLogFormat.maxBatchSize`, so a + /// commit can land in several batches. private func writeStagedBytes() { ioLock.assertOwned() let taken = bufferLock.withLock { () -> (bytes: Data, events: Int)? in @@ -565,25 +570,58 @@ extension EventStore { guard let taken else { return } - if append(taken.bytes) { + let base = taken.bytes.startIndex + var written = 0 + var writtenEvents = 0 + while written < taken.bytes.count { + let room = EventLogFormat.maxBatchSize - EventLogFormat.fileHeaderSize - bufferLock.withLock { committedBytes } + if EventLogFormat.frameSize(in: taken.bytes, at: written) > room { + guard closeFullLog() + else { break } + continue + } + let (end, landing) = EventLogFormat.framesFitting(in: taken.bytes, from: written, room: room) + let isWhole = written == 0 && end == taken.bytes.count + guard append(isWhole ? taken.bytes : taken.bytes.subdata(in: (base + written)..<(base + end))) + else { break } + // Counted where they landed in the same critical section that stops counting them on the way, so a + // capacity check never sees them in both places or in neither. bufferLock.withLock { - writingEvents -= taken.events - writingBytes -= taken.bytes.count - committedEvents += taken.events - committedBytes += taken.bytes.count + writingEvents -= landing + writingBytes -= end - written + committedEvents += landing + committedBytes += end - written } - } else { + written = end + writtenEvents += landing + } + + if written < taken.bytes.count { // Put back, to be delivered from memory rather than lost. Ahead of anything staged while the write was in // flight, so the events keep the order they were recorded in. bufferLock.withLock { - writingEvents -= taken.events - writingBytes -= taken.bytes.count - bufferData = taken.bytes + bufferData - bufferedEventCount += taken.events + writingEvents -= taken.events - writtenEvents + writingBytes -= taken.bytes.count - written + bufferData = taken.bytes.subdata(in: (base + written).. Bool { + ioLock.assertOwned() + guard let disk + else { return false } + let committed = bufferLock.withLock { EventTally(events: committedEvents, bytes: committedBytes) } + _ = closeOpenLog(of: disk, holding: committed) + return bufferLock.withLock { committedBytes == 0 } + } + /// Requires `ioLock`. private func append(_ bytes: Data) -> Bool { ioLock.assertOwned() @@ -623,11 +661,12 @@ private extension EventStore { bufferLock.withLock { persistEvents = false } } - /// Closes whatever is staged into a batch that never reaches a file. + /// Closes whatever is staged into batches that never reach a file, as many as it takes to keep each within + /// `EventLogFormat.maxBatchSize`. Returns them oldest first. /// /// Called under `ioLock` where there is a disk, so that it and the log are closed as one step, and on its own /// where there is not. - func closeInMemoryBatch() -> EventBatch? { + func closeInMemoryBatches() -> [EventBatch] { let (frames, events) = bufferLock.withLock { () -> (frames: Data, events: Int) in let staged = (frames: bufferData, events: bufferedEventCount) bufferData = Data() @@ -640,32 +679,43 @@ private extension EventStore { // Assembled outside `bufferLock`, because it copies every staged frame; `writingEvents` is what keeps them // counted meanwhile. Through the same reader a batch on disk goes through, so a held batch and a written one // produce byte-identical request bodies. - var log = EventLogFormat.fileHeader - log.append(frames) - guard let body = EventLogFormat.assembleBody(from: log) - else { - bufferLock.withLock { - writingEvents -= events - writingBytes -= frames.count + let base = frames.startIndex + var closed: [EventBatch] = [] + var held = 0 + var heldEvents = 0 + while held < frames.count { + let (end, batchEvents) = EventLogFormat.framesFitting(in: frames, from: held, + room: EventLogFormat.maxBatchSize - EventLogFormat.fileHeaderSize) + var log = EventLogFormat.fileHeader + log.append(frames.subdata(in: (base + held)..<(base + end))) + guard let body = EventLogFormat.assembleBody(from: log) + else { break } + + let payloadId = UUID().uuidString + heldLock.withLock { + inMemoryBatches.append(HeldBatch(payloadId: payloadId, body: body, eventCount: batchEvents)) + // Counted where they landed in the same critical section that stops counting them on the way, so a + // capacity check never sees them in both places or in neither. A held batch is counted as its body, + // which is what it keeps. + bufferLock.withLock { + writingEvents -= batchEvents + writingBytes -= end - held + closedEvents += batchEvents + closedBytes += body.count + } } - return nil + closed.append(EventBatch(payloadId: payloadId, eventCount: batchEvents)) + held = end + heldEvents += batchEvents } - let payloadId = UUID().uuidString - heldLock.withLock { - inMemoryBatches.append(HeldBatch(payloadId: payloadId, body: body, eventCount: events)) - // Counted where they landed in the same critical section that stops counting them on the way, so a - // capacity check never sees them in both places or in neither. A held batch is counted as its body, which - // is what it keeps. + if held < frames.count { bufferLock.withLock { - writingEvents -= events - writingBytes -= frames.count - closedEvents += events - closedBytes += body.count + writingEvents -= events - heldEvents + writingBytes -= frames.count - held } } - - return EventBatch(payloadId: payloadId, eventCount: events) + return closed } } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 510d61c73..d97406758 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -11,6 +11,7 @@ final class EventStoreSpec: QuickSpec { batchSpec() capacitySpec() byteCapacitySpec() + batchSizeSpec() expirySpec() fallbackSpec() optOutSpec() @@ -1098,6 +1099,87 @@ final class EventStoreSpec: QuickSpec { } } + /// Events framed in a quarter of a batch each: four of them and the file header are just past the batch size, so + /// three fit in each batch. + private func batchSizeSpec() { + describe("batch size") { + let quarter = EventLogFormat.maxBatchSize / 4 + var store: EventStore! + afterEach { + store.deleteEverything() + } + + func stage(_ indices: ClosedRange) { + for index in indices { + expect(store.stage(EventStoreSpec.payload(framedIn: quarter, key: "event-\(index)"))) == true + } + } + + func keysByBatch() -> [[String]] { + store.pendingBatches().map { batch -> [String] in + let body = try? store.body(of: batch) + expect(body?.count ?? 0) <= EventLogFormat.maxBatchSize + let keys = EventStoreSpec.keys(ofBody: body) + expect(keys.count) == batch.eventCount + return keys + } + // Batches closed within the same instant list in either order. + .sorted { $0.first ?? "" < $1.first ?? "" } + } + + func batchFileSizes() throws -> [Int] { + try FileManager.default + .contentsOfDirectory(at: store.directory!, includingPropertiesForKeys: [.fileSizeKey]) + .filter { $0.lastPathComponent.hasPrefix("ready-") } + .map { try $0.resourceValues(forKeys: [.fileSizeKey]).fileSize ?? 0 } + } + + it("closes a log a commit would take past the batch size, and carries on in a new one") { + store = EventStore.temporary() + stage(1...5) + store.commit() + _ = store.closeBatch() + + expect(keysByBatch()) == [["event-1", "event-2", "event-3"], ["event-4", "event-5"]] + expect(try batchFileSizes().allSatisfy { $0 <= EventLogFormat.maxBatchSize }) == true + expect(store.pendingEventCount) == 5 + } + + it("closes a log a later commit would take past the batch size") { + store = EventStore.temporary() + stage(1...2) + store.commit() + stage(3...5) + store.commit() + _ = store.closeBatch() + + expect(keysByBatch()) == [["event-1", "event-2", "event-3"], ["event-4", "event-5"]] + expect(try batchFileSizes().allSatisfy { $0 <= EventLogFormat.maxBatchSize }) == true + } + + it("splits what it holds in memory at the batch size too") { + store = EventStore.temporary(persistEvents: false) + stage(1...5) + _ = store.closeBatch() + + expect(keysByBatch()) == [["event-1", "event-2", "event-3"], ["event-4", "event-5"]] + expect(store.pendingEventCount) == 5 + } + + it("takes an event that fills a batch on its own, and refuses any larger") { + store = EventStore.temporary() + let largest = EventLogFormat.maxFrameSize + EventLogFormat.frameHeaderSize + expect(store.stage(EventStoreSpec.payload(framedIn: largest + 1, key: "too-large"))) == false + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-1"))) == true + expect(store.stage(EventStoreSpec.payload(framedIn: largest, key: "largest"))) == true + _ = store.closeBatch() + + expect(keysByBatch()) == [["event-1"], ["largest"]] + expect(try batchFileSizes().allSatisfy { $0 <= EventLogFormat.maxBatchSize }) == true + } + } + } + /// A byte capacity of 1000 lets full events take the store to 800 bytes and summaries to 1000. Every event here is /// framed in exactly 100 bytes, so eight full events fill it and two summaries fill the rest. private func byteCapacitySpec() { From 95957b9bf50d319f41a302a5b5a839b2b5dc22c3 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 20 Oct 2026 22:56:52 -0700 Subject: [PATCH 075/158] Hold a batch being delivered twice at most, and stop copying every frame Walking a log copied each payload into a Data of its own, even to count events or check for damage, and the request body grew as payloads were appended to it. The walk now hands over where each payload is, and the body is allocated once at its exact size, which leaves the file and the body. Co-authored-by: Cursor --- .../ServiceObjects/EventLogFormat.swift | 40 +++++++++++-------- .../ServiceObjects/EventStore.swift | 5 ++- .../ServiceObjects/EventStoreSpec.swift | 12 ++++++ 3 files changed, 39 insertions(+), 18 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift index 6b989f425..d609c69b4 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift @@ -62,6 +62,14 @@ enum EventLogFormat { /// log this version reads. @discardableResult static func forEachFrame(in file: Data, _ visit: (UInt16, Data) -> Void) -> FrameWalk? { + forEachFramePayload(in: file) { type, payload in + visit(type, file.subdata(in: (file.startIndex + payload.lowerBound)..<(file.startIndex + payload.upperBound))) + } + } + + /// `forEachFrame(in:_:)` handing over where each payload is, as offsets into the file, rather than a copy of it. + @discardableResult + static func forEachFramePayload(in file: Data, _ visit: (UInt16, Range) -> Void) -> FrameWalk? { guard file.count >= fileHeaderSize, file.prefix(magic.count) == magic, readUInt16(file, at: magic.count) == version @@ -83,7 +91,7 @@ enum EventLogFormat { guard checksum.checksum == readUInt32(bytes, at: cursor + 6) else { break } - visit(type, Data(payload)) + visit(type, start..<(start + length)) cursor = start + length } return FrameWalk(reachedEnd: cursor == bytes.count) @@ -97,7 +105,7 @@ enum EventLogFormat { /// How many events a log holds, or nil where it is not one this version reads. static func eventCount(in file: Data) -> Int? { var events = 0 - guard forEachFrame(in: file, { type, _ in + guard forEachFramePayload(in: file, { type, _ in if type == eventFrame { events += 1 } @@ -109,7 +117,7 @@ enum EventLogFormat { /// Whether a log holds frames that cannot be read back: torn, or failing their checksum. False for a log this /// version does not read at all. static func isDamaged(_ file: Data) -> Bool { - forEachFrame(in: file) { _, _ in }?.reachedEnd == false + forEachFramePayload(in: file) { _, _ in }?.reachedEnd == false } /// Assembles the JSON array LaunchDarkly expects out of the frames, without parsing the events themselves: they were @@ -117,24 +125,24 @@ enum EventLogFormat { /// /// Only the first `maxEvents` events go into it; any after them are left out. static func assembleBody(from file: Data, maxEvents: Int = .max) -> Data? { - var body = Data("[".utf8) - var isFirst = true - var events = 0 - let readable = forEachFrame(in: file) { type, payload in - guard type == eventFrame, events < maxEvents + // Found first so the body is allocated once at its exact size rather than grown as it is appended to. + var payloads: [Range] = [] + let readable = forEachFramePayload(in: file) { type, payload in + guard type == eventFrame, payloads.count < maxEvents else { return } - events += 1 - if !isFirst { - body.append(UInt8(ascii: ",")) - } - body.append(payload) - isFirst = false + payloads.append(payload) } - guard readable != nil, !isFirst + guard readable != nil, !payloads.isEmpty else { return nil } - body.append(UInt8(ascii: "]")) + // The "[", and then each payload with the "," or "]" after it. + var body = Data(capacity: payloads.reduce(1) { $0 + $1.count + 1 }) + body.append(UInt8(ascii: "[")) + for (index, payload) in payloads.enumerated() { + body.append(file[(file.startIndex + payload.lowerBound)..<(file.startIndex + payload.upperBound)]) + body.append(UInt8(ascii: index == payloads.count - 1 ? "]" : ",")) + } return body } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index 9428ee555..ac5f819c2 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -686,8 +686,9 @@ private extension EventStore { while held < frames.count { let (end, batchEvents) = EventLogFormat.framesFitting(in: frames, from: held, room: EventLogFormat.maxBatchSize - EventLogFormat.fileHeaderSize) - var log = EventLogFormat.fileHeader - log.append(frames.subdata(in: (base + held)..<(base + end))) + var log = Data(capacity: EventLogFormat.fileHeaderSize + end - held) + log.append(EventLogFormat.fileHeader) + log.append(frames[(base + held)..<(base + end)]) guard let body = EventLogFormat.assembleBody(from: log) else { break } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index d97406758..c42f25cef 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -697,6 +697,18 @@ final class EventStoreSpec: QuickSpec { expect(body) == Data("[{\"kind\":\"custom\",\"key\":\"first\"},{\"kind\":\"custom\",\"key\":\"second\"}]".utf8) } + it("assembles one event, and events held in memory, into the same body") { + _ = store.stage(EventStoreSpec.payload("only")) + expect(try store.body(of: store.closeBatch()!)) == Data("[{\"kind\":\"custom\",\"key\":\"only\"}]".utf8) + + let inMemory = EventStore.temporary(persistEvents: false) + defer { inMemory.deleteEverything() } + _ = inMemory.stage(EventStoreSpec.payload("first")) + _ = inMemory.stage(EventStoreSpec.payload("second")) + expect(try inMemory.body(of: inMemory.closeBatch()!)) + == Data("[{\"kind\":\"custom\",\"key\":\"first\"},{\"kind\":\"custom\",\"key\":\"second\"}]".utf8) + } + it("frames an event as the Android SDK does") { // The checksum is zlib's CRC-32, so these bytes are what the Android SDK writes for the same event too. let frame = EventLogFormat.frame(for: Data("{\"kind\":\"custom\",\"key\":\"k\"}".utf8)) From edb95e7e6b9692c73a7c6f8e891bd2595f4f721f Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 20 Oct 2026 23:16:01 -0700 Subject: [PATCH 076/158] Tell the next flush about a batch lost where nobody was waiting A batch that cannot be read, or that the service refuses, is removed. Count its events as lost, so a flush that did not wait on that delivery still learns events were dropped. Co-authored-by: Cursor --- .../ServiceObjects/DirectEventProcessor.swift | 10 +++++ .../DirectEventProcessorSpec.swift | 37 +++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index fef5c5672..470c6fbc9 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -298,6 +298,14 @@ class DirectEventProcessor: EventProcessor { } } + /// Records a batch lost for good. The pass that lost it tells its caller, but a delivery nobody waits on -- the + /// periodic one, a `flush()`, the one going online starts -- has no one to tell, so the loss is also counted for the + /// next `flushReportingOutcome` to hear about. A caller waiting on this pass can then hear about it from the next + /// one too. + private func reportBatchLost(_ batch: EventBatch) { + stateLock.withLock { lostEvents += UInt64(batch.eventCount) } + } + /// Records the loss of an event the SDK accepted and then could not keep, which no later attempt recovers. private func reportLost() { os_log("%s dropping an event the store would not take", log: service.config.logger, type: .error, typeName(and: #function)) @@ -707,6 +715,7 @@ extension DirectEventProcessor { // the caller hears that this pass lost events. os_log("%s dropping %d stored event(s) this version cannot read", log: service.config.logger, type: .error, typeName(and: #function), batch.eventCount) store.remove(batch) + reportBatchLost(batch) deliver(remaining, lostAnything: true, completion) return } catch { @@ -733,6 +742,7 @@ extension DirectEventProcessor { case .refused: // The rest are still attempted, so that batches LaunchDarkly will never take do not sit on the device // for the rest of the session, taking up the room the capacity limit leaves for new events. + self.reportBatchLost(batch) self.deliver(remaining, lostAnything: true, completion) case .retryable: completion?(false) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 581bf19db..f8d75283a 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -2040,6 +2040,43 @@ extension DirectEventProcessorSpec { expect(store.pendingEventCount) == 0 } + it("tells the next flush about a stored batch a delivery nobody waited on dropped") { + let store = EventStore.temporary() + testContext = TestContext(store: store) + testContext.eventProcessor.isOnline = true + waitUntil(timeout: Constants.deliveryTimeout) { done in + testContext.eventProcessor.flushReportingOutcome { _ in done() } + } + + testContext.recordEvents(1) + testContext.eventProcessor.commitRecordedEvents() + let damaged = store.closeBatch()! + let damagedUrl = store.directory!.appendingPathComponent("ready-\(damaged.payloadId)-\(damaged.eventCount)") + try Data("not an event log".utf8).write(to: damagedUrl) + + testContext.eventProcessor.flush(completion: nil) + expect(FileManager.default.fileExists(atPath: damagedUrl.path)).toEventually(beFalse(), timeout: Constants.deliveryTimeout) + + var delivered: Bool? + testContext.recordEvents(1) + waitUntil(timeout: Constants.deliveryTimeout) { done in + testContext.eventProcessor.flushReportingOutcome { result in + delivered = result + done() + } + } + expect(delivered) == false + + testContext.recordEvents(1) + waitUntil(timeout: Constants.deliveryTimeout) { done in + testContext.eventProcessor.flushReportingOutcome { result in + delivered = result + done() + } + } + expect(delivered) == true + } + it("delivers what memory was left holding beside what the disk took, and reports true") { var writes = 0 let store = EventStore.temporary(writeLog: { file, bytes in From 91a301a92f6970c54bad995913fba837e14974d7 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 20 Oct 2026 23:16:02 -0700 Subject: [PATCH 077/158] Say when a read that failed is tried again "Later" meant the next delivery; say so, and log it at info like the cases left for the next launch, since events wait on it. Co-authored-by: Cursor --- LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index 0907a0276..238067883 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -374,7 +374,7 @@ final class EventDirectory { case .failed(let error): // Left for a later listing rather than deleted: the read may pass then, and the events in it may have been // intact all along. Without a count it cannot be delivered now. - os_log("%s could not read stored events, will try again later: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) + os_log("%s could not read stored events, will try again at the next delivery: %s", log: logger, type: .info, typeName(and: #function), String(describing: error)) return nil case .bytes(let log): guard let events = EventLogFormat.eventCount(in: log) @@ -427,7 +427,7 @@ final class EventDirectory { // first manages to read it is what counts it. let uncounted = disk.directory.appendingPathComponent("\(EventDirectory.batchPrefix)\(UUID().uuidString)") if (try? FileManager.default.moveItem(at: leftOpen, to: uncounted)) != nil { - os_log("%s could not read events from a previous run, will try again later: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) + os_log("%s could not read events from a previous run, will try again at the next delivery: %s", log: logger, type: .info, typeName(and: #function), String(describing: error)) } else { // Left where it is, unlocked and appended to by no one, for the next launch to recover. os_log("%s could not read events from a previous run, will try again at the next launch: %s", log: logger, type: .info, typeName(and: #function), String(describing: error)) From e3c64b8bb6f56971c995d08964086ae3495e8bf1 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Wed, 21 Oct 2026 10:22:54 -0700 Subject: [PATCH 078/158] Leave a batch out of the count only while it is still counted startedSending could leave out a batch remove() had already forgotten, or one a listing dropped because another process delivered it, taking its events from the batches still counted. Leave a batch out only while it is counted, with the tally it is counted under, so the clamp that hid this is no longer needed. Co-authored-by: Cursor --- .../ServiceObjects/EventDirectory.swift | 7 ++- .../ServiceObjects/EventStore.swift | 56 +++++++++++-------- .../ServiceObjects/EventStoreSpec.swift | 38 +++++++++++++ 3 files changed, 76 insertions(+), 25 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index 238067883..94a4e933e 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -258,10 +258,11 @@ final class EventDirectory { return stored.values.reduce(0) { $0 + $1.bytes } } - /// The bytes in a batch, as far as this process has counted them. Requires `ioLock`. - func bytes(of payloadId: String) -> Int { + /// What this process counts for a batch, or nil if it counts nothing for it: one removed, or dropped by a listing it + /// was no longer in. Requires `ioLock`. + func counted(_ payloadId: String) -> EventTally? { ioLock.assertOwned() - return stored[payloadId]?.bytes ?? 0 + return stored[payloadId]?.tally } /// Whether a batch has already been delivered and only its file is left, trying the delete that failed once more diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index ac5f819c2..09ee2a070 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -17,7 +17,8 @@ protocol EventStoring { /// /// A batch being sent is left out so that a flush makes room as soon as it sends, as it did when events were only /// held in memory; otherwise an event recorded while the request is out is refused for a batch LaunchDarkly may - /// already have. A send that fails puts the batch back, so a full store can briefly hold capacity plus that batch. + /// already have. A send that fails puts the batch back, so a full store can briefly hold capacity plus that batch: + /// one at a time, since deliveries run one after another, and no more than `EventLogFormat.maxBatchSize`. var pendingEventCount: Int { get } /// Whether a full event of any size could still be staged within the byte capacity. @@ -154,8 +155,9 @@ final class EventStore: EventStoring { /// critical section that changes one of those, so that it cannot drift from them; a total worked out from them and /// written afterwards would overwrite a close or a removal in between. private var closedEvents = 0 - /// Closed batches whose request is out, by payload ID, and what is in them. Every one is also in `closedEvents`, - /// and leaves both in the same critical section. + /// Closed batches whose request is out, by payload ID, and what is in them as `closedEvents` counts it. Every one is + /// also counted there: one is only added while it is counted, and leaves in the same critical section that stops + /// counting it, so `sendingEvents` can never exceed `closedEvents`. private var sendingBatches: [String: EventTally] = [:] private var sendingEvents = 0 @@ -257,15 +259,14 @@ final class EventStore: EventStoring { /// Every event counted against capacity. Only to be used while holding `bufferLock`. private var countedEvents: Int { - // Clamped because a batch another process delivered can be marked as sending after a listing stopped counting it. - reservedEvents + bufferedEventCount + writingEvents + committedEvents + max(0, closedEvents - sendingEvents) + reservedEvents + bufferedEventCount + writingEvents + committedEvents + (closedEvents - sendingEvents) } /// Every byte counted against the byte capacity. Only to be used while holding `bufferLock`. /// /// Events reserved while being encoded have no bytes yet; they are counted when they are staged. private var countedBytes: Int { - bufferData.count + writingBytes + committedBytes + max(0, closedBytes - sendingBytes) + bufferData.count + writingBytes + committedBytes + (closedBytes - sendingBytes) } var hasRoomForEventBytes: Bool { @@ -512,14 +513,35 @@ final class EventStore: EventStoring { // MARK: Sending extension EventStore { + /// Stops counting a batch against capacity while its request is out. Ended by `remove(_:)` or `stoppedSending(_:)`. + /// + /// Does nothing for a batch this store no longer counts - one `remove(_:)` forgot, or a listing dropped because + /// another process delivered it - since leaving it out would take its events from the batches still counted. func startedSending(_ batch: EventBatch) { - let bytes = bytes(of: batch.payloadId) - bufferLock.withLock { - guard sendingBatches[batch.payloadId] == nil + guard let disk + else { + startedSending(batch.payloadId, disk: nil) + return + } + // Held across the check and the change, so that the batch cannot stop being counted in between. + ioLock.withLock { startedSending(batch.payloadId, disk: disk) } + } + + /// Requires `ioLock` where there is a disk. + private func startedSending(_ payloadId: String, disk: EventDirectory?) { + heldLock.withLock { + let counted = inMemoryBatches.first { $0.payloadId == payloadId } + .map { EventTally(events: $0.eventCount, bytes: $0.body.count) } + ?? disk?.counted(payloadId) + guard let counted else { return } - sendingBatches[batch.payloadId] = EventTally(events: batch.eventCount, bytes: bytes) - sendingEvents += batch.eventCount - sendingBytes += bytes + bufferLock.withLock { + guard sendingBatches[payloadId] == nil + else { return } + sendingBatches[payloadId] = counted + sendingEvents += counted.events + sendingBytes += counted.bytes + } } } @@ -527,16 +549,6 @@ extension EventStore { bufferLock.withLock { forgetSending(batch.payloadId) } } - /// The bytes a closed batch is counted as, held or on the disk. - private func bytes(of payloadId: String) -> Int { - if let held = heldLock.withLock({ inMemoryBatches.first { $0.payloadId == payloadId } }) { - return held.body.count - } - guard let disk - else { return 0 } - return ioLock.withLock { disk.bytes(of: payloadId) } - } - /// Requires `bufferLock`. fileprivate func forgetSending(_ payloadId: String) { if let sending = sendingBatches.removeValue(forKey: payloadId) { diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index c42f25cef..f9c9e7842 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -956,6 +956,44 @@ final class EventStoreSpec: QuickSpec { expect(store.pendingEventCount) == 1 } + // As when another process delivered it and a listing dropped it before this one got to it: leaving it out + // would take its events from the batches still counted, and admit that many more. + for persistEvents in [true, false] { + it("does not leave a batch already removed out of the count when its send starts, persisting: \(persistEvents)") { + store = EventStore.temporary(capacity: 2, persistEvents: persistEvents) + expect(store.stage(EventStoreSpec.payload("first"))) == true + let removed = store.closeBatch()! + expect(store.stage(EventStoreSpec.payload("second"))) == true + _ = store.closeBatch() + store.remove(removed) + + store.startedSending(removed) + expect(store.pendingEventCount) == 1 + expect(store.stage(EventStoreSpec.payload("third"))) == true + expect(store.stage(EventStoreSpec.payload("fourth"))) == false + + store.stoppedSending(removed) + expect(store.pendingEventCount) == 2 + } + } + + it("does not leave a batch another process delivered out of the count when its send starts") { + store = EventStore.temporary(capacity: 2) + expect(store.stage(EventStoreSpec.payload("first"))) == true + let delivered = store.closeBatch()! + expect(store.stage(EventStoreSpec.payload("second"))) == true + _ = store.closeBatch() + let other = EventStore.sharing(store.directory!) + _ = other.pendingBatches() + other.remove(delivered) + _ = store.pendingBatches() + + store.startedSending(delivered) + expect(store.pendingEventCount) == 1 + expect(store.stage(EventStoreSpec.payload("third"))) == true + expect(store.stage(EventStoreSpec.payload("fourth"))) == false + } + // The bound `LDConfig.eventCapacity` documents: the reporter reads the disk in the background after starting, // so what an earlier run left only counts from that listing on. it("counts what an earlier run left undelivered from the first listing on") { From 2fa7afccace70d22c06d9a214bf768fe95da3863 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Wed, 21 Oct 2026 16:50:43 -0700 Subject: [PATCH 079/158] Assemble a batch's payload in the bytes its file was read into The body is always shorter than the log it comes from - the file header becomes the "[" and each frame header a "," or "]" - so each payload can move towards the start of the same bytes as its frame is checked. A delivery holds the batch once rather than as the file and a copy of it. Co-authored-by: Cursor --- .../ServiceObjects/EventDirectory.swift | 27 +++-- .../ServiceObjects/EventLogFormat.swift | 110 +++++++++++------- .../ServiceObjects/EventStore.swift | 8 +- .../ServiceObjects/EventStoreSpec.swift | 15 +++ 4 files changed, 102 insertions(+), 58 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index 94a4e933e..fbabdea28 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -277,7 +277,8 @@ final class EventDirectory { return true } - /// The `[...]` request body for a batch, or nil if it is gone or holds no events. + /// The `[...]` payload for a batch, assembled in the bytes the file was read into; or nil if it is gone or holds no + /// events. /// /// Requires `ioLock`. /// @@ -285,20 +286,24 @@ final class EventDirectory { /// could not be read this time, which a later attempt may manage. func body(of batch: EventBatch) throws -> Data? { ioLock.assertOwned() + // Taken out of the read's result rather than bound inside the switch, so that nothing else refers to it and it + // is assembled where it is rather than copied first. + var file: Data switch read(url(of: batch.payloadId, holding: batch.eventCount)) { case .missing: return nil case .failed(let error): throw error - case .bytes(let file): - // No more than the batch was closed holding. A write that failed and could not be truncated leaves whole - // frames after them, whose events are delivered from memory instead, and would otherwise be sent twice. - if let body = EventLogFormat.assembleBody(from: file, maxEvents: batch.eventCount > 0 ? batch.eventCount : .max) { - return body - } - // A log this version reads that holds no events has nothing to lose, unless its first frame is damaged. - guard EventLogFormat.eventCount(in: file) == 0, !EventLogFormat.isDamaged(file) - else { throw UnreadableBatch(payloadId: batch.payloadId) } - return nil + case .bytes(let bytes): file = bytes + } + // No more than the batch was closed holding. A write that failed and could not be truncated leaves whole frames + // after them, whose events are delivered from memory instead, and would otherwise be sent twice. + if EventLogFormat.assembleBody(inPlace: &file, maxEvents: batch.eventCount > 0 ? batch.eventCount : .max) { + return file } + // A log this version reads that holds no events has nothing to lose, unless its first frame is damaged. Where + // there is no payload the log is as it was read. + guard EventLogFormat.eventCount(in: file) == 0, !EventLogFormat.isDamaged(file) + else { throw UnreadableBatch(payloadId: batch.payloadId) } + return nil } /// Forgets a batch. Requires `ioLock`. diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift index d609c69b4..773ba4a90 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift @@ -70,32 +70,37 @@ enum EventLogFormat { /// `forEachFrame(in:_:)` handing over where each payload is, as offsets into the file, rather than a copy of it. @discardableResult static func forEachFramePayload(in file: Data, _ visit: (UInt16, Range) -> Void) -> FrameWalk? { - guard file.count >= fileHeaderSize, - file.prefix(magic.count) == magic, - readUInt16(file, at: magic.count) == version + guard isReadable(file) else { return nil } + return file.withUnsafeBytes { walkFrames($0, visit) } + } - return file.withUnsafeBytes { bytes in - var cursor = fileHeaderSize - while cursor + frameHeaderSize <= bytes.count { - let type = UInt16(bytes[cursor]) << 8 | UInt16(bytes[cursor + 1]) - let length = Int(readUInt32(bytes, at: cursor + 2)) - let start = cursor + frameHeaderSize - guard length > 0, length <= maxFrameSize, start + length <= bytes.count - else { break } - - let payload = UnsafeRawBufferPointer(rebasing: bytes[start..<(start + length)]) - var checksum = Crc32() - checksum.advance(withBytes: UnsafeRawBufferPointer(rebasing: bytes[cursor..<(cursor + 6)])) - checksum.advance(withBytes: payload) - guard checksum.checksum == readUInt32(bytes, at: cursor + 6) - else { break } - - visit(type, start..<(start + length)) - cursor = start + length - } - return FrameWalk(reachedEnd: cursor == bytes.count) + private static func isReadable(_ file: Data) -> Bool { + file.count >= fileHeaderSize && file.prefix(magic.count) == magic && readUInt16(file, at: magic.count) == version + } + + /// Hands over each frame once it has been checked, and before the next one is read, so that `visit` may write + /// anywhere before where the frame ends. + private static func walkFrames(_ bytes: UnsafeRawBufferPointer, _ visit: (UInt16, Range) -> Void) -> FrameWalk { + var cursor = fileHeaderSize + while cursor + frameHeaderSize <= bytes.count { + let type = UInt16(bytes[cursor]) << 8 | UInt16(bytes[cursor + 1]) + let length = Int(readUInt32(bytes, at: cursor + 2)) + let start = cursor + frameHeaderSize + guard length > 0, length <= maxFrameSize, start + length <= bytes.count + else { break } + + let payload = UnsafeRawBufferPointer(rebasing: bytes[start..<(start + length)]) + var checksum = Crc32() + checksum.advance(withBytes: UnsafeRawBufferPointer(rebasing: bytes[cursor..<(cursor + 6)])) + checksum.advance(withBytes: payload) + guard checksum.checksum == readUInt32(bytes, at: cursor + 6) + else { break } + + visit(type, start..<(start + length)) + cursor = start + length } + return FrameWalk(reachedEnd: cursor == bytes.count) } private static func readUInt32(_ bytes: UnsafeRawBufferPointer, at offset: Int) -> UInt32 { @@ -120,30 +125,49 @@ enum EventLogFormat { forEachFramePayload(in: file) { _, _ in }?.reachedEnd == false } - /// Assembles the JSON array LaunchDarkly expects out of the frames, without parsing the events themselves: they were - /// serialized on the way in and are shipped exactly as they were recorded. + /// Turns a log into the JSON array LaunchDarkly expects, in the log's own bytes, without parsing the events + /// themselves: they were serialized on the way in and are shipped exactly as they were recorded. + /// + /// There is always room: the file header becomes the "[", and each frame header the "," or "]" after its payload. + /// Each payload moves towards the start only once its frame has been checked, and never past where that frame + /// began, so no frame still to be checked is written over. A log nothing else refers to is changed where it is, + /// rather than copied first. /// /// Only the first `maxEvents` events go into it; any after them are left out. - static func assembleBody(from file: Data, maxEvents: Int = .max) -> Data? { - // Found first so the body is allocated once at its exact size rather than grown as it is appended to. - var payloads: [Range] = [] - let readable = forEachFramePayload(in: file) { type, payload in - guard type == eventFrame, payloads.count < maxEvents - else { return } - payloads.append(payload) + /// + /// - Returns: whether the log held events. Where it did not, or is not one this version reads, it is left as it was. + static func assembleBody(inPlace file: inout Data, maxEvents: Int = .max) -> Bool { + guard isReadable(file) + else { return false } + + let length = file.withUnsafeMutableBytes { bytes -> Int in + var cursor = 1 + var events = 0 + _ = walkFrames(UnsafeRawBufferPointer(bytes)) { type, payload in + guard type == eventFrame, events < maxEvents + else { return } + memmove(bytes.baseAddress! + cursor, bytes.baseAddress! + payload.lowerBound, payload.count) + cursor += payload.count + bytes[cursor] = UInt8(ascii: ",") + cursor += 1 + events += 1 + } + guard events > 0 + else { return 0 } + bytes[0] = UInt8(ascii: "[") + bytes[cursor - 1] = UInt8(ascii: "]") + return cursor } + guard length > 0 + else { return false } + file.count = length + return true + } - guard readable != nil, !payloads.isEmpty - else { return nil } - - // The "[", and then each payload with the "," or "]" after it. - var body = Data(capacity: payloads.reduce(1) { $0 + $1.count + 1 }) - body.append(UInt8(ascii: "[")) - for (index, payload) in payloads.enumerated() { - body.append(file[(file.startIndex + payload.lowerBound)..<(file.startIndex + payload.upperBound)]) - body.append(UInt8(ascii: index == payloads.count - 1 ? "]" : ",")) - } - return body + /// `assembleBody(inPlace:maxEvents:)` into bytes of its own, leaving the log as it was. + static func assembleBody(from file: Data, maxEvents: Int = .max) -> Data? { + var body = file + return assembleBody(inPlace: &body, maxEvents: maxEvents) ? body : nil } static func bigEndian(_ value: UInt16) -> Data { diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index 09ee2a070..6d3c79a77 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -698,10 +698,10 @@ private extension EventStore { while held < frames.count { let (end, batchEvents) = EventLogFormat.framesFitting(in: frames, from: held, room: EventLogFormat.maxBatchSize - EventLogFormat.fileHeaderSize) - var log = Data(capacity: EventLogFormat.fileHeaderSize + end - held) - log.append(EventLogFormat.fileHeader) - log.append(frames[(base + held)..<(base + end)]) - guard let body = EventLogFormat.assembleBody(from: log) + var body = Data(capacity: EventLogFormat.fileHeaderSize + end - held) + body.append(EventLogFormat.fileHeader) + body.append(frames[(base + held)..<(base + end)]) + guard EventLogFormat.assembleBody(inPlace: &body) else { break } let payloadId = UUID().uuidString diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index f9c9e7842..2cddfa425 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -1216,6 +1216,21 @@ final class EventStoreSpec: QuickSpec { expect(store.pendingEventCount) == 5 } + it("assembles a stored batch in the bytes it was read into, rather than in a copy") { + var readInto: UnsafeRawPointer? + store = EventStore.temporary(readFile: { url in + let file = try Data(contentsOf: url) + readInto = file.withUnsafeBytes { $0.baseAddress } + return file + }) + stage(1...3) + let batch = store.closeBatch()! + + let body = try? store.body(of: batch) + expect(body?.withUnsafeBytes { $0.baseAddress }) == readInto + expect(EventStoreSpec.keys(ofBody: body)) == ["event-1", "event-2", "event-3"] + } + it("takes an event that fills a batch on its own, and refuses any larger") { store = EventStore.temporary() let largest = EventLogFormat.maxFrameSize + EventLogFormat.frameHeaderSize From 1e7616c0739634e2d6a890dab9c797078287448c Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Wed, 21 Oct 2026 17:04:55 -0700 Subject: [PATCH 080/158] Name batches ready-., read by position A dot cannot occur in a payload ID, so the count no longer needs telling apart from a UUID group by its digits. Fields after the count are ignored; a missing or unparseable count is read once to count it, which is what a recovery that cannot read a log leaves. Co-authored-by: Cursor --- .../ServiceObjects/EventDirectory.swift | 34 ++++++++++--------- .../DirectEventProcessorSpec.swift | 4 +-- .../ServiceObjects/EventStoreSpec.swift | 34 ++++++++++++++++--- 3 files changed, 49 insertions(+), 23 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index fbabdea28..c1803861a 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -28,7 +28,8 @@ struct EventTally: Equatable { /// that would have to be ordered against the others. final class EventDirectory { /// Names a log that has been closed off for delivery. The rest of the name is the batch's payload ID and how many - /// events are in it, `ready--`. + /// events are in it, `ready-.`. Anything more a batch needs to say belongs in a frame type readers + /// skip, rather than in the name. /// /// The count is in the name so that listing a batch this process did not close costs a directory entry rather than /// a read of the whole file. That is what every batch is at startup: the ones a previous run left behind, and the @@ -37,6 +38,7 @@ final class EventDirectory { /// /// A batch is never appended to once it is closed, so the count is as good as the file is. private static let batchPrefix = "ready-" + private static let nameFieldSeparator: Character = "." private let locateDirectory: () -> URL private let ioLock: UnfairLock @@ -140,25 +142,25 @@ final class EventDirectory { /// Requires `ioLock`. private func newUrl(of payloadId: String, holding events: Int) -> URL { - disk.directory.appendingPathComponent("\(EventDirectory.batchPrefix)\(payloadId)-\(events)") + disk.directory.appendingPathComponent( + "\(EventDirectory.batchPrefix)\(payloadId)\(EventDirectory.nameFieldSeparator)\(events)") } - /// A batch as its filename gives it: everything between the prefix and the count is the payload ID. + /// A batch as its name, `ready-.`, gives it: each field by its position. /// - /// - Returns: the payload ID, and the count the name ends with, which is nil where it has none -- how a version - /// that did not write one named its batches. A payload ID cannot be mistaken for a count: it is a UUID, whose - /// last group is twelve characters. Twelve digits are more than an `Int32` holds unless they start with a zero, - /// which a count never does. + /// Fields are split at `nameFieldSeparator`, which a payload ID never contains, so nothing about the count's own + /// characters is needed to tell it apart. Fields after it, which a later version might add, are ignored. + /// + /// - Returns: the payload ID, and the count, which is nil where the name has none or it does not parse -- a batch a + /// recovery could not read, which is read once to count it. private static func parse(batchName name: String) -> (payloadId: String, events: Int?) { - let rest = String(name.dropFirst(batchPrefix.count)) - guard let split = rest.lastIndex(of: "-") - else { return (rest, nil) } - let suffix = rest[rest.index(after: split)...] - guard !suffix.isEmpty, suffix.allSatisfy({ $0.isASCII && $0.isNumber }), - suffix == "0" || suffix.first != "0", - let events = Int32(suffix) - else { return (rest, nil) } - return (String(rest[.. 1, !fields[1].isEmpty, fields[1].allSatisfy({ $0.isASCII && $0.isNumber }), + let events = Int32(fields[1]) + else { return (payloadId, nil) } + return (payloadId, Int(events)) } // MARK: Writing diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index f8d75283a..b875b71d0 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -2024,7 +2024,7 @@ extension DirectEventProcessorSpec { testContext.eventProcessor.commitRecordedEvents() let damaged = store.closeBatch()! // Its name still counts it, so it is listed without being read; the damage is only found on sending. - let damagedUrl = store.directory!.appendingPathComponent("ready-\(damaged.payloadId)-\(damaged.eventCount)") + let damagedUrl = store.directory!.appendingPathComponent("ready-\(damaged.payloadId).\(damaged.eventCount)") try Data("not an event log".utf8).write(to: damagedUrl) testContext.recordEvents(1) @@ -2051,7 +2051,7 @@ extension DirectEventProcessorSpec { testContext.recordEvents(1) testContext.eventProcessor.commitRecordedEvents() let damaged = store.closeBatch()! - let damagedUrl = store.directory!.appendingPathComponent("ready-\(damaged.payloadId)-\(damaged.eventCount)") + let damagedUrl = store.directory!.appendingPathComponent("ready-\(damaged.payloadId).\(damaged.eventCount)") try Data("not an event log".utf8).write(to: damagedUrl) testContext.eventProcessor.flush(completion: nil) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 2cddfa425..fe22cb4dd 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -741,7 +741,7 @@ final class EventStoreSpec: QuickSpec { // Nothing appends to a batch once it is closed, so the count its name carries still holds and // listing it has no reason to go to the disk. Damaging the file is how the test tells the two apart: // a listing that read would refuse this batch. - let batchUrl = store.directory!.appendingPathComponent("ready-\(batch.payloadId)-2") + let batchUrl = store.directory!.appendingPathComponent("ready-\(batch.payloadId).2") try Data("not an event log".utf8).write(to: batchUrl) // The store that closed it and one that did not both count it from the name alone. @@ -757,9 +757,9 @@ final class EventStoreSpec: QuickSpec { _ = store.stage(EventStoreSpec.payload("left-by-a-previous-version")) let batch = store.closeBatch()! - // The name a version that did not count them in the name gave it, which is what such a version - // leaves behind for this one to deliver. - let counted = store.directory!.appendingPathComponent("ready-\(batch.payloadId)-1") + // A name without a count, which is what a recovery that could not read a log leaves behind for a + // listing to count. + let counted = store.directory!.appendingPathComponent("ready-\(batch.payloadId).1") let uncounted = store.directory!.appendingPathComponent("ready-\(batch.payloadId)") try FileManager.default.moveItem(at: counted, to: uncounted) @@ -777,13 +777,37 @@ final class EventStoreSpec: QuickSpec { // Its last group fits an Int32 once the leading zeros are dropped. let payloadId = "6F9619FF-8B86-D011-B42D-000012345678" - let counted = store.directory!.appendingPathComponent("ready-\(batch.payloadId)-1") + let counted = store.directory!.appendingPathComponent("ready-\(batch.payloadId).1") let uncounted = store.directory!.appendingPathComponent("ready-\(payloadId)") try FileManager.default.moveItem(at: counted, to: uncounted) expect(EventStoreSpec.reader(sharing: store).pendingBatches()) == [EventBatch(payloadId: payloadId, eventCount: 1)] } + it("reads a name with a field after the count by position") { + _ = store.stage(EventStoreSpec.payload("first")) + _ = store.stage(EventStoreSpec.payload("second")) + let batch = store.closeBatch()! + + // What a later version might add, which this one has no use for. + let counted = store.directory!.appendingPathComponent("ready-\(batch.payloadId).2") + let extended = store.directory!.appendingPathComponent("ready-\(batch.payloadId).2.v2-something") + try FileManager.default.moveItem(at: counted, to: extended) + + expect(EventStoreSpec.reader(sharing: store).pendingBatches()) == [batch] + } + + it("counts a batch whose count does not parse by reading it") { + _ = store.stage(EventStoreSpec.payload("only")) + let batch = store.closeBatch()! + + let counted = store.directory!.appendingPathComponent("ready-\(batch.payloadId).1") + let unparseable = store.directory!.appendingPathComponent("ready-\(batch.payloadId).not-a-count") + try FileManager.default.moveItem(at: counted, to: unparseable) + + expect(EventStoreSpec.reader(sharing: store).pendingBatches()) == [batch] + } + it("gives a batch its own payload id, so a retry is not read as a new delivery") { _ = store.stage(EventStoreSpec.payload("first")) let first = store.closeBatch()! From 8004c28bbde9eb53f40e45fc445373aece480e1a Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Wed, 21 Oct 2026 17:31:43 -0700 Subject: [PATCH 081/158] Test that a flush of more than a batch sends no request over the batch size A hundred 64 KiB events through the processor: every body is at most maxBatchSize and all but the last within an event of it, every event arrives once, and each request has its own payload ID. Co-authored-by: Cursor --- .../DirectEventProcessorSpec.swift | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index b875b71d0..45f5a683d 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -2148,6 +2148,38 @@ extension DirectEventProcessorSpec { expect(testContext.serviceMock.publishedEventPayloads.flatMap(customKeys)) == ["fine"] } + it("sends a flush of more than a batch as requests no larger than a batch, every event once") { + // Small enough that a batch closed at the limit is within one event of it, so a body that is short of + // the limit by more than that shows the split happened somewhere else. + let dataCharacters = EventLogFormat.maxBatchSize / 32 + let eventCount = 100 + testContext = TestContext(eventCapacity: eventCount) + let padding = LDValue.string(String(repeating: "x", count: dataCharacters)) + for index in 1...eventCount { + testContext.eventProcessor.record(CustomEvent(key: "event-\(index)", context: LDContext.stub(), + data: padding)) + } + testContext.eventProcessor.isOnline = true + var delivered: Bool? + waitUntil(timeout: .seconds(30)) { done in + testContext.eventProcessor.flushReportingOutcome { result in + delivered = result + done() + } + } + expect(delivered) == true + + // Before the service compresses it, so the larger of the two sizes. + let bodies = testContext.serviceMock.publishedEventPayloads + let sizes = bodies.map(\.count).sorted() + expect(sizes.count) > 1 + expect(sizes.last ?? .max) <= EventLogFormat.maxBatchSize + // Only the batch the events ran out in may be short of full. + expect(sizes.dropFirst().allSatisfy { $0 > EventLogFormat.maxBatchSize - dataCharacters - 4096 }) == true + expect(Set(testContext.serviceMock.publishedPayloadIds).count) == bodies.count + expect(bodies.flatMap(customKeys).sorted()) == (1...eventCount).map { "event-\($0)" }.sorted() + } + it("reports false when a retryable failure leaves the batch on disk") { testContext = TestContext(stubResponseSuccess: false) testContext.recordEvents(1) From 0586bdfdd9b3d348cc0fd1e7bbf00c1913e61b52 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 22 Oct 2026 07:53:40 -0700 Subject: [PATCH 082/158] Report a flush as failed when the stored events cannot be listed A directory that could not be listed read as an empty one: the store stopped counting every batch on disk, including the one the flush had just closed, and the flush answered true having sent nothing. A directory that is there and cannot be listed now answers with the batches this process last counted, which include every batch it closed itself, and leaves their counts alone. The flush sends those and answers false. Co-authored-by: Cursor --- .../ServiceObjects/DirectEventProcessor.swift | 24 +++++++---- .../ServiceObjects/EventDirectory.swift | 42 ++++++++++++++----- .../ServiceObjects/EventStore.swift | 29 ++++++++++--- .../DirectEventProcessorSpec.swift | 25 +++++++++-- .../ServiceObjects/EventStoreSpec.swift | 30 +++++++++++++ 5 files changed, 123 insertions(+), 27 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 470c6fbc9..f6525e08d 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -597,11 +597,16 @@ extension DirectEventProcessor { let lossMark = commitAndCloseBatch() - let (batches, waitingOn) = claimDeliverableBatches() + let (batches, waitingOn, listedEverything) = claimDeliverableBatches() + // Events in a batch a listing that failed could not see are still on the disk for a later delivery. This one + // delivers what it has, and cannot say the caller's events got out. + if !listedEverything { + os_log("%s will report failure: the stored events could not be listed", log: service.config.logger, type: .debug, typeName(and: #function)) + } // This delivery's own outcome, plus that of every batch another delivery is already sending. The caller's // events may be in either: a delivery that started after they were recorded closed them into its batch. let answer = DeliveryAnswer(outstanding: waitingOn.count + 1) { delivered in - completion?(delivered, lossMark) + completion?(delivered && listedEverything, lossMark) } for claim in waitingOn { whenSettled(claim, answer.report) @@ -624,19 +629,20 @@ extension DirectEventProcessor { /// Takes the batches this delivery will send, leaving out any another delivery is still waiting on a response for. /// - /// - Returns: the batches to send, and the claims on the ones left out. A batch left out may hold this caller's - /// events -- `commitRecordedEvents` only stages them, and whichever delivery runs next is what closes them into - /// a batch -- so the answer has to wait for those claims rather than assume they are someone else's business. - private func claimDeliverableBatches() -> (deliverable: [EventBatch], waitingOn: [Claim]) { + /// - Returns: the batches to send, the claims on the ones left out, and whether the listing was complete. A batch + /// left out may hold this caller's events -- `commitRecordedEvents` only stages them, and whichever delivery runs + /// next is what closes them into a batch -- so the answer has to wait for those claims rather than assume they + /// are someone else's business. + private func claimDeliverableBatches() -> (deliverable: [EventBatch], waitingOn: [Claim], isComplete: Bool) { // Listed before the lock is taken, because listing reads the disk. Taking what appeared after it is the next // delivery's business; all this has to be atomic is deciding which of these batches are this delivery's. // Expired first, so that a batch too old to report is not sent. reportDiscarded(store.discardExpired()) - let pending = store.pendingBatches() + let listing = store.listPendingBatches() return claimLock.withLock { var deliverable: [EventBatch] = [] var waitingOn: [Claim] = [] - for batch in pending { + for batch in listing.batches { if let claimed = batchesInFlight[batch.payloadId] { waitingOn.append(claimed) } else { @@ -644,7 +650,7 @@ extension DirectEventProcessor { deliverable.append(batch) } } - return (deliverable, waitingOn) + return (deliverable, waitingOn, listing.isComplete) } } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index c1803861a..0e33813dd 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -51,11 +51,13 @@ final class EventDirectory { /// Only to be used while holding `ioLock`. private var located: (directory: URL, log: DiskEventLog)? - /// A batch in the directory: the file it is, how many events it holds, and how big it is. + /// A batch in the directory: the file it is, how many events it holds, how big it is, and when it was last written, + /// which is what orders batches oldest first. private struct Stored { let url: URL let events: Int let bytes: Int + let modified: Date var tally: EventTally { EventTally(events: events, bytes: bytes) } } @@ -198,7 +200,7 @@ final class EventDirectory { return .failed(logSurvives: disk.log.exists) } - stored[payloadId] = Stored(url: file, events: events, bytes: held.bytes) + stored[payloadId] = Stored(url: file, events: events, bytes: held.bytes, modified: Date()) return .closed(EventBatch(payloadId: payloadId, eventCount: events)) } @@ -215,11 +217,29 @@ final class EventDirectory { /// The batches in the directory, oldest first. Requires `ioLock`. /// /// Sizes come from the same listing, so that counting the directory's bytes reads no file either. - func listBatches() -> [EventBatch] { + /// + /// A directory that is not there holds nothing. One that is there and cannot be listed -- out of file descriptors, + /// or no longer readable -- is not empty, so it answers with the batches the last listing or close counted, as they + /// were counted, and says the listing is incomplete. + func listBatches() -> BatchListing { ioLock.assertOwned() - let contents = (try? FileManager.default.contentsOfDirectory(at: disk.directory, - includingPropertiesForKeys: EventDirectory.listedKeys, - options: [.skipsHiddenFiles])) ?? [] + let contents: [URL] + do { + contents = try FileManager.default.contentsOfDirectory(at: disk.directory, + includingPropertiesForKeys: EventDirectory.listedKeys, + options: [.skipsHiddenFiles]) + } catch { + guard FileManager.default.fileExists(atPath: disk.directory.path) + else { + stored = [:] + return BatchListing(batches: [], isComplete: true) + } + os_log("%s could not list stored events, will try again at the next delivery: %s", log: logger, type: .info, typeName(and: #function), String(describing: error)) + let known = stored + .sorted { $0.value.modified < $1.value.modified } + .map { EventBatch(payloadId: $0.key, eventCount: $0.value.events) } + return BatchListing(batches: known, isComplete: false) + } let batches = contents .filter { $0.lastPathComponent.hasPrefix(EventDirectory.batchPrefix) } @@ -232,8 +252,10 @@ final class EventDirectory { guard let events = counted ?? stored[payloadId]?.events ?? countOfEvents(in: file) else { return nil } let listed = EventDirectory.listed(file) - stored[payloadId] = Stored(url: file, events: events, bytes: listed.bytes ?? stored[payloadId]?.bytes ?? 0) - return (EventBatch(payloadId: payloadId, eventCount: events), listed.modified ?? Date.distantPast) + let modified = listed.modified ?? Date.distantPast + stored[payloadId] = Stored(url: file, events: events, bytes: listed.bytes ?? stored[payloadId]?.bytes ?? 0, + modified: modified) + return (EventBatch(payloadId: payloadId, eventCount: events), modified) } .sorted { $0.1 < $1.1 } .map { $0.0 } @@ -243,7 +265,7 @@ final class EventDirectory { let listed = Set(batches.map { $0.payloadId }) stored = stored.filter { listed.contains($0.key) } - return batches + return BatchListing(batches: batches, isComplete: true) } private static let listedKeys: [URLResourceKey] = [.contentModificationDateKey, .fileSizeKey] @@ -466,7 +488,7 @@ final class EventDirectory { return EventTally() } - let recovered = Stored(url: file, events: events, bytes: bytes) + let recovered = Stored(url: file, events: events, bytes: bytes, modified: EventDirectory.listed(file).modified ?? Date()) stored[payloadId] = recovered return recovered.tally } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index 6d3c79a77..b2a5efa8a 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -10,6 +10,14 @@ struct EventBatch: Equatable { let eventCount: Int } +/// The batches awaiting delivery, and whether that is all of them. +struct BatchListing: Equatable { + let batches: [EventBatch] + /// False where the directory could not be listed. `batches` are then the ones this process last knew to be there, + /// which includes every batch it closed itself, but can miss one another process closed or a previous run left. + let isComplete: Bool +} + protocol EventStoring { /// Events counted against capacity: those recorded and not yet accepted by LaunchDarkly, whether they are reserved /// while being encoded, staged in memory, written to the open log, or sitting in a closed batch -- except a batch @@ -63,7 +71,10 @@ protocol EventStoring { func closeBatch() -> EventBatch? /// Batches awaiting delivery, oldest first, including any recovered from a previous run of the application. - func pendingBatches() -> [EventBatch] + /// + /// A directory that could not be listed leaves the batches this process last counted, and what it counts for them, + /// as they were. It does not empty them. + func listPendingBatches() -> BatchListing /// The `[...]` request body for a batch, or nil if it is gone or holds no events, neither of which loses anything. /// @@ -94,6 +105,13 @@ protocol EventStoring { func discardExpired() -> Int } +extension EventStoring { + /// The batches `listPendingBatches()` lists, whether or not the listing was complete. + func pendingBatches() -> [EventBatch] { + listPendingBatches().batches + } +} + /// A batch that is there to read and holds nothing this version can send: damaged, or written in a format it does not /// read. Unlike a batch that is gone, its events are lost. struct UnreadableBatch: Error { @@ -386,14 +404,15 @@ final class EventStore: EventStoring { // MARK: Delivery - func pendingBatches() -> [EventBatch] { + func listPendingBatches() -> BatchListing { // Listed in one critical section where there is a disk, so that the files and what is held cannot be counted // either side of a close that moves events between them. guard let disk - else { return heldBatchesAndRecount(after: [], storedBytes: 0) } + else { return BatchListing(batches: heldBatchesAndRecount(after: [], storedBytes: 0), isComplete: true) } return ioLock.withLock { - let listed = disk.listBatches() - return heldBatchesAndRecount(after: listed, storedBytes: disk.storedBytes) + let listing = disk.listBatches() + return BatchListing(batches: heldBatchesAndRecount(after: listing.batches, storedBytes: disk.storedBytes), + isComplete: listing.isComplete) } } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 45f5a683d..4a109c334 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -1473,6 +1473,25 @@ extension DirectEventProcessorSpec { expect(testContext.serviceMock.publishedEventPayloads.flatMap(customKeys)).toNot(contain("large")) } + itPersisting("reports a flush as failed when the stored events cannot be listed, and still sends the batch it closed") { + testContext = TestContext() + testContext.eventProcessor.isOnline = true + testContext.eventProcessor.record(CustomEvent(key: "tracked", context: testContext.context)) + testContext.eventProcessor.commitRecordedEvents() + + let directory = testContext.store.directory! + // Writable and searchable but not readable, so the batch can be closed and read and not listed. + try FileManager.default.setAttributes([.posixPermissions: 0o300], ofItemAtPath: directory.path) + defer { try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: directory.path) } + + expect(testContext.flushOutcome()) == false + expect(testContext.serviceMock.publishedEventPayloads.flatMap(customKeys)) == ["tracked"] + + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: directory.path) + expect(testContext.flushOutcome()) == true + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + } + it("does not bring a delivery forward when a previous run left nothing behind") { testContext = TestContext() @@ -2581,7 +2600,7 @@ private final class ClosingHookStore: EventStoring { func stageReserved(_ encodedEvent: Data) -> Bool { inner.stageReserved(encodedEvent) } func releaseReservations() { inner.releaseReservations() } func commit() { inner.commit() } - func pendingBatches() -> [EventBatch] { inner.pendingBatches() } + func listPendingBatches() -> BatchListing { inner.listPendingBatches() } func body(of batch: EventBatch) throws -> Data? { try inner.body(of: batch) } func remove(_ batch: EventBatch) { inner.remove(batch) } func startedSending(_ batch: EventBatch) { inner.startedSending(batch) } @@ -2668,7 +2687,7 @@ private final class SlowListingStore: EventStoring { func discardExpired() -> Int { inner.discardExpired() } /// Every listing but the first is held, the first being the one the reporter makes as it starts up. - func pendingBatches() -> [EventBatch] { + func listPendingBatches() -> BatchListing { let held = lock.withLock { () -> Bool in listings += 1 return listings > 1 @@ -2679,7 +2698,7 @@ private final class SlowListingStore: EventStoring { // Once opened the gate stays open. release.signal() } - return inner.pendingBatches() + return inner.listPendingBatches() } } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index fe22cb4dd..307115fbe 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -843,6 +843,36 @@ final class EventStoreSpec: QuickSpec { expect(store.pendingBatches()) == [other] expect(store.pendingEventCount) == 1 } + + it("keeps every batch it counted when the directory cannot be listed, and says the listing is incomplete") { + _ = store.stage(EventStoreSpec.payload("first")) + let first = store.closeBatch()! + _ = store.stage(EventStoreSpec.payload("second")) + let second = store.closeBatch()! + _ = store.pendingBatches() + + let directory = store.directory! + // Writable and searchable but not readable: its files can be created, renamed and read by name, and + // the directory cannot be listed. + let unlistable: [FileAttributeKey: Any] = [.posixPermissions: 0o300] + let listable: [FileAttributeKey: Any] = [.posixPermissions: 0o755] + try FileManager.default.setAttributes(unlistable, ofItemAtPath: directory.path) + defer { try? FileManager.default.setAttributes(listable, ofItemAtPath: directory.path) } + + _ = store.stage(EventStoreSpec.payload("third")) + let third = store.closeBatch()! + + // Not empty: the batch just closed is still there to send, and nothing stops counting. + let listing = store.listPendingBatches() + expect(listing.isComplete) == false + expect(listing.batches) == [first, second, third] + expect(store.pendingEventCount) == 3 + expect(EventStoreSpec.keys(ofBody: try? store.body(of: third))) == ["third"] + + try FileManager.default.setAttributes(listable, ofItemAtPath: directory.path) + expect(store.listPendingBatches()) == BatchListing(batches: [first, second, third], isComplete: true) + } + } } From f4adfafb46b44c774ce0f5ce63241119376cb899 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 22 Oct 2026 07:53:51 -0700 Subject: [PATCH 083/158] Keep a log whose rename failed open, and report the flush as failed A failed rename closed and forgot the log anyway. Its events were left for a later launch, the next append started a second log under the same committed count, and that log's batch was later named for more events than it held. The flush answered true. A log that is still there now stays open and this store's: the next append continues it, the next close moves all of it, and the flush that could not close it answers false. Co-authored-by: Cursor --- .../ServiceObjects/DirectEventProcessor.swift | 23 ++++++++------- .../ServiceObjects/DiskEventLog.swift | 9 ++++-- .../ServiceObjects/EventDirectory.swift | 15 ++++++---- .../ServiceObjects/EventStore.swift | 13 +++++++-- .../DirectEventProcessorSpec.swift | 22 ++++++++++++++ .../ServiceObjects/EventStoreSpec.swift | 29 +++++++++++++++++++ 6 files changed, 91 insertions(+), 20 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index f6525e08d..c96f4c554 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -384,13 +384,14 @@ class DirectEventProcessor: EventProcessor { /// Closing takes everything staged so far, and a commit stages its run one event at a time, so a commit running /// between the two would be closed partway: an evaluation's event in this batch and its summary in the next. /// - /// - Returns: the loss count as of the close, which is the mark this delivery answers for. Taken under the same - /// lock, so it covers every loss the closed batch could have suffered and nothing a later commit suffers. - private func commitAndCloseBatch() -> UInt64 { + /// - Returns: the loss count as of the close, which is the mark this delivery answers for, and whether the close + /// took every committed event into a batch. Both taken under the same lock, so they cover the closed batch and + /// nothing a later commit does. + private func commitAndCloseBatch() -> (lossMark: UInt64, closedEverything: Bool) { commitLock.withLock { encodeAndCommitPending() _ = store.closeBatch() - return stateLock.withLock { lostEvents } + return (stateLock.withLock { lostEvents }, !store.holdsUnclosedEvents) } } @@ -595,18 +596,20 @@ extension DirectEventProcessor { return } - let lossMark = commitAndCloseBatch() + let (lossMark, closedEverything) = commitAndCloseBatch() let (batches, waitingOn, listedEverything) = claimDeliverableBatches() - // Events in a batch a listing that failed could not see are still on the disk for a later delivery. This one - // delivers what it has, and cannot say the caller's events got out. - if !listedEverything { - os_log("%s will report failure: the stored events could not be listed", log: service.config.logger, type: .debug, typeName(and: #function)) + // Events left in a log that would not close, or in a batch a listing that failed could not see, are still on + // the disk for a later delivery. This one delivers what it has, and cannot say the caller's events got out. + let accountsForEverything = closedEverything && listedEverything + if !accountsForEverything { + os_log("%s will report failure: %s", log: service.config.logger, type: .debug, typeName(and: #function), + closedEverything ? "the stored events could not be listed" : "the event log could not be closed") } // This delivery's own outcome, plus that of every batch another delivery is already sending. The caller's // events may be in either: a delivery that started after they were recorded closed them into its batch. let answer = DeliveryAnswer(outstanding: waitingOn.count + 1) { delivered in - completion?(delivered && listedEverything, lossMark) + completion?(delivered && accountsForEverything, lossMark) } for claim in waitingOn { whenSettled(claim, answer.report) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift index b5e5fff2b..fd0088954 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift @@ -124,16 +124,19 @@ final class DiskEventLog { return true } - /// Moves the log to `destination` and closes it, so that the next append starts a new one. The descriptor is closed - /// whether or not the move succeeds. + /// Moves the log to `destination` and closes it, so that the next append starts a new one. + /// + /// A move that fails leaves the log open, held and this store's, so that the next append continues it and the next + /// close moves all of it. Closed and forgotten instead, it would be a log recovery may take at any moment, and the + /// next append would start another beside it: one tally of committed events for two files. /// /// Requires `ioLock`. func close(movingTo destination: URL) throws { ioLock.assertOwned() // Closed after the move rather than before it, so that the log is still held while it moves. Another store // recovering logs could otherwise take it in between. - defer { closeDescriptor() } try FileManager.default.moveItem(at: url, to: destination) + closeDescriptor() } /// Closes the descriptor, leaving the log where it is. Requires `ioLock`. diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index 0e33813dd..9a1eaad8e 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -179,9 +179,9 @@ final class EventDirectory { /// What closing the open log produced. enum LogClose { case closed(EventBatch) - /// The log is still there to try again with, unless `logSurvives` is false -- a purged cache directory on - /// tvOS, say. Then its events are gone, and they have to stop counting against capacity or the store would - /// refuse events for the rest of the session. + /// The log is still there and still open, for the next append to continue and the next close to try again, + /// unless `logSurvives` is false -- a purged cache directory on tvOS, say. Then its events are gone, and they + /// have to stop counting against capacity or the store would refuse events for the rest of the session. case failed(logSurvives: Bool) } @@ -196,8 +196,13 @@ final class EventDirectory { do { try disk.log.close(movingTo: file) } catch { - os_log("%s could not close the event log: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) - return .failed(logSurvives: disk.log.exists) + os_log("%s could not close the event log, will try again at the next delivery: %s", log: logger, type: .info, typeName(and: #function), String(describing: error)) + guard disk.log.exists + else { + disk.log.closeDescriptor() + return .failed(logSurvives: false) + } + return .failed(logSurvives: true) } stored[payloadId] = Stored(url: file, events: events, bytes: held.bytes, modified: Date()) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index b2a5efa8a..8a7217c54 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -70,6 +70,10 @@ protocol EventStoring { /// one. func closeBatch() -> EventBatch? + /// Whether committed events are outside every batch. Right after `closeBatch()`, that means it could not close the + /// log holding them; they stay in that log for the next close to try again. + var holdsUnclosedEvents: Bool { get } + /// Batches awaiting delivery, oldest first, including any recovered from a previous run of the application. /// /// A directory that could not be listed leaves the batches this process last counted, and what it counts for them, @@ -382,8 +386,9 @@ final class EventStore: EventStoring { let closed = EventTally(events: committed.events, bytes: committed.bytes + EventLogFormat.fileHeader.count) switch disk.closeLog(holding: closed) { case .failed(let logSurvives): - // Where the log went with its events, they have to stop counting against capacity or the store would - // refuse events for the rest of the session. + // A log that is still there is still the open one, and its events go on counting as committed until a + // later close manages it. Where the log went with its events, they have to stop counting against capacity + // or the store would refuse events for the rest of the session. if !logSurvives { bufferLock.withLock { committedEvents = 0 @@ -404,6 +409,10 @@ final class EventStore: EventStoring { // MARK: Delivery + var holdsUnclosedEvents: Bool { + bufferLock.withLock { committedEvents > 0 } + } + func listPendingBatches() -> BatchListing { // Listed in one critical section where there is a disk, so that the files and what is held cannot be counted // either side of a close that moves events between them. diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 4a109c334..883d19163 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -1492,6 +1492,26 @@ extension DirectEventProcessorSpec { expect(testContext.serviceMock.publishEventDataCallCount) == 1 } + itPersisting("reports a flush as failed while the event log will not close, and delivers all of it once it does") { + testContext = TestContext() + testContext.eventProcessor.isOnline = true + testContext.eventProcessor.record(CustomEvent(key: "first", context: testContext.context)) + testContext.eventProcessor.commitRecordedEvents() + + let directory = testContext.store.directory! + // No write permission, so the log cannot be renamed into a batch. + try FileManager.default.setAttributes([.posixPermissions: 0o555], ofItemAtPath: directory.path) + defer { try? FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: directory.path) } + + expect(testContext.flushOutcome()) == false + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + + try FileManager.default.setAttributes([.posixPermissions: 0o755], ofItemAtPath: directory.path) + testContext.eventProcessor.record(CustomEvent(key: "second", context: testContext.context)) + expect(testContext.flushOutcome()) == true + expect(testContext.serviceMock.publishedEventPayloads.flatMap(customKeys)) == ["first", "second"] + } + it("does not bring a delivery forward when a previous run left nothing behind") { testContext = TestContext() @@ -2600,6 +2620,7 @@ private final class ClosingHookStore: EventStoring { func stageReserved(_ encodedEvent: Data) -> Bool { inner.stageReserved(encodedEvent) } func releaseReservations() { inner.releaseReservations() } func commit() { inner.commit() } + var holdsUnclosedEvents: Bool { inner.holdsUnclosedEvents } func listPendingBatches() -> BatchListing { inner.listPendingBatches() } func body(of batch: EventBatch) throws -> Data? { try inner.body(of: batch) } func remove(_ batch: EventBatch) { inner.remove(batch) } @@ -2679,6 +2700,7 @@ private final class SlowListingStore: EventStoring { func releaseReservations() { inner.releaseReservations() } func commit() { inner.commit() } func closeBatch() -> EventBatch? { inner.closeBatch() } + var holdsUnclosedEvents: Bool { inner.holdsUnclosedEvents } func body(of batch: EventBatch) throws -> Data? { try inner.body(of: batch) } func remove(_ batch: EventBatch) { inner.remove(batch) } func startedSending(_ batch: EventBatch) { inner.startedSending(batch) } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 307115fbe..ed8c35c02 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -873,6 +873,35 @@ final class EventStoreSpec: QuickSpec { expect(store.listPendingBatches()) == BatchListing(batches: [first, second, third], isComplete: true) } + it("keeps a log it could not close open, and closes all of it the next time") { + _ = store.stage(EventStoreSpec.payload("first")) + store.commit() + + let directory = store.directory! + // A directory with no write permission is one nothing in it can be renamed in. + let readOnly: [FileAttributeKey: Any] = [.posixPermissions: 0o555] + let writable: [FileAttributeKey: Any] = [.posixPermissions: 0o755] + try FileManager.default.setAttributes(readOnly, ofItemAtPath: directory.path) + defer { try? FileManager.default.setAttributes(writable, ofItemAtPath: directory.path) } + + expect(store.closeBatch()).to(beNil()) + expect(store.holdsUnclosedEvents) == true + expect(store.pendingEventCount) == 1 + + try FileManager.default.setAttributes(writable, ofItemAtPath: directory.path) + _ = store.stage(EventStoreSpec.payload("second")) + store.commit() + + // One batch holding both, named for what it holds, and no log left behind for a later launch. + let batch = store.closeBatch()! + expect(batch.eventCount) == 2 + expect(EventStoreSpec.keys(ofBody: try? store.body(of: batch))) == ["first", "second"] + expect(store.holdsUnclosedEvents) == false + expect(store.pendingBatches()) == [batch] + let logs = try FileManager.default.contentsOfDirectory(atPath: directory.path) + .filter { $0.hasPrefix(DiskEventLog.prefix) } + expect(logs).to(beEmpty()) + } } } From 2b1e380688519bb7888013362b2e49a8672f83c0 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 22 Oct 2026 10:19:09 -0700 Subject: [PATCH 084/158] Report events a damaged batch could not send as lost A batch damaged after its first frame was sent with the events before the damage, removed once accepted, and the flush answered true: the rest were gone and nothing said so. Reading a batch now also says how many events made it into the body. Where the batch was named for more, the shortfall is counted as lost once the batch is removed, and the pass answers false. It is not counted at the read, since a batch kept for a retry is read short again. Co-authored-by: Cursor --- .../ServiceObjects/DirectEventProcessor.swift | 33 +++++++++++++----- .../ServiceObjects/EventDirectory.swift | 7 ++-- .../ServiceObjects/EventLogFormat.swift | 24 +++++++------ .../ServiceObjects/EventStore.swift | 23 ++++++++++--- .../DirectEventProcessorSpec.swift | 34 +++++++++++++++++-- .../ServiceObjects/EventStoreSpec.swift | 15 ++++++++ 6 files changed, 106 insertions(+), 30 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index c96f4c554..2d318ab9e 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -303,7 +303,12 @@ class DirectEventProcessor: EventProcessor { /// next `flushReportingOutcome` to hear about. A caller waiting on this pass can then hear about it from the next /// one too. private func reportBatchLost(_ batch: EventBatch) { - stateLock.withLock { lostEvents += UInt64(batch.eventCount) } + reportStoredEventsLost(batch.eventCount) + } + + /// `reportBatchLost(_:)` for part of a batch. + private func reportStoredEventsLost(_ events: Int) { + stateLock.withLock { lostEvents += UInt64(events) } } /// Records the loss of an event the SDK accepted and then could not keep, which no later attempt recovers. @@ -716,9 +721,9 @@ extension DirectEventProcessor { } let batch = remaining.removeFirst() - let read: Data? + let read: BatchPayload? do { - read = try store.body(of: batch) + read = try store.payload(of: batch) } catch is UnreadableBatch { // Lost like a refused batch: nothing will ever send it, so it is removed, the rest are still attempted, and // the caller hears that this pass lost events. @@ -734,7 +739,7 @@ extension DirectEventProcessor { return } - guard let body = read + guard let payload = read else { // Gone, which is usually another process having delivered it, or holding no events. Either way nothing is // lost and there is nothing to keep. @@ -743,11 +748,21 @@ extension DirectEventProcessor { return } - service.diagnosticCache?.recordEventsInLastBatch(eventsInLastBatch: batch.eventCount) - publish(batch, body) { outcome in + // Damage after the first frame: what came before it is sent, and the rest is gone with the file. Counted once + // the batch is removed rather than now, since one kept for a retry is read, and found short, again. + let leftOut = max(0, batch.eventCount - payload.eventCount) + service.diagnosticCache?.recordEventsInLastBatch(eventsInLastBatch: payload.eventCount) + publish(batch, payload) { outcome in switch outcome { case .accepted: - self.deliver(remaining, lostAnything: lostAnything, completion) + guard leftOut > 0 + else { + self.deliver(remaining, lostAnything: lostAnything, completion) + return + } + os_log("%s dropped %d damaged stored event(s) from a batch it delivered", log: self.service.config.logger, type: .error, self.typeName(and: #function), leftOut) + self.reportStoredEventsLost(leftOut) + self.deliver(remaining, lostAnything: true, completion) case .refused: // The rest are still attempted, so that batches LaunchDarkly will never take do not sit on the device // for the rest of the session, taking up the room the capacity limit leaves for new events. @@ -761,11 +776,11 @@ extension DirectEventProcessor { /// Sends one batch and acts on what LaunchDarkly made of it, before handing the outcome on. private func publish(_ batch: EventBatch, - _ body: Data, + _ payload: BatchPayload, _ completion: @escaping (AnalyticsEventSender.Outcome) -> Void) { store.startedSending(batch) // The store is settled before observers are told, so that one looking at it on hearing the outcome sees it. - sender.send(body, payloadId: batch.payloadId, eventCount: batch.eventCount) { outcome in + sender.send(payload.body, payloadId: batch.payloadId, eventCount: payload.eventCount) { outcome in switch outcome { case .accepted(let serverTime): self.stateLock.withLock { diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index 9a1eaad8e..71ddbff55 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -313,7 +313,7 @@ final class EventDirectory { /// /// - throws: `UnreadableBatch` if the batch is there but holds nothing this version can send; anything else if it /// could not be read this time, which a later attempt may manage. - func body(of batch: EventBatch) throws -> Data? { + func payload(of batch: EventBatch) throws -> BatchPayload? { ioLock.assertOwned() // Taken out of the read's result rather than bound inside the switch, so that nothing else refers to it and it // is assembled where it is rather than copied first. @@ -325,8 +325,9 @@ final class EventDirectory { } // No more than the batch was closed holding. A write that failed and could not be truncated leaves whole frames // after them, whose events are delivered from memory instead, and would otherwise be sent twice. - if EventLogFormat.assembleBody(inPlace: &file, maxEvents: batch.eventCount > 0 ? batch.eventCount : .max) { - return file + let assembled = EventLogFormat.assembleBody(inPlace: &file, maxEvents: batch.eventCount > 0 ? batch.eventCount : .max) + if assembled > 0 { + return BatchPayload(body: file, eventCount: assembled) } // A log this version reads that holds no events has nothing to lose, unless its first frame is damaged. Where // there is no payload the log is as it was read. diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift index 773ba4a90..f22b017e4 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift @@ -133,14 +133,16 @@ enum EventLogFormat { /// began, so no frame still to be checked is written over. A log nothing else refers to is changed where it is, /// rather than copied first. /// - /// Only the first `maxEvents` events go into it; any after them are left out. + /// Only the first `maxEvents` events go into it; any after them are left out. So is everything from a frame that is + /// torn or fails its checksum on, which is why the count it returns can be fewer than the log was closed holding. /// - /// - Returns: whether the log held events. Where it did not, or is not one this version reads, it is left as it was. - static func assembleBody(inPlace file: inout Data, maxEvents: Int = .max) -> Bool { + /// - Returns: how many events went into it. Where none did, or the log is not one this version reads, it is left as + /// it was. + static func assembleBody(inPlace file: inout Data, maxEvents: Int = .max) -> Int { guard isReadable(file) - else { return false } + else { return 0 } - let length = file.withUnsafeMutableBytes { bytes -> Int in + let (length, assembled) = file.withUnsafeMutableBytes { bytes -> (Int, Int) in var cursor = 1 var events = 0 _ = walkFrames(UnsafeRawBufferPointer(bytes)) { type, payload in @@ -153,21 +155,21 @@ enum EventLogFormat { events += 1 } guard events > 0 - else { return 0 } + else { return (0, 0) } bytes[0] = UInt8(ascii: "[") bytes[cursor - 1] = UInt8(ascii: "]") - return cursor + return (cursor, events) } - guard length > 0 - else { return false } + guard assembled > 0 + else { return 0 } file.count = length - return true + return assembled } /// `assembleBody(inPlace:maxEvents:)` into bytes of its own, leaving the log as it was. static func assembleBody(from file: Data, maxEvents: Int = .max) -> Data? { var body = file - return assembleBody(inPlace: &body, maxEvents: maxEvents) ? body : nil + return assembleBody(inPlace: &body, maxEvents: maxEvents) > 0 ? body : nil } static func bigEndian(_ value: UInt16) -> Data { diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index 8a7217c54..e6653703f 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -18,6 +18,14 @@ struct BatchListing: Equatable { let isComplete: Bool } +/// The request body for a batch, and how many events made it in. +struct BatchPayload: Equatable { + let body: Data + /// Fewer than the batch was closed holding where a frame after the first is torn or fails its checksum: the + /// reader stops there, and the events from it on are not in `body`. + let eventCount: Int +} + protocol EventStoring { /// Events counted against capacity: those recorded and not yet accepted by LaunchDarkly, whether they are reserved /// while being encoded, staged in memory, written to the open log, or sitting in a closed batch -- except a batch @@ -85,7 +93,7 @@ protocol EventStoring { /// - throws: `UnreadableBatch` if the batch is there but holds nothing this version can send, which no later /// attempt changes and which loses its events; anything else if it could not be read this time, which a later /// attempt may manage. - func body(of batch: EventBatch) throws -> Data? + func payload(of batch: EventBatch) throws -> BatchPayload? /// Forgets a batch, which is only correct once LaunchDarkly has accepted it or permanently refused it. func remove(_ batch: EventBatch) @@ -114,6 +122,11 @@ extension EventStoring { func pendingBatches() -> [EventBatch] { listPendingBatches().batches } + + /// The body `payload(of:)` reads, however many events made it in. + func body(of batch: EventBatch) throws -> Data? { + try payload(of: batch)?.body + } } /// A batch that is there to read and holds nothing this version can send: damaged, or written in a format it does not @@ -446,9 +459,9 @@ final class EventStore: EventStoring { } } - func body(of batch: EventBatch) throws -> Data? { + func payload(of batch: EventBatch) throws -> BatchPayload? { if let held = heldLock.withLock({ inMemoryBatches.first { $0.payloadId == batch.payloadId } }) { - return held.body + return BatchPayload(body: held.body, eventCount: held.eventCount) } // A batch is either held or on the disk, never both and never moved between them, so a batch that is not held @@ -456,7 +469,7 @@ final class EventStore: EventStoring { guard let disk else { return nil } - return try ioLock.withLock { try disk.body(of: batch) } + return try ioLock.withLock { try disk.payload(of: batch) } } func remove(_ batch: EventBatch) { @@ -729,7 +742,7 @@ private extension EventStore { var body = Data(capacity: EventLogFormat.fileHeaderSize + end - held) body.append(EventLogFormat.fileHeader) body.append(frames[(base + held)..<(base + end)]) - guard EventLogFormat.assembleBody(inPlace: &body) + guard EventLogFormat.assembleBody(inPlace: &body) > 0 else { break } let payloadId = UUID().uuidString diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 883d19163..bba8d856c 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -2079,6 +2079,36 @@ extension DirectEventProcessorSpec { expect(store.pendingEventCount) == 0 } + it("reports false when a stored batch damaged after its first frame is sent short") { + let store = EventStore.temporary() + testContext = TestContext(store: store) + testContext.eventProcessor.isOnline = true + waitUntil(timeout: Constants.deliveryTimeout) { done in + testContext.eventProcessor.flushReportingOutcome { _ in done() } + } + + testContext.recordEvents(3) + testContext.eventProcessor.commitRecordedEvents() + let damaged = store.closeBatch()! + let damagedUrl = store.directory!.appendingPathComponent("ready-\(damaged.payloadId).\(damaged.eventCount)") + var bytes = try Data(contentsOf: damagedUrl) + bytes[bytes.count - 1] ^= 0x01 + try bytes.write(to: damagedUrl) + + var delivered: Bool? + waitUntil(timeout: Constants.deliveryTimeout) { done in + testContext.eventProcessor.flushReportingOutcome { result in + delivered = result + done() + } + } + expect(delivered) == false + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == damaged.eventCount - 1 + expect(FileManager.default.fileExists(atPath: damagedUrl.path)) == false + expect(store.pendingEventCount) == 0 + } + it("tells the next flush about a stored batch a delivery nobody waited on dropped") { let store = EventStore.temporary() testContext = TestContext(store: store) @@ -2622,7 +2652,7 @@ private final class ClosingHookStore: EventStoring { func commit() { inner.commit() } var holdsUnclosedEvents: Bool { inner.holdsUnclosedEvents } func listPendingBatches() -> BatchListing { inner.listPendingBatches() } - func body(of batch: EventBatch) throws -> Data? { try inner.body(of: batch) } + func payload(of batch: EventBatch) throws -> BatchPayload? { try inner.payload(of: batch) } func remove(_ batch: EventBatch) { inner.remove(batch) } func startedSending(_ batch: EventBatch) { inner.startedSending(batch) } func stoppedSending(_ batch: EventBatch) { inner.stoppedSending(batch) } @@ -2701,7 +2731,7 @@ private final class SlowListingStore: EventStoring { func commit() { inner.commit() } func closeBatch() -> EventBatch? { inner.closeBatch() } var holdsUnclosedEvents: Bool { inner.holdsUnclosedEvents } - func body(of batch: EventBatch) throws -> Data? { try inner.body(of: batch) } + func payload(of batch: EventBatch) throws -> BatchPayload? { try inner.payload(of: batch) } func remove(_ batch: EventBatch) { inner.remove(batch) } func startedSending(_ batch: EventBatch) { inner.startedSending(batch) } func stoppedSending(_ batch: EventBatch) { inner.stoppedSending(batch) } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index ed8c35c02..2d04e8811 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -566,6 +566,21 @@ final class EventStoreSpec: QuickSpec { expect { try store.body(of: batch) }.to(throwError(errorType: UnreadableBatch.self)) } + it("says how many events made it into the body of a batch damaged after its first frame") { + store = EventStore.temporary() + ["first", "second", "third"].forEach { _ = store.stage(EventStoreSpec.payload($0)) } + let batch = store.closeBatch()! + let file = try EventStoreSpec.onlyBatchFile(in: store) + var damaged = try Data(contentsOf: file) + damaged[damaged.count - 1] ^= 0x01 + try damaged.write(to: file) + + let payload = try store.payload(of: batch) + expect(batch.eventCount) == 3 + expect(payload?.eventCount) == 2 + expect(EventStoreSpec.keys(ofBody: payload?.body)) == ["first", "second"] + } + it("sends no frames a failed write left behind where it could not be cut back") { store = EventStore.temporary() _ = store.stage(EventStoreSpec.payload("written")) From cc200a6cda11e7378d74e7a10b5c20a15be39048 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 22 Oct 2026 10:19:17 -0700 Subject: [PATCH 085/158] Read the logs a previous run left without holding the store's I/O lock Recovery held the I/O lock while it read and checked every log a crash left, so a commit at a commit point, which at .immediate is a track or an identify, waited for all of them: ten 2 MiB logs held one up for about 11 ms. Each log is now claimed, read and counted without the lock, which is taken again only to move it into a batch and count it. A commit waits for one rename at most. Co-authored-by: Cursor --- .../ServiceObjects/DiskEventLog.swift | 42 +++++++--- .../ServiceObjects/EventDirectory.swift | 79 +++++++++++++------ .../ServiceObjects/EventStore.swift | 27 +++++-- .../ServiceObjects/EventStoreSpec.swift | 39 +++++++++ 4 files changed, 144 insertions(+), 43 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift index fd0088954..a833c053c 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift @@ -152,25 +152,45 @@ final class DiskEventLog { DiskEventLog.forget(url) } - /// Hands `recover` every log no live process holds, which it has to move or remove: whoever wrote it is gone. + /// Every log no store in this process has open, for recovery to claim with `claimForRecovery(_:)`. + /// + /// None of them becomes open here later: a log is opened under a name of its own and never reopened once closed. /// /// Requires `ioLock`. - func forEachUnheldLog(_ recover: (URL) -> Void) { + func logsNotOpenInThisProcess() -> [URL] { ioLock.assertOwned() let contents = (try? FileManager.default.contentsOfDirectory(at: directory, includingPropertiesForKeys: nil, options: [.skipsHiddenFiles])) ?? [] - for log in contents where log.lastPathComponent.hasPrefix(DiskEventLog.prefix) { - guard !DiskEventLog.isOpenInThisProcess(log) - else { continue } + return contents.filter { $0.lastPathComponent.hasPrefix(DiskEventLog.prefix) && !DiskEventLog.isOpenInThisProcess($0) } + } + + /// Claims a log whose writer is gone, for recovery to read and then move or remove; nil where a live process holds + /// it. Needs no lock, so that recovery can read the log without holding `ioLock`. + /// + /// On a Mac the claim is the log's lock, held until `release()`, so that another instance recovering at the same + /// time leaves it be. Elsewhere every log not open in this process is its writer's no longer. + static func claimForRecovery(_ log: URL) -> RecoveryClaim? { + #if os(macOS) || targetEnvironment(macCatalyst) + guard let held = EventLogFile.openLocked(log, flags: O_RDONLY) + else { return nil } + return RecoveryClaim(url: log, held: held) + #else + return RecoveryClaim(url: log) + #endif + } + + /// A log recovery has claimed; see `claimForRecovery(_:)`. + struct RecoveryClaim { + let url: URL + #if os(macOS) || targetEnvironment(macCatalyst) + fileprivate let held: EventLogFile + #endif + + /// To be called once the log has been moved or removed, or left for a later run. + func release() { #if os(macOS) || targetEnvironment(macCatalyst) - // Locked for as long as it is recovered, so that another instance recovering at the same time leaves it be. - guard let held = EventLogFile.openLocked(log, flags: O_RDONLY) - else { continue } - recover(log) held.close() - #else - recover(log) #endif } } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index 71ddbff55..c29fbd743 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -426,33 +426,60 @@ final class EventDirectory { // MARK: Recovery - /// Closes off every log no live process holds, so its events join the batches to deliver. + /// A log whose writer is gone, and what reading it found. + struct InterruptedLog { + fileprivate enum Contents { + case missing + case failed(Error) + /// Not a log this version reads. + case unreadable + case events(EventTally) + } + + fileprivate let url: URL + fileprivate let contents: Contents + } + + /// The logs recovery is to close off, so their events join the batches to deliver: every one no store in this + /// process has open, of which `DiskEventLog.claimForRecovery(_:)` then takes those no live process holds. /// - /// The events in it were recorded by a store that is gone, so there is no one left to add to it: a previous run of - /// this process, or another process that has since ended, whatever it was called. A log a live process is still - /// appending to is left alone. + /// The events in them were recorded by a store that is gone, so there is no one left to add to them: a previous run + /// of this process, or another process that has since ended, whatever it was called. /// /// Requires `ioLock`. - /// - /// - Returns: what it closed off. - func preparePayloadsFromInterruptedLogs() -> EventTally { + func interruptedLogs() -> [URL] { ioLock.assertOwned() - var recovered = EventTally() - disk.log.forEachUnheldLog { recovered += preparePayload(fromInterruptedLog: $0) } - if recovered.events > 0 { - os_log("%s prepared %d event(s) from a previous run of this application for delivery", log: logger, type: .debug, typeName(and: #function), recovered.events) + return disk.log.logsNotOpenInThisProcess() + } + + /// Reads and counts a log recovery has claimed. + /// + /// Needs no lock, and is meant to be called without `ioLock`: a log can be as large as a batch, and a commit at a + /// commit point would otherwise wait for every log a crash left. Nothing here is this directory's state, and the + /// log's writer is gone, so it is not changing either. + func read(interruptedLog leftOpen: URL) -> InterruptedLog { + switch read(leftOpen) { + case .missing: + return InterruptedLog(url: leftOpen, contents: .missing) + case .failed(let error): + return InterruptedLog(url: leftOpen, contents: .failed(error)) + case .bytes(let log): + guard let events = EventLogFormat.eventCount(in: log) + else { return InterruptedLog(url: leftOpen, contents: .unreadable) } + return InterruptedLog(url: leftOpen, contents: .events(EventTally(events: events, bytes: log.count))) } - return recovered } - /// Closes off a log nobody is writing to any more. Requires `ioLock`. + /// Closes off a log `read(interruptedLog:)` read. Requires `ioLock`, which is what keeps a listing from counting + /// the batch it becomes on its own as well. /// /// - Returns: what it closed off, or nothing where there was nothing to close off. - private func preparePayload(fromInterruptedLog leftOpen: URL) -> EventTally { + func preparePayload(from interrupted: InterruptedLog) -> EventTally { ioLock.assertOwned() + let leftOpen = interrupted.url let events: Int let bytes: Int - switch read(leftOpen) { + switch interrupted.contents { case .missing: return EventTally() case .failed(let error): @@ -469,15 +496,13 @@ final class EventDirectory { os_log("%s could not read events from a previous run, will try again at the next launch: %s", log: logger, type: .info, typeName(and: #function), String(describing: error)) } return EventTally() - case .bytes(let log): - guard let counted = EventLogFormat.eventCount(in: log) - else { - // Unreadable, and a log that cannot be read cannot be appended to either. - try? FileManager.default.removeItem(at: leftOpen) - return EventTally() - } - events = counted - bytes = log.count + case .unreadable: + // A log that cannot be read cannot be appended to either. + try? FileManager.default.removeItem(at: leftOpen) + return EventTally() + case .events(let counted): + events = counted.events + bytes = counted.bytes } guard events > 0 @@ -490,7 +515,11 @@ final class EventDirectory { let file = newUrl(of: payloadId, holding: events) guard (try? FileManager.default.moveItem(at: leftOpen, to: file)) != nil else { - os_log("%s could not prepare %d event(s) from a previous run, will try again at the next launch", log: logger, type: .info, typeName(and: #function), events) + // Gone since it was read is another store recovering it first, which where logs are not locked can be one + // in this process. + if FileManager.default.fileExists(atPath: leftOpen.path) { + os_log("%s could not prepare %d event(s) from a previous run, will try again at the next launch", log: logger, type: .info, typeName(and: #function), events) + } return EventTally() } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index e6653703f..6fb708c73 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -507,14 +507,27 @@ final class EventStore: EventStoring { func preparePayloadsFromInterruptedLogs() { guard let disk else { return } - ioLock.withLock { - let recovered = disk.preparePayloadsFromInterruptedLogs() - guard recovered.events > 0 - else { return } - bufferLock.withLock { - closedEvents += recovered.events - closedBytes += recovered.bytes + // Each log is read without `ioLock`, one at a time, and the lock taken only to move it and count it. A commit + // at a commit point then waits for at most one rename rather than for every log a crash left to be read. + var recovered = 0 + for leftOpen in ioLock.withLock({ disk.interruptedLogs() }) { + guard let claim = DiskEventLog.claimForRecovery(leftOpen) + else { continue } + let interrupted = disk.read(interruptedLog: leftOpen) + ioLock.withLock { + let prepared = disk.preparePayload(from: interrupted) + guard prepared.events > 0 + else { return } + recovered += prepared.events + bufferLock.withLock { + closedEvents += prepared.events + closedBytes += prepared.bytes + } } + claim.release() + } + if recovered > 0 { + os_log("%s prepared %d event(s) from a previous run of this application for delivery", log: logger, type: .debug, typeName(and: #function), recovered) } } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 2d04e8811..79072601a 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -477,6 +477,45 @@ final class EventStoreSpec: QuickSpec { expect(bodies) == [["previous-0"], ["this-run", "later"]] expect(next.pendingEventCount) == 3 } + + it("reads the log without holding up a commit") { + store = EventStore.temporary() + _ = store.stage(EventStoreSpec.payload("from-the-previous-run")) + store.commit() + + let reading = DispatchSemaphore(value: 0) + let release = DispatchSemaphore(value: 0) + let next = store.nextRun(readFile: { url in + if url.lastPathComponent.hasPrefix(DiskEventLog.prefix) { + reading.signal() + _ = release.wait(timeout: .now() + 5) + } + return try Data(contentsOf: url) + }) + let recovered = DispatchSemaphore(value: 0) + DispatchQueue.global().async { + next.preparePayloadsFromInterruptedLogs() + recovered.signal() + } + expect(reading.wait(timeout: .now() + 5)) == .success + + // Recovery is partway through reading the log, which a commit does not wait for. + let committed = DispatchSemaphore(value: 0) + DispatchQueue.global().async { + _ = next.stage(EventStoreSpec.payload("this-run")) + next.commit() + committed.signal() + } + expect(committed.wait(timeout: .now() + 2)) == .success + release.signal() + expect(recovered.wait(timeout: .now() + 5)) == .success + + expect(next.pendingEventCount) == 2 + let batches = next.pendingBatches() + expect(batches.count) == 1 + expect(EventStoreSpec.keys(ofBody: try next.body(of: batches[0]))) == ["from-the-previous-run"] + expect(EventStoreSpec.keys(in: next.payloadsOnDisk)) == ["from-the-previous-run", "this-run"] + } } } From 4c3025766d2fb6fddbbd6879519ee912f3b9a75f Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 22 Oct 2026 10:30:54 -0700 Subject: [PATCH 086/158] Describe the pending threshold as a trigger, not a ceiling The threshold only schedules a background commit, and recording does not wait for it. Where evaluations arrive faster than that commit encodes them, the run grows up to the event capacity, and the track or identify that follows at .immediate encodes all of it. The comments said the threshold bounded what a commit point costs; they now say it bounds the common case, and the public .immediate documentation says a burst can make the next call take milliseconds. Co-authored-by: Cursor --- LaunchDarkly/LaunchDarkly/Models/LDConfig.swift | 7 +++++-- .../ServiceObjects/DirectEventProcessor.swift | 10 +++++++--- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift index 51a7af690..bb436320a 100644 --- a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift @@ -50,8 +50,11 @@ public enum LDStreamingMode { /// Events are written to disk before `track` and `identify` return. /// /// The write happens on the calling thread, costing tens of microseconds depending on how many evaluations are - /// waiting to be encoded alongside it. In exchange there is no window: a process that dies the instant after - /// `track` returns still reports that event on the next launch. Evaluating a flag stays in memory regardless. + /// waiting to be encoded alongside it. Evaluations are usually written in the background before many build up. A + /// burst of them faster than that can leave up to `eventCapacity` waiting, and the `track` or `identify` that + /// follows encodes them all, which can take milliseconds. In exchange there is no window: a process that dies the + /// instant after `track` returns still reports that event on the next launch. Evaluating a flag stays in memory + /// regardless. /// /// This level is experimental; see `EventPersistence`. case immediate diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 2d318ab9e..33b79e7dd 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -29,9 +29,13 @@ class DirectEventProcessor: EventProcessor { /// terminations that run nothing on the way out, such as the system reclaiming a backgrounded process. Recording a /// custom or identify event closes it too, because that commits. /// - /// The second is the worst a commit point can cost, since it encodes whatever is held before returning. The common - /// case is far below the bound, because the commit queue keeps the run drained; raising this trades that tail - /// against the number of writes. + /// The second is what a commit point usually costs, since it encodes whatever is held before returning. The commit + /// queue keeps the run drained, so the common case is far below this; raising it trades that cost against the + /// number of writes. + /// + /// Neither is a ceiling. Reaching this only schedules a commit, and recording does not wait for it, so where events + /// arrive faster than the commit queue encodes them, or the queue is starved, the run grows up to the event + /// capacity, and the next commit point encodes all of it. /// /// It is the only thing that writes events no caller asked to have written. The store writes only when told to, so /// every staged byte reaches the disk through a commit that started here, at a commit point, or at a delivery. From 1b0a80d6f44b41e4c28e476e0b74e30548fca214 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 22 Oct 2026 10:57:49 -0700 Subject: [PATCH 087/158] Keep the disk out of flushAndWait's caller thread flushAndWait computed its deadline only after it had found the clients and each had committed on the caller's thread, so a stalled write held the caller past its timeout: a flush waited the full five seconds of a write blocked on the disk. A flush that reports its outcome now queues its commit on the commit queue instead, the deadline is taken on entry, and flush() no longer commits every environment under the instances queue's barrier. The delivery still commits before it sends, so true still means the events were written. flush() keeps its caller-thread commit. Co-authored-by: Cursor --- LaunchDarkly/LaunchDarkly/LDClientFlush.swift | 14 ++++--- .../ServiceObjects/DirectEventProcessor.swift | 18 ++++++++- .../ServiceObjects/EventProcessor.swift | 4 ++ .../DirectEventProcessorSpec.swift | 38 +++++++++++++++++++ 4 files changed, 67 insertions(+), 7 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/LDClientFlush.swift b/LaunchDarkly/LaunchDarkly/LDClientFlush.swift index 4cb7f6eb7..abb37317f 100644 --- a/LaunchDarkly/LaunchDarkly/LDClientFlush.swift +++ b/LaunchDarkly/LaunchDarkly/LDClientFlush.swift @@ -24,9 +24,10 @@ extension LDClient { last write. What was written before then is delivered on the next launch. */ public func flush() { - LDClient.instancesQueue.sync(flags: .barrier) { - LDClient.instances?.forEach { $1.internalFlush() } - } + // Taken under the queue and flushed after it is released, since each flush writes, and every other client call + // in the process that reads the instances would otherwise wait behind every environment's write. + let clients = LDClient.instancesQueue.sync { LDClient.instances.map { Array($0.values) } } ?? [] + clients.forEach { $0.internalFlush() } } /** @@ -119,6 +120,8 @@ extension LDClient { */ @discardableResult public func flushAndWait(timeout: TimeInterval) -> Bool { + // Taken first, so that the timeout covers the whole call, finding the clients included. + let deadline = DispatchTime.now() + max(0, timeout) if timeout > LDClient.longTimeoutInterval { os_log("%s LDClient.flushAndWait was called with a timeout greater than %f seconds. We recommend a timeout of less than %f seconds.", log: config.logger, type: .info, self.typeName(and: #function), LDClient.longTimeoutInterval, LDClient.longTimeoutInterval) } @@ -131,7 +134,6 @@ extension LDClient { os_log("%s called on a closed client", log: config.logger, type: .debug, self.typeName(and: #function)) return false } - let deadline = DispatchTime.now() + max(0, timeout) // Counts answers rather than tracking each delivery, so that a reporter answering twice costs a spare permit // no one reads. A DispatchGroup would fit as well, but its `leave` traps without a matching `enter`, which // turns a broken reporter into a crash in the application. @@ -153,8 +155,8 @@ extension LDClient { } // Started, then given up on. The deliveries are what makes a zero timeout worth calling at all: wherever events - // persist, they are on disk by the time each `flushReportingOutcome` returns. Waiting is what is - // skipped, and a reporter that answered during the loop above must not turn that into a `true`. + // persist, each `flushReportingOutcome` has queued their write, which no longer waits on this thread. Waiting + // is what is skipped, and a reporter that answered during the loop above must not turn that into a `true`. guard timeout > 0 else { return false } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 33b79e7dd..4b49e9ec3 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -513,7 +513,8 @@ extension DirectEventProcessor { } func flushReportingOutcome(completion: @escaping FlushOutcomeClosure) { - commitThenQueueDelivery(completion) + queueCommitWherePersisting() + queueDelivery(completion) } /// Queues a delivery the way a caller asking for one expects, which is as a commit point. @@ -539,6 +540,21 @@ extension DirectEventProcessor { } } + /// `commitWherePersisting()` queued on `commitQueue` rather than written on the caller's thread, for a flush whose + /// caller has a deadline. + /// + /// `flushAndWait` can only hold to its timeout once every environment has handed back, and each step of a commit + /// can block for as long as the device takes: on `commitLock` behind a commit stalled on the disk, on the store's + /// I/O lock, and on the write itself. Queued rather than left to the delivery, because `deliveryQueue` can be held + /// by a request and its retry long after the caller stopped waiting, while `commitQueue` never waits on the + /// network. The answer still covers the write: the delivery commits again before it sends, so `true` means the + /// events were written first. A commit still running when the caller stops waiting is left to finish. + private func queueCommitWherePersisting() { + if service.config.eventPersistence == .immediate || store.isPersisting { + scheduleCommit() + } + } + private func queueScheduledDelivery() { queueDelivery(nil) } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift index 52c13c191..964c05da4 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift @@ -31,6 +31,10 @@ protocol EventProcessor { /// The completion, like `flush`'s, runs on a background queue, and never on the reporter's own: a completion is /// free to record events or flush again without waiting on the queue it is running on. It is not the caller's /// thread either, and two completions may run at once, so anything it touches needs to expect that. + /// + /// Unlike `flush`, it writes nothing on the caller's thread: wherever events persist, it queues that write, so a + /// caller with a deadline is not held by the disk before it can start waiting. A `true` still means the events + /// were written, since the delivery writes them before it sends. func flushReportingOutcome(completion: @escaping FlushOutcomeClosure) /// Makes everything recorded so far outlive the process, without waiting for a delivery. diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index bba8d856c..c4f1d3931 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -1378,6 +1378,44 @@ extension DirectEventProcessorSpec { .toEventually(equal(400), timeout: Constants.deliveryTimeout, pollInterval: .milliseconds(200)) } + itPersisting("hands back a flush that reports its outcome while a write is stalled on the disk") { + let lock = UnfairLock() + var stalling = false + let stalled = DispatchSemaphore(value: 0) + let release = DispatchSemaphore(value: 0) + let store = EventStore.temporary(writeLog: { file, bytes in + if lock.withLock({ stalling }) { + stalled.signal() + _ = release.wait(timeout: .now() + 5) + } + return file.writeAll(bytes) + }) + testContext = TestContext(store: store) + let processor = testContext.eventProcessor! + processor.isOnline = true + waitUntil(timeout: Constants.deliveryTimeout) { done in + processor.flushReportingOutcome { _ in done() } + } + + let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: true) + processor.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: testContext.context, includeReason: false) + lock.withLock { stalling = true } + // A commit stalled in its write, holding the commit lock, as one on a slow disk does. + DispatchQueue.global().async { processor.commitRecordedEvents() } + expect(stalled.wait(timeout: .now() + 5)) == .success + lock.withLock { stalling = false } + + let answer = Answer() + let started = Date() + processor.flushReportingOutcome(completion: answer.record) + expect(Date().timeIntervalSince(started)) < 1 + expect(answer.value()).to(beNil()) + + release.signal() + expect(answer.value()).toEventually(equal(true), timeout: Constants.deliveryTimeout) + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + } + itPersisting("reports the events a previous run left behind") { testContext = TestContext() testContext.recordEvents(2) From 4e049fdfae4b5e59fd18e73b12c1db562ef75c24 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 22 Oct 2026 11:04:22 -0700 Subject: [PATCH 088/158] Keep expiry away from batches any live store or process is sending Expiry only spared the batches its own store had a request out for, so another store in this process, or another process of a Mac application on the same environment, could delete a batch mid-request and leave nothing to retry from. The sweep of other environments spared nothing, and could remove the directory of one open in this process. A sender now holds a shared lock on the batch for as long as its request is out, and expiry deletes a batch only under a nonblocking exclusive lock, opening it without creating it. Locks are taken on a Mac, where logs are locked too; the batches sent in this process are also kept in a registry that expiry here asks instead. The sweep of other environments skips the directory of any store open in this process, and a batch already gone when expiry gets to it is no longer counted as dropped. Every send now ends with stoppedSending, delivered or not. Co-authored-by: Cursor --- .../ServiceObjects/DirectEventProcessor.swift | 2 + .../ServiceObjects/EventDirectory.swift | 187 +++++++++++++++++- .../ServiceObjects/EventLogFile.swift | 9 + .../ServiceObjects/EventStore.swift | 31 ++- .../ServiceObjects/EventStoreSpec.swift | 45 +++++ 5 files changed, 259 insertions(+), 15 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 4b49e9ec3..6f83614b2 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -809,10 +809,12 @@ extension DirectEventProcessor { } } self.store.remove(batch) + self.store.stoppedSending(batch) self.reportSyncComplete(nil) case .refused(let response): self.isOnline = false self.store.remove(batch) + self.store.stoppedSending(batch) self.reportSyncComplete(.response(response)) case .retryable(let error): self.store.stoppedSending(batch) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index c29fbd743..9250e4596 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -85,6 +85,24 @@ final class EventDirectory { /// Only to be used while holding `ioLock`. private var deliveredButUndeleted: Set = [] + /// The batches this store has a request out for, by payload ID: the file each was locked under, so the lock is + /// released from that file even once the batch has been removed or dropped from a listing, and how many of this + /// store's deliveries are sending it. + /// + /// Only to be used while holding `ioLock`. + private var sending: [String: (url: URL, senders: Int)] = [:] + + /// What `openInThisProcess` counts this store under, from locating the directory until closing the log, kept so + /// that the count is taken back under the same path even once the directory is gone. Only to be used while holding + /// `ioLock`. + private var countedAsOpen: String? + + /// The directories of the stores open in this process, by resolved path, with how many stores have each open. Another + /// environment's sweep leaves these alone, however long since they were written: a store that is open may be about + /// to write, or be sending a batch older than the cutoff. Guarded by `openInThisProcessLock`. + private static var openInThisProcess: [String: Int] = [:] + private static let openInThisProcessLock = UnfairLock() + #if os(macOS) || targetEnvironment(macCatalyst) /// Whether the open log is locked while it is open. Only to be changed before the store first touches the disk. /// @@ -124,9 +142,17 @@ final class EventDirectory { #endif let disk = (directory: directory, log: log) located = disk + let path = directory.resolvingSymlinksInPath().path + countedAsOpen = path + EventDirectory.openInThisProcessLock.withLock { EventDirectory.openInThisProcess[path, default: 0] += 1 } return disk } + private static func isOpenInThisProcess(_ directory: URL) -> Bool { + let path = directory.resolvingSymlinksInPath().path + return openInThisProcessLock.withLock { openInThisProcess[path] != nil } + } + /// Where the batches are kept. Requires `ioLock`. var url: URL { disk.directory } @@ -209,12 +235,42 @@ final class EventDirectory { return .closed(EventBatch(payloadId: payloadId, eventCount: events)) } - /// Closes the descriptor, leaving everything where it is. Requires `ioLock`. + /// Closes the descriptor, leaving everything where it is, and stops counting the directory as open in this process. + /// Requires `ioLock`. func closeOpenLog() { ioLock.assertOwned() // Not `disk`: a store that never wrote has never worked out where it lives, and shutting down or giving up on // the disk is no reason to start. located?.log.closeDescriptor() + guard let path = countedAsOpen + else { return } + countedAsOpen = nil + EventDirectory.openInThisProcessLock.withLock { + let stores = EventDirectory.openInThisProcess[path] ?? 0 + EventDirectory.openInThisProcess[path] = stores > 1 ? stores - 1 : nil + } + } + + /// Locks a batch against expiry, in this process and every other, for as long as a request for it is out. Each + /// call is ended by one `stoppedSending(_:)`. Requires `ioLock`. + /// + /// - Parameter events: what the caller holding the batch knows it to contain, as for `payload(of:)`. + func startedSending(_ payloadId: String, holding events: Int) { + ioLock.assertOwned() + var batch = sending[payloadId] ?? (url: url(of: payloadId, holding: events), senders: 0) + batch.senders += 1 + sending[payloadId] = batch + BatchLocks.startedSending(batch.url) + } + + /// Ends one `startedSending(_:holding:)`. Requires `ioLock`. + func stoppedSending(_ payloadId: String) { + ioLock.assertOwned() + guard var batch = sending[payloadId] + else { return } + BatchLocks.stoppedSending(batch.url) + batch.senders -= 1 + sending[payloadId] = batch.senders > 0 ? batch : nil } // MARK: Listing and reading @@ -540,9 +596,12 @@ extension EventDirectory { /// nothing is appending to it, however long ago it was written: a live process may have gone that long without an /// event. /// - /// - Parameter sending: batches whose request is out, which are left alone however old they are. - /// - Returns: how many events were discarded, and how much of that this process had counted, which is what stops - /// counting against capacity. + /// A batch whose request is out is left alone however old it is, whichever store or process sent it; see + /// `BatchLocks`. + /// + /// - Parameter sending: batches whose request is out from this store. + /// - Returns: how many events were discarded, and how much this process had counted of the batches that left the + /// directory, which is what stops counting against capacity. func discard(lastWrittenBefore cutoff: Date, sparing sending: Set) -> (events: Int, counted: EventTally) { ioLock.assertOwned() let contents = (try? FileManager.default.contentsOfDirectory(at: disk.directory, @@ -554,11 +613,18 @@ extension EventDirectory { let name = file.lastPathComponent if name.hasPrefix(EventDirectory.batchPrefix) { let (payloadId, named) = EventDirectory.parse(batchName: name) - guard !sending.contains(payloadId), !deliveredButUndeleted.contains(payloadId) + guard !sending.contains(payloadId), !deliveredButUndeleted.contains(payloadId), + !BatchLocks.isSentInThisProcess(file) else { continue } let held = named ?? stored[payloadId]?.events ?? EventDirectory.eventsReading(file) - guard EventDirectory.delete(file) - else { continue } + guard BatchLocks.deleteUnlessBeingSent(file) + else { + if !FileManager.default.fileExists(atPath: file.path) { + // Delivered by another process since the listing: no longer counted, and not dropped. + counted += stored.removeValue(forKey: payloadId)?.tally ?? EventTally() + } + continue + } events += held counted += stored.removeValue(forKey: payloadId)?.tally ?? EventTally() } else { @@ -579,6 +645,9 @@ extension EventDirectory { /// Only directories carrying this application's bundle identifier are considered. An unsandboxed Mac application /// shares Application Support with every other one, whose events are not this application's to delete. /// + /// The directory of a store open in this process is left alone, and so is a batch some live process is sending; + /// see `BatchLocks`. + /// /// - Returns: how many events were discarded. static func discardAbandonedEnvironments(besides current: URL, lastWrittenBefore cutoff: Date) -> Int { let root = current.deletingLastPathComponent() @@ -596,7 +665,8 @@ extension EventDirectory { options: [.skipsHiddenFiles])) ?? [] var events = 0 for directory in siblings where directory.lastPathComponent != ownName && isSameApplication(directory.lastPathComponent) { - guard (try? directory.resourceValues(forKeys: [.isDirectoryKey]))?.isDirectory == true + guard (try? directory.resourceValues(forKeys: [.isDirectoryKey]))?.isDirectory == true, + !isOpenInThisProcess(directory) else { continue } let contents = (try? FileManager.default.contentsOfDirectory(at: directory, includingPropertiesForKeys: listedKeys, @@ -604,8 +674,10 @@ extension EventDirectory { for file in contents where listed(file).modified.map({ $0 < cutoff }) ?? false { let name = file.lastPathComponent if name.hasPrefix(batchPrefix) { + guard !BatchLocks.isSentInThisProcess(file) + else { continue } let held = parse(batchName: name).events ?? eventsReading(file) - if delete(file) { + if BatchLocks.deleteUnlessBeingSent(file) { events += held } } else { @@ -643,6 +715,103 @@ extension EventDirectory { } } +/// What tells expiry which batches have a request out, so that it never deletes a batch someone is still sending: were +/// that request to fail, there would be no file left to retry from. +/// +/// On a Mac, where processes of one application can share an environment, a sender holds a shared lock on the batch +/// for as long as its request is out, and expiry deletes a batch only while holding an exclusive lock on it, which it +/// cannot have while anyone holds the shared one. The kernel drops a lock when its process dies, so a batch whose +/// sender is gone is expired like any other. Elsewhere a process has its directory to itself, as it does its logs. +/// +/// The batches sent in this process are also remembered, and expiry here asks that rather than opening the file. The +/// lock is taken once the batch has been read for sending; a batch another process expires in the moment between is +/// still sent from what was read, but has no file for a retry. +enum BatchLocks { + /// A batch this process is sending, the descriptor its lock is held through, and how many send it. + private final class Held { + var file: EventLogFile? + var senders = 0 + } + + /// The batches this process has a request out for, by `key(_:)`. Guarded by `lock`, which covers the probes and + /// deletions below as well, so that a store in this process cannot start sending a batch another is expiring. + private static var heldInThisProcess: [String: Held] = [:] + private static let lock = UnfairLock() + + /// One name for a batch however its URL was come by, from a listing or put together from the directory: only the + /// directory is resolved, since the batch itself may be gone by the time its send stops. + private static func key(_ batch: URL) -> String { + batch.deletingLastPathComponent().resolvingSymlinksInPath().appendingPathComponent(batch.lastPathComponent).path + } + + /// Locks a batch for as long as a request for it is out. Each call is ended by one `stoppedSending(_:)`; a batch + /// two deliveries send at once stays locked until both have stopped. + static func startedSending(_ batch: URL) { + let path = key(batch) + lock.withLock { + let held = heldInThisProcess[path] ?? Held() + heldInThisProcess[path] = held + held.senders += 1 + #if os(macOS) || targetEnvironment(macCatalyst) + // Only fails to open where the batch is gone already, leaving nothing for expiry to take. Still remembered + // where it cannot be locked, which keeps expiry in this process away from it; only other processes cannot + // tell. + guard held.file == nil, let opened = EventLogFile.open(at: batch, flags: O_RDONLY) + else { return } + if opened.tryLock(shared: true) { + held.file = opened + } else { + opened.close() + } + #endif + } + } + + /// Ends one `startedSending(_:)`, releasing the lock once the last sender has stopped. + static func stoppedSending(_ batch: URL) { + let path = key(batch) + lock.withLock { + guard let held = heldInThisProcess[path] + else { return } + held.senders -= 1 + guard held.senders <= 0 + else { return } + heldInThisProcess[path] = nil + // Releases the lock with it. + held.file?.close() + } + } + + /// Whether a store in this process is sending a batch, answered without opening the file. + static func isSentInThisProcess(_ batch: URL) -> Bool { + let path = key(batch) + return lock.withLock { heldInThisProcess[path] != nil } + } + + /// Deletes a batch for expiry, unless some live process is sending it. + /// + /// - Returns: whether this call deleted it; false for one being sent, one that could not be looked at, and one + /// already gone, whose events are someone else's to count. + static func deleteUnlessBeingSent(_ batch: URL) -> Bool { + let path = key(batch) + return lock.withLock { + guard heldInThisProcess[path] == nil + else { return false } + #if os(macOS) || targetEnvironment(macCatalyst) + // Opened without being created: a batch delivered since it was listed has to stay gone, since a delivery + // reading one brought back empty would report its events lost. + guard let probe = EventLogFile.open(at: batch, flags: O_RDONLY) + else { return false } + defer { probe.close() } + guard probe.tryLock(shared: false) + else { return false } + #endif + // Deleted while still locked, so no sender can take its lock between the look and the delete. + return (try? FileManager.default.removeItem(at: batch)) != nil + } + } +} + extension EventDirectory { /// The directory the store for a mobile key belongs in. /// diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFile.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFile.swift index 491b095b3..0dff684b0 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFile.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFile.swift @@ -57,6 +57,15 @@ struct EventLogFile { opened.close() return false } + + /// Takes the file's lock without waiting: shared, which any number can hold at once, or exclusive, which nobody + /// else can hold alongside it. False where someone else's lock is in the way. + /// + /// The lock belongs to this open file rather than to the process, so closing another descriptor on the same file + /// leaves it in place, and another descriptor in this process contends for it as another process would. + func tryLock(shared: Bool) -> Bool { + flock(descriptor, (shared ? LOCK_SH : LOCK_EX) | LOCK_NB) == 0 + } #endif /// Removes a directory only if it is empty, so that nothing put there since it was last looked at goes with it. diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index 6fb708c73..e693035cb 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -98,10 +98,13 @@ protocol EventStoring { /// Forgets a batch, which is only correct once LaunchDarkly has accepted it or permanently refused it. func remove(_ batch: EventBatch) - /// Stops counting a batch against capacity while its request is out. Ended by `remove(_:)` or `stoppedSending(_:)`. + /// Stops counting a batch against capacity while its request is out, and keeps expiry, in this process or any + /// other, from deleting it meanwhile. Each call is ended by one `stoppedSending(_:)`, after `remove(_:)` where the + /// batch was delivered or refused. func startedSending(_ batch: EventBatch) - /// Counts a batch against capacity again after a send that will be retried later. + /// Ends one `startedSending(_:)`: counts a batch against capacity again after a send that will be retried later, + /// and lets expiry have it once nobody is sending it. func stoppedSending(_ batch: EventBatch) /// Closes every log no live process holds, so its events join the batches to deliver: one a previous run left open, @@ -567,10 +570,15 @@ final class EventStore: EventStoring { // MARK: Sending extension EventStore { - /// Stops counting a batch against capacity while its request is out. Ended by `remove(_:)` or `stoppedSending(_:)`. + /// Stops counting a batch against capacity while its request is out. Each call is ended by one + /// `stoppedSending(_:)`. /// - /// Does nothing for a batch this store no longer counts - one `remove(_:)` forgot, or a listing dropped because - /// another process delivered it - since leaving it out would take its events from the batches still counted. + /// Does nothing to the count for a batch this store no longer counts - one `remove(_:)` forgot, or a listing + /// dropped because another process delivered it - since leaving it out would take its events from the batches + /// still counted. + /// + /// A batch on the disk is also locked against expiry, by this process and any other, until `stoppedSending(_:)`: + /// one deleted while its request is out would leave nothing to retry from. func startedSending(_ batch: EventBatch) { guard let disk else { @@ -578,7 +586,12 @@ extension EventStore { return } // Held across the check and the change, so that the batch cannot stop being counted in between. - ioLock.withLock { startedSending(batch.payloadId, disk: disk) } + ioLock.withLock { + startedSending(batch.payloadId, disk: disk) + if !heldLock.withLock({ inMemoryBatches.contains { $0.payloadId == batch.payloadId } }) { + disk.startedSending(batch.payloadId, holding: batch.eventCount) + } + } } /// Requires `ioLock` where there is a disk. @@ -599,7 +612,13 @@ extension EventStore { } } + /// Counts a batch against capacity again, after a request that did not end with it removed. Does nothing to the + /// count for a batch `remove(_:)` already forgot. Ends every `startedSending(_:)`, removed or not, so that the lock + /// against expiry is released either way. func stoppedSending(_ batch: EventBatch) { + if let disk { + ioLock.withLock { disk.stoppedSending(batch.payloadId) } + } bufferLock.withLock { forgetSending(batch.payloadId) } } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 79072601a..1af856541 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -1538,6 +1538,51 @@ final class EventStoreSpec: QuickSpec { expect(try store.body(of: batch)).toNot(beNil()) } + it("leaves a batch another store in this process is sending until it stops") { + let sender = EventStore.sharing(store.directory!) + _ = sender.stage(EventStoreSpec.payload("sending")) + let batch = sender.closeBatch()! + let file = try EventStoreSpec.onlyBatchFile(in: sender) + try EventStoreSpec.age(file, byDays: 30) + sender.startedSending(batch) + + expect(store.discardExpired()) == 0 + expect(FileManager.default.fileExists(atPath: file.path)) == true + + sender.stoppedSending(batch) + expect(store.discardExpired()) == 1 + expect(FileManager.default.fileExists(atPath: file.path)) == false + } + + #if os(macOS) || targetEnvironment(macCatalyst) + it("leaves a batch another process is sending until it stops") { + _ = store.stage(EventStoreSpec.payload("sending")) + _ = store.closeBatch() + let file = try EventStoreSpec.onlyBatchFile(in: store) + try EventStoreSpec.age(file, byDays: 30) + // A lock taken through a descriptor of its own contends as another process's would. + let otherProcess = EventLogFile.open(at: file, flags: O_RDONLY)! + expect(otherProcess.tryLock(shared: true)) == true + + expect(store.discardExpired()) == 0 + expect(FileManager.default.fileExists(atPath: file.path)) == true + + otherProcess.close() + expect(store.discardExpired()) == 1 + expect(FileManager.default.fileExists(atPath: file.path)) == false + } + #endif + + it("leaves the directory of another environment open in this process") { + let open = EventStore.sharing(root.appendingPathComponent("app.open", isDirectory: true)) + _ = open.stage(EventStoreSpec.payload("open")) + _ = open.closeBatch() + try EventStoreSpec.age(contentsOf: open.directory!, byDays: 30) + + expect(store.discardExpired()) == 0 + expect(open.pendingBatches().count) == 1 + } + it("discards a log nobody holds once nothing has written to it for the maximum age, and leaves a younger one") { func logLeftBy(_ key: String) -> URL { // Going away closes its log, as its process ending would. From d506d3481aaef6df83b63d800d782c4621e5df70 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 22 Oct 2026 11:36:04 -0700 Subject: [PATCH 089/158] Close no batch behind one that is blocked Every delivery committed, closed a new batch, and only then sent the batch at the head. For as long as an outage lasted, each flush, periodic delivery and trip to the background left one more batch: another file on the disk, or another batch held in memory, each with the summary its commit staged. A delivery now sends the batches already waiting first, and closes what was committed since only once none of its own is left blocked. Until then it still commits, into the open log, which closes once it is full, and answers false. Batches another delivery is sending still count toward the answer but do not hold the close back, so the store keeps draining while a request and its retry are out. Co-authored-by: Cursor --- .../ServiceObjects/DeliveryClaims.swift | 27 ++-- .../ServiceObjects/DirectEventProcessor.swift | 123 ++++++++++++++---- .../DirectEventProcessorSpec.swift | 26 ++++ 3 files changed, 135 insertions(+), 41 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DeliveryClaims.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DeliveryClaims.swift index 18ebd386a..c517e33f4 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DeliveryClaims.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DeliveryClaims.swift @@ -4,8 +4,8 @@ import Foundation /// /// A commit only stages events; whichever delivery runs next is what closes them into a batch. So the batch holding a /// caller's events may belong to a delivery already under way, and answering that caller means hearing from it. These -/// are the two small pieces that arrange it: a `Claim` is a batch somebody is waiting on, and a `DeliveryAnswer` -/// gathers the claims one delivery's answer depends on. +/// are the two small pieces that arrange it: a `Claim` is a batch somebody is waiting on, and a `PassAnswer` gathers +/// the claims one pass of a delivery depends on. extension DirectEventProcessor { /// A batch one delivery has taken, which another delivery's answer may depend on. final class Claim { @@ -16,32 +16,33 @@ extension DirectEventProcessor { var waiters: [FlushOutcomeClosure] = [] } - /// Gathers the outcomes one delivery's answer depends on, and reports success only where all of them succeeded. + /// Gathers the outcomes one pass of a delivery depends on, and reports the worst of them: blocked where any batch + /// was, lost where any was lost, and delivered only where all of them were. /// /// How many there are is settled before the first can arrive, so this needs no notion of being complete beyond /// counting down. - final class DeliveryAnswer { + final class PassAnswer { private let lock = UnfairLock() /// Outcomes still to come, and the answer to make once there are none. Only to be used while holding `lock`. private var outstanding: Int - private var delivered = true - private var answer: FlushOutcomeClosure? + private var outcome = PassOutcome.delivered + private var answer: ((PassOutcome) -> Void)? - init(outstanding: Int, _ answer: @escaping FlushOutcomeClosure) { + init(outstanding: Int, _ answer: @escaping (PassOutcome) -> Void) { self.outstanding = outstanding self.answer = answer } - func report(_ delivered: Bool) { - let (answer, outcome) = lock.withLock { () -> (FlushOutcomeClosure?, Bool) in - self.delivered = self.delivered && delivered + func report(_ outcome: PassOutcome) { + let (answer, worst) = lock.withLock { () -> (((PassOutcome) -> Void)?, PassOutcome) in + self.outcome = max(self.outcome, outcome) outstanding -= 1 guard outstanding <= 0 - else { return (nil, self.delivered) } + else { return (nil, self.outcome) } defer { self.answer = nil } - return (self.answer, self.delivered) + return (self.answer, self.outcome) } - answer?(outcome) + answer?(worst) } } } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 6f83614b2..691285cd0 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -621,51 +621,103 @@ extension DirectEventProcessor { return } - let (lossMark, closedEverything) = commitAndCloseBatch() - - let (batches, waitingOn, listedEverything) = claimDeliverableBatches() - // Events left in a log that would not close, or in a batch a listing that failed could not see, are still on - // the disk for a later delivery. This one delivers what it has, and cannot say the caller's events got out. - let accountsForEverything = closedEverything && listedEverything - if !accountsForEverything { - os_log("%s will report failure: %s", log: service.config.logger, type: .debug, typeName(and: #function), - closedEverything ? "the stored events could not be listed" : "the event log could not be closed") - } + // The batches already waiting go first, and what has been committed since is closed into a batch of its own + // only once none of them is left blocked. Closing first would add a batch for every pass a failure lasts: every + // flush, periodic delivery and trip to the background of an outage would leave one more file, or one more batch + // held in memory, each with the summary its commit staged. Held back, what is committed meanwhile goes on into + // the open log, which closes once it is full or once the batches ahead of it are gone, so an outage costs a + // batch per `EventLogFormat.maxBatchSize` rather than one per pass. + let waiting = claimDeliverableBatches(discardingExpired: true) + // Settled before this delivery's own outcome is reported, which is what the answer waits for last. + var settled = (lossMark: UInt64(0), accountsForEverything: false) // This delivery's own outcome, plus that of every batch another delivery is already sending. The caller's // events may be in either: a delivery that started after they were recorded closed them into its batch. - let answer = DeliveryAnswer(outstanding: waitingOn.count + 1) { delivered in - completion?(delivered && accountsForEverything, lossMark) + let answer = PassAnswer(outstanding: waiting.waitingOn.count + 1) { outcome in + completion?(outcome == .delivered && settled.accountsForEverything, settled.lossMark) } - for claim in waitingOn { - whenSettled(claim, answer.report) + // Waited on for the answer, but not before closing: holding back until another delivery answers, which through + // a retry can take twice the connection timeout, would leave the store filling with nothing draining it. + waitOn(waiting.waitingOn, answer) + + deliverOwn(waiting.deliverable) { waitingOutcome in + guard waitingOutcome != .blocked + else { + // Committed all the same, since a delivery is a commit point. Nothing was closed, so the caller's events + // did not go, and the mark is as of this commit, the last point anything the caller recorded was staged. + settled = (self.commitLeavingTheLogOpen(), false) + os_log("%s will report failure: an earlier batch could not be delivered", log: self.service.config.logger, type: .debug, self.typeName(and: #function)) + answer.report(.blocked) + return + } + + let (lossMark, closedEverything) = self.commitAndCloseBatch() + let new = self.claimDeliverableBatches(discardingExpired: false) + // Events left in a log that would not close, or in a batch a listing that failed could not see, are still on + // the disk for a later delivery. This one delivers what it has, and cannot say the caller's events got out. + settled = (lossMark, closedEverything && waiting.isComplete && new.isComplete) + if !settled.accountsForEverything { + os_log("%s will report failure: %s", log: self.service.config.logger, type: .debug, self.typeName(and: #function), + closedEverything ? "the stored events could not be listed" : "the event log could not be closed") + } + if waiting.deliverable.isEmpty && waiting.waitingOn.isEmpty && new.deliverable.isEmpty && new.waitingOn.isEmpty { + os_log("%s aborted. Event store is empty", log: self.service.config.logger, type: .debug, self.typeName(and: #function)) + self.reportSyncComplete(nil) + } + + let newAnswer = PassAnswer(outstanding: new.waitingOn.count + 1) { newOutcome in + answer.report(max(waitingOutcome, newOutcome)) + } + self.waitOn(new.waitingOn, newAnswer) + self.deliverOwn(new.deliverable, newAnswer.report) } + } + /// Delivers the batches this delivery claimed, releasing the claims once it has an outcome. + private func deliverOwn(_ batches: [EventBatch], _ completion: @escaping (PassOutcome) -> Void) { guard !batches.isEmpty else { - os_log("%s aborted. Event store is empty", log: service.config.logger, type: .debug, typeName(and: #function)) - reportSyncComplete(nil) - answer.report(true) + completion(.delivered) return } - os_log("%s starting", log: service.config.logger, type: .debug, typeName(and: #function)) - deliver(batches) { [weak self] delivered in - self?.releaseClaim(on: batches, delivered: delivered) - answer.report(delivered) + deliver(batches) { outcome in + self.releaseClaim(on: batches, delivered: outcome == .delivered) + completion(outcome) + } + } + + /// Reports the outcome of each batch another delivery is sending to `answer`, once it has one. + private func waitOn(_ claims: [Claim], _ answer: PassAnswer) { + for claim in claims { + whenSettled(claim) { answer.report($0 ? .delivered : .blocked) } + } + } + + /// `commitAndCloseBatch()` for a pass that found an earlier batch blocked, and so closes nothing. + /// + /// - Returns: the loss count as of the commit. + private func commitLeavingTheLogOpen() -> UInt64 { + commitLock.withLock { + encodeAndCommitPending() + return stateLock.withLock { lostEvents } } } /// Takes the batches this delivery will send, leaving out any another delivery is still waiting on a response for. /// + /// - Parameter discardingExpired: whether to discard what is too old to report first, which a second pass moments + /// after the first has no need to do again. /// - Returns: the batches to send, the claims on the ones left out, and whether the listing was complete. A batch /// left out may hold this caller's events -- `commitRecordedEvents` only stages them, and whichever delivery runs /// next is what closes them into a batch -- so the answer has to wait for those claims rather than assume they /// are someone else's business. - private func claimDeliverableBatches() -> (deliverable: [EventBatch], waitingOn: [Claim], isComplete: Bool) { + private func claimDeliverableBatches(discardingExpired: Bool) -> (deliverable: [EventBatch], waitingOn: [Claim], isComplete: Bool) { // Listed before the lock is taken, because listing reads the disk. Taking what appeared after it is the next // delivery's business; all this has to be atomic is deciding which of these batches are this delivery's. // Expired first, so that a batch too old to report is not sent. - reportDiscarded(store.discardExpired()) + if discardingExpired { + reportDiscarded(store.discardExpired()) + } let listing = store.listPendingBatches() return claimLock.withLock { var deliverable: [EventBatch] = [] @@ -721,7 +773,22 @@ extension DirectEventProcessor { } } - private func deliver(_ batches: [EventBatch], _ completion: FlushOutcomeClosure?) { + /// How a pass over some batches went, from best to worst. + enum PassOutcome: Int, Comparable { + /// Every batch was accepted. + case delivered + /// Every batch was attempted, and at least one was lost for good: refused, unreadable, or damaged. + case lost + /// A batch is still there to be sent again: its request failed in a way worth retrying, or it could not be read + /// this time. The batches after it were not attempted. + case blocked + + static func < (lhs: PassOutcome, rhs: PassOutcome) -> Bool { + lhs.rawValue < rhs.rawValue + } + } + + private func deliver(_ batches: [EventBatch], _ completion: @escaping (PassOutcome) -> Void) { deliver(batches, lostAnything: false, completion) } @@ -732,11 +799,11 @@ extension DirectEventProcessor { /// /// `lostAnything` carries whether a batch already taken in this pass was lost for good, so that the caller hears /// about it even where the batches after it were accepted. - private func deliver(_ batches: [EventBatch], lostAnything: Bool, _ completion: FlushOutcomeClosure?) { + private func deliver(_ batches: [EventBatch], lostAnything: Bool, _ completion: @escaping (PassOutcome) -> Void) { var remaining = batches guard !remaining.isEmpty else { - completion?(!lostAnything) + completion(lostAnything ? .lost : .delivered) return } @@ -755,7 +822,7 @@ extension DirectEventProcessor { } catch { // Kept: the batch is still there, and a read that failed may not fail next time. os_log("%s could not read stored events: %s", log: service.config.logger, type: .debug, typeName(and: #function), String(describing: error)) - completion?(false) + completion(.blocked) return } @@ -789,7 +856,7 @@ extension DirectEventProcessor { self.reportBatchLost(batch) self.deliver(remaining, lostAnything: true, completion) case .retryable: - completion?(false) + completion(.blocked) } } } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index c4f1d3931..52ff33a36 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -2558,6 +2558,32 @@ extension DirectEventProcessorSpec { expect(testContext.serviceMock.publishedEventPayloads.dropFirst(4).map(customKeys)) == [["first"], ["second"]] } + // A batch per flush would grow without bound for as long as an outage lasts: files on the disk, or batches + // held in memory, each with a summary of its own. + for persistence in [EventPersistence.immediate, .disabled] { + it("closes no batch behind the one blocked during an outage, persistence: \(persistence)") { + testContext = TestContext(store: EventStore.temporary(persistEvents: persistence != .disabled), + eventPersistence: persistence) + testContext.serviceMock.respondToEvents(with: 503) + testContext.eventProcessor.isOnline = true + testContext.eventProcessor.record(CustomEvent(key: "blocked", context: testContext.context)) + for flush in 0..<3 { + expect(testContext.flushOutcome()) == false + testContext.eventProcessor.record(CustomEvent(key: "during-\(flush)", context: testContext.context)) + } + // The attempt and its retry for each flush, all of the batch at the head. + expect(testContext.serviceMock.publishedEventPayloads.map(customKeys)) == Array(repeating: ["blocked"], count: 6) + expect(testContext.store.listPendingBatches().batches.count) == 1 + + testContext.serviceMock.stubEventResponse(success: true) + expect(testContext.flushOutcome()) == true + + expect(testContext.serviceMock.publishedEventPayloads.dropFirst(6).map(customKeys)) + == [["blocked"], ["during-0", "during-1", "during-2"]] + expect(testContext.store.listPendingBatches().batches).to(beEmpty()) + } + } + it("does not tell a flush its events arrived when an earlier delivery nobody waited on failed to send them") { testContext = TestContext() testContext.serviceMock.respondToEvents(with: 503) From 5d04131cc6f96602c569e52cb5b50e30e558df7e Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 22 Oct 2026 11:38:26 -0700 Subject: [PATCH 090/158] Bound summaries by bytes where events stay off the disk Capacity never refuses a summary, and the byte capacity applied only where events were persisted, so at .disabled nothing bounded them: a delivery that kept failing left the heap growing for the life of the process. The byte capacity now refuses a summary either way. Full events kept off the disk are still bounded by count alone, and the public eventCapacityBytes documentation says so. Co-authored-by: Cursor --- LaunchDarkly/LaunchDarkly/Models/LDConfig.swift | 3 ++- .../LaunchDarkly/ServiceObjects/EventStore.swift | 8 ++++++-- .../ServiceObjects/EventStoreSpec.swift | 16 ++++++++++++++++ 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift index bb436320a..20a726cee 100644 --- a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift @@ -389,7 +389,8 @@ public struct LDConfig { /// of flag evaluations, so that evaluations are still counted after full events start being discarded. A value of zero or less /// means the default. (Default: 20 MiB) /// - /// It has no effect where `eventPersistence` is `.disabled`. + /// Where `eventPersistence` is `.disabled`, full events are bounded by `eventCapacity` alone, and this bounds only the summaries, + /// which `eventCapacity` never refuses: a delivery that keeps failing would otherwise leave them growing for the life of the process. public var eventCapacityBytes: Int = Defaults.eventCapacityBytes /// How long the SDK keeps events it has persisted but not yet reported, where `eventPersistence` keeps events across launches. Events diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index e693035cb..e291fce09 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -54,7 +54,8 @@ protocol EventStoring { /// outlive the process, and deciding when to call it is the caller's. /// /// - Parameter bypassingCapacity: true for a summary, which is not refused for `capacity` and may use the part of - /// the byte capacity held back from full events, but is still refused past the byte capacity itself. + /// the byte capacity held back from full events, but is still refused past the byte capacity itself, whether or + /// not events are persisted. func stage(_ encodedEvent: Data, bypassingCapacity: Bool) -> Bool /// Counts events that have been accepted but are still being encoded, so capacity sees them until they are staged. @@ -346,7 +347,10 @@ final class EventStore: EventStoring { let frame = EventLogFormat.frame(for: encodedEvent) // Only a summary bypasses capacity without a reservation, and only a summary may use the bytes held back. let isSummary = bypassingCapacity && !usingReservation - let boundedByBytes = disk != nil + // Kept off the disk, full events are bounded by count alone, as they were before persistence. A summary bypasses + // the count wherever it is kept, so there its bytes are all that bounds it: a delivery that keeps failing would + // otherwise go on holding one more for the life of the process. + let boundedByBytes = disk != nil || isSummary let staged = bufferLock.withLock { () -> Bool in // In the same critical section as the append, so the event is never counted twice or not at all. if usingReservation { diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 1af856541..7670f10c9 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -1461,6 +1461,22 @@ final class EventStoreSpec: QuickSpec { expect(store.pendingEventCount) == 20 } + it("bounds summaries by bytes even where it keeps events off the disk") { + // Nothing else bounds them: capacity never refuses a summary, and a delivery that keeps failing keeps + // every one it was given. + store = EventStore.temporary(capacity: 100, byteCapacity: 1000, persistEvents: false) + for index in 1...8 { + _ = store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-\(index)")) + } + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "summary-1"), bypassingCapacity: true)) == true + expect(store.closeBatch()?.eventCount) == 9 + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "summary-2"), bypassingCapacity: true)) == true + + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "summary-3"), bypassingCapacity: true)) == false + // Full events are still bounded by count alone. + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-9"))) == true + } + it("counts committed bytes as well as staged ones") { store = EventStore.temporary(capacity: 100, byteCapacity: 1000) for index in 1...8 { From 051ff2b531e4b49cae6d669068f37e20617e3841 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 22 Oct 2026 11:54:21 -0700 Subject: [PATCH 091/158] Leave summaries out of the event count Every commit point stages a summary of the evaluations since the last, and the store counted each one as an event. Over an outage, where every flush is a commit point, the summaries alone filled the capacity and a track was refused for room nothing but summaries had taken. The store now keeps a summary's frame apart from full events until it is committed, and follows how many summaries each write, commit, closed batch and batch being sent holds, leaving them out of the event count in every state. Their bytes still count against the byte capacity. A batch recovered from an earlier process counts all its frames as events, since which ones were summaries is not recorded. Co-authored-by: Cursor --- .../ServiceObjects/EventDirectory.swift | 28 ++- .../ServiceObjects/EventLogFormat.swift | 11 ++ .../ServiceObjects/EventStore.swift | 182 +++++++++++++----- .../DirectEventProcessorSpec.swift | 38 +++- .../ServiceObjects/EventStoreSpec.swift | 59 +++++- 5 files changed, 255 insertions(+), 63 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index 9250e4596..fbd9a3db8 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -1,14 +1,18 @@ import Foundation import OSLog -/// How many events, and how many bytes, something stored holds: the two things capacity counts. +/// How many events, and how many bytes, something stored holds: the two things capacity counts. With how many of the +/// events are summaries, which capacity leaves out, where this store wrote them. struct EventTally: Equatable { var events = 0 var bytes = 0 + /// How many of `events` are known to be summaries; none for a batch this store did not close. + var summaries = 0 static func += (lhs: inout EventTally, rhs: EventTally) { lhs.events += rhs.events lhs.bytes += rhs.bytes + lhs.summaries += rhs.summaries } } @@ -51,15 +55,16 @@ final class EventDirectory { /// Only to be used while holding `ioLock`. private var located: (directory: URL, log: DiskEventLog)? - /// A batch in the directory: the file it is, how many events it holds, how big it is, and when it was last written, - /// which is what orders batches oldest first. + /// A batch in the directory: the file it is, how many events it holds, how big it is, when it was last written, + /// which is what orders batches oldest first, and how many of its events this store knows to be summaries. private struct Stored { let url: URL let events: Int let bytes: Int let modified: Date + var summaries = 0 - var tally: EventTally { EventTally(events: events, bytes: bytes) } + var tally: EventTally { EventTally(events: events, bytes: bytes, summaries: summaries) } } /// What this process knows about each batch in the directory, by payload ID, which is how everything outside this @@ -231,7 +236,7 @@ final class EventDirectory { return .failed(logSurvives: true) } - stored[payloadId] = Stored(url: file, events: events, bytes: held.bytes, modified: Date()) + stored[payloadId] = Stored(url: file, events: events, bytes: held.bytes, modified: Date(), summaries: held.summaries) return .closed(EventBatch(payloadId: payloadId, eventCount: events)) } @@ -314,8 +319,11 @@ final class EventDirectory { else { return nil } let listed = EventDirectory.listed(file) let modified = listed.modified ?? Date.distantPast - stored[payloadId] = Stored(url: file, events: events, bytes: listed.bytes ?? stored[payloadId]?.bytes ?? 0, - modified: modified) + // What the close knew of its summaries is kept; nothing in the file says. + let previous = stored[payloadId] + stored[payloadId] = Stored(url: file, events: events, bytes: listed.bytes ?? previous?.bytes ?? 0, + modified: modified, + summaries: previous.map { $0.events == events ? $0.summaries : 0 } ?? 0) return (EventBatch(payloadId: payloadId, eventCount: events), modified) } .sorted { $0.1 < $1.1 } @@ -343,6 +351,12 @@ final class EventDirectory { return stored.values.reduce(0) { $0 + $1.bytes } } + /// The summaries known to be in the batches the last listing or close counted. Requires `ioLock`. + var storedSummaries: Int { + ioLock.assertOwned() + return stored.values.reduce(0) { $0 + $1.summaries } + } + /// What this process counts for a batch, or nil if it counts nothing for it: one removed, or dropped by a listing it /// was no longer in. Requires `ioLock`. func counted(_ payloadId: String) -> EventTally? { diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift index f22b017e4..797cbbf2d 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift @@ -200,6 +200,17 @@ enum EventLogFormat { return (end, count) } + /// How many whole frames lie from `start` up to `end`, in frames this process wrote. + static func frameCount(in frames: Data, from start: Int, to end: Int) -> Int { + var count = 0 + var at = start + while at < end { + at += frameSize(in: frames, at: at) + count += 1 + } + return count + } + static func readUInt16(_ data: Data, at offset: Int) -> UInt16 { let start = data.startIndex + offset return UInt16(data[start]) << 8 | UInt16(data[start + 1]) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index e291fce09..804e42a9a 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -182,8 +182,23 @@ final class EventStore: EventStoring { private let ioLock: UnfairLock /// Only to be used while holding `bufferLock`. + /// + /// Full events staged, `bufferedEventCount` of them. private var bufferData = Data() private var bufferedEventCount = 0 + /// Summaries staged, `bufferedSummaryCount` of them, apart from the full events because capacity does not count + /// them. Everything staged is taken as the full events followed by the summaries, so that any part of it can say + /// how many of its frames are summaries from where the summaries start. + private var summaryData = Data() + private var bufferedSummaryCount = 0 + /// How many of the frames counted in each state below are summaries, state for state, so capacity can leave them + /// out. Changed in the same critical sections as the counts beside them. A batch this store did not close -- one a + /// previous run or another process left -- counts none: its name says how many frames it holds, not what they are, + /// so every one counts as an event, which errs towards refusing. + private var writingSummaries = 0 + private var committedSummaries = 0 + private var closedSummaries = 0 + private var sendingSummaries = 0 private var reservedEvents = 0 /// Taken out of the buffer to be written to the log, or assembled into a batch held in memory, and not yet /// counted where they land. Both happen outside `bufferLock`, and for as long as they take the events are in @@ -298,14 +313,17 @@ final class EventStore: EventStoring { /// Every event counted against capacity. Only to be used while holding `bufferLock`. private var countedEvents: Int { - reservedEvents + bufferedEventCount + writingEvents + committedEvents + (closedEvents - sendingEvents) + // Summaries left out: they are not events but the record of evaluations already counted, and counting them + // would let the ones a long outage writes at every commit point refuse full events. + reservedEvents + bufferedEventCount + (writingEvents - writingSummaries) + (committedEvents - committedSummaries) + + ((closedEvents - closedSummaries) - (sendingEvents - sendingSummaries)) } - /// Every byte counted against the byte capacity. Only to be used while holding `bufferLock`. + /// Every byte counted against the byte capacity, summaries included. Only to be used while holding `bufferLock`. /// /// Events reserved while being encoded have no bytes yet; they are counted when they are staged. private var countedBytes: Int { - bufferData.count + writingBytes + committedBytes + (closedBytes - sendingBytes) + bufferData.count + summaryData.count + writingBytes + committedBytes + (closedBytes - sendingBytes) } var hasRoomForEventBytes: Bool { @@ -360,8 +378,13 @@ final class EventStore: EventStoring { !boundedByBytes || countedBytes + frame.count <= (isSummary ? byteCapacity : eventByteCapacity) else { return false } - bufferData.append(frame) - bufferedEventCount += 1 + if isSummary { + summaryData.append(frame) + bufferedSummaryCount += 1 + } else { + bufferData.append(frame) + bufferedEventCount += 1 + } return true } if !staged { @@ -387,7 +410,8 @@ final class EventStore: EventStoring { writeStagedBytes() let (committed, stillBuffered) = bufferLock.withLock { - (EventTally(events: committedEvents, bytes: committedBytes), bufferedEventCount) + (EventTally(events: committedEvents, bytes: committedBytes, summaries: committedSummaries), + bufferedEventCount + bufferedSummaryCount) } let fromLog = committed.events > 0 ? closeOpenLog(of: disk, holding: committed) : nil @@ -403,7 +427,8 @@ final class EventStore: EventStoring { private func closeOpenLog(of disk: EventDirectory, holding committed: EventTally) -> EventBatch? { ioLock.assertOwned() // The file is the frames behind its header. A listing counts it from the file itself. - let closed = EventTally(events: committed.events, bytes: committed.bytes + EventLogFormat.fileHeader.count) + let closed = EventTally(events: committed.events, bytes: committed.bytes + EventLogFormat.fileHeader.count, + summaries: committed.summaries) switch disk.closeLog(holding: closed) { case .failed(let logSurvives): // A log that is still there is still the open one, and its events go on counting as committed until a @@ -412,6 +437,7 @@ final class EventStore: EventStoring { if !logSurvives { bufferLock.withLock { committedEvents = 0 + committedSummaries = 0 committedBytes = 0 } } @@ -419,8 +445,10 @@ final class EventStore: EventStoring { case .closed(let batch): bufferLock.withLock { committedEvents = 0 + committedSummaries = 0 committedBytes = 0 closedEvents += closed.events + closedSummaries += closed.summaries closedBytes += closed.bytes } return batch @@ -437,10 +465,13 @@ final class EventStore: EventStoring { // Listed in one critical section where there is a disk, so that the files and what is held cannot be counted // either side of a close that moves events between them. guard let disk - else { return BatchListing(batches: heldBatchesAndRecount(after: [], storedBytes: 0), isComplete: true) } + else { + return BatchListing(batches: heldBatchesAndRecount(after: [], storedSummaries: 0, storedBytes: 0), isComplete: true) + } return ioLock.withLock { let listing = disk.listBatches() - return BatchListing(batches: heldBatchesAndRecount(after: listing.batches, storedBytes: disk.storedBytes), + return BatchListing(batches: heldBatchesAndRecount(after: listing.batches, storedSummaries: disk.storedSummaries, + storedBytes: disk.storedBytes), isComplete: listing.isComplete) } } @@ -451,12 +482,15 @@ final class EventStore: EventStoring { /// /// Requires `ioLock` where there is a disk, so that the count is of the listing as it stands. /// - /// - Parameter storedBytes: the bytes in the batches listed from the disk. - private func heldBatchesAndRecount(after onDisk: [EventBatch], storedBytes: Int) -> [EventBatch] { + /// - Parameters: + /// - storedSummaries: the summaries this store knows of in the batches listed from the disk. + /// - storedBytes: the bytes in the batches listed from the disk. + private func heldBatchesAndRecount(after onDisk: [EventBatch], storedSummaries: Int, storedBytes: Int) -> [EventBatch] { heldLock.withLock { let batches = onDisk + inMemoryBatches.map { EventBatch(payloadId: $0.payloadId, eventCount: $0.eventCount) } bufferLock.withLock { closedEvents = batches.reduce(0) { $0 + $1.eventCount } + closedSummaries = inMemoryBatches.reduce(storedSummaries) { $0 + $1.summaries } closedBytes = inMemoryBatches.reduce(storedBytes) { $0 + $1.body.count } // One no longer listed is no longer counted, so it cannot be left out of the count either. let listed = Set(batches.map { $0.payloadId }) @@ -488,6 +522,7 @@ final class EventStore: EventStoring { let held = inMemoryBatches.remove(at: index) bufferLock.withLock { closedEvents -= held.eventCount + closedSummaries -= held.summaries closedBytes -= held.body.count forgetSending(batch.payloadId) } @@ -505,6 +540,7 @@ final class EventStore: EventStoring { let counted = disk.remove(batch.payloadId, holding: batch.eventCount) bufferLock.withLock { closedEvents -= counted.events + closedSummaries -= counted.summaries closedBytes -= counted.bytes forgetSending(batch.payloadId) } @@ -528,6 +564,7 @@ final class EventStore: EventStoring { recovered += prepared.events bufferLock.withLock { closedEvents += prepared.events + closedSummaries += prepared.summaries closedBytes += prepared.bytes } } @@ -550,6 +587,7 @@ final class EventStore: EventStoring { // them, so the store cannot go on counting what is gone. bufferLock.withLock { closedEvents -= counted.events + closedSummaries -= counted.summaries closedBytes -= counted.bytes } defer { hasDiscardedAbandonedEnvironments = true } @@ -602,7 +640,7 @@ extension EventStore { private func startedSending(_ payloadId: String, disk: EventDirectory?) { heldLock.withLock { let counted = inMemoryBatches.first { $0.payloadId == payloadId } - .map { EventTally(events: $0.eventCount, bytes: $0.body.count) } + .map { EventTally(events: $0.eventCount, bytes: $0.body.count, summaries: $0.summaries) } ?? disk?.counted(payloadId) guard let counted else { return } @@ -611,6 +649,7 @@ extension EventStore { else { return } sendingBatches[payloadId] = counted sendingEvents += counted.events + sendingSummaries += counted.summaries sendingBytes += counted.bytes } } @@ -630,6 +669,7 @@ extension EventStore { fileprivate func forgetSending(_ payloadId: String) { if let sending = sendingBatches.removeValue(forKey: payloadId) { sendingEvents -= sending.events + sendingSummaries -= sending.summaries sendingBytes -= sending.bytes } } @@ -644,59 +684,83 @@ extension EventStore { /// commit can land in several batches. private func writeStagedBytes() { ioLock.assertOwned() - let taken = bufferLock.withLock { () -> (bytes: Data, events: Int)? in + let taken = bufferLock.withLock { () -> StagedFrames? in // Staged bytes are left where they are. With nowhere durable to put them, memory is better than dropping // them: a delivery can still close them into a batch and send them. - guard persistEvents, bufferedEventCount > 0 + guard persistEvents, bufferedEventCount + bufferedSummaryCount > 0 else { return nil } - let staged = (bytes: bufferData, events: bufferedEventCount) - bufferData = Data() - bufferedEventCount = 0 - writingEvents += staged.events - writingBytes += staged.bytes.count - return staged + return takeStaged() } guard let taken else { return } - let base = taken.bytes.startIndex + let bytes = taken.frames + let base = bytes.startIndex var written = 0 - var writtenEvents = 0 - while written < taken.bytes.count { + while written < bytes.count { let room = EventLogFormat.maxBatchSize - EventLogFormat.fileHeaderSize - bufferLock.withLock { committedBytes } - if EventLogFormat.frameSize(in: taken.bytes, at: written) > room { + if EventLogFormat.frameSize(in: bytes, at: written) > room { guard closeFullLog() else { break } continue } - let (end, landing) = EventLogFormat.framesFitting(in: taken.bytes, from: written, room: room) - let isWhole = written == 0 && end == taken.bytes.count - guard append(isWhole ? taken.bytes : taken.bytes.subdata(in: (base + written)..<(base + end))) + let (end, landing) = EventLogFormat.framesFitting(in: bytes, from: written, room: room) + let landingSummaries = taken.summaries(from: written, to: end) + let isWhole = written == 0 && end == bytes.count + guard append(isWhole ? bytes : bytes.subdata(in: (base + written)..<(base + end))) else { break } // Counted where they landed in the same critical section that stops counting them on the way, so a // capacity check never sees them in both places or in neither. bufferLock.withLock { writingEvents -= landing + writingSummaries -= landingSummaries writingBytes -= end - written committedEvents += landing + committedSummaries += landingSummaries committedBytes += end - written } written = end - writtenEvents += landing } - if written < taken.bytes.count { - // Put back, to be delivered from memory rather than lost. Ahead of anything staged while the write was in - // flight, so the events keep the order they were recorded in. - bufferLock.withLock { - writingEvents -= taken.events - writtenEvents - writingBytes -= taken.bytes.count - written - bufferData = taken.bytes.subdata(in: (base + written).. StagedFrames { + let staged = StagedFrames(frames: summaryData.isEmpty ? bufferData : bufferData + summaryData, + summariesFrom: bufferData.count, + events: bufferedEventCount + bufferedSummaryCount, + summaries: bufferedSummaryCount) + bufferData = Data() + summaryData = Data() + bufferedEventCount = 0 + bufferedSummaryCount = 0 + writingEvents += staged.events + writingSummaries += staged.summaries + writingBytes += staged.frames.count + return staged + } + + /// Puts back what of `staged` from `start` on never landed, to be delivered from memory rather than lost. Ahead of + /// anything staged while the write was in flight, so the events keep the order they were recorded in. Requires + /// `bufferLock`. + private func putBack(_ staged: StagedFrames, from start: Int) { + let frames = staged.frames + let base = frames.startIndex + let summariesFrom = max(start, staged.summariesFrom) + let events = EventLogFormat.frameCount(in: frames, from: start, to: summariesFrom) + let summaries = EventLogFormat.frameCount(in: frames, from: summariesFrom, to: frames.count) + writingEvents -= events + summaries + writingSummaries -= summaries + writingBytes -= frames.count - start + bufferData = frames.subdata(in: (base + start)..<(base + summariesFrom)) + bufferData + summaryData = frames.subdata(in: (base + summariesFrom).. Int { + end <= summariesFrom ? 0 : EventLogFormat.frameCount(in: frames, from: max(start, summariesFrom), to: end) + } } /// What the store does once the filesystem has refused it. @@ -756,14 +839,8 @@ private extension EventStore { /// Called under `ioLock` where there is a disk, so that it and the log are closed as one step, and on its own /// where there is not. func closeInMemoryBatches() -> [EventBatch] { - let (frames, events) = bufferLock.withLock { () -> (frames: Data, events: Int) in - let staged = (frames: bufferData, events: bufferedEventCount) - bufferData = Data() - bufferedEventCount = 0 - writingEvents += staged.events - writingBytes += staged.frames.count - return staged - } + let staged = bufferLock.withLock { takeStaged() } + let frames = staged.frames // Assembled outside `bufferLock`, because it copies every staged frame; `writingEvents` is what keeps them // counted meanwhile. Through the same reader a batch on disk goes through, so a held batch and a written one @@ -772,9 +849,11 @@ private extension EventStore { var closed: [EventBatch] = [] var held = 0 var heldEvents = 0 + var heldSummaries = 0 while held < frames.count { let (end, batchEvents) = EventLogFormat.framesFitting(in: frames, from: held, room: EventLogFormat.maxBatchSize - EventLogFormat.fileHeaderSize) + let batchSummaries = staged.summaries(from: held, to: end) var body = Data(capacity: EventLogFormat.fileHeaderSize + end - held) body.append(EventLogFormat.fileHeader) body.append(frames[(base + held)..<(base + end)]) @@ -783,25 +862,30 @@ private extension EventStore { let payloadId = UUID().uuidString heldLock.withLock { - inMemoryBatches.append(HeldBatch(payloadId: payloadId, body: body, eventCount: batchEvents)) + inMemoryBatches.append(HeldBatch(payloadId: payloadId, body: body, eventCount: batchEvents, + summaries: batchSummaries)) // Counted where they landed in the same critical section that stops counting them on the way, so a // capacity check never sees them in both places or in neither. A held batch is counted as its body, // which is what it keeps. bufferLock.withLock { writingEvents -= batchEvents + writingSummaries -= batchSummaries writingBytes -= end - held closedEvents += batchEvents + closedSummaries += batchSummaries closedBytes += body.count } } closed.append(EventBatch(payloadId: payloadId, eventCount: batchEvents)) held = end heldEvents += batchEvents + heldSummaries += batchSummaries } if held < frames.count { bufferLock.withLock { - writingEvents -= events - heldEvents + writingEvents -= staged.events - heldEvents + writingSummaries -= staged.summaries - heldSummaries writingBytes -= frames.count - held } } @@ -828,7 +912,7 @@ extension EventStore { payloads.append(contentsOf: events.compactMap { try? JSONEncoder().encode($0) }) } var staged = EventLogFormat.fileHeader - staged.append(bufferLock.withLock { bufferData }) + staged.append(bufferLock.withLock { bufferData + summaryData }) _ = EventLogFormat.forEachFrame(in: staged) { type, payload in if type == EventLogFormat.eventFrame { payloads.append(payload) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 52ff33a36..03ba9d5d6 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -842,8 +842,9 @@ class DirectEventProcessorSpec: QuickSpec { it("keeps the events to retry after the failure") { erOnline() expect(testContext.serviceMock.publishEventDataCallCount) == 2 // 1 retry attempt - // The events are not lost by a failed delivery; they stay in the log, summary included. - expect(testContext.store.pendingEventCount) == testContext.events.count + 1 + // The events are not lost by a failed delivery; they stay in the log, summary included, + // though only the full events count against capacity. + expect(testContext.store.pendingEventCount) == testContext.events.count let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) valueIsArray(published) { valueArray in expect(valueArray.count) == testContext.events.count + 1 @@ -888,8 +889,9 @@ class DirectEventProcessorSpec: QuickSpec { it("keeps the events to retry after the failure") { erOnline() expect(testContext.serviceMock.publishEventDataCallCount) == 2 // 1 retry attempt - // The events are not lost by a failed delivery; they stay in the log, summary included. - expect(testContext.store.pendingEventCount) == testContext.events.count + 1 + // The events are not lost by a failed delivery; they stay in the log, summary included, + // though only the full events count against capacity. + expect(testContext.store.pendingEventCount) == testContext.events.count let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) valueIsArray(published) { valueArray in expect(valueArray.count) == testContext.events.count + 1 @@ -937,8 +939,9 @@ class DirectEventProcessorSpec: QuickSpec { it("keeps the events to retry after the failure") { erOnline() expect(testContext.serviceMock.publishEventDataCallCount) == 2 // 1 retry attempt - // The events are not lost by a failed delivery; they stay in the log, summary included. - expect(testContext.store.pendingEventCount) == testContext.events.count + 1 + // The events are not lost by a failed delivery; they stay in the log, summary included, + // though only the full events count against capacity. + expect(testContext.store.pendingEventCount) == testContext.events.count let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) valueIsArray(published) { valueArray in expect(valueArray.count) == testContext.events.count + 1 @@ -2582,6 +2585,29 @@ extension DirectEventProcessorSpec { == [["blocked"], ["during-0", "during-1", "during-2"]] expect(testContext.store.listPendingBatches().batches).to(beEmpty()) } + + // Every flush of an outage is a commit point, and each stages a summary of the evaluations since the + // last. Counted as events, they would leave a long outage refusing every track. + it("leaves room for events beside the summaries an outage writes, persistence: \(persistence)") { + testContext = TestContext(lastEventResponseDate: Date(), + store: EventStore.temporary(capacity: 2, persistEvents: persistence != .disabled), + eventCapacity: 2, eventPersistence: persistence) + testContext.serviceMock.respondToEvents(with: 503) + testContext.eventProcessor.isOnline = true + testContext.eventProcessor.record(CustomEvent(key: "blocked", context: testContext.context)) + expect(testContext.flushOutcome()) == false + let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: false) + for _ in 0..<3 { + testContext.eventProcessor.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: testContext.context, includeReason: false) + expect(testContext.flushOutcome()) == false + } + + testContext.eventProcessor.record(CustomEvent(key: "after-the-summaries", context: testContext.context)) + testContext.serviceMock.stubEventResponse(success: true) + expect(testContext.flushOutcome()) == true + expect(testContext.serviceMock.publishedEventPayloads.flatMap(customKeys)) + .to(contain("after-the-summaries"), description: "refused for room the summaries took") + } } it("does not tell a flush its events arrived when an earlier delivery nobody waited on failed to send them") { diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 7670f10c9..5667e7402 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -1186,6 +1186,62 @@ final class EventStoreSpec: QuickSpec { expect(store.stage(EventStoreSpec.payload("first"))) == true expect(store.stage(EventStoreSpec.payload("summary"), bypassingCapacity: true)) == true + // Kept, but not counted as an event. + expect(store.pendingEventCount) == 1 + } + + // Written at every commit point, summaries would otherwise fill capacity during an outage, and full events + // would be refused for the summaries of evaluations already counted. + for persistEvents in [true, false] { + it("leaves summaries out of the event count in every state, persisting: \(persistEvents)") { + store = EventStore.temporary(capacity: 2, persistEvents: persistEvents) + expect(store.stage(EventStoreSpec.payload("summary-1"), bypassingCapacity: true)) == true + expect(store.stage(EventStoreSpec.payload("event-1"))) == true + expect(store.pendingEventCount).to(equal(1), description: "staged") + store.commit() + expect(store.pendingEventCount).to(equal(1), description: "committed") + + let batch = store.closeBatch()! + expect(batch.eventCount).to(equal(2), description: "the batch holds both frames") + expect(store.pendingEventCount).to(equal(1), description: "closed") + expect(store.pendingBatches()) == [batch] + expect(store.pendingEventCount).to(equal(1), description: "listed") + + expect(store.stage(EventStoreSpec.payload("summary-2"), bypassingCapacity: true)) == true + expect(store.stage(EventStoreSpec.payload("event-2"))) == true + expect(store.stage(EventStoreSpec.payload("event-3"))).to(equal(false), description: "capacity still counts the events") + + store.startedSending(batch) + expect(store.pendingEventCount).to(equal(1), description: "sending") + store.stoppedSending(batch) + expect(store.pendingEventCount).to(equal(2), description: "sent and failed") + store.remove(batch) + expect(store.pendingEventCount).to(equal(1), description: "delivered") + } + } + + it("does not count as events the summaries a failed write left in memory") { + let lock = UnfairLock() + var failing = false + store = EventStore.temporary(writeLog: { file, bytes in + lock.withLock { failing } ? ENOSPC : file.writeAll(bytes) + }) + _ = store.stage(EventStoreSpec.payload("written")) + _ = store.stage(EventStoreSpec.payload("summary-written"), bypassingCapacity: true) + store.commit() + lock.withLock { failing = true } + _ = store.stage(EventStoreSpec.payload("put-back")) + _ = store.stage(EventStoreSpec.payload("summary-put-back"), bypassingCapacity: true) + store.commit() + expect(store.isPersisting) == false + expect(store.pendingEventCount) == 2 + + let fromLog = store.closeBatch()! + let fromMemory = store.pendingBatches().last! + expect(EventStoreSpec.keys(ofBody: try store.body(of: fromLog))) == ["written", "summary-written"] + expect(EventStoreSpec.keys(ofBody: try store.body(of: fromMemory))) == ["put-back", "summary-put-back"] + expect(store.pendingEventCount) == 2 + _ = store.pendingBatches() expect(store.pendingEventCount) == 2 } @@ -1411,7 +1467,8 @@ final class EventStoreSpec: QuickSpec { expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "summary-1"), bypassingCapacity: true)) == true expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "summary-2"), bypassingCapacity: true)) == true expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "summary-3"), bypassingCapacity: true)) == false - expect(store.pendingEventCount) == 10 + // Their bytes count, but they are not events. + expect(store.pendingEventCount) == 8 } it("refuses a reserved event that turns out not to fit, and gives up its reservation") { From 288a27be87a7d128709e175f15eaef8cc36e1423 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 22 Oct 2026 14:34:33 -0700 Subject: [PATCH 092/158] Back off failed event deliveries per RETRY instead of dropping and going offline MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A refused delivery deleted the batch and took the processor offline, and every other failure was posted again on each flush. Now each environment keeps an in-memory DeliveryRetryState bound as durability §9 specifies: the wait starts at a second and doubles up to the flush interval (at least a second); a refusal unlikely to pass soon (a 4xx other than 400/408/429, or a TLS failure) jumps to about eight and a half minutes and doubles up to an hour; Retry-After sets the wait, capped at an hour and jittered; one accepted delivery resets it. Every batch is kept whatever the failure. Nothing is posted during a wait. The sender posts once, and the processor makes the quick retry itself: a pass sits out a wait no longer than the initial delay, so a flush hears from its retry, and otherwise answers at once, with a delivery set for when the wait ends. Co-authored-by: Cursor --- .../Networking/HTTPURLResponse.swift | 14 ++ .../ServiceObjects/AnalyticsEventSender.swift | 72 ++++---- .../ServiceObjects/DirectEventProcessor.swift | 149 ++++++++++++---- .../ServiceObjects/RetryState.swift | 101 +++++++++++ .../DirectEventProcessorSpec.swift | 165 +++++++++++++----- .../ServiceObjects/RetryStateSpec.swift | 80 +++++++++ 6 files changed, 470 insertions(+), 111 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/Networking/HTTPURLResponse.swift b/LaunchDarkly/LaunchDarkly/Networking/HTTPURLResponse.swift index 2f7c07ad0..72e542c8f 100644 --- a/LaunchDarkly/LaunchDarkly/Networking/HTTPURLResponse.swift +++ b/LaunchDarkly/LaunchDarkly/Networking/HTTPURLResponse.swift @@ -5,6 +5,7 @@ extension HTTPURLResponse { struct HeaderKeys { static let date = "Date" static let etag = "Etag" + static let retryAfter = "Retry-After" } struct StatusCodes { @@ -38,4 +39,17 @@ extension HTTPURLResponse { var headerEtag: String? { self.allHeaderFields[HeaderKeys.etag] as? String } + + /// The wait a `Retry-After` header asks for, in delta-seconds or as an HTTP date; nil where there is none, or it + /// cannot be read. A date already past is no wait at all. + var headerRetryAfter: TimeInterval? { + guard let value = value(forHTTPHeaderField: HeaderKeys.retryAfter)?.trimmingCharacters(in: .whitespaces) + else { return nil } + if let seconds = Int(value) { + return seconds >= 0 ? TimeInterval(seconds) : nil + } + guard let date = DateFormatter.httpUrlHeaderFormatter.date(from: value) + else { return nil } + return max(0, date.timeIntervalSinceNow) + } } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/AnalyticsEventSender.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/AnalyticsEventSender.swift index 6b27e2786..21f77607e 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/AnalyticsEventSender.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/AnalyticsEventSender.swift @@ -1,44 +1,50 @@ import Foundation import OSLog -/// Posts a batch of events to LaunchDarkly, retrying once a second later where the failure may pass, and says what the -/// response means. Android's `AnalyticsEventSender` plays the same part. +/// Posts a batch of events to LaunchDarkly once and says what the response means. Android's `AnalyticsEventSender` +/// plays the same part. /// -/// It keeps no state. What an outcome means for the SDK -- the server's time, going offline, removing the batch -- is -/// left to `DirectEventProcessor`, which owns that state. +/// It keeps no state. What an outcome means for the SDK -- the server's time, when to post again, removing the batch -- +/// is left to `DirectEventProcessor`, which owns that state. final class AnalyticsEventSender { enum Outcome { /// LaunchDarkly took the events. Carries the time the response gave, if it gave one. case accepted(serverTime: Date?) - /// LaunchDarkly will never take these events, so keeping them only wastes space. - case refused(HTTPURLResponse?) - /// The events may still be deliverable. Carries why the retry failed. - case retryable(SynchronizingError) + /// The post failed. The events are kept whatever the failure; it decides only how long to wait. + case failed(Failure) } + struct Failure { + let error: SynchronizingError + /// The response's status, or nil where there was no response. + let statusCode: Int? + /// Whether the failure is unlikely to pass soon: a 4xx other than 400, 408 and 429, or a TLS or certificate + /// failure. + let unexpected: Bool + /// The wait the response asked for, or nil where it gave none. + let retryAfter: TimeInterval? + } + + private static let tlsErrorCodes: Set = [ + .secureConnectionFailed, + .serverCertificateHasBadDate, + .serverCertificateUntrusted, + .serverCertificateHasUnknownRoot, + .serverCertificateNotYetValid, + .clientCertificateRejected, + .clientCertificateRequired + ] + private let service: DarklyServiceProvider init(service: DarklyServiceProvider) { self.service = service } - /// Posts `body` under `payloadId`, and once more after a second if that failed in a way that may pass. The - /// completion runs once, with the outcome of the last attempt. + /// Posts `body` under `payloadId`. The completion runs once, with the outcome. func send(_ body: Data, payloadId: String, eventCount: Int, completion: @escaping (Outcome) -> Void) { service.publishEventData(body, payloadId) { response in - let outcome = self.outcome(sentEvents: eventCount, response: response.urlResponse as? HTTPURLResponse, error: response.error) - guard case .retryable = outcome - else { - completion(outcome) - return - } - - os_log("%s Retrying event post after delay.", log: self.service.config.logger, type: .debug, self.typeName(and: #function)) - DispatchQueue.global().asyncAfter(deadline: DispatchTime.now() + 1.0) { - self.service.publishEventData(body, payloadId) { retryResponse in - completion(self.outcome(sentEvents: eventCount, response: retryResponse.urlResponse as? HTTPURLResponse, error: retryResponse.error)) - } - } + completion(self.outcome(sentEvents: eventCount, response: response.urlResponse as? HTTPURLResponse, error: response.error)) } } @@ -48,16 +54,20 @@ final class AnalyticsEventSender { return .accepted(serverTime: response?.headerDate) } - if let statusCode = response?.statusCode, HTTPURLResponse.StatusCodes.isTerminalStatusCode(statusCode) { - os_log("%s dropping events due to non-retriable response: %s", log: service.config.logger, type: .debug, typeName(and: #function), String(describing: response)) - return .refused(response) - } - os_log("%s Sending events failed with error: %s response: %s", log: service.config.logger, type: .debug, typeName(and: #function), String(describing: error), String(describing: response)) - if let error = error { - return .retryable(.request(error)) + let statusCode = response?.statusCode + let unexpected: Bool + if let statusCode, (400..<600).contains(statusCode) { + unexpected = HTTPURLResponse.StatusCodes.isTerminalStatusCode(statusCode) + } else if let urlError = error as? URLError { + unexpected = AnalyticsEventSender.tlsErrorCodes.contains(urlError.code) + } else { + unexpected = false } - return .retryable(.response(response)) + return .failed(Failure(error: error.map { .request($0) } ?? .response(response), + statusCode: statusCode, + unexpected: unexpected, + retryAfter: response?.headerRetryAfter)) } } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 691285cd0..4ae36b30c 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -56,6 +56,9 @@ class DirectEventProcessor: EventProcessor { private let sender: AnalyticsEventSender + /// When a post is allowed after a failure. Nothing is posted while it says to wait. + private let retryState: DeliveryRetryState + /// Its summarizer is guarded by `recordLock`, and its encoders are reached only under `commitLock`. let eventBuffer: OutboundEventBuffer @@ -108,10 +111,10 @@ class DirectEventProcessor: EventProcessor { /// How many events the SDK accepted and then dropped for good. Only to be used while holding `stateLock`. /// - /// A batch LaunchDarkly refuses in a way that may pass is kept and retried, so it is not a loss. What is lost is - /// what the store never received: an event or a summary that could not be serialized when it was committed, or - /// that serialized too large to store. A delivery that finds nothing to send cannot tell those apart from events - /// that arrived, so without this it would report success for them. + /// A batch LaunchDarkly refuses is kept and retried, so it is not a loss. What is lost is what the store never + /// received: an event or a summary that could not be serialized when it was committed, or that serialized too large + /// to store. A delivery that finds nothing to send cannot tell those apart from events that arrived, so without this + /// it would report success for them. /// /// A count rather than a flag because deliveries overlap. Each takes the count as it closes its batch, and /// answers for the losses up to that mark, so a loss is told to the caller whose delivery it happened before @@ -124,10 +127,14 @@ class DirectEventProcessor: EventProcessor { // MARK: Guarded by deliveryLock - /// Guards `queuedDelivery`. Taken on a caller's thread and on `deliveryQueue`, and never held across either. + /// Guards `queuedDelivery` and `isRetryScheduled`. Taken on a caller's thread and on `deliveryQueue`, and never held + /// across either. private let deliveryLock = UnfairLock() /// Callers waiting on a delivery that is queued and has not taken the batches yet, or nil where none is queued. private var queuedDelivery: [FlushOutcomeClosure]? + /// Whether a delivery is already set to run when the backoff's wait ends, so a run of passes inside the wait sets + /// one rather than one each. + private var isRetryScheduled = false #if DEBUG /// How many deliveries have started, which is fewer than the flushes asked for where they were joined. @@ -181,10 +188,12 @@ class DirectEventProcessor: EventProcessor { onSyncComplete: EventSyncCompleteClosure?, store: EventStoring? = nil, encoding: OutboundEventBuffer.Encoding = .handWritten, - commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.DirectEventProcessor.commitQueue", qos: .userInitiated)) { + commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.DirectEventProcessor.commitQueue", qos: .userInitiated), + retryState: DeliveryRetryState? = nil) { self.service = service self.onSyncComplete = onSyncComplete self.sender = AnalyticsEventSender(service: service) + self.retryState = retryState ?? DeliveryRetryState(flushInterval: service.config.eventFlushInterval) self.responseDate = Date() self.eventCapacity = service.config.eventCapacity self.commitQueue = commitQueue @@ -777,10 +786,10 @@ extension DirectEventProcessor { enum PassOutcome: Int, Comparable { /// Every batch was accepted. case delivered - /// Every batch was attempted, and at least one was lost for good: refused, unreadable, or damaged. + /// Every batch was attempted, and at least one was lost for good: unreadable, or damaged. case lost - /// A batch is still there to be sent again: its request failed in a way worth retrying, or it could not be read - /// this time. The batches after it were not attempted. + /// A batch is still there to be sent again: its request failed, the backoff did not allow it to be posted yet, + /// or it could not be read this time. The batches after it were not attempted. case blocked static func < (lhs: PassOutcome, rhs: PassOutcome) -> Bool { @@ -789,17 +798,20 @@ extension DirectEventProcessor { } private func deliver(_ batches: [EventBatch], _ completion: @escaping (PassOutcome) -> Void) { - deliver(batches, lostAnything: false, completion) + deliver(batches, lostAnything: false, retried: false, completion) } - /// Delivers batches oldest first, stopping at the first one that failed in a way worth retrying. + /// Delivers batches oldest first, stopping at the first one that failed. /// /// Stopping matters: the batches that are left keep their place in the log, and a later delivery attempts them /// again rather than the SDK spending the rest of the session's requests on a service that is refusing them. /// + /// A failure whose wait is no longer than the initial delay is retried here, once a pass, so a caller waiting on + /// the pass hears from the retry; a longer wait ends the pass at once, with a delivery set for when the wait ends. + /// /// `lostAnything` carries whether a batch already taken in this pass was lost for good, so that the caller hears - /// about it even where the batches after it were accepted. - private func deliver(_ batches: [EventBatch], lostAnything: Bool, _ completion: @escaping (PassOutcome) -> Void) { + /// about it even where the batches after it were accepted. `retried` whether this pass has had its retry. + private func deliver(_ batches: [EventBatch], lostAnything: Bool, retried: Bool, _ completion: @escaping (PassOutcome) -> Void) { var remaining = batches guard !remaining.isEmpty else { @@ -807,17 +819,33 @@ extension DirectEventProcessor { return } + let wait = retryState.remainingWait + guard wait <= 0 + else { + // Not an attempt, so it moves nothing in the backoff. A wait no longer than a retry's is sat out, as this + // pass's retry; a longer one ends the pass. + if !retried && wait <= retryState.initialDelay { + DispatchQueue.global().asyncAfter(deadline: .now() + wait) { + self.deliver(batches, lostAnything: lostAnything, retried: true, completion) + } + } else { + scheduleDeliveryWhenWaitEnds() + completion(.blocked) + } + return + } + let batch = remaining.removeFirst() let read: BatchPayload? do { read = try store.payload(of: batch) } catch is UnreadableBatch { - // Lost like a refused batch: nothing will ever send it, so it is removed, the rest are still attempted, and - // the caller hears that this pass lost events. + // Lost: nothing will ever send it, so it is removed, the rest are still attempted, and the caller hears + // that this pass lost events. os_log("%s dropping %d stored event(s) this version cannot read", log: service.config.logger, type: .error, typeName(and: #function), batch.eventCount) store.remove(batch) reportBatchLost(batch) - deliver(remaining, lostAnything: true, completion) + deliver(remaining, lostAnything: true, retried: retried, completion) return } catch { // Kept: the batch is still there, and a read that failed may not fail next time. @@ -831,7 +859,7 @@ extension DirectEventProcessor { // Gone, which is usually another process having delivered it, or holding no events. Either way nothing is // lost and there is nothing to keep. store.remove(batch) - deliver(remaining, lostAnything: lostAnything, completion) + deliver(remaining, lostAnything: lostAnything, retried: retried, completion) return } @@ -839,28 +867,80 @@ extension DirectEventProcessor { // the batch is removed rather than now, since one kept for a retry is read, and found short, again. let leftOut = max(0, batch.eventCount - payload.eventCount) service.diagnosticCache?.recordEventsInLastBatch(eventsInLastBatch: payload.eventCount) + post(batch, payload, leftOut: leftOut, remaining: remaining, lostAnything: lostAnything, retried: retried, completion) + } + + /// Posts a batch already read, then goes on to the rest. Requires the backoff to allow a post. + // swiftlint:disable:next function_parameter_count + private func post(_ batch: EventBatch, + _ payload: BatchPayload, + leftOut: Int, + remaining: [EventBatch], + lostAnything: Bool, + retried: Bool, + _ completion: @escaping (PassOutcome) -> Void) { publish(batch, payload) { outcome in switch outcome { case .accepted: guard leftOut > 0 else { - self.deliver(remaining, lostAnything: lostAnything, completion) + self.deliver(remaining, lostAnything: lostAnything, retried: retried, completion) return } os_log("%s dropped %d damaged stored event(s) from a batch it delivered", log: self.service.config.logger, type: .error, self.typeName(and: #function), leftOut) self.reportStoredEventsLost(leftOut) - self.deliver(remaining, lostAnything: true, completion) - case .refused: - // The rest are still attempted, so that batches LaunchDarkly will never take do not sit on the device - // for the rest of the session, taking up the room the capacity limit leaves for new events. - self.reportBatchLost(batch) - self.deliver(remaining, lostAnything: true, completion) - case .retryable: - completion(.blocked) + self.deliver(remaining, lostAnything: true, retried: retried, completion) + case .failed(let failure): + let wait = self.retryState.recordFailure(unexpected: failure.unexpected, retryAfter: failure.retryAfter) + guard !retried, wait <= self.retryState.initialDelay + else { + // Told once the pass gives the batch up, not for a failure its retry may still answer. + self.reportSyncComplete(failure.error) + self.scheduleDeliveryWhenWaitEnds() + completion(.blocked) + return + } + os_log("%s Retrying event post after %.1fs", log: self.service.config.logger, type: .debug, self.typeName(and: #function), wait) + DispatchQueue.global().asyncAfter(deadline: .now() + wait) { + // Another delivery's failure may have set a longer wait meanwhile. + guard self.retryState.remainingWait <= 0 + else { + self.reportSyncComplete(failure.error) + self.scheduleDeliveryWhenWaitEnds() + completion(.blocked) + return + } + self.post(batch, payload, leftOut: leftOut, remaining: remaining, lostAnything: lostAnything, retried: true, completion) + } } } } + /// Sets a delivery for when the backoff's wait ends, unless one is set already, so a batch kept through a failure + /// goes out then rather than at the next report interval after it. + /// + /// A delivery that fails again sets the next one, each a longer wait away, which is what keeps this from posting + /// in a loop. Going offline leaves the one set to find nothing to do. + private func scheduleDeliveryWhenWaitEnds() { + let wait = retryState.remainingWait + guard wait > 0 + else { return } + let alreadyScheduled = deliveryLock.withLock { () -> Bool in + defer { isRetryScheduled = true } + return isRetryScheduled + } + guard !alreadyScheduled + else { return } + + os_log("%s Will retry posting events in %.1fs after a failure", log: service.config.logger, type: .debug, typeName(and: #function), wait) + DispatchQueue.global().asyncAfter(deadline: .now() + wait) { [weak self] in + guard let self + else { return } + self.deliveryLock.withLock { self.isRetryScheduled = false } + self.queueScheduledDelivery() + } + } + /// Sends one batch and acts on what LaunchDarkly made of it, before handing the outcome on. private func publish(_ batch: EventBatch, _ payload: BatchPayload, @@ -870,6 +950,7 @@ extension DirectEventProcessor { sender.send(payload.body, payloadId: batch.payloadId, eventCount: payload.eventCount) { outcome in switch outcome { case .accepted(let serverTime): + self.retryState.recordSuccess() self.stateLock.withLock { if let serverTime = serverTime, serverTime > self.responseDate { self.responseDate = serverTime @@ -878,17 +959,13 @@ extension DirectEventProcessor { self.store.remove(batch) self.store.stoppedSending(batch) self.reportSyncComplete(nil) - case .refused(let response): - self.isOnline = false - self.store.remove(batch) - self.store.stoppedSending(batch) - self.reportSyncComplete(.response(response)) - case .retryable(let error): + case .failed: self.store.stoppedSending(batch) - self.reportSyncComplete(error) - // The batch stays on disk. A later delivery, in this run of the application or the next one, sends it - // under the same payload ID, so LaunchDarkly can tell it is not new. - os_log("%s Keeping %d event(s) on disk to retry later", log: self.service.config.logger, type: .debug, self.typeName(and: #function), batch.eventCount) + // Kept whatever the failure, a 401 included: a valid key is sometimes refused for minutes, and deleting + // on a refusal would lose every event recorded meanwhile. A later delivery, in this run of the + // application or the next one, sends it under the same payload ID, so LaunchDarkly can tell it is not + // new. + os_log("%s Keeping %d event(s) to retry later", log: self.service.config.logger, type: .debug, self.typeName(and: #function), batch.eventCount) } completion(outcome) } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/RetryState.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/RetryState.swift index 749198b44..a03e98118 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/RetryState.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/RetryState.swift @@ -7,6 +7,10 @@ private struct Constants { static let extendedMaxDelay: TimeInterval = 60 * 60 // The consecutive successful polls that clear the backoff. static let pollingResetThreshold = 2 + static let deliveryInitialDelay: TimeInterval = 1 + // Where the attempt count goes at least to on an unexpected failure, so its first wait is 2^9 initial delays. + static let deliveryAttemptsAfterUnexpected = 10 + static let maxServerDirectedWait: TimeInterval = 60 * 60 } /// The delay bounds for one retry regime. @@ -98,3 +102,100 @@ final class PollingRetryState { return max(pollInterval, jitteredBackoff(extended, attempts: attempts)) } } + +/// When event delivery may post again after a failure, following the SDK Retry Behavior specification as event +/// durability §9 binds it. Android's `DeliveryRetryState` plays the same part. +/// +/// A failure never stops delivery, it only makes the next post wait. An ordinary one, such as a 503 or no network, +/// waits from a second up to the flush interval, so it is never later than the schedule already would be. A refusal +/// that is unlikely to pass soon, such as a 401, waits about eight and a half minutes, doubling up to an hour, rather +/// than deleting the batch: a valid key is sometimes refused for minutes during an incident, and every event the fleet +/// recorded meanwhile would be lost. A `Retry-After` sets the wait instead. One accepted delivery puts everything back. +/// +/// One per event processor, so each environment backs off on its own. Held only in memory, so a new run of the +/// application posts at once. Thread safe: a delivery's responses arrive on whichever thread the session answers on. +final class DeliveryRetryState { + /// The wait after a first failure, which a flush waiting on the delivery that failed sits out. + let initialDelay: TimeInterval + private let normalCeiling: TimeInterval + private let extendedCeiling: TimeInterval + private let random: () -> Double + + private let lock = UnfairLock() + // Guarded by lock. + private var attempts = 0 + private var maxDelay: TimeInterval + private var nextAttempt: DispatchTime? + + /// - Parameter flushInterval: the normal ceiling; never below the initial delay, so a short interval does not make + /// a failing service be posted to more often than once a second. + convenience init(flushInterval: TimeInterval) { + self.init(initialDelay: Constants.deliveryInitialDelay, + normalCeiling: max(flushInterval, Constants.deliveryInitialDelay), + extendedCeiling: Constants.extendedMaxDelay) + } + + init(initialDelay: TimeInterval, + normalCeiling: TimeInterval, + extendedCeiling: TimeInterval, + random: @escaping () -> Double = { Double.random(in: 0..<1) }) { + self.initialDelay = initialDelay + self.normalCeiling = normalCeiling + self.extendedCeiling = extendedCeiling + self.random = random + self.maxDelay = normalCeiling + } + + /// An accepted delivery, which is all it takes for the next failure to start from a second again. + func recordSuccess() { + lock.withLock { + attempts = 0 + maxDelay = normalCeiling + nextAttempt = nil + } + } + + /// - Parameters: + /// - unexpected: whether the failure is unlikely to pass soon: a 4xx other than 400, 408 and 429, or a TLS or + /// certificate failure. + /// - retryAfter: the wait the response asked for, or nil where it gave none. + /// - Returns: how long nothing may be posted. + @discardableResult + func recordFailure(unexpected: Bool, retryAfter: TimeInterval?) -> TimeInterval { + lock.withLock { + if unexpected { + attempts = max(attempts + 1, Constants.deliveryAttemptsAfterUnexpected) + maxDelay = extendedCeiling + } else { + attempts += 1 + } + let wait: TimeInterval + if let retryAfter, retryAfter >= 0 { + // Every instance hears the same value at the same moment, so it is jittered to keep them from posting + // together when it runs out. + let directed = min(retryAfter, Constants.maxServerDirectedWait) + wait = directed - random() * (directed / 2) + } else { + // Not jittered: each instance's failures follow its own history, which already spreads them. + wait = min(maxDelay, initialDelay * pow(2, Double(min(attempts - 1, 40)))) + } + nextAttempt = .now() + wait + return wait + } + } + + /// How long until a post is allowed; zero once it is. + var remainingWait: TimeInterval { + lock.withLock { + guard let nextAttempt + else { return 0 } + let now = DispatchTime.now() + guard now < nextAttempt + else { + self.nextAttempt = nil + return 0 + } + return TimeInterval(nextAttempt.uptimeNanoseconds - now.uptimeNanoseconds) / 1_000_000_000 + } + } +} diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 03ba9d5d6..d6ecefdaf 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -23,6 +23,12 @@ class DirectEventProcessorSpec: QuickSpec { static let deliveryTimeout: DispatchTimeInterval = .seconds(10) } + /// Retries at once and never waits, so every flush posts: for tests about what is kept through failures rather than + /// when it is posted again. + static func withoutBackoff() -> DeliveryRetryState { + DeliveryRetryState(initialDelay: 0, normalCeiling: 0, extendedCeiling: 0) + } + struct TestContext { var eventProcessor: DirectEventProcessor! var config: LDConfig! @@ -46,7 +52,8 @@ class DirectEventProcessorSpec: QuickSpec { store: EventStore? = nil, eventCapacity: Int = Event.Kind.allKinds.count, eventPersistence: EventPersistence? = nil, - commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.tests.DirectEventProcessor.commitQueue")) { + commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.tests.DirectEventProcessor.commitQueue"), + retryState: DeliveryRetryState? = nil) { config = LDConfig.stub config.eventCapacity = eventCapacity @@ -70,7 +77,7 @@ class DirectEventProcessorSpec: QuickSpec { self.lastEventResponseDate = lastEventResponseDate.adjustedForHttpUrlHeaderUse self.store = store ?? EventStore.temporary(capacity: config.eventCapacity, persistEvents: persistence != .disabled) eventProcessor = DirectEventProcessor(service: serviceMock, onSyncComplete: onSyncComplete, - store: self.store, commitQueue: commitQueue) + store: self.store, commitQueue: commitQueue, retryState: retryState) (0..= 3 - expect(posts) <= Int(window / interval) + 1 + Int(window / 1.0) + 1 + expect(posts) >= 2 + expect(posts) <= Int(window / 1.0) + 1 // Offline stops it entirely, kept batch or not. Waits out a retry already scheduled before counting. testContext.eventProcessor.isOnline = false diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/RetryStateSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/RetryStateSpec.swift index c3cec2665..d3ed8ed3b 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/RetryStateSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/RetryStateSpec.swift @@ -101,4 +101,84 @@ final class RetryStateSpec: XCTestCase { retry.recordFailure(unexpected: true) assertDelay(retry, inClosedRange: 600...600) } + + func testDeliveryNothingWaitsBeforeTheFirstFailure() { + XCTAssertEqual(DeliveryRetryState(flushInterval: 30).remainingWait, 0) + } + + func testDeliveryOrdinaryFailuresDoubleTheWaitUpToTheFlushInterval() { + let retry = DeliveryRetryState(flushInterval: 30) + for expected in [1, 2, 4, 8, 16, 30, 30] as [TimeInterval] { + XCTAssertEqual(retry.recordFailure(unexpected: false, retryAfter: nil), expected) + } + } + + func testDeliveryNormalCeilingIsNeverBelowASecond() { + let retry = DeliveryRetryState(flushInterval: 0.2) + retry.recordFailure(unexpected: false, retryAfter: nil) + XCTAssertEqual(retry.recordFailure(unexpected: false, retryAfter: nil), 1) + } + + func testDeliveryRefusalWaitsAboutEightAndAHalfMinutesAndDoublesToAnHour() { + let retry = DeliveryRetryState(flushInterval: 30) + for expected in [512, 1024, 2048, 3600] as [TimeInterval] { + XCTAssertEqual(retry.recordFailure(unexpected: true, retryAfter: nil), expected) + } + } + + func testDeliveryOrdinaryFailureAfterARefusalKeepsTheLongerCeiling() { + let retry = DeliveryRetryState(flushInterval: 30) + retry.recordFailure(unexpected: true, retryAfter: nil) + XCTAssertEqual(retry.recordFailure(unexpected: false, retryAfter: nil), 1024) + } + + func testDeliveryAcceptedDeliveryPutsEverythingBack() { + let retry = DeliveryRetryState(flushInterval: 30) + retry.recordFailure(unexpected: true, retryAfter: nil) + retry.recordFailure(unexpected: false, retryAfter: nil) + retry.recordSuccess() + XCTAssertEqual(retry.remainingWait, 0) + XCTAssertEqual(retry.recordFailure(unexpected: false, retryAfter: nil), 1) + } + + func testDeliveryRetryAfterSetsTheWaitJitteredToNoLessThanHalfOfIt() { + let retry = DeliveryRetryState(flushInterval: 30) + for _ in 0..<100 { + let wait = retry.recordFailure(unexpected: false, retryAfter: 10) + XCTAssertTrue((5...10).contains(wait), "\(wait) is outside 5...10") + } + } + + func testDeliveryRetryAfterIsCappedAtAnHour() { + let retry = DeliveryRetryState(initialDelay: 1, normalCeiling: 30, extendedCeiling: 3600, random: { 0 }) + XCTAssertEqual(retry.recordFailure(unexpected: false, retryAfter: 5 * 3600), 3600) + } + + func testDeliveryFailureWithARetryAfterStillCountsTowardTheBackoff() { + let retry = DeliveryRetryState(flushInterval: 30) + retry.recordFailure(unexpected: false, retryAfter: nil) + retry.recordFailure(unexpected: false, retryAfter: 0) + XCTAssertEqual(retry.recordFailure(unexpected: false, retryAfter: nil), 4) + } + + func testDeliveryWaitRunsOut() { + let retry = DeliveryRetryState(initialDelay: 0.02, normalCeiling: 0.02, extendedCeiling: 0.02) + retry.recordFailure(unexpected: false, retryAfter: nil) + XCTAssertGreaterThan(retry.remainingWait, 0) + Thread.sleep(forTimeInterval: 0.04) + XCTAssertEqual(retry.remainingWait, 0) + } + + func testRetryAfterHeaderIsReadAsSecondsOrAnHttpDate() { + func response(_ retryAfter: String?) -> HTTPURLResponse { + HTTPURLResponse(url: URL(string: "https://example.com")!, statusCode: 503, httpVersion: "1.1", + headerFields: retryAfter.map { ["retry-after": $0] })! + } + XCTAssertNil(response(nil).headerRetryAfter) + XCTAssertEqual(response("120").headerRetryAfter, 120) + XCTAssertNil(response("soon").headerRetryAfter) + XCTAssertEqual(response(DateFormatter.httpUrlHeaderFormatter.string(from: Date(timeIntervalSinceNow: -60))).headerRetryAfter, 0) + let date = DateFormatter.httpUrlHeaderFormatter.string(from: Date(timeIntervalSinceNow: 600)) + XCTAssertEqual(response(date).headerRetryAfter ?? 0, 600, accuracy: 2) + } } From b55c839c221f84913b6f68331853f13267578d1d Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 22 Oct 2026 15:14:01 -0700 Subject: [PATCH 093/158] Drop a batch after two 400s in a row so it cannot block the rest A 400 is a payload the service cannot read. Kept for good, the batch is the oldest and so the next one posted, which held every batch behind it for the rest of the process. Now a second consecutive 400 for the same batch removes it and counts its events as lost, so the next flush a caller waits on hears about it; delivery of the batches behind it goes on once the backoff allows. Any other response for the batch starts the count again. The count is held only in memory. Co-authored-by: Cursor --- .../ServiceObjects/DirectEventProcessor.swift | 51 ++++++++++++++++--- .../DirectEventProcessorSpec.swift | 39 ++++++++++++-- 2 files changed, 81 insertions(+), 9 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 4ae36b30c..b57476835 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -104,17 +104,17 @@ class DirectEventProcessor: EventProcessor { stateLock.withLock { responseDate } } - /// Guards the last response date and the record of what has been lost. + /// Guards the last response date, the record of what has been lost, and the count of 400s. private let stateLock = UnfairLock() /// Only to be used while holding `stateLock`. private var responseDate: Date /// How many events the SDK accepted and then dropped for good. Only to be used while holding `stateLock`. /// - /// A batch LaunchDarkly refuses is kept and retried, so it is not a loss. What is lost is what the store never - /// received: an event or a summary that could not be serialized when it was committed, or that serialized too large - /// to store. A delivery that finds nothing to send cannot tell those apart from events that arrived, so without this - /// it would report success for them. + /// A batch LaunchDarkly refuses is kept and retried, so it is not a loss, unless the refusal is a second 400 in a + /// row. Otherwise what is lost is what the store never received: an event or a summary that could not be serialized + /// when it was committed, or that serialized too large to store. A delivery that finds nothing to send cannot tell + /// those apart from events that arrived, so without this it would report success for them. /// /// A count rather than a flag because deliveries overlap. Each takes the count as it closes its batch, and /// answers for the losses up to that mark, so a loss is told to the caller whose delivery it happened before @@ -125,6 +125,16 @@ class DirectEventProcessor: EventProcessor { /// used while holding `stateLock`. private var reportedLostEvents: UInt64 = 0 + /// How many 400s in a row drop a batch. A 400 is a payload the service cannot read, which posting it again will not + /// change, and a kept batch is the oldest and so the next posted: kept for good, it would hold every batch behind + /// it. Two rather than one, so a 400 from a proxy or an incident costs nothing. + static let badRequestsBeforeDropping = 2 + + /// The batch that last got a 400 and how many in a row: delivery goes oldest first and stops at a kept failure, so + /// the next post is that batch's unless capacity dropped it. Held only in memory, so a new run of the application + /// counts again. Only to be used while holding `stateLock`. + private var badRequests: (payloadId: String, inARow: Int)? + // MARK: Guarded by deliveryLock /// Guards `queuedDelivery` and `isRetryScheduled`. Taken on a caller's thread and on `deliveryQueue`, and never held @@ -786,7 +796,8 @@ extension DirectEventProcessor { enum PassOutcome: Int, Comparable { /// Every batch was accepted. case delivered - /// Every batch was attempted, and at least one was lost for good: unreadable, or damaged. + /// Every batch was attempted, and at least one was lost for good: unreadable, damaged, or refused as malformed + /// `badRequestsBeforeDropping` times in a row. case lost /// A batch is still there to be sent again: its request failed, the backoff did not allow it to be posted yet, /// or it could not be read this time. The batches after it were not attempted. @@ -882,6 +893,7 @@ extension DirectEventProcessor { publish(batch, payload) { outcome in switch outcome { case .accepted: + _ = self.countBadRequest(batch, statusCode: nil) guard leftOut > 0 else { self.deliver(remaining, lostAnything: lostAnything, retried: retried, completion) @@ -892,6 +904,15 @@ extension DirectEventProcessor { self.deliver(remaining, lostAnything: true, retried: retried, completion) case .failed(let failure): let wait = self.retryState.recordFailure(unexpected: failure.unexpected, retryAfter: failure.retryAfter) + if self.countBadRequest(batch, statusCode: failure.statusCode) >= DirectEventProcessor.badRequestsBeforeDropping { + // The rest are still attempted, once the backoff allows. + os_log("%s dropping %d event(s) the service refused as malformed %d times in a row", log: self.service.config.logger, type: .error, self.typeName(and: #function), batch.eventCount, DirectEventProcessor.badRequestsBeforeDropping) + self.store.remove(batch) + self.reportBatchLost(batch) + self.reportSyncComplete(failure.error) + self.deliver(remaining, lostAnything: true, retried: retried, completion) + return + } guard !retried, wait <= self.retryState.initialDelay else { // Told once the pass gives the batch up, not for a failure its retry may still answer. @@ -916,6 +937,24 @@ extension DirectEventProcessor { } } + /// - Returns: how many 400s in a row this batch has had, counting this response; zero for any other status, which + /// ends the run, and once the batch is dropped. + private func countBadRequest(_ batch: EventBatch, statusCode: Int?) -> Int { + stateLock.withLock { + guard statusCode == HTTPURLResponse.StatusCodes.badRequest + else { + badRequests = nil + return 0 + } + var inARow = 1 + if let previous = badRequests, previous.payloadId == batch.payloadId { + inARow = previous.inARow + 1 + } + badRequests = inARow >= DirectEventProcessor.badRequestsBeforeDropping ? nil : (batch.payloadId, inARow) + return inARow + } + } + /// Sets a delivery for when the backoff's wait ends, unless one is set already, so a batch kept through a failure /// goes out then rather than at the next report interval after it. /// diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index d6ecefdaf..f670ade80 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -2529,8 +2529,9 @@ extension DirectEventProcessorSpec { expectKeptThroughRefusals(status: 503, persistence: .disabled, refusedFlushes: 3) } - // Each has to keep the batch, not only 503: a refusal such as a 401 only waits longer. - for status in [400, 401, 403, 404, 408, 413, 429, 500, 502, 504] { + // Each has to keep the batch, not only 503: a refusal such as a 401 only waits longer. Not a 400, which drops + // the batch when it comes twice in a row; see below. + for status in [401, 403, 404, 408, 413, 429, 500, 502, 504] { it("keeps a batch refused with HTTP \(status) past the retry") { expectKeptThroughRefusals(status: status, persistence: .immediate, refusedFlushes: 2) } @@ -2539,7 +2540,7 @@ extension DirectEventProcessorSpec { // Kept is not the same as resent on its own: a batch the service always refuses is posted a second after the // first failure and then twice as long after each one, and a flush in between posts nothing, or it would be // posted in a loop. - for status in [400, 503] { + for status in [503, 429] { it("posts a batch refused with HTTP \(status) only as the backoff allows") { testContext = TestContext() testContext.serviceMock.respondToEvents(with: status) @@ -2564,6 +2565,38 @@ extension DirectEventProcessorSpec { } } + it("drops a batch refused with HTTP 400 twice in a row, and no longer holds the ones behind it") { + testContext = TestContext(retryState: DirectEventProcessorSpec.withoutBackoff()) + testContext.serviceMock.respondToEvents(with: HTTPURLResponse.StatusCodes.badRequest) + testContext.eventProcessor.record(CustomEvent(key: "malformed", context: testContext.context)) + testContext.eventProcessor.isOnline = true + expect(testContext.flushOutcome()) == false + expect(testContext.serviceMock.publishedEventPayloads.map(customKeys)) == [["malformed"], ["malformed"]] + expect(Set(testContext.serviceMock.publishedPayloadIds).count) == 1 + expect(testContext.store.pendingBatches()).to(beEmpty()) + waitLettingTimersFire(0.3) + expect(testContext.serviceMock.publishEventDataCallCount) == 2 + testContext.serviceMock.stubEventResponse(success: true) + + // What was recorded since goes out alone, and that flush is the first a caller waits on since the loss, + // so it hears about it. + testContext.eventProcessor.record(CustomEvent(key: "next", context: testContext.context)) + expect(testContext.flushOutcome()) == false + expect(testContext.serviceMock.publishedEventPayloads.dropFirst(2).map(customKeys)) == [["next"]] + expect(testContext.flushOutcome()) == true + } + + it("drops a batch only for two 400s in a row") { + testContext = TestContext(retryState: DirectEventProcessorSpec.withoutBackoff()) + testContext.serviceMock.eventStatusForPost = { post in [1: 400, 2: 503, 3: 400][post] ?? 202 } + testContext.eventProcessor.record(CustomEvent(key: "kept", context: testContext.context)) + testContext.eventProcessor.isOnline = true + expect(testContext.flushOutcome()) == false + expect(testContext.flushOutcome()) == true + expect(testContext.serviceMock.publishedEventPayloads.map(customKeys)) == Array(repeating: ["kept"], count: 4) + expect(Set(testContext.serviceMock.publishedPayloadIds).count) == 1 + } + it("keeps a refused batch and posts it again once the longer wait is over") { // Cut to a second and a half here from the eight and a half minutes the SDK waits. testContext = TestContext(retryState: DeliveryRetryState(initialDelay: 1, normalCeiling: 1, extendedCeiling: 1.5)) From 0124fc850d645330ec58b5daf02e50614dcb1a56 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 22 Oct 2026 16:42:15 -0700 Subject: [PATCH 094/158] Deliver one batch at a time MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A delivery returned from the delivery queue as soon as its first request was out, so a flush arriving before the response started a second delivery beside it. Each batch whose request is out stops counting against both capacities, so a run of flushes, each held up by a slow response, took the store a batch past capacity per flush, against the spec's single delivery (Event Durability §10.1, C9a). A delivery now runs until it has an answer for everything it sent, its retry included. One asked for meanwhile is queued, joined by any after it, and starts once the running one ends: it sends what that one left and what was recorded since, so its caller hears true only if both got out. No thread waits for it. Co-authored-by: Cursor --- .../ServiceObjects/DeliveryClaims.swift | 10 +-- .../ServiceObjects/DirectEventProcessor.swift | 61 ++++++++++------ .../DirectEventProcessorSpec.swift | 73 ++++++++++++++----- 3 files changed, 98 insertions(+), 46 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DeliveryClaims.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DeliveryClaims.swift index c517e33f4..c5b7aef56 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DeliveryClaims.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DeliveryClaims.swift @@ -1,11 +1,11 @@ import Foundation -/// What one delivery's answer is made of, where deliveries overlap. +/// What one delivery's answer is made of. /// -/// A commit only stages events; whichever delivery runs next is what closes them into a batch. So the batch holding a -/// caller's events may belong to a delivery already under way, and answering that caller means hearing from it. These -/// are the two small pieces that arrange it: a `Claim` is a batch somebody is waiting on, and a `PassAnswer` gathers -/// the claims one pass of a delivery depends on. +/// A commit only stages events; whichever delivery runs next is what closes them into a batch. Deliveries run one at a +/// time, but should a batch holding a caller's events belong to a delivery still under way, answering that caller means +/// hearing from it. These are the two small pieces that arrange it: a `Claim` is a batch somebody is waiting on, and a +/// `PassAnswer` gathers the outcomes one pass of a delivery depends on. extension DirectEventProcessor { /// A batch one delivery has taken, which another delivery's answer may depend on. final class Claim { diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index b57476835..0f60eafdd 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -116,9 +116,9 @@ class DirectEventProcessor: EventProcessor { /// when it was committed, or that serialized too large to store. A delivery that finds nothing to send cannot tell /// those apart from events that arrived, so without this it would report success for them. /// - /// A count rather than a flag because deliveries overlap. Each takes the count as it closes its batch, and - /// answers for the losses up to that mark, so a loss is told to the caller whose delivery it happened before - /// rather than to whichever response arrived first. + /// A count rather than a flag. Each delivery takes the count as it closes its batch, and answers for the losses up + /// to that mark, so a loss is told to the caller whose delivery it happened before rather than to whichever + /// delivery answers next. private var lostEvents: UInt64 = 0 /// The mark up to which losses have already been reported to a caller, so that none is told twice. Only to be @@ -137,11 +137,17 @@ class DirectEventProcessor: EventProcessor { // MARK: Guarded by deliveryLock - /// Guards `queuedDelivery` and `isRetryScheduled`. Taken on a caller's thread and on `deliveryQueue`, and never held - /// across either. + /// Guards `queuedDelivery`, `isDelivering` and `isRetryScheduled`. Taken on a caller's thread and on `deliveryQueue`, + /// and never held across either. private let deliveryLock = UnfairLock() /// Callers waiting on a delivery that is queued and has not taken the batches yet, or nil where none is queued. private var queuedDelivery: [FlushOutcomeClosure]? + /// Whether a delivery has started and not yet had an answer for everything it sent, its retry included. + /// + /// A delivery returns from `deliveryQueue` as soon as its first request is out, so the queue alone does not keep two + /// apart. Two at once would each leave the batch it is sending out of both capacities, and a run of flushes, each + /// held up by a slow response, could take the store that many batches past them. + private var isDelivering = false /// Whether a delivery is already set to run when the backoff's wait ends, so a run of passes inside the wait sets /// one rather than one each. private var isRetryScheduled = false @@ -162,14 +168,9 @@ class DirectEventProcessor: EventProcessor { /// The batches a delivery has taken and not yet had an answer for, by payload ID. Only to be used while holding /// `claimLock`. /// - /// `deliveryQueue` serializes the *start* of a delivery but not the round trip it waits on, and a batch stays on - /// disk until its response arrives. Without this, a delivery beginning inside that window would list the same batch - /// again and send it a second time -- under the same payload ID, leaving it to LaunchDarkly to notice that two - /// requests arriving at once are the same one. - /// - /// This bounds the overlap rather than preventing it, which is the point: a later delivery still runs, and still - /// sends what it closed for itself. Only the batches already on their way are left out of it, and a caller waiting - /// on that later delivery hears their outcome too. + /// A batch stays on disk until its response arrives. `isDelivering` keeps a second delivery from listing it inside + /// that window; this is what keeps it from being sent twice, under the same payload ID, should two ever overlap. + /// A delivery that finds a batch claimed waits on that claim's outcome rather than sending it. private var batchesInFlight: [String: Claim] = [:] // MARK: Confined to timerQueue @@ -586,8 +587,9 @@ extension DirectEventProcessor { /// them. /// /// A delivery that has already started is not joined. It has closed and taken its batches, so it cannot speak for - /// anything recorded since, and holding later deliveries back until it answers would leave the store growing - /// through a retry while nothing drained it. + /// anything recorded since. Nor does another start beside it: the one queued waits until the running one has an + /// answer, then sends what that one left behind as well as what was recorded since, so its caller hears `true` + /// only if both got out. private func queueDelivery(_ completion: FlushOutcomeClosure?) { deliveryLock.lock() if queuedDelivery != nil { @@ -598,9 +600,13 @@ extension DirectEventProcessor { return } queuedDelivery = completion.map { [$0] } ?? [] + let starts = !isDelivering + isDelivering = true deliveryLock.unlock() - deliveryQueue.async { self.runDelivery() } + if starts { + deliveryQueue.async { self.runDelivery() } + } } private func runDelivery() { @@ -615,11 +621,14 @@ extension DirectEventProcessor { deliveryLock.unlock() reportEvents { delivered, lossMark in + // Taken once for the delivery rather than once per waiter, so that every caller joined to it hears about + // a loss instead of only whichever happened to be answered first. Taken before the next delivery starts, + // which could otherwise answer first and take this one's loss with its own. + let kept = waiters.isEmpty || !self.takeEventsLost(upTo: lossMark) + // Before the waiters are answered, so a completion that flushes again finds this delivery over. + self.startQueuedDeliveryOrStop() guard !waiters.isEmpty else { return } - // Taken once for the delivery rather than once per waiter, so that every caller joined to it hears about - // a loss instead of only whichever happened to be answered first. - let kept = !self.takeEventsLost(upTo: lossMark) // Answered away from `deliveryQueue`, and from whichever thread the delivery ended on, so that every // completion reaches its caller the same way. A completion is free to flush again: it is never running // on the queue the delivery it asks for would be queued on. @@ -631,6 +640,17 @@ extension DirectEventProcessor { } } + /// Ends the delivery that just had its answer, starting the one queued behind it if there is one. + private func startQueuedDeliveryOrStop() { + let next = deliveryLock.withLock { () -> Bool in + isDelivering = queuedDelivery != nil + return isDelivering + } + if next { + deliveryQueue.async { self.runDelivery() } + } + } + private func reportEvents(completion: ((_ delivered: Bool, _ lossMark: UInt64) -> Void)?) { guard isOnline else { @@ -654,8 +674,7 @@ extension DirectEventProcessor { let answer = PassAnswer(outstanding: waiting.waitingOn.count + 1) { outcome in completion?(outcome == .delivered && settled.accountsForEverything, settled.lossMark) } - // Waited on for the answer, but not before closing: holding back until another delivery answers, which through - // a retry can take twice the connection timeout, would leave the store filling with nothing draining it. + // Waited on for the answer, but not before closing. waitOn(waiting.waitingOn, answer) deliverOwn(waiting.deliverable) { waitingOutcome in diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index f670ade80..1d6e4fd10 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -1613,8 +1613,8 @@ extension DirectEventProcessorSpec { testContext.serviceMock.holdsEventCompletions = false testContext.serviceMock.releaseHeldEventCompletions() - // The second caller finds the only batch claimed by the first delivery, so it waits on that claim - // rather than sending anything. The batch goes out once in total. + // The second caller's delivery starts once the first is answered, and finds nothing left to send. The + // batch goes out once in total. expect(first.value()).toEventually(beTrue(), timeout: Constants.deliveryTimeout) expect(second.value()).toEventually(beTrue(), timeout: Constants.deliveryTimeout) expect(testContext.serviceMock.publishEventDataCallCount) == 1 @@ -2336,7 +2336,7 @@ extension DirectEventProcessorSpec { expect(testContext.serviceMock.publishEventDataCallCount) == 1 } - it("keeps draining while a delivery is in flight, and sends each batch once") { + it("starts no second request while a delivery is in flight, and sends what was recorded meanwhile after it") { testContext = TestContext() testContext.recordEvents(1) testContext.eventProcessor.isOnline = true @@ -2345,18 +2345,46 @@ extension DirectEventProcessorSpec { testContext.eventProcessor.flushReportingOutcome { _ in } expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) - // Holding this back until the first delivery answers would let the store fill through a retry, which - // can run for twice the connection timeout, and events recorded past the capacity are lost. testContext.recordEvents(1) - testContext.eventProcessor.flushReportingOutcome { _ in } - expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(2), timeout: Constants.deliveryTimeout) + let second = Answer() + testContext.eventProcessor.flushReportingOutcome(completion: second.record) + Thread.sleep(forTimeInterval: 0.3) + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + expect(second.value()).to(beNil()) - // The first batch is still awaiting its response, so the second delivery left it alone rather than - // listing it again and sending it a second time under the same payload ID. + testContext.serviceMock.holdsEventCompletions = false + testContext.serviceMock.releaseHeldEventCompletions() + expect(second.value()).toEventually(beTrue(), timeout: Constants.deliveryTimeout) + expect(testContext.serviceMock.publishEventDataCallCount) == 2 expect(Set(testContext.serviceMock.publishedPayloadIds).count) == 2 + } + + // Every batch whose request is out stops counting against capacity. With a request per flush, a run of + // flushes each held up by a slow response would take the store a batch past capacity per flush. + it("holds the store to its capacity through flushes that each find a delivery in flight") { + testContext = TestContext(eventCapacity: 2) + testContext.eventProcessor.isOnline = true + testContext.serviceMock.holdsEventCompletions = true + testContext.eventProcessor.record(CustomEvent(key: "a-0", context: testContext.context)) + testContext.eventProcessor.record(CustomEvent(key: "a-1", context: testContext.context)) + testContext.eventProcessor.flush(completion: nil) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + + for round in 0..<3 { + testContext.eventProcessor.record(CustomEvent(key: "b\(round)-0", context: testContext.context)) + testContext.eventProcessor.record(CustomEvent(key: "b\(round)-1", context: testContext.context)) + testContext.eventProcessor.flush(completion: nil) + } + Thread.sleep(forTimeInterval: 0.3) + expect(testContext.serviceMock.publishEventDataCallCount) == 1 testContext.serviceMock.holdsEventCompletions = false testContext.serviceMock.releaseHeldEventCompletions() + waitUntil(timeout: Constants.deliveryTimeout) { done in + testContext.eventProcessor.flushReportingOutcome { _ in done() } + } + // The batch out made room for one store's worth, and nothing past it was taken. + expect(testContext.serviceMock.publishedEventPayloads.flatMap(customKeys)) == ["a-0", "a-1", "b0-0", "b0-1"] } it("inherits the outcome of a delivery holding the batch it would have sent") { @@ -2394,22 +2422,21 @@ extension DirectEventProcessorSpec { testContext.eventProcessor.flushReportingOutcome(completion: first.record) expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) - // B closes a batch of its own, dropping an event the store will never take as it does, so the loss - // falls between the two closes. B has a post of its own outstanding, which is what leaves A to be - // answered first. + // B is asked for while A's post is out, with an event the store will never take among what it will + // close. A's batch closed before that event was dropped, so the loss is B's. testContext.eventProcessor.record(CustomEvent(key: "poison", context: LDContext.stub(), metricValue: .nan)) testContext.recordEvents(1) let second = Answer() testContext.eventProcessor.flushReportingOutcome(completion: second.record) - expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(2), timeout: Constants.deliveryTimeout) - // Responses land in the order they were made, so A is answered while B's post is still out. With - // one shared flag A would take B's loss here, and B would report success for the event it dropped. + // With one shared flag, whichever delivery answered first would take the loss, and B would report + // success for the event it dropped. testContext.serviceMock.holdsEventCompletions = false testContext.serviceMock.releaseHeldEventCompletions() expect(first.value()).toEventually(beTrue(), timeout: Constants.deliveryTimeout) expect(second.value()).toEventually(beFalse(), timeout: Constants.deliveryTimeout) + expect(testContext.serviceMock.publishEventDataCallCount) == 2 } it("answers away from the queue a delivery runs on, after a delivery") { @@ -2736,7 +2763,7 @@ extension DirectEventProcessorSpec { expect(testContext.flushOutcome()) == false } - it("starts another delivery while a refused post waits to retry") { + it("starts a delivery asked for while a refused post waits to retry once that retry is answered") { testContext = TestContext() testContext.serviceMock.respondToEvents(with: 503) testContext.eventProcessor.record(CustomEvent(key: "kept", context: testContext.context)) @@ -2744,12 +2771,18 @@ extension DirectEventProcessorSpec { testContext.eventProcessor.flush(completion: nil) expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) - // Well inside the one-second retry delay: a delivery queue sleeping through it would not get to this. - testContext.eventProcessor.flush(completion: nil) - expect(testContext.eventProcessor.startedDeliveryCount).toEventually(equal(2), timeout: .milliseconds(500)) + // Well inside the one-second retry delay. The retry belongs to the delivery still running, so this + // one waits for it rather than posting beside it. + let answer = Answer() + testContext.eventProcessor.flushReportingOutcome(completion: answer.record) + Thread.sleep(forTimeInterval: 0.3) + expect(testContext.eventProcessor.startedDeliveryCount) == 1 - // One retry and no more; the batch then waits for the next flush. + // One retry and no more; the delivery behind it finds the backoff still waiting, and the batch then + // waits for the delivery that wait sets. expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(2), timeout: .seconds(3)) + expect(testContext.eventProcessor.startedDeliveryCount).toEventually(equal(2), timeout: Constants.deliveryTimeout) + expect(answer.value()).toEventually(beFalse(), timeout: Constants.deliveryTimeout) waitLettingTimersFire(1.5) expect(testContext.serviceMock.publishEventDataCallCount) == 2 } From 307eaa71a34c4aaa653e39c7b64dbea73f863e40 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 22 Oct 2026 16:48:08 -0700 Subject: [PATCH 095/158] Pin that a delivery's own loss is told to its caller A delivery waiting on a batch another delivery had claimed was answered from inside that delivery's release of the claim, before the delivery itself answered. Holding the same loss mark, it took the loss first, and the caller whose commit dropped the event heard true. Deliveries now run one at a time, and each takes its losses before the next starts, so the later one never answers first. This pins it: the first delivery drops an event as it closes its batch, a second with nothing of its own is asked for while the first is out, and the loss is told to the first. It fails where deliveries overlap. Co-authored-by: Cursor --- .../DirectEventProcessorSpec.swift | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 1d6e4fd10..69552d797 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -2439,6 +2439,29 @@ extension DirectEventProcessorSpec { expect(testContext.serviceMock.publishEventDataCallCount) == 2 } + it("tells the caller whose delivery lost an event, not a later one with nothing of its own to send") { + testContext = TestContext() + testContext.eventProcessor.record(CustomEvent(key: "poison", context: LDContext.stub(), metricValue: .nan)) + testContext.recordEvents(1) + testContext.eventProcessor.isOnline = true + testContext.serviceMock.holdsEventCompletions = true + + // A drops the event the store will never take as it closes its batch, so the loss is A's. + let first = Answer() + testContext.eventProcessor.flushReportingOutcome(completion: first.record) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + + // B has nothing of its own. Answered as soon as A's batch is, it would take A's loss and leave A + // reporting success for the event A dropped. + let second = Answer() + testContext.eventProcessor.flushReportingOutcome(completion: second.record) + testContext.serviceMock.holdsEventCompletions = false + testContext.serviceMock.releaseHeldEventCompletions() + + expect(first.value()).toEventually(beFalse(), timeout: Constants.deliveryTimeout) + expect(second.value()).toEventually(beTrue(), timeout: Constants.deliveryTimeout) + } + it("answers away from the queue a delivery runs on, after a delivery") { testContext = TestContext() testContext.recordEvents(1) From d5ca597bb91d0bf6de504a142842b74c8f5c3f83 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 22 Oct 2026 16:52:28 -0700 Subject: [PATCH 096/158] Remove a log whose first write failed and could not be cut back Memory delivers every frame of a failed write. Where the log held no events before it, the log is never closed into a batch, so a later run would recover the frames that landed and send them a second time. Co-authored-by: Cursor --- .../ServiceObjects/DiskEventLog.swift | 15 ++++++++ .../ServiceObjects/EventStoreSpec.swift | 38 +++++++++++++++++++ 2 files changed, 53 insertions(+) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift index a833c053c..ad7c21165 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift @@ -118,6 +118,11 @@ final class DiskEventLog { // batch this log is closed into still delivers only the events it was closed holding. if let start, !file.truncate(to: start) { os_log("%s could not cut back a failed write: errno %d", log: logger, type: .error, typeName(and: #function), errno) + // A log with no events before this write is never closed into a batch, and a later run recovering it + // would send what landed under no count that leaves it out. Memory holds all of it. + if start <= EventLogFormat.fileHeader.count { + discardFailedFirstWrite() + } } return false } @@ -255,6 +260,16 @@ final class DiskEventLog { return opened } + /// Closes and removes a log whose only events are those of a first write that failed. Requires `ioLock`. + private func discardFailedFirstWrite() { + do { + try FileManager.default.removeItem(at: url) + } catch { + os_log("%s could not remove a log holding only a failed write: %s", log: logger, type: .error, typeName(and: #function), String(describing: error)) + } + closeDescriptor() + } + /// Closes and removes a log that never got the name recovery looks for. Requires `ioLock`. private func abandon(_ opened: EventLogFile, at opening: URL) -> EventLogFile? { opened.close() diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 5667e7402..a93e7e702 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -332,6 +332,19 @@ final class EventStoreSpec: QuickSpec { } } + /// Puts a read-only open of the same file behind the log's descriptor, so that cutting it back fails while the file + /// itself can still be renamed or removed. + private static func refuseTruncating(_ file: EventLogFile) { + var path = [CChar](repeating: 0, count: Int(MAXPATHLEN)) + guard fcntl(file.descriptor, F_GETPATH, &path) == 0 + else { return fail("could not find the log's path") } + let readOnly = open(path, O_RDONLY) + guard readOnly >= 0 + else { return fail("could not reopen the log") } + _ = dup2(readOnly, file.descriptor) + _ = close(readOnly) + } + private static func payload(_ key: String) -> Data { Data("{\"kind\":\"custom\",\"key\":\"\(key)\"}".utf8) } @@ -1886,6 +1899,31 @@ final class EventStoreSpec: QuickSpec { expect(next.pendingBatches()).to(beEmpty()) } + it("leaves the next run nothing to recover from a first write that failed partway and could not be cut back") { + var writes = 0 + let interrupted = EventStore.temporary(writeLog: { file, bytes in + writes += 1 + guard writes > 1 + else { return file.writeAll(bytes) } + _ = file.writeAll(bytes.dropLast(3)) + EventStoreSpec.refuseTruncating(file) + return ENOSPC + }) + defer { interrupted.deleteEverything() } + + _ = interrupted.stage(EventStoreSpec.payload("landed")) + _ = interrupted.stage(EventStoreSpec.payload("torn")) + interrupted.commit() + let fromMemory = interrupted.closeBatch() + expect(EventStoreSpec.keys(ofBody: fromMemory.flatMap { try? interrupted.body(of: $0) })) == ["landed", "torn"] + + // The whole frame that landed is in the log as well as in memory, under no count that would leave it + // out, so a later run recovering the log would send it a second time under another payload ID. + let next = interrupted.nextRun() + next.preparePayloadsFromInterruptedLogs() + expect(next.pendingBatches()).to(beEmpty()) + } + it("holds a batch until it is removed, so a failed delivery can be retried") { _ = store.stage(EventStoreSpec.payload("retry-me")) let batch = store.closeBatch()! From 6506c92070d4ed1a16dd19920d4d741818a3288f Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 22 Oct 2026 17:51:59 -0700 Subject: [PATCH 097/158] Count a batch deleted unsent as lost where no other process shares the directory A missing batch was always taken to have been delivered by another process. Only a Mac runs a second instance over the same directory; elsewhere, a counted batch that is gone and that no store in this process removed was deleted unsent, as when tvOS reclaims its caches. Its events now count as lost, whether the listing or the read finds it gone, so the flush a caller waits on answers false. Co-authored-by: Cursor --- .../ServiceObjects/DirectEventProcessor.swift | 64 +++++++++++----- .../ServiceObjects/EventDirectory.swift | 64 ++++++++++++++-- .../ServiceObjects/EventStore.swift | 28 +++++-- .../DirectEventProcessorSpec.swift | 31 ++++++++ .../ServiceObjects/EventStoreSpec.swift | 75 +++++++++++++++++++ 5 files changed, 230 insertions(+), 32 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 0f60eafdd..b28742dce 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -217,7 +217,9 @@ class DirectEventProcessor: EventProcessor { deliveryQueue.async { [weak self] in store.preparePayloadsFromInterruptedLogs() self?.reportDiscarded(store.discardExpired()) - self?.hasEventsFromPreviousRun = !store.pendingBatches().isEmpty + let listing = store.listPendingBatches() + self?.reportVanished(listing.vanishedEvents) + self?.hasEventsFromPreviousRun = !listing.batches.isEmpty } } @@ -677,7 +679,7 @@ extension DirectEventProcessor { // Waited on for the answer, but not before closing. waitOn(waiting.waitingOn, answer) - deliverOwn(waiting.deliverable) { waitingOutcome in + deliverOwn(waiting.deliverable, lostAnything: waiting.lostAnything) { waitingOutcome in guard waitingOutcome != .blocked else { // Committed all the same, since a delivery is a commit point. Nothing was closed, so the caller's events @@ -706,19 +708,21 @@ extension DirectEventProcessor { answer.report(max(waitingOutcome, newOutcome)) } self.waitOn(new.waitingOn, newAnswer) - self.deliverOwn(new.deliverable, newAnswer.report) + self.deliverOwn(new.deliverable, lostAnything: new.lostAnything, newAnswer.report) } } /// Delivers the batches this delivery claimed, releasing the claims once it has an outcome. - private func deliverOwn(_ batches: [EventBatch], _ completion: @escaping (PassOutcome) -> Void) { + /// + /// - Parameter lostAnything: whether the listing that found them found other batches deleted unsent. + private func deliverOwn(_ batches: [EventBatch], lostAnything: Bool, _ completion: @escaping (PassOutcome) -> Void) { guard !batches.isEmpty else { - completion(.delivered) + completion(lostAnything ? .lost : .delivered) return } os_log("%s starting", log: service.config.logger, type: .debug, typeName(and: #function)) - deliver(batches) { outcome in + deliver(batches, lostAnything: lostAnything, retried: false) { outcome in self.releaseClaim(on: batches, delivered: outcome == .delivered) completion(outcome) } @@ -741,15 +745,23 @@ extension DirectEventProcessor { } } + /// What `claimDeliverableBatches(discardingExpired:)` took. + private struct ClaimedBatches { + let deliverable: [EventBatch] + /// The claims on the batches left out. One may hold this caller's events -- `commitRecordedEvents` only stages + /// them, and whichever delivery runs next is what closes them into a batch -- so the answer has to wait for + /// those claims rather than assume they are someone else's business. + let waitingOn: [Claim] + let isComplete: Bool + /// Whether the listing found batches deleted unsent. + let lostAnything: Bool + } + /// Takes the batches this delivery will send, leaving out any another delivery is still waiting on a response for. /// /// - Parameter discardingExpired: whether to discard what is too old to report first, which a second pass moments /// after the first has no need to do again. - /// - Returns: the batches to send, the claims on the ones left out, and whether the listing was complete. A batch - /// left out may hold this caller's events -- `commitRecordedEvents` only stages them, and whichever delivery runs - /// next is what closes them into a batch -- so the answer has to wait for those claims rather than assume they - /// are someone else's business. - private func claimDeliverableBatches(discardingExpired: Bool) -> (deliverable: [EventBatch], waitingOn: [Claim], isComplete: Bool) { + private func claimDeliverableBatches(discardingExpired: Bool) -> ClaimedBatches { // Listed before the lock is taken, because listing reads the disk. Taking what appeared after it is the next // delivery's business; all this has to be atomic is deciding which of these batches are this delivery's. // Expired first, so that a batch too old to report is not sent. @@ -757,6 +769,7 @@ extension DirectEventProcessor { reportDiscarded(store.discardExpired()) } let listing = store.listPendingBatches() + reportVanished(listing.vanishedEvents) return claimLock.withLock { var deliverable: [EventBatch] = [] var waitingOn: [Claim] = [] @@ -768,10 +781,19 @@ extension DirectEventProcessor { deliverable.append(batch) } } - return (deliverable, waitingOn, listing.isComplete) + return ClaimedBatches(deliverable: deliverable, waitingOn: waitingOn, isComplete: listing.isComplete, + lostAnything: listing.vanishedEvents > 0) } } + /// Counts as lost the events of batches a listing found deleted unsent. + private func reportVanished(_ events: Int) { + guard events > 0 + else { return } + os_log("%s %d stored event(s) were deleted before they could be sent", log: service.config.logger, type: .error, typeName(and: #function), events) + reportStoredEventsLost(events) + } + /// Releases the claims once the delivery has an answer, whether or not every batch was sent: `deliver` stops at the /// first retryable failure, and what it did not reach has to be deliverable again. /// @@ -815,8 +837,8 @@ extension DirectEventProcessor { enum PassOutcome: Int, Comparable { /// Every batch was accepted. case delivered - /// Every batch was attempted, and at least one was lost for good: unreadable, damaged, or refused as malformed - /// `badRequestsBeforeDropping` times in a row. + /// Every batch was attempted, and at least one was lost for good: unreadable, damaged, deleted unsent, or refused + /// as malformed `badRequestsBeforeDropping` times in a row. case lost /// A batch is still there to be sent again: its request failed, the backoff did not allow it to be posted yet, /// or it could not be read this time. The batches after it were not attempted. @@ -827,10 +849,6 @@ extension DirectEventProcessor { } } - private func deliver(_ batches: [EventBatch], _ completion: @escaping (PassOutcome) -> Void) { - deliver(batches, lostAnything: false, retried: false, completion) - } - /// Delivers batches oldest first, stopping at the first one that failed. /// /// Stopping matters: the batches that are left keep their place in the log, and a later delivery attempts them @@ -877,6 +895,12 @@ extension DirectEventProcessor { reportBatchLost(batch) deliver(remaining, lostAnything: true, retried: retried, completion) return + } catch is VanishedBatch { + os_log("%s %d stored event(s) were deleted before they could be sent", log: service.config.logger, type: .error, typeName(and: #function), batch.eventCount) + store.remove(batch) + reportBatchLost(batch) + deliver(remaining, lostAnything: true, retried: retried, completion) + return } catch { // Kept: the batch is still there, and a read that failed may not fail next time. os_log("%s could not read stored events: %s", log: service.config.logger, type: .debug, typeName(and: #function), String(describing: error)) @@ -886,8 +910,8 @@ extension DirectEventProcessor { guard let payload = read else { - // Gone, which is usually another process having delivered it, or holding no events. Either way nothing is - // lost and there is nothing to keep. + // Delivered by another process or store, or holding no events. Either way nothing is lost and there is + // nothing to keep. store.remove(batch) deliver(remaining, lostAnything: lostAnything, retried: retried, completion) return diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index fbd9a3db8..1be0957cb 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -118,6 +118,36 @@ final class EventDirectory { } #endif + /// Whether other processes of the application may use this directory, and so deliver its batches. Only a Mac runs + /// a second instance of an application over the same directory; elsewhere a batch that is gone, and that no store + /// in this process removed, was deleted unsent. Only to be used while holding `ioLock`. + #if os(macOS) || targetEnvironment(macCatalyst) + var sharesDirectoryWithOtherProcesses = true + #else + var sharesDirectoryWithOtherProcesses = false + #endif + + /// The payload IDs of the batches stores in this process last removed, delivered or expired, oldest first, so that + /// another store on the same directory -- one opened after `LDClient` was closed, say -- does not take a batch it + /// listed and then found gone for one deleted unsent. Only the most recent are kept: what matters is a removal + /// between another store's listing and its read, which is never many batches ago. Guarded by `removedLock`. + private static var removedInThisProcess: [String] = [] + private static let removedInThisProcessLimit = 1024 + private static let removedLock = UnfairLock() + + private static func rememberRemoved(_ payloadId: String) { + removedLock.withLock { + removedInThisProcess.append(payloadId) + if removedInThisProcess.count > removedInThisProcessLimit { + removedInThisProcess.removeFirst(removedInThisProcess.count - removedInThisProcessLimit) + } + } + } + + private static func wasRemovedInThisProcess(_ payloadId: String) -> Bool { + removedLock.withLock { removedInThisProcess.contains(payloadId) } + } + /// - Parameter locatingDirectory: answers where the events go. Asked once, and only on first use. init(locatingDirectory: @escaping () -> URL, ioLock: UnfairLock, @@ -297,8 +327,9 @@ final class EventDirectory { } catch { guard FileManager.default.fileExists(atPath: disk.directory.path) else { + let vanished = vanishedEvents(listed: []) stored = [:] - return BatchListing(batches: [], isComplete: true) + return BatchListing(batches: [], isComplete: true, vanishedEvents: vanished) } os_log("%s could not list stored events, will try again at the next delivery: %s", log: logger, type: .info, typeName(and: #function), String(describing: error)) let known = stored @@ -332,9 +363,21 @@ final class EventDirectory { // A batch that left the directory without going through `remove` -- a purged cache directory on tvOS, say -- // would otherwise keep its count for the rest of the session. let listed = Set(batches.map { $0.payloadId }) + let vanished = vanishedEvents(listed: listed) stored = stored.filter { listed.contains($0.key) } - return BatchListing(batches: batches, isComplete: true) + return BatchListing(batches: batches, isComplete: true, vanishedEvents: vanished) + } + + /// The events in the batches counted here that are not in `listed` and that nothing could have delivered: no other + /// process uses the directory, and no store in this one removed them. Requires `ioLock`. + private func vanishedEvents(listed: Set) -> Int { + ioLock.assertOwned() + guard !sharesDirectoryWithOtherProcesses + else { return 0 } + return stored + .filter { !listed.contains($0.key) && !EventDirectory.wasRemovedInThisProcess($0.key) } + .reduce(0) { $0 + $1.value.events } } private static let listedKeys: [URLResourceKey] = [.contentModificationDateKey, .fileSizeKey] @@ -376,20 +419,25 @@ final class EventDirectory { return true } - /// The `[...]` payload for a batch, assembled in the bytes the file was read into; or nil if it is gone or holds no - /// events. + /// The `[...]` payload for a batch, assembled in the bytes the file was read into; or nil if it holds no events, or + /// is gone where another process or store may have delivered it. /// /// Requires `ioLock`. /// - /// - throws: `UnreadableBatch` if the batch is there but holds nothing this version can send; anything else if it - /// could not be read this time, which a later attempt may manage. + /// - throws: `UnreadableBatch` if the batch is there but holds nothing this version can send; `VanishedBatch` if it + /// held events and is gone where nothing could have delivered it; anything else if it could not be read this + /// time, which a later attempt may manage. func payload(of batch: EventBatch) throws -> BatchPayload? { ioLock.assertOwned() // Taken out of the read's result rather than bound inside the switch, so that nothing else refers to it and it // is assembled where it is rather than copied first. var file: Data switch read(url(of: batch.payloadId, holding: batch.eventCount)) { - case .missing: return nil + case .missing: + guard batch.eventCount > 0, !sharesDirectoryWithOtherProcesses, + !EventDirectory.wasRemovedInThisProcess(batch.payloadId) + else { return nil } + throw VanishedBatch(payloadId: batch.payloadId) case .failed(let error): throw error case .bytes(let bytes): file = bytes } @@ -414,6 +462,7 @@ final class EventDirectory { ioLock.assertOwned() let file = url(of: payloadId, holding: events) try? FileManager.default.removeItem(at: file) + EventDirectory.rememberRemoved(payloadId) if FileManager.default.fileExists(atPath: file.path) { // Its events stop counting all the same: LaunchDarkly has them, and holding capacity for a file this // process may never be allowed to delete would cost the session events it could still have recorded. @@ -639,6 +688,7 @@ extension EventDirectory { } continue } + EventDirectory.rememberRemoved(payloadId) events += held counted += stored.removeValue(forKey: payloadId)?.tally ?? EventTally() } else { diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index 804e42a9a..f7dd95685 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -16,6 +16,9 @@ struct BatchListing: Equatable { /// False where the directory could not be listed. `batches` are then the ones this process last knew to be there, /// which includes every batch it closed itself, but can miss one another process closed or a previous run left. let isComplete: Bool + /// The events in batches this process counted that have left the directory where nothing could have delivered + /// them; see `VanishedBatch`. No longer counted from this listing on, so no later listing reports them again. + var vanishedEvents = 0 } /// The request body for a batch, and how many events made it in. @@ -89,11 +92,12 @@ protocol EventStoring { /// as they were. It does not empty them. func listPendingBatches() -> BatchListing - /// The `[...]` request body for a batch, or nil if it is gone or holds no events, neither of which loses anything. + /// The `[...]` request body for a batch, or nil if it holds no events, or is gone where another process or store + /// may have delivered it, neither of which loses anything. /// /// - throws: `UnreadableBatch` if the batch is there but holds nothing this version can send, which no later - /// attempt changes and which loses its events; anything else if it could not be read this time, which a later - /// attempt may manage. + /// attempt changes and which loses its events; `VanishedBatch` if it is gone and nothing could have delivered it; + /// anything else if it could not be read this time, which a later attempt may manage. func payload(of batch: EventBatch) throws -> BatchPayload? /// Forgets a batch, which is only correct once LaunchDarkly has accepted it or permanently refused it. @@ -134,11 +138,18 @@ extension EventStoring { } /// A batch that is there to read and holds nothing this version can send: damaged, or written in a format it does not -/// read. Unlike a batch that is gone, its events are lost. +/// read. Unlike a batch another process delivered, its events are lost. struct UnreadableBatch: Error { let payloadId: String } +/// A batch that held events and is gone where nothing could have delivered it: no other process uses the directory, +/// and no store in this one removed it. Something deleted it unsent, such as tvOS reclaiming its caches, so its events +/// are lost. +struct VanishedBatch: Error { + let payloadId: String +} + /// A crash-durable append log of encoded analytics events. /// /// Events are appended to an open log, which is closed into a batch when the reporter is ready to deliver one. A batch @@ -264,6 +275,12 @@ final class EventStore: EventStoring { } #endif + /// Whether other processes of the application may use this directory, and so deliver its batches. + var sharesDirectoryWithOtherProcesses: Bool { + get { ioLock.withLock { disk?.sharesDirectoryWithOtherProcesses ?? false } } + set { ioLock.withLock { disk?.sharesDirectoryWithOtherProcesses = newValue } } + } + convenience init(directory: URL, capacity: Int, byteCapacity: Int = LDConfig.Defaults.eventCapacityBytes, @@ -472,7 +489,8 @@ final class EventStore: EventStoring { let listing = disk.listBatches() return BatchListing(batches: heldBatchesAndRecount(after: listing.batches, storedSummaries: disk.storedSummaries, storedBytes: disk.storedBytes), - isComplete: listing.isComplete) + isComplete: listing.isComplete, + vanishedEvents: listing.vanishedEvents) } } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 69552d797..a77da4a7d 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -2159,6 +2159,37 @@ extension DirectEventProcessorSpec { expect(store.pendingEventCount) == 0 } + for shared in [false, true] { + it("reports \(shared ? "true" : "false") for a stored batch gone before it was sent, where other processes \(shared ? "share" : "do not share") the directory") { + let store = EventStore.temporary() + store.sharesDirectoryWithOtherProcesses = shared + testContext = TestContext(store: store) + testContext.eventProcessor.isOnline = true + waitUntil(timeout: Constants.deliveryTimeout) { done in + testContext.eventProcessor.flushReportingOutcome { _ in done() } + } + + testContext.recordEvents(1) + testContext.eventProcessor.commitRecordedEvents() + let gone = store.closeBatch()! + try FileManager.default.removeItem( + at: store.directory!.appendingPathComponent("ready-\(gone.payloadId).\(gone.eventCount)")) + testContext.recordEvents(1) + + var delivered: Bool? + waitUntil(timeout: Constants.deliveryTimeout) { done in + testContext.eventProcessor.flushReportingOutcome { result in + delivered = result + done() + } + } + // Where another process may have delivered it, it is taken to have been; otherwise its events are lost. + expect(delivered) == shared + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + expect(store.pendingEventCount) == 0 + } + } + it("tells the next flush about a stored batch a delivery nobody waited on dropped") { let store = EventStore.temporary() testContext = TestContext(store: store) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index a93e7e702..f0a9e0e38 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -1318,6 +1318,81 @@ final class EventStoreSpec: QuickSpec { expect(store.pendingEventCount) == 1 } + it("finds a batch deleted unsent lost where no other process shares the directory") { + store = EventStore.temporary() + store.sharesDirectoryWithOtherProcesses = false + _ = store.stage(EventStoreSpec.payload("deleted")) + let batch = store.closeBatch()! + + try FileManager.default.removeItem(at: EventStoreSpec.onlyBatchFile(in: store)) + + expect { try store.body(of: batch) }.to(throwError(errorType: VanishedBatch.self)) + } + + it("finds a batch another store in this process removed delivered, where no other process shares the directory") { + store = EventStore.temporary() + store.sharesDirectoryWithOtherProcesses = false + _ = store.stage(EventStoreSpec.payload("delivered")) + let batch = store.closeBatch()! + + EventStoreSpec.reader(sharing: store).remove(batch) + + expect(try store.body(of: batch)).to(beNil()) + } + + it("lists the events of a batch deleted unsent as vanished, once, where no other process shares the directory") { + store = EventStore.temporary() + store.sharesDirectoryWithOtherProcesses = false + _ = store.stage(EventStoreSpec.payload("deleted")) + _ = store.stage(EventStoreSpec.payload("with it")) + let deleted = store.closeBatch()! + _ = store.stage(EventStoreSpec.payload("delivered")) + let delivered = store.closeBatch()! + + try FileManager.default.removeItem( + at: store.directory!.appendingPathComponent("ready-\(deleted.payloadId).\(deleted.eventCount)")) + EventStoreSpec.reader(sharing: store).remove(delivered) + + expect(store.listPendingBatches().vanishedEvents) == 2 + expect(store.listPendingBatches().vanishedEvents) == 0 + expect(store.pendingEventCount) == 0 + } + + it("lists the events of a directory deleted whole as vanished, where no other process shares it") { + store = EventStore.temporary() + store.sharesDirectoryWithOtherProcesses = false + _ = store.stage(EventStoreSpec.payload("deleted")) + _ = store.closeBatch() + + try FileManager.default.removeItem(at: store.directory!) + + expect(store.listPendingBatches().vanishedEvents) == 1 + expect(store.pendingEventCount) == 0 + } + + it("lists nothing as vanished from a directory another process shares") { + store = EventStore.temporary() + store.sharesDirectoryWithOtherProcesses = true + _ = store.stage(EventStoreSpec.payload("delivered")) + _ = store.closeBatch() + + try FileManager.default.removeItem(at: EventStoreSpec.onlyBatchFile(in: store)) + + expect(store.listPendingBatches().vanishedEvents) == 0 + expect(store.pendingEventCount) == 0 + } + + it("finds a batch gone from a directory another process shares delivered") { + store = EventStore.temporary() + store.sharesDirectoryWithOtherProcesses = true + _ = store.stage(EventStoreSpec.payload("delivered")) + let batch = store.closeBatch()! + + try FileManager.default.removeItem(at: EventStoreSpec.onlyBatchFile(in: store)) + + expect(try store.body(of: batch)).to(beNil()) + } + for persistEvents in [true, false] { it("neither loses nor doubles a batch closed while a listing runs (persisting: \(persistEvents))") { store = EventStore.temporary(capacity: 100_000, persistEvents: persistEvents) From 60935488f8e1357d98de346834ce865e024fc316 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 22 Oct 2026 19:23:43 -0700 Subject: [PATCH 098/158] Pin context redaction in the payload the processor posts Android's EventProcessorPrivacyTest checks redaction through the event processor, with privacy configured as an application configures it. iOS only checked it at the encoders, and mostly as the two encoders agreeing. These run through DirectEventProcessor with each encoder, and cover what had no iOS test at all: an identify event keeping an anonymous context's attributes, a summary redacting the private attributes of the context it inlines, every kind of a multi-context, and no private value anywhere in the request body. Co-authored-by: Cursor --- .../DirectEventProcessorSpec.swift | 214 ++++++++++++++++++ 1 file changed, 214 insertions(+) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index a77da4a7d..2a96f5b51 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -153,6 +153,7 @@ class DirectEventProcessorSpec: QuickSpec { testRecordFlagEvaluationEvents() reportEventsSpec() unserializableEventSpec() + privacySpec() reportTimerSpec() durabilitySpec() commitSpec() @@ -633,6 +634,219 @@ class DirectEventProcessorSpec: QuickSpec { } } + /// Context redaction as it reaches the payload the processor posts, configured through `LDConfig` as an application + /// configures it, with each encoder. + /// + /// The encoder tests pin what redaction produces. These pin that the configuration reaches the encoder, which + /// events inline a context and which of those also redact an anonymous context's attributes, and that a summary, + /// which inlines the context it counted, is redacted like any other event. Android's `EventProcessorPrivacyTest` + /// covers the same ground. + private func privacySpec() { + describe("context privacy") { + var serviceMock: DarklyServiceMock! + var reporter: DirectEventProcessor! + var store: EventStore! + + let trackedFlag = FeatureFlag(flagKey: "flag-key", value: true, variation: 1, flagVersion: 2, trackEvents: true) + + func person() -> LDContext { + var builder = LDContextBuilder(key: "user-key") + builder.name("Sandy") + _ = builder.trySetValue("email", "sandy@example.com") + _ = builder.trySetValue("address", ["city": "Oakland", "street": "123 Main St"]) + return try! builder.build().get() + } + + func makeReporter(_ encoding: OutboundEventBuffer.Encoding, + configure: (inout LDConfig) -> Void = { _ in }) -> DirectEventProcessor { + var config = LDConfig.stub + config.eventCapacity = 100 + configure(&config) + serviceMock = DarklyServiceMock() + serviceMock.config = config + serviceMock.stubEventResponse(success: true) + store = EventStore.temporary(capacity: config.eventCapacity) + reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: encoding) + reporter.isOnline = true + return reporter + } + + /// Records, flushes, and answers the events in the one request the flush posted. + func posted(_ record: (DirectEventProcessor) -> Void) -> [[String: LDValue]] { + record(reporter) + waitUntil(timeout: Constants.deliveryTimeout) { done in reporter.flush(completion: done) } + expect(serviceMock.publishEventDataCallCount) == 1 + guard let data = serviceMock.publishedEventData, + case .array(let events)? = try? JSONDecoder().decode(LDValue.self, from: data) + else { return [] } + return events.map(object) + } + + func object(_ value: LDValue?) -> [String: LDValue] { + guard case .object(let members)? = value + else { return [:] } + return members + } + + func inlinedContext(ofKind kind: String, in events: [[String: LDValue]]) -> [String: LDValue] { + object(events.first { $0["kind"] == .string(kind) }?["context"]) + } + + /// What a context reports as redacted, sorted so the comparison is stable. + func redacted(_ context: [String: LDValue]) -> [String] { + guard case .array(let names)? = object(context["_meta"])["redactedAttributes"] + else { return [] } + return names.compactMap { name -> String? in + guard case .string(let string) = name + else { return nil } + return string + }.sorted() + } + + afterEach { + reporter?.isOnline = false + store?.deleteEverything() + } + + for encoding in [OutboundEventBuffer.Encoding.codable, .handWritten] { + context("\(encoding)") { + it("redacts and reports an attribute the configuration names private") { + _ = makeReporter(encoding) { $0.privateContextAttributes = [Reference("email")] } + let delivered = inlinedContext(ofKind: "identify", in: posted { $0.record(IdentifyEvent(context: person())) }) + + expect(delivered["key"]) == "user-key" + expect(delivered["email"]).to(beNil()) + expect(delivered["name"]) == "Sandy" + expect(redacted(delivered)) == ["email"] + } + + it("leaves only the key and kind where every attribute is private") { + _ = makeReporter(encoding) { $0.allContextAttributesPrivate = true } + let delivered = inlinedContext(ofKind: "identify", in: posted { $0.record(IdentifyEvent(context: person())) }) + + expect(delivered["key"]) == "user-key" + expect(delivered["kind"]) == "user" + expect(delivered["name"]).to(beNil()) + expect(delivered["email"]).to(beNil()) + expect(delivered["address"]).to(beNil()) + expect(redacted(delivered)) == ["address", "email", "name"] + } + + it("reports each name as the reference that names it where every attribute is private") { + var builder = LDContextBuilder(key: "user-key") + _ = builder.trySetValue("/ssn", "123-45-6789") + _ = builder.trySetValue("/a~b", "secret") + _ = builder.trySetValue("c/d", "value") + let slashed = try! builder.build().get() + _ = makeReporter(encoding) { $0.allContextAttributesPrivate = true } + let delivered = inlinedContext(ofKind: "identify", in: posted { $0.record(IdentifyEvent(context: slashed)) }) + + expect(delivered["/ssn"]).to(beNil()) + expect(redacted(delivered)) == ["/~1a~0b", "/~1ssn", "c/d"] + } + + it("redacts only the property a private sub-attribute names") { + _ = makeReporter(encoding) { $0.privateContextAttributes = [Reference("/address/street")] } + let delivered = inlinedContext(ofKind: "identify", in: posted { $0.record(IdentifyEvent(context: person())) }) + + expect(object(delivered["address"])["city"]) == "Oakland" + expect(object(delivered["address"])["street"]).to(beNil()) + expect(redacted(delivered)) == ["/address/street"] + } + + it("redacts an attribute the context itself marks private") { + var builder = LDContextBuilder(key: "user-key") + builder.name("Sandy") + _ = builder.trySetValue("email", "sandy@example.com") + builder.addPrivateAttribute(Reference("email")) + let marked = try! builder.build().get() + _ = makeReporter(encoding) + let delivered = inlinedContext(ofKind: "identify", in: posted { $0.record(IdentifyEvent(context: marked)) }) + + expect(delivered["email"]).to(beNil()) + expect(delivered["name"]) == "Sandy" + expect(redacted(delivered)) == ["email"] + } + + it("redacts an anonymous context's attributes from a feature event but not an identify event") { + var builder = LDContextBuilder(key: "anon-key") + builder.anonymous(true) + _ = builder.trySetValue("email", "sandy@example.com") + let anonymous = try! builder.build().get() + _ = makeReporter(encoding) + let events = posted { + $0.record(IdentifyEvent(context: anonymous)) + $0.recordFlagEvaluationEvents(flagKey: "flag-key", value: true, defaultValue: false, + featureFlag: trackedFlag, context: anonymous, includeReason: false) + } + + // An identify event is the application deliberately registering this context. + expect(inlinedContext(ofKind: "identify", in: events)["email"]) == "sandy@example.com" + // A feature event redacts every attribute of an anonymous context unasked. + let feature = inlinedContext(ofKind: "feature", in: events) + expect(feature["key"]) == "anon-key" + expect(feature["email"]).to(beNil()) + expect(redacted(feature)) == ["email"] + } + + it("redacts the private attributes of the context a summary inlines") { + _ = makeReporter(encoding) { $0.privateContextAttributes = [Reference("email")] } + let events = posted { + $0.recordFlagEvaluationEvents(flagKey: "flag-key", value: true, defaultValue: false, + featureFlag: nil, context: person(), includeReason: false) + } + + let summarized = inlinedContext(ofKind: "summary", in: events) + expect(summarized["key"]) == "user-key" + expect(summarized["email"]).to(beNil()) + expect(redacted(summarized)) == ["email"] + expect(object(summarized["address"])["city"]) == "Oakland" + } + + it("redacts every kind of a multi-context") { + var multi = LDMultiContextBuilder() + for (kind, email) in [("user", "sandy@example.com"), ("org", "billing@example.com")] { + var part = LDContextBuilder(key: "\(kind)-key") + part.kind(kind) + _ = part.trySetValue("email", .string(email)) + multi.addContext(try! part.build().get()) + } + let both = try! multi.build().get() + _ = makeReporter(encoding) { $0.privateContextAttributes = [Reference("email")] } + let delivered = inlinedContext(ofKind: "identify", in: posted { $0.record(IdentifyEvent(context: both)) }) + + expect(delivered["kind"]) == "multi" + for kind in ["user", "org"] { + expect(object(delivered[kind])["email"]).to(beNil(), description: kind) + expect(redacted(object(delivered[kind]))).to(equal(["email"]), description: kind) + } + } + + it("posts no private value in any event") { + // Over the whole request body rather than one field, so that a private value cannot slip + // through in an event kind the tests above do not name. + _ = makeReporter(encoding) { $0.privateContextAttributes = [Reference("email")] } + let events = posted { + $0.record(IdentifyEvent(context: person())) + $0.record(CustomEvent(key: "an-event", context: person())) + $0.recordFlagEvaluationEvents(flagKey: "flag-key", value: true, defaultValue: false, + featureFlag: trackedFlag, context: person(), includeReason: false) + } + + let kinds = events.compactMap { event -> String? in + guard case .string(let kind)? = event["kind"] + else { return nil } + return kind + } + expect(Set(kinds)) == ["identify", "custom", "feature", "summary"] + let body = String(data: serviceMock.publishedEventData ?? Data(), encoding: .utf8) ?? "" + expect(body.contains("sandy@example.com")) == false + } + } + } + } + } + private func reportEventsSpec() { describe("reportEvents") { var testContext: TestContext! From bd3b796a84d1e5dfa77684a27356a66e2e5f736e Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 22 Oct 2026 19:49:27 -0700 Subject: [PATCH 099/158] Count the events of an open log deleted unsent as lost A close that found the open log gone stopped counting its events and reported nothing, so a flush could answer true for events the SDK had accepted and then lost. Where the log was still held and no other process could have recovered it, the store now hands those events to the processor, which counts them as lost before taking the flush's loss mark. Co-authored-by: Cursor --- .../ServiceObjects/DirectEventProcessor.swift | 5 +- .../ServiceObjects/EventDirectory.swift | 27 ++++++-- .../ServiceObjects/EventStore.swift | 31 ++++++--- .../DirectEventProcessorSpec.swift | 38 +++++++++++ .../ServiceObjects/EventStoreSpec.swift | 67 +++++++++++++++++++ 5 files changed, 153 insertions(+), 15 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index b28742dce..4f28659f1 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -422,6 +422,7 @@ class DirectEventProcessor: EventProcessor { commitLock.withLock { encodeAndCommitPending() _ = store.closeBatch() + reportVanished(store.takeVanishedLogEvents()) return (stateLock.withLock { lostEvents }, !store.holdsUnclosedEvents) } } @@ -741,6 +742,8 @@ extension DirectEventProcessor { private func commitLeavingTheLogOpen() -> UInt64 { commitLock.withLock { encodeAndCommitPending() + // A commit that fills the log closes it, and can find it gone. + reportVanished(store.takeVanishedLogEvents()) return stateLock.withLock { lostEvents } } } @@ -786,7 +789,7 @@ extension DirectEventProcessor { } } - /// Counts as lost the events of batches a listing found deleted unsent. + /// Counts as lost the events of batches a listing found deleted unsent, or of an open log a close found gone. private func reportVanished(_ events: Int) { guard events > 0 else { return } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index 1be0957cb..b97809f11 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -127,6 +127,16 @@ final class EventDirectory { var sharesDirectoryWithOtherProcesses = false #endif + /// Whether another process could recover the log this store holds open, and so deliver it: only one sharing the + /// directory, and on a Mac only where the log is not locked. Requires `ioLock`. + private var otherProcessesCanTakeTheOpenLog: Bool { + #if os(macOS) || targetEnvironment(macCatalyst) + sharesDirectoryWithOtherProcesses && !locksOpenLog + #else + sharesDirectoryWithOtherProcesses + #endif + } + /// The payload IDs of the batches stores in this process last removed, delivered or expired, oldest first, so that /// another store on the same directory -- one opened after `LDClient` was closed, say -- does not take a batch it /// listed and then found gone for one deleted unsent. Only the most recent are kept: what matters is a removal @@ -240,10 +250,12 @@ final class EventDirectory { /// What closing the open log produced. enum LogClose { case closed(EventBatch) - /// The log is still there and still open, for the next append to continue and the next close to try again, - /// unless `logSurvives` is false -- a purged cache directory on tvOS, say. Then its events are gone, and they - /// have to stop counting against capacity or the store would refuse events for the rest of the session. - case failed(logSurvives: Bool) + /// The log is still there and still open, for the next append to continue and the next close to try again. + case failed + /// The log is gone with its events -- a purged cache directory on tvOS, say -- and they have to stop counting + /// against capacity or the store would refuse events for the rest of the session. `deletedUnsent` where + /// nothing could have delivered them: the log was still held, and no other process could have taken it. + case vanished(deletedUnsent: Bool) } /// Closes the open log into a batch, so the next append starts a new one. Requires `ioLock`. @@ -260,10 +272,13 @@ final class EventDirectory { os_log("%s could not close the event log, will try again at the next delivery: %s", log: logger, type: .info, typeName(and: #function), String(describing: error)) guard disk.log.exists else { + // One no longer held -- persistence given up on closes it where it is -- may have been moved into a + // batch by recovery, which delivers it. + let deletedUnsent = disk.log.isOpen && !otherProcessesCanTakeTheOpenLog disk.log.closeDescriptor() - return .failed(logSurvives: false) + return .vanished(deletedUnsent: deletedUnsent) } - return .failed(logSurvives: true) + return .failed } stored[payloadId] = Stored(url: file, events: events, bytes: held.bytes, modified: Date(), summaries: held.summaries) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index f7dd95685..c8a173ead 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -86,6 +86,10 @@ protocol EventStoring { /// log holding them; they stay in that log for the next close to try again. var holdsUnclosedEvents: Bool { get } + /// The committed events lost since this was last asked because the open log was deleted unsent, which a close then + /// finds gone: they are in no batch, and no longer counted as unclosed either. + func takeVanishedLogEvents() -> Int + /// Batches awaiting delivery, oldest first, including any recovered from a previous run of the application. /// /// A directory that could not be listed leaves the batches this process last counted, and what it counts for them, @@ -225,6 +229,8 @@ final class EventStore: EventStoring { /// counting it, so `sendingEvents` can never exceed `closedEvents`. private var sendingBatches: [String: EventTally] = [:] private var sendingEvents = 0 + /// Committed events that left the count with an open log deleted unsent, until `takeVanishedLogEvents()` takes them. + private var vanishedLogEvents = 0 /// The bytes of the events counted above, state for state. Staged bytes are `bufferData` itself. Changed in the /// same critical sections as the event counts beside them, so that bytes and events are never counted in @@ -447,16 +453,18 @@ final class EventStore: EventStoring { let closed = EventTally(events: committed.events, bytes: committed.bytes + EventLogFormat.fileHeader.count, summaries: committed.summaries) switch disk.closeLog(holding: closed) { - case .failed(let logSurvives): + case .failed: // A log that is still there is still the open one, and its events go on counting as committed until a - // later close manages it. Where the log went with its events, they have to stop counting against capacity - // or the store would refuse events for the rest of the session. - if !logSurvives { - bufferLock.withLock { - committedEvents = 0 - committedSummaries = 0 - committedBytes = 0 + // later close manages it. + return nil + case .vanished(let deletedUnsent): + bufferLock.withLock { + if deletedUnsent { + vanishedLogEvents += committedEvents } + committedEvents = 0 + committedSummaries = 0 + committedBytes = 0 } return nil case .closed(let batch): @@ -478,6 +486,13 @@ final class EventStore: EventStoring { bufferLock.withLock { committedEvents > 0 } } + func takeVanishedLogEvents() -> Int { + bufferLock.withLock { + defer { vanishedLogEvents = 0 } + return vanishedLogEvents + } + } + func listPendingBatches() -> BatchListing { // Listed in one critical section where there is a disk, so that the files and what is held cannot be counted // either side of a close that moves events between them. diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 2a96f5b51..80f03dba0 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -2402,6 +2402,42 @@ extension DirectEventProcessorSpec { expect(testContext.serviceMock.publishEventDataCallCount) == 1 expect(store.pendingEventCount) == 0 } + + it("reports \(shared ? "true" : "false") for events whose open log was gone before it was closed, where other processes \(shared ? "may take" : "cannot take") it") { + let store = EventStore.temporary() + store.sharesDirectoryWithOtherProcesses = shared + testContext = TestContext(store: store) + testContext.eventProcessor.isOnline = true + waitUntil(timeout: Constants.deliveryTimeout) { done in + testContext.eventProcessor.flushReportingOutcome { _ in done() } + } + + testContext.recordEvents(1) + testContext.eventProcessor.commitRecordedEvents() + try FileManager.default.removeItem(at: store.openLogUrl!) + + var delivered: Bool? + waitUntil(timeout: Constants.deliveryTimeout) { done in + testContext.eventProcessor.flushReportingOutcome { result in + delivered = result + done() + } + } + expect(delivered) == shared + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + expect(store.pendingEventCount) == 0 + + // Told once: the next flush answers for its own events. + testContext.recordEvents(1) + waitUntil(timeout: Constants.deliveryTimeout) { done in + testContext.eventProcessor.flushReportingOutcome { result in + delivered = result + done() + } + } + expect(delivered) == true + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + } } it("tells the next flush about a stored batch a delivery nobody waited on dropped") { @@ -3152,6 +3188,7 @@ private final class ClosingHookStore: EventStoring { func releaseReservations() { inner.releaseReservations() } func commit() { inner.commit() } var holdsUnclosedEvents: Bool { inner.holdsUnclosedEvents } + func takeVanishedLogEvents() -> Int { inner.takeVanishedLogEvents() } func listPendingBatches() -> BatchListing { inner.listPendingBatches() } func payload(of batch: EventBatch) throws -> BatchPayload? { try inner.payload(of: batch) } func remove(_ batch: EventBatch) { inner.remove(batch) } @@ -3232,6 +3269,7 @@ private final class SlowListingStore: EventStoring { func commit() { inner.commit() } func closeBatch() -> EventBatch? { inner.closeBatch() } var holdsUnclosedEvents: Bool { inner.holdsUnclosedEvents } + func takeVanishedLogEvents() -> Int { inner.takeVanishedLogEvents() } func payload(of batch: EventBatch) throws -> BatchPayload? { try inner.payload(of: batch) } func remove(_ batch: EventBatch) { inner.remove(batch) } func startedSending(_ batch: EventBatch) { inner.startedSending(batch) } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index f0a9e0e38..c24fb56c2 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -1393,6 +1393,73 @@ final class EventStoreSpec: QuickSpec { expect(try store.body(of: batch)).to(beNil()) } + it("hands over the events of an open log deleted unsent as vanished, once, where no other process shares the directory") { + store = EventStore.temporary() + store.sharesDirectoryWithOtherProcesses = false + _ = store.stage(EventStoreSpec.payload("deleted")) + _ = store.stage(EventStoreSpec.payload("with it")) + store.commit() + + try FileManager.default.removeItem(at: store.openLogUrl!) + + expect(store.closeBatch()).to(beNil()) + expect(store.holdsUnclosedEvents) == false + expect(store.pendingEventCount) == 0 + expect(store.takeVanishedLogEvents()) == 2 + expect(store.takeVanishedLogEvents()) == 0 + } + + #if os(macOS) || targetEnvironment(macCatalyst) + it("hands over the events of a locked open log deleted unsent as vanished, though another process shares the directory") { + store = EventStore.temporary() + store.sharesDirectoryWithOtherProcesses = true + store.locksOpenLog = true + _ = store.stage(EventStoreSpec.payload("deleted")) + store.commit() + + try FileManager.default.removeItem(at: store.openLogUrl!) + + expect(store.closeBatch()).to(beNil()) + expect(store.takeVanishedLogEvents()) == 1 + } + + it("hands over nothing as vanished from an unlocked open log gone from a directory another process shares") { + store = EventStore.temporary() + store.sharesDirectoryWithOtherProcesses = true + _ = store.stage(EventStoreSpec.payload("recovered elsewhere")) + store.commit() + + try FileManager.default.removeItem(at: store.openLogUrl!) + + expect(store.closeBatch()).to(beNil()) + expect(store.pendingEventCount) == 0 + expect(store.takeVanishedLogEvents()) == 0 + } + #endif + + it("hands over nothing as vanished from a log persistence gave up on, which recovery may have taken") { + var writes = 0 + store = EventStore.temporary(writeLog: { file, bytes in + writes += 1 + // The file header and the first event land; the device is full from then on. + return writes > 2 ? ENOSPC : file.writeAll(bytes) + }) + store.sharesDirectoryWithOtherProcesses = false + _ = store.stage(EventStoreSpec.payload("written")) + store.commit() + _ = store.stage(EventStoreSpec.payload("held")) + store.commit() + expect(store.isPersisting) == false + + // No longer held, the log is one recovery takes, as it would one a process that died left behind. + EventStoreSpec.reader(sharing: store).preparePayloadsFromInterruptedLogs() + + _ = store.closeBatch() + expect(store.takeVanishedLogEvents()) == 0 + expect(store.pendingBatches().flatMap { EventStoreSpec.keys(ofBody: try? store.body(of: $0)) } + .sorted()) == ["held", "written"] + } + for persistEvents in [true, false] { it("neither loses nor doubles a batch closed while a listing runs (persisting: \(persistEvents))") { store = EventStore.temporary(capacity: 100_000, persistEvents: persistEvents) From b94ce8ea95f35ae49da3397fef1a73c4979329b9 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Fri, 23 Oct 2026 11:27:22 -0700 Subject: [PATCH 100/158] Put each frame's checksum after its payload A frame is now type, size, payload, then the CRC-32 of everything before it, so the checksum covers one contiguous run of bytes. The checksum values do not change, only where they sit. The format stays version 1: no release wrote the earlier layout. Android writes the same bytes. Co-authored-by: Cursor --- .../ServiceObjects/EventLogFormat.swift | 45 ++++++++++--------- .../DirectEventProcessorSpec.swift | 4 +- .../ServiceObjects/EventStoreSpec.swift | 22 ++++++--- 3 files changed, 43 insertions(+), 28 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift index 797cbbf2d..e1345af17 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift @@ -4,13 +4,14 @@ import Foundation /// /// A log opens with a magic and a format version, and holds a sequence of frames: /// -/// +- 4 bytes -+- 2 bytes -+ +- 2 bytes -+- 4 bytes -+- 4 bytes -+- n bytes -+ -/// | LDEV | version | ... | type | size (n) | CRC-32 | payload | +/// +- 4 bytes -+- 2 bytes -+ +- 2 bytes -+- 4 bytes -+- n bytes -+- 4 bytes -+ +/// | LDEV | version | ... | type | size (n) | payload | CRC-32 | /// +-----------+-----------+ +-----------+-----------+-----------+-----------+ /// /// The size prefix is what makes a log recoverable: a reader can tell a frame the writer finished from one it did not, -/// without having to parse the event to find where it ends. The checksum, over the type, the size and the payload, is -/// what tells a frame the writer finished from one the storage has since damaged; a reader treats either the same way. +/// without having to parse the event to find where it ends. The checksum, over the type, the size and the payload -- +/// everything before it in the frame -- is what tells a frame the writer finished from one the storage has since +/// damaged; a reader treats either the same way. /// The version is what makes an upgrade safe -- a log this version does not understand is discarded rather than /// misread -- and the frame type leaves room for a later version to write something new into a log this one still /// reads. @@ -19,7 +20,12 @@ enum EventLogFormat { static let version: UInt16 = 1 static let magic = Data("LDEV".utf8) static let fileHeaderSize = 6 - static let frameHeaderSize = 10 + /// The type and the size, before a frame's payload. + static let frameHeaderSize = 6 + /// The checksum, after a frame's payload. + static let frameTrailerSize = 4 + /// What a frame takes beyond its payload. + static let frameOverhead = frameHeaderSize + frameTrailerSize /// The only frame type written today; a reader skips a frame whose type it does not know. static let eventFrame: UInt16 = 1 /// The most a batch may hold, file header included, so that no request it becomes is larger than the events @@ -27,7 +33,7 @@ enum EventLogFormat { static let maxBatchSize = 2 * 1024 * 1024 /// A ceiling on a single frame's payload: the most that still fits in a batch on its own. It is also what keeps a /// corrupt size from making recovery allocate wildly. - static let maxFrameSize = maxBatchSize - fileHeaderSize - frameHeaderSize + static let maxFrameSize = maxBatchSize - fileHeaderSize - frameOverhead static var fileHeader: Data { var header = magic @@ -36,14 +42,13 @@ enum EventLogFormat { } static func frame(for encodedEvent: Data) -> Data { - var frame = Data(capacity: frameHeaderSize + encodedEvent.count) + var frame = Data(capacity: frameOverhead + encodedEvent.count) withUnsafeBytes(of: eventFrame.bigEndian) { frame.append(contentsOf: $0) } withUnsafeBytes(of: UInt32(encodedEvent.count).bigEndian) { frame.append(contentsOf: $0) } + frame.append(encodedEvent) var checksum = Crc32() frame.withUnsafeBytes { checksum.advance(withBytes: $0) } - encodedEvent.withUnsafeBytes { checksum.advance(withBytes: $0) } withUnsafeBytes(of: checksum.checksum.bigEndian) { frame.append(contentsOf: $0) } - frame.append(encodedEvent) return frame } @@ -83,22 +88,21 @@ enum EventLogFormat { /// anywhere before where the frame ends. private static func walkFrames(_ bytes: UnsafeRawBufferPointer, _ visit: (UInt16, Range) -> Void) -> FrameWalk { var cursor = fileHeaderSize - while cursor + frameHeaderSize <= bytes.count { + while cursor + frameOverhead <= bytes.count { let type = UInt16(bytes[cursor]) << 8 | UInt16(bytes[cursor + 1]) let length = Int(readUInt32(bytes, at: cursor + 2)) let start = cursor + frameHeaderSize - guard length > 0, length <= maxFrameSize, start + length <= bytes.count + let end = start + length + guard length > 0, length <= maxFrameSize, end + frameTrailerSize <= bytes.count else { break } - let payload = UnsafeRawBufferPointer(rebasing: bytes[start..<(start + length)]) var checksum = Crc32() - checksum.advance(withBytes: UnsafeRawBufferPointer(rebasing: bytes[cursor..<(cursor + 6)])) - checksum.advance(withBytes: payload) - guard checksum.checksum == readUInt32(bytes, at: cursor + 6) + checksum.advance(withBytes: UnsafeRawBufferPointer(rebasing: bytes[cursor.. Int { - frameHeaderSize + Int(readUInt32(frames, at: start + 2)) + frameOverhead + Int(readUInt32(frames, at: start + 2)) } /// Where a run of whole frames from `start`, in frames this process wrote, ends if it is to take no more than diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 80f03dba0..7fdbc4ac4 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -1711,7 +1711,7 @@ extension DirectEventProcessorSpec { itPersisting("refuses an event without encoding it once the stored bytes leave no room for full events") { let store = EventStore.temporary(capacity: 100, byteCapacity: 1000) // One frame of 800 bytes, which is all the room full events have. - expect(store.stage(Data(repeating: UInt8(ascii: "x"), count: 800 - EventLogFormat.frameHeaderSize))) == true + expect(store.stage(Data(repeating: UInt8(ascii: "x"), count: 800 - EventLogFormat.frameOverhead))) == true testContext = TestContext(store: store, eventCapacity: 100) let refused = EncodingGate(key: "refused", context: testContext.context) @@ -1724,7 +1724,7 @@ extension DirectEventProcessorSpec { itPersisting("reports a flush as failed when an accepted event turns out too large for the bytes left") { let store = EventStore.temporary(capacity: 100, byteCapacity: 1000) - expect(store.stage(Data(repeating: UInt8(ascii: "x"), count: 700 - EventLogFormat.frameHeaderSize))) == true + expect(store.stage(Data(repeating: UInt8(ascii: "x"), count: 700 - EventLogFormat.frameOverhead))) == true testContext = TestContext(store: store, eventCapacity: 100) testContext.eventProcessor.isOnline = true diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index c24fb56c2..43f7faf78 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -353,9 +353,9 @@ final class EventStoreSpec: QuickSpec { private static func frameOffsets(in log: Data) -> [Int] { var offsets: [Int] = [] var cursor = EventLogFormat.fileHeaderSize - while cursor + EventLogFormat.frameHeaderSize <= log.count { + while cursor + EventLogFormat.frameOverhead <= log.count { offsets.append(cursor) - cursor += EventLogFormat.frameHeaderSize + Int(EventLogFormat.readUInt32(log, at: cursor + 2)) + cursor += EventLogFormat.frameOverhead + Int(EventLogFormat.readUInt32(log, at: cursor + 2)) } return offsets } @@ -363,7 +363,7 @@ final class EventStoreSpec: QuickSpec { /// An event whose frame is exactly `frameBytes` long, padded out in a `data` field. private static func payload(framedIn frameBytes: Int, key: String) -> Data { let bare = "{\"kind\":\"custom\",\"key\":\"\(key)\",\"data\":\"\"}" - let padding = frameBytes - EventLogFormat.frameHeaderSize - bare.utf8.count + let padding = frameBytes - EventLogFormat.frameOverhead - bare.utf8.count precondition(padding >= 0, "\(key) does not fit in \(frameBytes) bytes") return Data("{\"kind\":\"custom\",\"key\":\"\(key)\",\"data\":\"\(String(repeating: "x", count: padding))\"}".utf8) } @@ -595,6 +595,15 @@ final class EventStoreSpec: QuickSpec { expect(EventStoreSpec.keys(in: store.payloadsOnDisk)) == ["event-0"] } + it("keeps every event before a damaged checksum") { + var damaged = try Data(contentsOf: log) + let size = Int(EventLogFormat.readUInt32(damaged, at: second + 2)) + damaged[second + EventLogFormat.frameHeaderSize + size + 1] ^= 0x10 + try damaged.write(to: log) + + expect(EventStoreSpec.keys(in: store.payloadsOnDisk)) == ["event-0"] + } + it("keeps every event before a size that still fits in the file") { // A size that is merely wrong rather than impossible passes every bound a torn tail is caught by, // and would have the reader start the next frame partway through this one's payload. @@ -778,8 +787,9 @@ final class EventStoreSpec: QuickSpec { it("frames an event as the Android SDK does") { // The checksum is zlib's CRC-32, so these bytes are what the Android SDK writes for the same event too. - let frame = EventLogFormat.frame(for: Data("{\"kind\":\"custom\",\"key\":\"k\"}".utf8)) - expect(frame.prefix(EventLogFormat.frameHeaderSize)) == Data([0x00, 0x01, 0x00, 0x00, 0x00, 0x1B, 0xCB, 0x2B, 0x5E, 0x6F]) + let event = Data("{\"kind\":\"custom\",\"key\":\"k\"}".utf8) + let frame = EventLogFormat.frame(for: event) + expect(frame) == Data([0x00, 0x01, 0x00, 0x00, 0x00, 0x1B]) + event + Data([0xCB, 0x2B, 0x5E, 0x6F]) } it("has nothing to close when nothing was recorded") { @@ -1581,7 +1591,7 @@ final class EventStoreSpec: QuickSpec { it("takes an event that fills a batch on its own, and refuses any larger") { store = EventStore.temporary() - let largest = EventLogFormat.maxFrameSize + EventLogFormat.frameHeaderSize + let largest = EventLogFormat.maxFrameSize + EventLogFormat.frameOverhead expect(store.stage(EventStoreSpec.payload(framedIn: largest + 1, key: "too-large"))) == false expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-1"))) == true expect(store.stage(EventStoreSpec.payload(framedIn: largest, key: "largest"))) == true From c438c9e60973d10b0b99a536df440354442cfafd Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 27 Oct 2026 11:22:11 -0700 Subject: [PATCH 101/158] fix(events): let the delivery at close post through the backoff close() made an ordinary flush, held by the backoff like any other: after a refusal, a wait of minutes to an hour, it posted nothing, and with persistence off those events went with the process. It then went offline straight away, and a delivery that had not started by then found the reporter offline and posted nothing either. close() now calls flushAtClose(), whose delivery posts whatever wait the backoff is in and runs although closing went offline after asking. Only for its own length: the delivery a failure of its own sets for the end of the wait is held, and finds the reporter offline, as before. A client already offline at close still sends nothing. Co-authored-by: Cursor --- .../GeneratedCode/mocks.generated.swift | 7 ++++ LaunchDarkly/LaunchDarkly/LDClient.swift | 2 +- .../ServiceObjects/DirectEventProcessor.swift | 36 +++++++++++++++---- .../ServiceObjects/EventProcessor.swift | 8 +++++ .../ServiceObjects/NullEventProcessor.swift | 3 ++ .../LaunchDarklyTests/LDClientSpec.swift | 9 +++-- .../DirectEventProcessorSpec.swift | 31 ++++++++++++++++ 7 files changed, 85 insertions(+), 11 deletions(-) diff --git a/LaunchDarkly/GeneratedCode/mocks.generated.swift b/LaunchDarkly/GeneratedCode/mocks.generated.swift index 2ea9cd722..61d9c7835 100644 --- a/LaunchDarkly/GeneratedCode/mocks.generated.swift +++ b/LaunchDarkly/GeneratedCode/mocks.generated.swift @@ -231,6 +231,13 @@ final class EventProcessorMock: EventProcessor { try! flushReportingOutcomeCallback?() } + var flushAtCloseCallCount = 0 + var flushAtCloseCallback: (() throws -> Void)? + func flushAtClose() { + flushAtCloseCallCount += 1 + try! flushAtCloseCallback?() + } + var commitRecordedEventsCallCount = 0 var commitRecordedEventsCallback: (() throws -> Void)? func commitRecordedEvents() { diff --git a/LaunchDarkly/LaunchDarkly/LDClient.swift b/LaunchDarkly/LaunchDarkly/LDClient.swift index b7d1d1424..7b1449a4c 100644 --- a/LaunchDarkly/LaunchDarkly/LDClient.swift +++ b/LaunchDarkly/LaunchDarkly/LDClient.swift @@ -255,7 +255,7 @@ public class LDClient { private func internalClose() { os_log("%s stopping", log: config.logger, type: .debug, typeName(and: #function)) - internalFlush() + eventProcessor.flushAtClose() internalSetOnline(false) hasStarted = false os_log("%s stopped", log: config.logger, type: .debug, typeName(and: #function)) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 4f28659f1..43ac9391d 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -137,8 +137,8 @@ class DirectEventProcessor: EventProcessor { // MARK: Guarded by deliveryLock - /// Guards `queuedDelivery`, `isDelivering` and `isRetryScheduled`. Taken on a caller's thread and on `deliveryQueue`, - /// and never held across either. + /// Guards `queuedDelivery`, `isDelivering`, `isRetryScheduled` and whether a delivery is close's. Taken on a caller's + /// thread, on `deliveryQueue` and where a delivery's posts are answered, and never held across any of them. private let deliveryLock = UnfairLock() /// Callers waiting on a delivery that is queued and has not taken the batches yet, or nil where none is queued. private var queuedDelivery: [FlushOutcomeClosure]? @@ -151,6 +151,12 @@ class DirectEventProcessor: EventProcessor { /// Whether a delivery is already set to run when the backoff's wait ends, so a run of passes inside the wait sets /// one rather than one each. private var isRetryScheduled = false + /// Whether the delivery queued is `flushAtClose()`'s, or has been joined by it. + private var queuedDeliveryIsClosing = false + /// Whether the delivery running is `flushAtClose()`'s, which posts whatever wait the backoff is in and runs although + /// closing has turned `isOnline` off since. For its length only: the delivery one of its failures sets for the end + /// of the wait is held by the backoff, and finds the reporter offline, as any other. + private var isClosingDelivery = false #if DEBUG /// How many deliveries have started, which is fewer than the flushes asked for where they were joined. @@ -540,6 +546,13 @@ extension DirectEventProcessor { queueDelivery(completion) } + func flushAtClose() { + commitWherePersisting() + // Offline, it is an ordinary delivery, which finds the reporter offline and sends nothing: offline is the + // application keeping the SDK off the network, and closing does not revoke that. + queueDelivery(nil, closing: isOnline) + } + /// Queues a delivery the way a caller asking for one expects, which is as a commit point. /// /// Wherever events persist, everything accepted before the call is on disk by the time it returns, even where the @@ -593,16 +606,20 @@ extension DirectEventProcessor { /// anything recorded since. Nor does another start beside it: the one queued waits until the running one has an /// answer, then sends what that one left behind as well as what was recorded since, so its caller hears `true` /// only if both got out. - private func queueDelivery(_ completion: FlushOutcomeClosure?) { + /// + /// - Parameter closing: whether it is `flushAtClose()`'s, which a queued delivery it joins becomes. + private func queueDelivery(_ completion: FlushOutcomeClosure?, closing: Bool = false) { deliveryLock.lock() if queuedDelivery != nil { if let completion { queuedDelivery?.append(completion) } + queuedDeliveryIsClosing = queuedDeliveryIsClosing || closing deliveryLock.unlock() return } queuedDelivery = completion.map { [$0] } ?? [] + queuedDeliveryIsClosing = closing let starts = !isDelivering isDelivering = true deliveryLock.unlock() @@ -618,12 +635,15 @@ extension DirectEventProcessor { deliveryLock.lock() let waiters = queuedDelivery ?? [] queuedDelivery = nil + let closing = queuedDeliveryIsClosing + queuedDeliveryIsClosing = false + isClosingDelivery = closing #if DEBUG deliveryRunCount += 1 #endif deliveryLock.unlock() - reportEvents { delivered, lossMark in + reportEvents(closing: closing) { delivered, lossMark in // Taken once for the delivery rather than once per waiter, so that every caller joined to it hears about // a loss instead of only whichever happened to be answered first. Taken before the next delivery starts, // which could otherwise answer first and take this one's loss with its own. @@ -646,6 +666,7 @@ extension DirectEventProcessor { /// Ends the delivery that just had its answer, starting the one queued behind it if there is one. private func startQueuedDeliveryOrStop() { let next = deliveryLock.withLock { () -> Bool in + isClosingDelivery = false isDelivering = queuedDelivery != nil return isDelivering } @@ -654,8 +675,9 @@ extension DirectEventProcessor { } } - private func reportEvents(completion: ((_ delivered: Bool, _ lossMark: UInt64) -> Void)?) { - guard isOnline + /// - Parameter closing: whether it is `flushAtClose()`'s, which was online when it was asked for. + private func reportEvents(closing: Bool = false, completion: ((_ delivered: Bool, _ lossMark: UInt64) -> Void)?) { + guard closing || isOnline else { os_log("%s aborted. DirectEventProcessor is offline", log: service.config.logger, type: .debug, typeName(and: #function)) reportSyncComplete(.isOffline) @@ -870,7 +892,7 @@ extension DirectEventProcessor { return } - let wait = retryState.remainingWait + let wait = deliveryLock.withLock { isClosingDelivery } ? 0 : retryState.remainingWait guard wait <= 0 else { // Not an attempt, so it moves nothing in the backoff. A wait no longer than a retry's is sat out, as this diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift index 964c05da4..75949de91 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift @@ -37,6 +37,14 @@ protocol EventProcessor { /// were written, since the delivery writes them before it sends. func flushReportingOutcome(completion: @escaping FlushOutcomeClosure) + /// The last delivery before the client closes: `flush`, except that it posts whatever wait the backoff is in, which + /// closing cuts short. After a refusal that wait is minutes to an hour, and where nothing persists the events would + /// not outlive it. + /// + /// Whether it posts is settled by whether the reporter is online when this is called, so the client going offline + /// as it closes, straight after, does not stop it. + func flushAtClose() + /// Makes everything recorded so far outlive the process, without waiting for a delivery. /// /// The SDK does this itself at the points where an application is most likely to be about to die. It is worth diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventProcessor.swift index 81cc0d82d..38cd660a2 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventProcessor.swift @@ -30,6 +30,9 @@ class NullEventProcessor: EventProcessor { DispatchQueue.global().async { completion(true) } } + func flushAtClose() { + } + func commitRecordedEvents() { } diff --git a/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift b/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift index 81e32d309..5b9134e2b 100644 --- a/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift @@ -900,7 +900,8 @@ final class LDClientSpec: QuickSpec { testContext.subject.close() expect(testContext.subject.isOnline) == false - expect(testContext.eventProcessorMock.flushCallCount) == 1 + expect(testContext.eventProcessorMock.flushAtCloseCallCount) == 1 + expect(testContext.eventProcessorMock.flushCallCount) == 0 } it("when started and offline") { let testContext = TestContext() @@ -908,7 +909,8 @@ final class LDClientSpec: QuickSpec { testContext.subject.close() expect(testContext.subject.isOnline) == false - expect(testContext.eventProcessorMock.flushCallCount) == 1 + expect(testContext.eventProcessorMock.flushAtCloseCallCount) == 1 + expect(testContext.eventProcessorMock.flushCallCount) == 0 } it("when already stopped") { let testContext = TestContext() @@ -917,7 +919,8 @@ final class LDClientSpec: QuickSpec { testContext.subject.close() expect(testContext.subject.isOnline) == false - expect(testContext.eventProcessorMock.flushCallCount) == 1 + expect(testContext.eventProcessorMock.flushAtCloseCallCount) == 1 + expect(testContext.eventProcessorMock.flushCallCount) == 0 } } } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 7fdbc4ac4..209501e37 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -2960,6 +2960,37 @@ extension DirectEventProcessorSpec { expect(testContext.flushOutcome()) == true } + it("posts what is buffered at close through the backoff's wait, though closing goes offline straight after") { + testContext = TestContext(retryState: DeliveryRetryState(initialDelay: 1, normalCeiling: 1, extendedCeiling: 3600)) + testContext.serviceMock.respondToEvents(with: HTTPURLResponse.StatusCodes.unauthorized) + testContext.eventProcessor.record(CustomEvent(key: "refused", context: testContext.context)) + testContext.eventProcessor.isOnline = true + // A refusal, after which the backoff allows nothing for about eight and a half minutes. + expect(testContext.flushOutcome()) == false + testContext.serviceMock.stubEventResponse(success: true) + testContext.serviceMock.eventStatusForPost = nil + testContext.eventProcessor.record(CustomEvent(key: "at-close", context: testContext.context)) + expect(testContext.flushOutcome()) == false + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + + // What LDClient.close() does. + testContext.eventProcessor.flushAtClose() + testContext.eventProcessor.isOnline = false + + expect(testContext.serviceMock.publishedEventPayloads.dropFirst().flatMap(customKeys)) + .toEventually(equal(["refused", "at-close"]), timeout: Constants.deliveryTimeout) + } + + it("posts nothing at close when the client was already offline") { + testContext = TestContext() + testContext.eventProcessor.record(CustomEvent(key: "an-event", context: testContext.context)) + + testContext.eventProcessor.flushAtClose() + waitLettingTimersFire(0.5) + + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + } + it("waits as long as a Retry-After asks before posting again") { testContext = TestContext() let busy = HTTPURLResponse(url: testContext.serviceMock.config.eventsUrl, From 455028bcd15a847e3a66dc5848e488022626fea5 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 27 Oct 2026 12:11:27 -0700 Subject: [PATCH 102/158] fix(events): drop a batch after two 413, 415 or 422 refusals, as for 400 A 413 was terminal: the batch was kept under the extended backoff and, being the oldest, held back every batch behind it until it passed the maximum age. The events service answers 413 to a body over its limit, which posting it again never changes, and 415 and 422 are refusals of the payload in the same way. All four now count together toward the drop that 400 had: two in a row for the same batch, in any mix, remove it and count its events as lost, and they wait the normal backoff a 400 does rather than the extended one. isTerminalStatusCode, which flag synchronization shares, is unchanged; the events-only check is StatusCodes.isEventPayloadRefused. Co-authored-by: Cursor --- .../Networking/HTTPURLResponse.swift | 10 ++++ .../ServiceObjects/AnalyticsEventSender.swift | 7 ++- .../ServiceObjects/DirectEventProcessor.swift | 29 ++++----- .../DirectEventProcessorSpec.swift | 59 +++++++++++++------ 4 files changed, 72 insertions(+), 33 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/Networking/HTTPURLResponse.swift b/LaunchDarkly/LaunchDarkly/Networking/HTTPURLResponse.swift index 72e542c8f..4e88b3989 100644 --- a/LaunchDarkly/LaunchDarkly/Networking/HTTPURLResponse.swift +++ b/LaunchDarkly/LaunchDarkly/Networking/HTTPURLResponse.swift @@ -17,6 +17,9 @@ extension HTTPURLResponse { static let unauthorized = 401 static let methodNotAllowed = 405 static let requestTimeout = 408 + static let contentTooLarge = 413 + static let unsupportedMediaType = 415 + static let unprocessableContent = 422 static let tooManyRequests = 429 static let internalServerError = 500 static let notImplemented = 501 @@ -28,6 +31,13 @@ extension HTTPURLResponse { static func isTerminalStatusCode(_ statusCode: Int) -> Bool { (400..<500).contains(statusCode) && !recoverableClientErrors.contains(statusCode) } + + /// Whether the events service refused an event payload for what it holds rather than for who sent it: one it + /// cannot read, too large for it, of a type it does not take, or read but not accepted. Posting the same body + /// again will not change the answer, though another body may be taken. + static func isEventPayloadRefused(_ statusCode: Int) -> Bool { + [badRequest, contentTooLarge, unsupportedMediaType, unprocessableContent].contains(statusCode) + } } var headerDate: Date? { diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/AnalyticsEventSender.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/AnalyticsEventSender.swift index 21f77607e..b31aebb64 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/AnalyticsEventSender.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/AnalyticsEventSender.swift @@ -18,8 +18,8 @@ final class AnalyticsEventSender { let error: SynchronizingError /// The response's status, or nil where there was no response. let statusCode: Int? - /// Whether the failure is unlikely to pass soon: a 4xx other than 400, 408 and 429, or a TLS or certificate - /// failure. + /// Whether the failure is unlikely to pass soon: a 4xx other than a refusal of the payload + /// (`isEventPayloadRefused`), 408 and 429, or a TLS or certificate failure. let unexpected: Bool /// The wait the response asked for, or nil where it gave none. let retryAfter: TimeInterval? @@ -58,7 +58,10 @@ final class AnalyticsEventSender { let statusCode = response?.statusCode let unexpected: Bool if let statusCode, (400..<600).contains(statusCode) { + // A refusal of the payload waits like a 400: the processor drops the batch after a second in a row, and + // the batches behind it are not to be held back an extended wait meanwhile. unexpected = HTTPURLResponse.StatusCodes.isTerminalStatusCode(statusCode) + && !HTTPURLResponse.StatusCodes.isEventPayloadRefused(statusCode) } else if let urlError = error as? URLError { unexpected = AnalyticsEventSender.tlsErrorCodes.contains(urlError.code) } else { diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 43ac9391d..264aa0593 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -104,15 +104,15 @@ class DirectEventProcessor: EventProcessor { stateLock.withLock { responseDate } } - /// Guards the last response date, the record of what has been lost, and the count of 400s. + /// Guards the last response date, the record of what has been lost, and the count of refused payloads. private let stateLock = UnfairLock() /// Only to be used while holding `stateLock`. private var responseDate: Date /// How many events the SDK accepted and then dropped for good. Only to be used while holding `stateLock`. /// - /// A batch LaunchDarkly refuses is kept and retried, so it is not a loss, unless the refusal is a second 400 in a - /// row. Otherwise what is lost is what the store never received: an event or a summary that could not be serialized + /// A batch LaunchDarkly refuses is kept and retried, so it is not a loss, unless the refusal is a second refusal of + /// the payload in a row. Otherwise what is lost is what the store never received: an event or a summary that could not be serialized /// when it was committed, or that serialized too large to store. A delivery that finds nothing to send cannot tell /// those apart from events that arrived, so without this it would report success for them. /// @@ -125,14 +125,15 @@ class DirectEventProcessor: EventProcessor { /// used while holding `stateLock`. private var reportedLostEvents: UInt64 = 0 - /// How many 400s in a row drop a batch. A 400 is a payload the service cannot read, which posting it again will not + /// How many refusals of the payload in a row drop a batch: a 400, 413, 415 or 422, in any mix + /// (`isEventPayloadRefused`). Each says the service will not take this body, which posting it again will not /// change, and a kept batch is the oldest and so the next posted: kept for good, it would hold every batch behind - /// it. Two rather than one, so a 400 from a proxy or an incident costs nothing. + /// it. Two rather than one, so one from a proxy or an incident costs nothing. static let badRequestsBeforeDropping = 2 - /// The batch that last got a 400 and how many in a row: delivery goes oldest first and stops at a kept failure, so - /// the next post is that batch's unless capacity dropped it. Held only in memory, so a new run of the application - /// counts again. Only to be used while holding `stateLock`. + /// The batch that last had its payload refused and how many times in a row: delivery goes oldest first and stops at + /// a kept failure, so the next post is that batch's unless capacity dropped it. Held only in memory, so a new run of + /// the application counts again. Only to be used while holding `stateLock`. private var badRequests: (payloadId: String, inARow: Int)? // MARK: Guarded by deliveryLock @@ -862,8 +863,8 @@ extension DirectEventProcessor { enum PassOutcome: Int, Comparable { /// Every batch was accepted. case delivered - /// Every batch was attempted, and at least one was lost for good: unreadable, damaged, deleted unsent, or refused - /// as malformed `badRequestsBeforeDropping` times in a row. + /// Every batch was attempted, and at least one was lost for good: unreadable, damaged, deleted unsent, or had its + /// payload refused `badRequestsBeforeDropping` times in a row. case lost /// A batch is still there to be sent again: its request failed, the backoff did not allow it to be posted yet, /// or it could not be read this time. The batches after it were not attempted. @@ -974,7 +975,7 @@ extension DirectEventProcessor { let wait = self.retryState.recordFailure(unexpected: failure.unexpected, retryAfter: failure.retryAfter) if self.countBadRequest(batch, statusCode: failure.statusCode) >= DirectEventProcessor.badRequestsBeforeDropping { // The rest are still attempted, once the backoff allows. - os_log("%s dropping %d event(s) the service refused as malformed %d times in a row", log: self.service.config.logger, type: .error, self.typeName(and: #function), batch.eventCount, DirectEventProcessor.badRequestsBeforeDropping) + os_log("%s dropping %d event(s) whose payload the service refused %d times in a row, last with HTTP %d", log: self.service.config.logger, type: .error, self.typeName(and: #function), batch.eventCount, DirectEventProcessor.badRequestsBeforeDropping, failure.statusCode ?? 0) self.store.remove(batch) self.reportBatchLost(batch) self.reportSyncComplete(failure.error) @@ -1005,11 +1006,11 @@ extension DirectEventProcessor { } } - /// - Returns: how many 400s in a row this batch has had, counting this response; zero for any other status, which - /// ends the run, and once the batch is dropped. + /// - Returns: how many refusals of the payload in a row this batch has had, counting this response; zero for any + /// other status, which ends the run, and once the batch is dropped. private func countBadRequest(_ batch: EventBatch, statusCode: Int?) -> Int { stateLock.withLock { - guard statusCode == HTTPURLResponse.StatusCodes.badRequest + guard let statusCode, HTTPURLResponse.StatusCodes.isEventPayloadRefused(statusCode) else { badRequests = nil return 0 diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 209501e37..60645d815 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -2860,9 +2860,9 @@ extension DirectEventProcessorSpec { expectKeptThroughRefusals(status: 503, persistence: .disabled, refusedFlushes: 3) } - // Each has to keep the batch, not only 503: a refusal such as a 401 only waits longer. Not a 400, which drops - // the batch when it comes twice in a row; see below. - for status in [401, 403, 404, 408, 413, 429, 500, 502, 504] { + // Each has to keep the batch, not only 503: a refusal such as a 401 only waits longer. Not a 400, 413, 415 or + // 422, which drop the batch when they come twice in a row; see below. + for status in [401, 403, 404, 408, 429, 500, 502, 504] { it("keeps a batch refused with HTTP \(status) past the retry") { expectKeptThroughRefusals(status: status, persistence: .immediate, refusedFlushes: 2) } @@ -2896,28 +2896,53 @@ extension DirectEventProcessorSpec { } } - it("drops a batch refused with HTTP 400 twice in a row, and no longer holds the ones behind it") { + for status in [400, 413, 415, 422] { + it("drops a batch refused with HTTP \(status) twice in a row, and no longer holds the ones behind it") { + testContext = TestContext(retryState: DirectEventProcessorSpec.withoutBackoff()) + testContext.serviceMock.respondToEvents(with: status) + testContext.eventProcessor.record(CustomEvent(key: "refused", context: testContext.context)) + testContext.eventProcessor.isOnline = true + expect(testContext.flushOutcome()) == false + expect(testContext.serviceMock.publishedEventPayloads.map(customKeys)) == [["refused"], ["refused"]] + expect(Set(testContext.serviceMock.publishedPayloadIds).count) == 1 + expect(testContext.store.pendingBatches()).to(beEmpty()) + waitLettingTimersFire(0.3) + expect(testContext.serviceMock.publishEventDataCallCount) == 2 + testContext.serviceMock.stubEventResponse(success: true) + + // What was recorded since goes out alone, and that flush is the first a caller waits on since the + // loss, so it hears about it. + testContext.eventProcessor.record(CustomEvent(key: "next", context: testContext.context)) + expect(testContext.flushOutcome()) == false + expect(testContext.serviceMock.publishedEventPayloads.dropFirst(2).map(customKeys)) == [["next"]] + expect(testContext.flushOutcome()) == true + } + } + + it("counts different refusals of the payload in a row together") { testContext = TestContext(retryState: DirectEventProcessorSpec.withoutBackoff()) - testContext.serviceMock.respondToEvents(with: HTTPURLResponse.StatusCodes.badRequest) - testContext.eventProcessor.record(CustomEvent(key: "malformed", context: testContext.context)) + testContext.serviceMock.eventStatusForPost = { post in [1: 413, 2: 400][post] ?? 202 } + testContext.eventProcessor.record(CustomEvent(key: "refused", context: testContext.context)) testContext.eventProcessor.isOnline = true expect(testContext.flushOutcome()) == false - expect(testContext.serviceMock.publishedEventPayloads.map(customKeys)) == [["malformed"], ["malformed"]] - expect(Set(testContext.serviceMock.publishedPayloadIds).count) == 1 expect(testContext.store.pendingBatches()).to(beEmpty()) - waitLettingTimersFire(0.3) - expect(testContext.serviceMock.publishEventDataCallCount) == 2 - testContext.serviceMock.stubEventResponse(success: true) - - // What was recorded since goes out alone, and that flush is the first a caller waits on since the loss, - // so it hears about it. testContext.eventProcessor.record(CustomEvent(key: "next", context: testContext.context)) expect(testContext.flushOutcome()) == false - expect(testContext.serviceMock.publishedEventPayloads.dropFirst(2).map(customKeys)) == [["next"]] - expect(testContext.flushOutcome()) == true + expect(testContext.serviceMock.publishedEventPayloads.map(customKeys)) == [["refused"], ["refused"], ["next"]] + } + + // The SDK's own backoff: a refusal such as a 401 would wait eight and a half minutes before the second post. + it("retries a batch refused with HTTP 413 a second later, as it does a 400") { + testContext = TestContext() + testContext.serviceMock.respondToEvents(with: HTTPURLResponse.StatusCodes.contentTooLarge) + testContext.eventProcessor.record(CustomEvent(key: "too-large", context: testContext.context)) + testContext.eventProcessor.isOnline = true + expect(testContext.flushOutcome()) == false + expect(testContext.serviceMock.publishEventDataCallCount) == 2 + expect(testContext.store.pendingBatches()).to(beEmpty()) } - it("drops a batch only for two 400s in a row") { + it("drops a batch only for two refusals of the payload in a row") { testContext = TestContext(retryState: DirectEventProcessorSpec.withoutBackoff()) testContext.serviceMock.eventStatusForPost = { post in [1: 400, 2: 503, 3: 400][post] ?? 202 } testContext.eventProcessor.record(CustomEvent(key: "kept", context: testContext.context)) From 68ce4c4757582cf3db5d3d645ec46432da159bfa Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 27 Oct 2026 12:29:47 -0700 Subject: [PATCH 103/158] Name and list every file in the event directory in EventDirectory EventDirectory says it decides what the files in the directory are called and how they are listed, but the log's two names, pending- and new-, were DiskEventLog's, which also built them and listed the logs recovery takes. The names are EventDirectory's now, beside ready-, and so is that listing; DiskEventLog takes a new log's name, and the one it is opened under, from there. Co-authored-by: Cursor --- .../ServiceObjects/DiskEventLog.swift | 44 +++++-------------- .../ServiceObjects/EventDirectory.swift | 31 +++++++++++-- .../ServiceObjects/EventStoreSpec.swift | 12 ++--- LaunchDarkly/LaunchDarklyTests/TestUtil.swift | 2 +- 4 files changed, 44 insertions(+), 45 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift index ad7c21165..0844aefb9 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift @@ -11,9 +11,9 @@ import Musl /// The file of events on disk a store appends to, not to be confused with diagnostic logging. /// -/// One per store rather than one per directory, so that only one writer ever appends to a given log. Each is named for a -/// UUID picked when it is opened, which is what keeps two writers apart: nothing about it comes from the application, and -/// no two processes, or two stores in one, ever pick the same one. +/// One per store rather than one per directory, so that only one writer ever appends to a given log. Each is named, by +/// `EventDirectory` as every file there is, for a UUID picked when it is opened, which is what keeps two writers apart: +/// nothing about it comes from the application, and no two processes, or two stores in one, ever pick the same one. /// /// A log nobody is appending to belongs to no one alive, whoever wrote it, and recovery takes it. The logs open in this /// process are remembered, which is all it takes where only this process uses the directory. On a Mac, which runs a @@ -28,13 +28,6 @@ import Musl /// Belongs to an `EventStore`, which makes every call holding its `ioLock`. The lock is handed in so that each call can /// check it is held, rather than this taking a lock of its own that would have to be ordered against the store's. final class DiskEventLog { - /// Names a log whose events are not yet in a batch: one its owner is appending to, or one left by an owner that is - /// gone, for recovery to close off. The rest of the name is the UUID it was opened under. - static let prefix = "pending-" - /// Names a log while it is being opened, before it has its header. Recovery never looks at these; one left by a - /// process that died opening it holds no events, and expiry removes it. - static let openingPrefix = "new-" - /// The paths of the logs open in this process, which recovery and expiry leave alone. Guarded by /// `openInThisProcessLock`. private static var openInThisProcess: Set = [] @@ -66,11 +59,7 @@ final class DiskEventLog { self.ioLock = ioLock self.logger = logger self.writeLog = writeLog - self.url = DiskEventLog.newUrl(in: directory) - } - - private static func newUrl(in directory: URL) -> URL { - directory.appendingPathComponent(prefix + UUID().uuidString) + self.url = EventDirectory.newLogUrl(in: directory) } /// Whether some live process is appending to a log, and recovery and expiry must leave it alone. @@ -157,19 +146,6 @@ final class DiskEventLog { DiskEventLog.forget(url) } - /// Every log no store in this process has open, for recovery to claim with `claimForRecovery(_:)`. - /// - /// None of them becomes open here later: a log is opened under a name of its own and never reopened once closed. - /// - /// Requires `ioLock`. - func logsNotOpenInThisProcess() -> [URL] { - ioLock.assertOwned() - let contents = (try? FileManager.default.contentsOfDirectory(at: directory, - includingPropertiesForKeys: nil, - options: [.skipsHiddenFiles])) ?? [] - return contents.filter { $0.lastPathComponent.hasPrefix(DiskEventLog.prefix) && !DiskEventLog.isOpenInThisProcess($0) } - } - /// Claims a log whose writer is gone, for recovery to read and then move or remove; nil where a live process holds /// it. Needs no lock, so that recovery can read the log without holding `ioLock`. /// @@ -217,7 +193,7 @@ final class DiskEventLog { if FileManager.default.fileExists(atPath: url.path) { // Left by an earlier open this session and closed since, so no longer this store's to write. - url = DiskEventLog.newUrl(in: directory) + url = EventDirectory.newLogUrl(in: directory) } guard let opened = open() else { return nil } @@ -232,12 +208,11 @@ final class DiskEventLog { return false } - /// Opens a new log: created under `openingPrefix`, locked where logs are, given its header, and only then moved to - /// the name recovery looks for. Requires `ioLock`. + /// Opens a new log: created under `EventDirectory.openingLogPrefix`, locked where logs are, given its header, and + /// only then moved to the name recovery looks for. Requires `ioLock`. private func open() -> EventLogFile? { ioLock.assertOwned() - let opening = directory.appendingPathComponent( - DiskEventLog.openingPrefix + url.lastPathComponent.dropFirst(DiskEventLog.prefix.count)) + let opening = EventDirectory.openingUrl(for: url) // O_APPEND is what makes each write land at the end of the file as one step, so that a commit racing a reader // cannot produce a spliced frame. let flags = O_WRONLY | O_APPEND | O_CREAT | O_EXCL @@ -288,7 +263,8 @@ final class DiskEventLog { return EventLogFile.open(at: url, flags: flags) } - private static func isOpenInThisProcess(_ log: URL) -> Bool { + /// Whether a store in this process has `log` open, which recovery and expiry leave alone. + static func isOpenInThisProcess(_ log: URL) -> Bool { openInThisProcessLock.withLock { openInThisProcess.contains(log.standardizedFileURL.path) } } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index b97809f11..087e11e8b 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -43,6 +43,23 @@ final class EventDirectory { /// A batch is never appended to once it is closed, so the count is as good as the file is. private static let batchPrefix = "ready-" private static let nameFieldSeparator: Character = "." + /// Names a log whose events are not yet in a batch: one its owner is appending to, or one left by an owner that is + /// gone, for recovery to close off. The rest of the name is the UUID it was opened under. + static let logPrefix = "pending-" + /// Names a log while it is being opened, before it has its header. Recovery never looks at these; one left by a + /// process that died opening it holds no events, and expiry removes it. + static let openingLogPrefix = "new-" + + /// A name for a new log in `directory`, under a UUID no other writer picks. + static func newLogUrl(in directory: URL) -> URL { + directory.appendingPathComponent(logPrefix + UUID().uuidString) + } + + /// Where `log` is created and given its header, before it is moved to its own name. + static func openingUrl(for log: URL) -> URL { + log.deletingLastPathComponent() + .appendingPathComponent(openingLogPrefix + log.lastPathComponent.dropFirst(logPrefix.count)) + } private let locateDirectory: () -> URL private let ioLock: UnfairLock @@ -578,12 +595,18 @@ final class EventDirectory { /// process has open, of which `DiskEventLog.claimForRecovery(_:)` then takes those no live process holds. /// /// The events in them were recorded by a store that is gone, so there is no one left to add to them: a previous run - /// of this process, or another process that has since ended, whatever it was called. + /// of this process, or another process that has since ended, whatever it was called. None of them becomes open here + /// later: a log is opened under a name of its own and never reopened once closed. /// /// Requires `ioLock`. func interruptedLogs() -> [URL] { ioLock.assertOwned() - return disk.log.logsNotOpenInThisProcess() + let contents = (try? FileManager.default.contentsOfDirectory(at: disk.directory, + includingPropertiesForKeys: nil, + options: [.skipsHiddenFiles])) ?? [] + return contents.filter { + $0.lastPathComponent.hasPrefix(EventDirectory.logPrefix) && !DiskEventLog.isOpenInThisProcess($0) + } } /// Reads and counts a log recovery has claimed. @@ -772,12 +795,12 @@ extension EventDirectory { /// held. Anything else is left alone. private static func discardIfAbandonedLog(_ file: URL) -> Int { let name = file.lastPathComponent - if name.hasPrefix(DiskEventLog.openingPrefix) { + if name.hasPrefix(EventDirectory.openingLogPrefix) { // Opening a log takes moments, so one this old was never finished, and holds no events. delete(file) return 0 } - guard name.hasPrefix(DiskEventLog.prefix), !DiskEventLog.isHeldByLiveProcess(file) + guard name.hasPrefix(EventDirectory.logPrefix), !DiskEventLog.isHeldByLiveProcess(file) else { return 0 } let held = eventsReading(file) return delete(file) ? held : 0 diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 43f7faf78..43ed80e00 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -184,11 +184,11 @@ final class EventStoreSpec: QuickSpec { app.commit() let logs = try FileManager.default.contentsOfDirectory(atPath: app.directory!.path) - .filter { $0.hasPrefix(DiskEventLog.prefix) } + .filter { $0.hasPrefix(EventDirectory.logPrefix) } expect(logs.count) == 1 - expect(logs.first.flatMap { UUID(uuidString: String($0.dropFirst(DiskEventLog.prefix.count))) }).toNot(beNil()) + expect(logs.first.flatMap { UUID(uuidString: String($0.dropFirst(EventDirectory.logPrefix.count))) }).toNot(beNil()) expect(try FileManager.default.contentsOfDirectory(atPath: app.directory!.path) - .filter { $0.hasPrefix(DiskEventLog.openingPrefix) }).to(beEmpty()) + .filter { $0.hasPrefix(EventDirectory.openingLogPrefix) }).to(beEmpty()) } } } @@ -499,7 +499,7 @@ final class EventStoreSpec: QuickSpec { let reading = DispatchSemaphore(value: 0) let release = DispatchSemaphore(value: 0) let next = store.nextRun(readFile: { url in - if url.lastPathComponent.hasPrefix(DiskEventLog.prefix) { + if url.lastPathComponent.hasPrefix(EventDirectory.logPrefix) { reading.signal() _ = release.wait(timeout: .now() + 5) } @@ -976,7 +976,7 @@ final class EventStoreSpec: QuickSpec { expect(store.holdsUnclosedEvents) == false expect(store.pendingBatches()) == [batch] let logs = try FileManager.default.contentsOfDirectory(atPath: directory.path) - .filter { $0.hasPrefix(DiskEventLog.prefix) } + .filter { $0.hasPrefix(EventDirectory.logPrefix) } expect(logs).to(beEmpty()) } } @@ -1853,7 +1853,7 @@ final class EventStoreSpec: QuickSpec { it("discards a log a process died while opening, which holds no events") { try FileManager.default.createDirectory(at: store.directory!, withIntermediateDirectories: true) - let opening = store.directory!.appendingPathComponent(DiskEventLog.openingPrefix + UUID().uuidString) + let opening = store.directory!.appendingPathComponent(EventDirectory.openingLogPrefix + UUID().uuidString) try EventLogFormat.fileHeader.write(to: opening) try EventStoreSpec.age(opening, byDays: 8) diff --git a/LaunchDarkly/LaunchDarklyTests/TestUtil.swift b/LaunchDarkly/LaunchDarklyTests/TestUtil.swift index 7217c7797..9d533c5ae 100644 --- a/LaunchDarkly/LaunchDarklyTests/TestUtil.swift +++ b/LaunchDarkly/LaunchDarklyTests/TestUtil.swift @@ -123,7 +123,7 @@ extension EventStore { var payloadsOnDisk: [Data] { let files = ((try? FileManager.default.contentsOfDirectory(atPath: directory!.path)) ?? []).sorted() let batches = files.filter { $0.hasPrefix("ready-") } - let logs = files.filter { $0.hasPrefix(DiskEventLog.prefix) } + let logs = files.filter { $0.hasPrefix(EventDirectory.logPrefix) } var payloads: [Data] = [] for name in batches + logs { guard let file = try? Data(contentsOf: directory!.appendingPathComponent(name)) From 24ba5b206a839cec76f1dab73585645ba282aa48 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 27 Oct 2026 15:36:01 -0700 Subject: [PATCH 104/158] test: pace the lister so closes are not starved of the store's lock The spec listed in a tight loop with a yield between listings. The locks are not fair, and a yield returns at once when the closing thread is blocked on a lock rather than runnable, so the lister took the lock straight back. On a busy runner the closes can starve, as they did on Android CI. A 200 microsecond pause lets each close in while a listing still overlaps it. Co-authored-by: Cursor --- .../LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 43ed80e00..d288ce147 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -1480,9 +1480,10 @@ final class EventStoreSpec: QuickSpec { DispatchQueue.global().async { while !doneLock.withLock({ done }) { _ = listing.pendingBatches() - // The locks are not fair, and without this the listings can hold them so continuously that - // the closes starve. - sched_yield() + // The locks are not fair, and without a pause the listings can hold them so continuously + // that the closes starve. A yield is not enough: it returns at once while the closing thread + // is blocked on a lock rather than runnable. + usleep(200) } listerFinished.signal() } From 8319688f367a4411a50c1c55a684886ea492fd62 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 27 Oct 2026 15:36:05 -0700 Subject: [PATCH 105/158] Drop the part of EventDirectory's comment about EventStore's concerns Co-authored-by: Cursor --- .../LaunchDarkly/ServiceObjects/EventDirectory.swift | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index 087e11e8b..1e065ff65 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -20,9 +20,8 @@ struct EventTally: Equatable { /// been closed off for delivery. /// /// Everything about the filesystem lives here -- where the directory is, what the files in it are called, and how they -/// are listed, read, moved and deleted -- so that `EventStore` is left with the buffer, the capacity arithmetic, and -/// the batches it holds in memory, none of which need a disk at all. A store the application told to keep events off -/// the disk has no `EventDirectory`, rather than one it is careful never to use. +/// are listed, read, moved and deleted. A store the application told to keep events off the disk has no +/// `EventDirectory`, rather than one it is careful never to use. /// /// Where the directory is, is worked out on first use rather than at construction, so that a store which never reaches /// for the disk makes no filesystem call at all, not even to find out where it would have written. From 5028134db55f5e8b40aeb6e50386008062440081 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 27 Oct 2026 15:37:45 -0700 Subject: [PATCH 106/158] fix: count a batch read to count it by the bytes that were read A listing that read a batch named without a count sized it from the size the directory listing came with, and with none and no earlier size to fall back on, counted it as 0 bytes against the byte capacity. It now takes the bytes it read, and failing a size of any kind, a header's six bytes, as on Android. The sizes are fetched with the listing, so unlike on Android a file gone after the read still has one, and the new spec passes without the fix too; it guards that a batch read to count it is sized. Co-authored-by: Cursor --- .../ServiceObjects/EventDirectory.swift | 16 +++++++++++----- .../ServiceObjects/EventStoreSpec.swift | 18 ++++++++++++++++++ 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index 1e065ff65..d72f58c2d 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -377,13 +377,18 @@ final class EventDirectory { else { return nil } // Reading is only for a batch whose name does not count it, which a version that named them without a // count may have left behind, and then only the once. - guard let events = counted ?? stored[payloadId]?.events ?? countOfEvents(in: file) + let read = counted == nil && stored[payloadId] == nil ? countOfEvents(in: file) : nil + guard let events = counted ?? stored[payloadId]?.events ?? read?.events else { return nil } let listed = EventDirectory.listed(file) let modified = listed.modified ?? Date.distantPast // What the close knew of its summaries is kept; nothing in the file says. let previous = stored[payloadId] - stored[payloadId] = Stored(url: file, events: events, bytes: listed.bytes ?? previous?.bytes ?? 0, + // A batch just read is as big as what was read. The size the listing came with can be missing, and no + // batch is empty: it has a header at least. + let bytes = read?.bytes ?? listed.bytes.flatMap { $0 > 0 ? $0 : nil } + ?? previous?.bytes ?? EventLogFormat.fileHeaderSize + stored[payloadId] = Stored(url: file, events: events, bytes: bytes, modified: modified, summaries: previous.map { $0.events == events ? $0.summaries : 0 } ?? 0) return (EventBatch(payloadId: payloadId, eventCount: events), modified) @@ -548,10 +553,11 @@ final class EventDirectory { } } - /// How many events a batch file holds, or nil when it is not one to list this time. + /// How many events a batch file holds, and how many bytes were read to count them, or nil when it is not one to + /// list this time. /// /// Requires `ioLock`. - private func countOfEvents(in file: URL) -> Int? { + private func countOfEvents(in file: URL) -> EventTally? { ioLock.assertOwned() switch read(file) { case .missing: @@ -570,7 +576,7 @@ final class EventDirectory { try? FileManager.default.removeItem(at: file) return nil } - return events + return EventTally(events: events, bytes: log.count) } } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index d288ce147..8c4fb08f2 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -1663,6 +1663,24 @@ final class EventStoreSpec: QuickSpec { expect(store.hasRoomForEventBytes) == true } + it("counts the bytes of a batch it read to count from what it read") { + store = EventStore.temporary(capacity: 100, byteCapacity: 1000) + for index in 1...8 { + _ = store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-\(index)")) + } + _ = store.closeBatch() + try EventStoreSpec.asUncountedName(EventStoreSpec.onlyBatchFile(in: store)) + + // Gone once it has been read, so that only what was read, or what the listing saw, can size it. + let reader = EventStore.sharing(store.directory!, byteCapacity: 1000, readFile: { url in + let read = try Data(contentsOf: url) + try? FileManager.default.removeItem(at: url) + return read + }) + expect(reader.pendingBatches().map { $0.eventCount }) == [8] + expect(reader.hasRoomForEventBytes) == false + } + it("bounds the batches it holds in memory once it has given up on the disk") { store = EventStore.temporary(capacity: 100, byteCapacity: 1000, writeLog: { _, _ in ENOSPC }) for index in 1...8 { From 0083526d32c264379f818606e32d34c3da6dc89f Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 27 Oct 2026 18:41:42 -0700 Subject: [PATCH 107/158] fix: never read a stored file larger than this version writes A file past twice the batch size is not one this version wrote, and read whole it could take the application's memory with it. Such a file is never read: a batch is unreadable and dropped as lost, a log a previous run left is deleted, and the listing and expiry count it by its name, or as one event where the name has none. Co-authored-by: Cursor --- .../ServiceObjects/EventDirectory.swift | 34 ++++++++++- .../ServiceObjects/EventStoreSpec.swift | 57 +++++++++++++++++++ 2 files changed, 89 insertions(+), 2 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index d72f58c2d..e2aedd133 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -60,6 +60,12 @@ final class EventDirectory { .appendingPathComponent(openingLogPrefix + log.lastPathComponent.dropFirst(logPrefix.count)) } + /// The largest file this version reads. It writes none larger than `EventLogFormat.maxBatchSize`, so one past this + /// is not its own, whatever it holds. Read anyway, it would be held in memory whole, and one large enough would + /// take the application down with it. Twice the batch size, so that a later version may raise its own a little + /// without an earlier one losing what it wrote. + static let maxReadSize = 2 * EventLogFormat.maxBatchSize + private let locateDirectory: () -> URL private let ioLock: UnfairLock private let logger: OSLog @@ -475,6 +481,7 @@ final class EventDirectory { else { return nil } throw VanishedBatch(payloadId: batch.payloadId) case .failed(let error): throw error + case .tooLarge: throw UnreadableBatch(payloadId: batch.payloadId) case .bytes(let bytes): file = bytes } // No more than the batch was closed holding. A write that failed and could not be truncated leaves whole frames @@ -523,7 +530,7 @@ final class EventDirectory { var payloads: [Data] = [] for log in logs { - guard let file = try? Data(contentsOf: log) + guard case .bytes(let file) = read(log) else { continue } _ = EventLogFormat.forEachFrame(in: file) { type, payload in if type == EventLogFormat.eventFrame { @@ -543,9 +550,14 @@ final class EventDirectory { case bytes(Data) case missing case failed(Error) + /// Larger than `maxReadSize`, and so never read: it can never be delivered. + case tooLarge(size: Int) } private func read(_ url: URL) -> LogRead { + if let size = EventDirectory.sizeIfTooLarge(url) { + return .tooLarge(size: size) + } do { return .bytes(try readFile(url)) } catch { @@ -553,6 +565,15 @@ final class EventDirectory { } } + /// The size of a file larger than `maxReadSize`, or nil for any other, including one whose size could not be + /// found, which the read then fails on or reads. + private static func sizeIfTooLarge(_ url: URL) -> Int? { + guard let size = (try? FileManager.default.attributesOfItem(atPath: url.path))?[.size] as? Int, + size > maxReadSize + else { return nil } + return size + } + /// How many events a batch file holds, and how many bytes were read to count them, or nil when it is not one to /// list this time. /// @@ -568,6 +589,10 @@ final class EventDirectory { // intact all along. Without a count it cannot be delivered now. os_log("%s could not read stored events, will try again at the next delivery: %s", log: logger, type: .info, typeName(and: #function), String(describing: error)) return nil + case .tooLarge(let size): + // Never read, so never counted. Listed as holding one event, the fewest it can, so that the delivery + // reaching it drops it and counts the loss. + return EventTally(events: 1, bytes: size) case .bytes(let log): guard let events = EventLogFormat.eventCount(in: log) else { @@ -625,6 +650,8 @@ final class EventDirectory { return InterruptedLog(url: leftOpen, contents: .missing) case .failed(let error): return InterruptedLog(url: leftOpen, contents: .failed(error)) + case .tooLarge: + return InterruptedLog(url: leftOpen, contents: .unreadable) case .bytes(let log): guard let events = EventLogFormat.eventCount(in: log) else { return InterruptedLog(url: leftOpen, contents: .unreadable) } @@ -812,7 +839,10 @@ extension EventDirectory { } private static func eventsReading(_ file: URL) -> Int { - (try? Data(contentsOf: file)).flatMap { EventLogFormat.eventCount(in: $0) } ?? 0 + // Never read, so held to be the one event it holds at the fewest. + guard sizeIfTooLarge(file) == nil + else { return 1 } + return (try? Data(contentsOf: file)).flatMap { EventLogFormat.eventCount(in: $0) } ?? 0 } @discardableResult diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 8c4fb08f2..5d177efda 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -750,6 +750,63 @@ final class EventStoreSpec: QuickSpec { expect(FileManager.default.fileExists(atPath: batchUrl.path)) == false } } + + describe("a file larger than this version reads") { + var store: EventStore! + beforeEach { + store = EventStore.temporary() + } + afterEach { + store.deleteEverything() + } + + it("is deleted where a previous run left it as a log") { + let log = try EventStoreSpec.tooLargeFile(EventDirectory.logPrefix + UUID().uuidString, in: store) + + store.preparePayloadsFromInterruptedLogs() + + expect(store.pendingBatches()).to(beEmpty()) + expect(FileManager.default.fileExists(atPath: log.path)) == false + } + + it("is listed by the count in its name, and unreadable") { + try EventStoreSpec.tooLargeFile("ready-\(UUID().uuidString).3", in: store) + + let batches = store.pendingBatches() + expect(batches.map { $0.eventCount }) == [3] + expect { try store.body(of: batches[0]) }.to(throwError(errorType: UnreadableBatch.self)) + } + + it("is listed as one event, the fewest it can hold, where its name has no count, and unreadable") { + try EventStoreSpec.tooLargeFile("ready-\(UUID().uuidString)", in: store) + + let batches = store.pendingBatches() + expect(batches.map { $0.eventCount }) == [1] + expect { try store.body(of: batches[0]) }.to(throwError(errorType: UnreadableBatch.self)) + } + + it("is discarded as one event once it expires, where nothing counts it") { + let uncounted = try EventStoreSpec.tooLargeFile("ready-\(UUID().uuidString)", in: store) + let log = try EventStoreSpec.tooLargeFile(EventDirectory.logPrefix + UUID().uuidString, in: store) + try EventStoreSpec.age(uncounted, byDays: 8) + try EventStoreSpec.age(log, byDays: 8) + + expect(store.discardExpired()) == 2 + expect(FileManager.default.fileExists(atPath: uncounted.path)) == false + expect(FileManager.default.fileExists(atPath: log.path)) == false + } + } + } + + /// A file in the store's directory one byte past the largest this version reads, starting as a log does. + @discardableResult + private static func tooLargeFile(_ name: String, in store: EventStore) throws -> URL { + try FileManager.default.createDirectory(at: store.directory!, withIntermediateDirectories: true) + let file = store.directory!.appendingPathComponent(name) + var contents = EventLogFormat.fileHeader + contents.append(Data(count: EventDirectory.maxReadSize + 1 - contents.count)) + try contents.write(to: file) + return file } private func batchSpec() { From 8676b9f3e71fd1b55da88e30f63a8003fb929c49 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 27 Oct 2026 19:04:23 -0700 Subject: [PATCH 108/158] feat: tell the event processor when a flag request starts and settles The flag synchronizer reports, to a listener the client sets, the start of its first flag request after going online and how it ended: loaded, failed, or stopped by going offline before it answered. A poll a fresh cache puts off makes no request at launch and reports nothing. Later answers and periodic polls report nothing either. Adds FlagTraffic, which the processor uses to give way to these requests. Groundwork for the launch gate and pacing (spec D16-D20). Co-authored-by: Cursor --- LaunchDarkly.xcodeproj/project.pbxproj | 10 ++ .../ServiceObjects/FlagSynchronizer.swift | 53 +++++- .../ServiceObjects/FlagTraffic.swift | 158 ++++++++++++++++++ .../ServiceObjects/FlagSynchronizerSpec.swift | 81 +++++++++ 4 files changed, 297 insertions(+), 5 deletions(-) create mode 100644 LaunchDarkly/LaunchDarkly/ServiceObjects/FlagTraffic.swift diff --git a/LaunchDarkly.xcodeproj/project.pbxproj b/LaunchDarkly.xcodeproj/project.pbxproj index 82a32c4d2..5ca0b5b49 100644 --- a/LaunchDarkly.xcodeproj/project.pbxproj +++ b/LaunchDarkly.xcodeproj/project.pbxproj @@ -21,6 +21,7 @@ A1B2C3D4E5F60718293A4B01 /* EventDirectory.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D4E5F60718293A4B5C /* EventDirectory.swift */; }; 3C55384C9E3485B493E1F86A /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; C3F7E502BD916C40AE238F64 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; + 713C1C0A71192DA31CD28E1F /* FlagTraffic.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34ED898C31FA1DC1E35C8266 /* FlagTraffic.swift */; }; AB4827D11084D58661E24B05 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; 3D2406142E0D90E000F91253 /* Plugin.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3D2406102E0D90E000F91253 /* Plugin.swift */; }; 3D2406152E0D90E000F91253 /* PluginMetadata.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3D2406112E0D90E000F91253 /* PluginMetadata.swift */; }; @@ -58,6 +59,7 @@ A1B2C3D4E5F60718293A4B02 /* EventDirectory.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D4E5F60718293A4B5C /* EventDirectory.swift */; }; DDA788E3056F8AC3CA203933 /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; D4080613CEA27D51BF349075 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; + 45B28E8795FFF24BDA23A37C /* FlagTraffic.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34ED898C31FA1DC1E35C8266 /* FlagTraffic.swift */; }; ECBE45806B470ACC5187B7D6 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; 7185A3C30E7FFC4F725769F1 /* BackgroundActivity.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9A771B316E19ED598CD4FB0B /* BackgroundActivity.swift */; }; 830BF933202D188E006DF9B1 /* HTTPURLRequest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 830BF932202D188E006DF9B1 /* HTTPURLRequest.swift */; }; @@ -412,6 +414,7 @@ A1B2C3D4E5F60718293A4B03 /* EventDirectory.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D4E5F60718293A4B5C /* EventDirectory.swift */; }; 44E0BE449714952A68628CEB /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; E5191724DFB38E62C0451086 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; + 8C5CDD837AA4B41BF2D94268 /* FlagTraffic.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34ED898C31FA1DC1E35C8266 /* FlagTraffic.swift */; }; CA2ADB3D20A229D70CA34F73 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; B40B419C249ADA6B00CD0726 /* DiagnosticCacheSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = B40B419B249ADA6B00CD0726 /* DiagnosticCacheSpec.swift */; }; B4265EB124E7390C001CFD2C /* TestUtil.swift in Sources */ = {isa = PBXBuildFile; fileRef = B4265EB024E7390C001CFD2C /* TestUtil.swift */; }; @@ -468,6 +471,7 @@ A1B2C3D4E5F60718293A4B04 /* EventDirectory.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D4E5F60718293A4B5C /* EventDirectory.swift */; }; F0F504E5EC32223905356098 /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; B2E6D4F1AC805B3F9D127E53 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; + CF5053295713576276009C0F /* FlagTraffic.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34ED898C31FA1DC1E35C8266 /* FlagTraffic.swift */; }; 625ADAC0CCFE3BE324C45752 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; D7A500FCA04E3412F931003F /* ContextEncodingCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = D93B7F5F6CF5C3DE79FB4286 /* ContextEncodingCache.swift */; }; DAF5695D58E28B8CF9221213 /* LDContextJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88DAB8D1864C14D9C7E06F11 /* LDContextJSONWriter.swift */; }; @@ -703,6 +707,7 @@ A1B2C3D4E5F60718293A4B5C /* EventDirectory.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventDirectory.swift; sourceTree = ""; }; 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventLogFile.swift; sourceTree = ""; }; A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = DeliveryClaims.swift; sourceTree = ""; }; + 34ED898C31FA1DC1E35C8266 /* FlagTraffic.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = FlagTraffic.swift; sourceTree = ""; }; B58551BB165F28784C785720 /* EventLogFormat.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventLogFormat.swift; sourceTree = ""; }; C408884623033B3600420721 /* ConnectionInformationStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConnectionInformationStore.swift; sourceTree = ""; }; C408884823033B7500420721 /* ConnectionInformation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConnectionInformation.swift; sourceTree = ""; }; @@ -1098,6 +1103,7 @@ A1B2C3D4E5F60718293A4B5C /* EventDirectory.swift */, 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */, A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */, + 34ED898C31FA1DC1E35C8266 /* FlagTraffic.swift */, B58551BB165F28784C785720 /* EventLogFormat.swift */, ); path = ServiceObjects; @@ -1635,6 +1641,7 @@ A1B2C3D4E5F60718293A4B04 /* EventDirectory.swift in Sources */, F0F504E5EC32223905356098 /* EventLogFile.swift in Sources */, B2E6D4F1AC805B3F9D127E53 /* DeliveryClaims.swift in Sources */, + CF5053295713576276009C0F /* FlagTraffic.swift in Sources */, 625ADAC0CCFE3BE324C45752 /* EventLogFormat.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; @@ -1740,6 +1747,7 @@ A1B2C3D4E5F60718293A4B01 /* EventDirectory.swift in Sources */, 3C55384C9E3485B493E1F86A /* EventLogFile.swift in Sources */, C3F7E502BD916C40AE238F64 /* DeliveryClaims.swift in Sources */, + 713C1C0A71192DA31CD28E1F /* FlagTraffic.swift in Sources */, AB4827D11084D58661E24B05 /* EventLogFormat.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; @@ -1844,6 +1852,7 @@ A1B2C3D4E5F60718293A4B02 /* EventDirectory.swift in Sources */, DDA788E3056F8AC3CA203933 /* EventLogFile.swift in Sources */, D4080613CEA27D51BF349075 /* DeliveryClaims.swift in Sources */, + 45B28E8795FFF24BDA23A37C /* FlagTraffic.swift in Sources */, ECBE45806B470ACC5187B7D6 /* EventLogFormat.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; @@ -2015,6 +2024,7 @@ A1B2C3D4E5F60718293A4B03 /* EventDirectory.swift in Sources */, 44E0BE449714952A68628CEB /* EventLogFile.swift in Sources */, E5191724DFB38E62C0451086 /* DeliveryClaims.swift in Sources */, + 8C5CDD837AA4B41BF2D94268 /* FlagTraffic.swift in Sources */, CA2ADB3D20A229D70CA34F73 /* EventLogFormat.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/FlagSynchronizer.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/FlagSynchronizer.swift index 8fb10d36a..a9e58a361 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/FlagSynchronizer.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/FlagSynchronizer.swift @@ -99,6 +99,14 @@ class FlagSynchronizer: LDFlagSynchronizing, EventHandler { private var lastCachedRequestedTime: Date? private var syncQueue = DispatchQueue(label: Constants.queueName, qos: .utility) + + /// Told when the first flag request after going online starts and how it ends, so that events can give way to it. + /// Set once the client exists, as `onSyncComplete` is. + weak var flagRequests: FlagRequestListening? + private let flagRequestLock = UnfairLock() + /// Whether that request has started and not yet been told settled. Only to be used while holding `flagRequestLock`. + private var flagRequestInFlight = false + private var eventSourceStarted: Date? // Only accessed on the event source callback queue. private var connectedAt: Date? @@ -127,14 +135,38 @@ class FlagSynchronizer: LDFlagSynchronizing, EventHandler { switch streamingMode { case .streaming: stopPolling() + if eventSource == nil { + flagRequestStarted() + } startEventSource() case .polling: stopEventSource() + // A poll a fresh cache puts off makes no request now, and events do not wait for the later one. + if flagRequestTimer == nil && firstPollDelay == 0 { + flagRequestStarted() + } startPolling() } } else { stopEventSource() stopPolling() + flagRequestSettled(.stopped) + } + } + + private func flagRequestStarted() { + flagRequestLock.withLock { flagRequestInFlight = true } + flagRequests?.flagRequestStarted() + } + + /// Tells the listener how the first request ended, once; any answer after it, and every later poll, says nothing. + private func flagRequestSettled(_ outcome: FlagRequestOutcome) { + let wasInFlight = flagRequestLock.withLock { () -> Bool in + defer { flagRequestInFlight = false } + return flagRequestInFlight + } + if wasInFlight { + flagRequests?.flagRequestSettled(outcome) } } @@ -192,10 +224,8 @@ class FlagSynchronizer: LDFlagSynchronizing, EventHandler { return } - var initialDelay: TimeInterval = 0 - if let lastTime = self.lastCachedRequestedTime { - // The cache is still fresh, so delay the first poll until the cached flags would go stale. - initialDelay = max(0, lastTime.addingTimeInterval(pollingInterval).timeIntervalSinceNow) + let initialDelay = firstPollDelay + if self.lastCachedRequestedTime != nil { // We loaded cached flags, so report completion immediately rather than blocking on the first poll. syncQueue.async { [self] in guard isOnline @@ -207,6 +237,13 @@ class FlagSynchronizer: LDFlagSynchronizing, EventHandler { os_log("%s", log: service.config.logger, type: .debug, typeName(and: #function)) } + /// How long the first poll waits: while the cache is still fresh, until the cached flags would go stale. + private var firstPollDelay: TimeInterval { + guard let lastTime = lastCachedRequestedTime + else { return 0 } + return max(0, lastTime.addingTimeInterval(pollingInterval).timeIntervalSinceNow) + } + private func stopPolling() { guard flagRequestTimer != nil else { @@ -286,7 +323,10 @@ class FlagSynchronizer: LDFlagSynchronizing, EventHandler { } var failed = false var unexpected = false - defer { pollDidComplete(failed: failed, unexpected: unexpected) } + defer { + flagRequestSettled(failed ? .failed : .loaded) + pollDidComplete(failed: failed, unexpected: unexpected) + } if let serviceResponseError = serviceResponse.error { os_log("%s error: %s", log: service.config.logger, type: .debug, typeName(and: #function), String(describing: serviceResponseError)) @@ -354,6 +394,7 @@ class FlagSynchronizer: LDFlagSynchronizing, EventHandler { streamingRetry.recordFailure(unexpected: SynchronizingError.streamError(error).isTerminal) let delay = streamingRetry.nextDelay() os_log("%s stream error; reconnecting in %.3fs. error: %s", log: service.config.logger, type: .debug, typeName(and: #function), delay, String(describing: error)) + flagRequestSettled(.failed) reportSyncComplete(.error(.streamError(error))) // Tear down the failed connection now, then reconnect after the backoff. @@ -416,10 +457,12 @@ class FlagSynchronizer: LDFlagSynchronizing, EventHandler { guard let data = messageEvent.data.data(using: .utf8), let flagCollection = try? JSONDecoder().decode(FeatureFlagCollection.self, from: data) else { + flagRequestSettled(.failed) reportDataError(messageEvent.data.data(using: .utf8)) return } + flagRequestSettled(.loaded) // NOTE: If you are adding e-tag support through the streaming // connection, make sure you read the documentation on the // FeatureFlagCaching.saveCachedData method. diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/FlagTraffic.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/FlagTraffic.swift new file mode 100644 index 000000000..f710dd72a --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/FlagTraffic.swift @@ -0,0 +1,158 @@ +import Foundation + +/// Hears when the flag synchronizer's first request after going online starts and how it ended, so that events can +/// give way to it. +protocol FlagRequestListening: AnyObject { + func flagRequestStarted() + func flagRequestSettled(_ outcome: FlagRequestOutcome) +} + +enum FlagRequestOutcome { + case loaded + case failed + /// The synchronizer went offline, or was replaced, before an answer came. + case stopped +} + +/// Whether a flag request is in flight, and the waits events make for it. +/// +/// Flag values decide what the application shows, and events only report on it afterwards. So at launch, and whenever +/// the SDK sends a backlog on its own, events wait for the flag request rather than compete with it for the network. +final class FlagTraffic { + /// How long events give way. + struct Timings { + static let specified = Timings(launchFloor: 2, launchCap: 10, gap: 1, flagRequestWait: 10) + /// No waits at all, for tests about something else. + static let noWaits = Timings(launchFloor: 0, launchCap: 0, gap: 0, flagRequestWait: 0) + + /// The soonest after first going online that a previous run's events are sent, whatever the flag request is + /// doing. It gives the application's own startup traffic the network first, and is short enough for an + /// application that dies a few seconds after every launch to still send them. + let launchFloor: TimeInterval + /// The longest after first going online that a previous run's events wait for the first flag request: the + /// default connection timeout, by which a request that has not answered has most likely failed. + let launchCap: TimeInterval + /// The least time between one paced batch's answer and the next batch's request. + let gap: TimeInterval + /// The longest a paced batch waits for a flag request in flight, so that a hung one cannot stop delivery. + let flagRequestWait: TimeInterval + } + + /// One batch waiting for its turn. + private final class Turn { + let gapEnds: DispatchTime + let flagWait: TimeInterval + let isStopped: () -> Bool + let proceed: () -> Void + /// When it stops waiting on the flag request, set once the gap has passed with one in flight. Only to be used + /// while holding the lock. + var flagWaitEnds: DispatchTime? + + init(gapEnds: DispatchTime, flagWait: TimeInterval, isStopped: @escaping () -> Bool, proceed: @escaping () -> Void) { + self.gapEnds = gapEnds + self.flagWait = flagWait + self.isStopped = isStopped + self.proceed = proceed + } + } + + private let lock = UnfairLock() + // Only to be used while holding `lock`. + private var inFlight = false + /// How the last flag request ended, or nil if none has. + private var lastOutcome: FlagRequestOutcome? + private var turns: [Turn] = [] + + func started() { + lock.withLock { inFlight = true } + } + + func settled(_ outcome: FlagRequestOutcome) { + lock.withLock { + inFlight = false + lastOutcome = outcome + } + checkTurns() + } + + /// Has the batches waiting look at whether they were stopped again. + func wake() { + checkTurns() + } + + /// Why the launch gate may open, once its floor has passed, or nil while it waits on the flag request. + /// + /// - Parameters: + /// - capReached: whether the cap has passed, after which it waits on nothing + /// - cap: the cap, for saying so + func launchGateReason(capReached: Bool, cap: TimeInterval) -> String? { + lock.withLock { + guard inFlight + else { + switch lastOutcome { + case .loaded: return "flags loaded" + case .failed: return "the flag request failed" + case .stopped, nil: return "no flag request at launch" + } + } + return capReached ? "waiting \(Int(cap * 1000))ms for flags" : nil + } + } + + /// Calls `proceed`, on a global queue, once `gapEnds` has passed and then no flag request is in flight, waiting on + /// one for no longer than `flagWait`; or at once when `isStopped` says so. `isStopped` is asked outside this + /// object's lock, so it may take a lock of its own. + func awaitTurn(gapEnds: DispatchTime, flagWait: TimeInterval, isStopped: @escaping () -> Bool, then proceed: @escaping () -> Void) { + let turn = Turn(gapEnds: gapEnds, flagWait: flagWait, isStopped: isStopped, proceed: proceed) + lock.withLock { turns.append(turn) } + DispatchQueue.global().asyncAfter(deadline: gapEnds) { [weak self] in self?.checkTurns() } + checkTurns() + } + + private func checkTurns() { + let now = DispatchTime.now() + var due: [Turn] = [] + var waiting: [Turn] = [] + var deadlines: [DispatchTime] = [] + lock.withLock { + for turn in turns { + guard now >= turn.gapEnds + else { + waiting.append(turn) + continue + } + guard inFlight + else { + due.append(turn) + continue + } + if turn.flagWaitEnds == nil { + let ends = now + turn.flagWait + turn.flagWaitEnds = ends + deadlines.append(ends) + } + if let ends = turn.flagWaitEnds, now >= ends { + due.append(turn) + } else { + waiting.append(turn) + } + } + } + due += waiting.filter { $0.isStopped() } + for deadline in deadlines { + DispatchQueue.global().asyncAfter(deadline: deadline) { [weak self] in self?.checkTurns() } + } + for turn in due { + // Two checks can find the same turn due; only the one that takes it proceeds. + let taken = lock.withLock { () -> Bool in + guard let index = turns.firstIndex(where: { $0 === turn }) + else { return false } + turns.remove(at: index) + return true + } + if taken { + DispatchQueue.global().async(execute: turn.proceed) + } + } + } +} diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/FlagSynchronizerSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/FlagSynchronizerSpec.swift index 3534f20f8..be7f40833 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/FlagSynchronizerSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/FlagSynchronizerSpec.swift @@ -41,6 +41,87 @@ final class FlagSynchronizerSpec: QuickSpec { pollingTimerFiresSpec() pollingBackoffSpec() flagRequestSpec() + flagRequestReportingSpec() + } + + /// Records what a synchronizer tells the event processor about its flag requests. + final class FlagRequestRecorder: FlagRequestListening { + private let lock = NSLock() + private var _reports: [String] = [] + var reports: [String] { + lock.lock() + defer { lock.unlock() } + return _reports + } + + func flagRequestStarted() { + record("started") + } + + func flagRequestSettled(_ outcome: FlagRequestOutcome) { + record("settled \(outcome)") + } + + private func record(_ report: String) { + lock.lock() + defer { lock.unlock() } + _reports.append(report) + } + } + + func flagRequestReportingSpec() { + describe("reporting flag requests") { + it("reports the stream starting, and that its first payload loaded flags, once") { + let testContext = TestContext(streamingMode: .streaming, useReport: false) + let recorder = FlagRequestRecorder() + testContext.flagSynchronizer.flagRequests = recorder + + testContext.flagSynchronizer.isOnline = true + expect(recorder.reports) == ["started"] + testContext.flagSynchronizer.testStreamOnMessage(event: "put", messageEvent: MessageEvent(data: "{\"flag1\": {}}")) + testContext.flagSynchronizer.testStreamOnMessage(event: "put", messageEvent: MessageEvent(data: "{\"flag1\": {}}")) + + expect(recorder.reports) == ["started", "settled loaded"] + testContext.flagSynchronizer.isOnline = false + expect(recorder.reports) == ["started", "settled loaded"] + } + + it("reports a request going offline before it answered as stopped") { + let testContext = TestContext(streamingMode: .streaming, useReport: false) + let recorder = FlagRequestRecorder() + testContext.flagSynchronizer.flagRequests = recorder + + testContext.flagSynchronizer.isOnline = true + testContext.flagSynchronizer.isOnline = false + + expect(recorder.reports) == ["started", "settled stopped"] + } + + it("reports the first poll and that it failed") { + let testContext = TestContext(streamingMode: .polling, useReport: false) + testContext.serviceMock.stubFlagResponse(statusCode: HTTPURLResponse.StatusCodes.internalServerError, responseOnly: true) + let recorder = FlagRequestRecorder() + testContext.flagSynchronizer.flagRequests = recorder + + testContext.flagSynchronizer.isOnline = true + + expect(recorder.reports).toEventually(equal(["started", "settled failed"])) + testContext.flagSynchronizer.isOnline = false + } + + it("reports nothing for a poll a fresh cache puts off") { + let serviceMock = DarklyServiceMock() + let flagSynchronizer = FlagSynchronizer(streamingMode: .polling, pollingInterval: 60, useReport: false, + lastUpdated: Date(), service: serviceMock, onSyncComplete: nil) + let recorder = FlagRequestRecorder() + flagSynchronizer.flagRequests = recorder + + flagSynchronizer.isOnline = true + flagSynchronizer.isOnline = false + + expect(recorder.reports) == [] + } + } } func initSpec() { From 874c11aead46c71d943dbd0ab2d6c3eeed108750 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 27 Oct 2026 19:04:23 -0700 Subject: [PATCH 109/158] feat: give flag requests the network ahead of a backlog of events Where events persist: - What a previous run left waits for the launch gate: at least 2 s after first going online, and until the first flag request settles, there was none, or 10 s have passed. The diagnostic init event waits for it too. - Deliveries the reporter starts on its own (the launch delivery and the report timer) send one batch at a time, at least 1 s after the last answered, and hold while a flag request is in flight, for up to 10 s. - A flush the application asks for, close, and going offline are never held, and end the pacing of a delivery under way. - Recovery of a previous run's logs runs at utility QoS. At .disabled nothing changes. Spec D16-D20. Co-authored-by: Cursor --- LaunchDarkly/LaunchDarkly/LDClient.swift | 19 +- .../ServiceObjects/DiagnosticReporter.swift | 10 +- .../ServiceObjects/DirectEventProcessor.swift | 181 ++++++++++++++++-- .../DirectEventProcessorSpec.swift | 176 +++++++++++++++-- 4 files changed, 354 insertions(+), 32 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/LDClient.swift b/LaunchDarkly/LaunchDarkly/LDClient.swift index 7b1449a4c..e5bf1d14e 100644 --- a/LaunchDarkly/LaunchDarkly/LDClient.swift +++ b/LaunchDarkly/LaunchDarkly/LDClient.swift @@ -982,7 +982,7 @@ public class LDClient { let networkReporter: NetworkReporting private(set) var flagCache: FeatureFlagCaching - private(set) var flagSynchronizer: LDFlagSynchronizing + private(set) var flagSynchronizer: LDFlagSynchronizing { didSet { reportFlagRequests() } } var flagChangeNotifier: FlagChangeNotifying private(set) var eventProcessor: EventProcessor private(set) var environmentReporter: EnvironmentReporting @@ -1083,6 +1083,7 @@ public class LDClient { service.resetFlagResponseCache(etag: cachedData.etag) // Reports to this client, so it could not be given at construction: `self` is only usable from here. flagSynchronizer.onSyncComplete = onFlagSyncComplete + giveFlagRequestsTheNetworkFirst() if let cachedFlags = cachedData.items, !cachedFlags.isEmpty { flagStore.replaceStore(newStoredItems: cachedFlags) @@ -1103,6 +1104,22 @@ public class LDClient { extension LDClient: TypeIdentifying { } +extension LDClient { + /// Has the flag synchronizer tell the event processor about its flag requests, so that events can give way to them. + fileprivate func reportFlagRequests() { + (flagSynchronizer as? FlagSynchronizer)?.flagRequests = eventProcessor as? FlagRequestListening + } + + /// Has events give way to flag requests: the synchronizer reports its requests to the event processor, and the + /// diagnostic init event waits for the processor's launch gate as a previous run's events do. + fileprivate func giveFlagRequestsTheNetworkFirst() { + reportFlagRequests() + if let eventProcessor = eventProcessor as? DirectEventProcessor { + (diagnosticReporter as? DiagnosticReporter)?.sendsInitAfter = eventProcessor.whenLaunchGateOpens + } + } +} + #if DEBUG extension LDClient { func setRunMode(_ runMode: LDClientRunMode) { diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DiagnosticReporter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DiagnosticReporter.swift index 92bdb1aba..55f792018 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DiagnosticReporter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DiagnosticReporter.swift @@ -18,6 +18,9 @@ class DiagnosticReporter: DiagnosticReporting { private var sentInit: Bool private let stateQueue = DispatchQueue(label: "com.launchdarkly.diagnosticReporter.state", qos: .utility) private let workQueue = DispatchQueue(label: "com.launchdarkly.diagnosticReporter.work", qos: .utility) + /// Where set, what the init event waits for: the event processor's launch gate, so that it does not compete with the + /// first flag request. Set once the client has both. + var sendsInitAfter: ((@escaping () -> Void) -> Void)? init(service: DarklyServiceProvider, environmentReporting: EnvironmentReporting) { self.service = service @@ -43,7 +46,12 @@ class DiagnosticReporter: DiagnosticReporting { environmentReporting: environmentReporting, diagnosticId: cache.getDiagnosticId(), creationDate: Date().millisSince1970) - sendDiagnosticEventAsync(diagnosticEvent: initEvent) + let send = { self.sendDiagnosticEventAsync(diagnosticEvent: initEvent) } + if let sendsInitAfter { + sendsInitAfter(send) + } else { + send() + } } timer = LDTimer(withTimeInterval: service.config.diagnosticRecordingInterval, fireQueue: workQueue) { diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 264aa0593..3c5f98c34 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -21,7 +21,12 @@ import OSLog /// Whether a commit point runs on the caller's thread is the application's choice, through /// `LDConfig.eventPersistence`. Only at `.immediate` can `track` promise the event is on disk by the time it returns. /// At `.disabled` nothing commits early: the held run waits for the delivery, which encodes it. -class DirectEventProcessor: EventProcessor { +/// +/// Flag values come first. Where events persist, what a previous run left waits at launch for the first flag request, +/// and the deliveries this reporter starts on its own send one batch at a time, giving way to any flag request in +/// flight. The flag synchronizer says what it is doing through `FlagRequestListening`. A delivery the application asks +/// for is never held. +class DirectEventProcessor: EventProcessor, FlagRequestListening { /// How many full events may be held unencoded before one of them pays for a commit. /// /// It sets two things at once. The first is how many evaluations a termination can take -- not everything a crash @@ -62,6 +67,12 @@ class DirectEventProcessor: EventProcessor { /// Its summarizer is guarded by `recordLock`, and its encoders are reached only under `commitLock`. let eventBuffer: OutboundEventBuffer + private let flagTraffic = FlagTraffic() + private let flagTimings: FlagTraffic.Timings + /// Whether events give way to flag values: only where they persist, like the rest of what persistence brings. + /// Without it there is no previous run to hold back. + private let givesWayToFlags: Bool + // MARK: Guarded by commitLock /// Held for the whole of a commit, so that only one runs at a time. @@ -158,6 +169,13 @@ class DirectEventProcessor: EventProcessor { /// closing has turned `isOnline` off since. For its length only: the delivery one of its failures sets for the end /// of the wait is held by the backoff, and finds the reporter offline, as any other. private var isClosingDelivery = false + /// Whether the delivery queued was asked for only by the report timer or by the launch gate, and so is paced. + private var queuedDeliveryIsPaced = false + /// Whether the delivery running is paced: until a delivery somebody asked for is queued, or the reporter goes + /// offline, after which it sends the rest back to back. + private var isPacedDelivery = false + /// When the paced delivery running last had an answer for a post, or nil before its first. + private var lastPacedPostAnswered: DispatchTime? #if DEBUG /// How many deliveries have started, which is fewer than the flushes asked for where they were joined. @@ -202,12 +220,23 @@ class DirectEventProcessor: EventProcessor { /// question and the delivery that acts on it are both enqueued there, in that order. private var hasEventsFromPreviousRun = false + /// Whether the launch gate has opened. It opens once the floor has passed since first going online, and the first + /// flag request has settled, or there was none, or the cap has passed too. Until then neither a previous run's events + /// nor the diagnostic init event are sent. Open from the start where events do not give way. + private var launchGateOpen: Bool + /// When the reporter first went online, which the gate counts from. + private var wentOnlineAt: DispatchTime? + /// What waits for the gate to open. + private var launchGateWaiters: [() -> Void] = [] + init(service: DarklyServiceProvider, onSyncComplete: EventSyncCompleteClosure?, store: EventStoring? = nil, encoding: OutboundEventBuffer.Encoding = .handWritten, commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.DirectEventProcessor.commitQueue", qos: .userInitiated), - retryState: DeliveryRetryState? = nil) { + retryState: DeliveryRetryState? = nil, + flagTimings: FlagTraffic.Timings = .specified) { + let eventStore = store ?? DirectEventProcessor.makeStore(config: service.config) self.service = service self.onSyncComplete = onSyncComplete self.sender = AnalyticsEventSender(service: service) @@ -216,12 +245,16 @@ class DirectEventProcessor: EventProcessor { self.eventCapacity = service.config.eventCapacity self.commitQueue = commitQueue self.eventBuffer = OutboundEventBuffer(config: service.config, encoding: encoding) - self.store = store ?? DirectEventProcessor.makeStore(config: service.config) + self.store = eventStore + self.flagTimings = flagTimings + self.givesWayToFlags = eventStore.isPersisting + self.launchGateOpen = !eventStore.isPersisting // A log left open by a previous run has to be closed before it can be delivered, but nothing waits on that: - // doing it on the caller's thread would put file I/O in the way of the client starting up. - let store = self.store - deliveryQueue.async { [weak self] in + // doing it on the caller's thread would put file I/O in the way of the client starting up. Below the queue's + // own priority, and so below the flag requests, which matter more at launch than what a previous run left. + let store = eventStore + deliveryQueue.async(qos: .utility, flags: .enforceQoS) { [weak self] in store.preparePayloadsFromInterruptedLogs() self?.reportDiscarded(store.discardExpired()) let listing = store.listPendingBatches() @@ -521,11 +554,19 @@ extension DirectEventProcessor { eventReportTimer = LDTimer(withTimeInterval: service.config.eventFlushInterval, fireQueue: deliveryQueue, execute: queueScheduledDelivery) // Events a previous run left behind are already late by however long the application was gone, so they do not - // wait out a report interval on top of that: an application that crashed reports it as soon as it is next able - // to. Only the first time online brings a delivery forward, so going online again later, as a network comes and - // goes, keeps to the ordinary cadence. + // wait out a report interval on top of that, only the launch gate: an application that crashed reports it as + // soon as it is next able to. Only the first time online brings a delivery forward, so going online again + // later, as a network comes and goes, keeps to the ordinary cadence. deliveryQueue.async { [weak self] in - guard let self, self.hasEventsFromPreviousRun + guard let self + else { return } + if self.wentOnlineAt == nil { + self.wentOnlineAt = .now() + if !self.launchGateOpen { + self.checkLaunchGate(after: self.flagTimings.launchFloor) + } + } + guard self.launchGateOpen, self.hasEventsFromPreviousRun else { return } self.hasEventsFromPreviousRun = false self.queueScheduledDelivery() @@ -535,6 +576,62 @@ extension DirectEventProcessor { private func stopReporting() { eventReportTimer?.cancel() eventReportTimer = nil + endPacing() + } + + func flagRequestStarted() { + flagTraffic.started() + } + + func flagRequestSettled(_ outcome: FlagRequestOutcome) { + flagTraffic.settled(outcome) + deliveryQueue.async { [weak self] in self?.checkLaunchGate() } + } + + /// Runs `block`, on a global queue, once the launch gate is open. + func whenLaunchGateOpens(_ block: @escaping () -> Void) { + deliveryQueue.async { [weak self] in + guard let self, !self.launchGateOpen + else { + DispatchQueue.global().async(execute: block) + return + } + self.launchGateWaiters.append(block) + } + } + + private func checkLaunchGate(after delay: TimeInterval) { + deliveryQueue.asyncAfter(deadline: .now() + max(0, delay)) { [weak self] in self?.checkLaunchGate() } + } + + /// Opens the launch gate if it may open now, and otherwise looks again when it next could: at the floor, then at + /// the cap, or sooner once the flag request settles. On `deliveryQueue`. + private func checkLaunchGate() { + guard !launchGateOpen, let wentOnlineAt + else { return } + let sinceOnline = TimeInterval(DispatchTime.now().uptimeNanoseconds - wentOnlineAt.uptimeNanoseconds) / 1_000_000_000 + guard sinceOnline >= flagTimings.launchFloor + else { + checkLaunchGate(after: flagTimings.launchFloor - sinceOnline) + return + } + guard let why = flagTraffic.launchGateReason(capReached: sinceOnline >= flagTimings.launchCap, cap: flagTimings.launchCap) + else { + checkLaunchGate(after: flagTimings.launchCap - sinceOnline) + return + } + launchGateOpen = true + let waiters = launchGateWaiters + launchGateWaiters = [] + waiters.forEach { DispatchQueue.global().async(execute: $0) } + guard hasEventsFromPreviousRun + else { return } + os_log("%s Sending events from a previous run after %s", log: service.config.logger, type: .debug, typeName(and: #function), why) + // Offline, they wait for the reporter to go online, which finds the gate open. + guard isOnline + else { return } + hasEventsFromPreviousRun = false + queueScheduledDelivery() } func flush(completion: CompletionClosure?) { @@ -592,8 +689,29 @@ extension DirectEventProcessor { } } + /// Queues the delivery the report timer and the launch gate start, which nobody asked for now and which is paced. private func queueScheduledDelivery() { - queueDelivery(nil) + queueDelivery(nil, paced: true) + } + + /// Ends the pacing of a delivery under way, which then sends the rest back to back. + private func endPacing() { + let wasPaced = deliveryLock.withLock { () -> Bool in + defer { isPacedDelivery = false } + return isPacedDelivery + } + if wasPaced { + flagTraffic.wake() + } + } + + /// When the next batch of the delivery running may be posted, or nil where it is not paced. + private func pacedTurn() -> DispatchTime? { + deliveryLock.withLock { + guard isPacedDelivery + else { return nil } + return lastPacedPostAnswered.map { $0 + flagTimings.gap } ?? .now() + } } /// Queues a delivery, or joins one that is queued and has not started. @@ -608,19 +726,27 @@ extension DirectEventProcessor { /// answer, then sends what that one left behind as well as what was recorded since, so its caller hears `true` /// only if both got out. /// - /// - Parameter closing: whether it is `flushAtClose()`'s, which a queued delivery it joins becomes. - private func queueDelivery(_ completion: FlushOutcomeClosure?, closing: Bool = false) { + /// - Parameters: + /// - closing: whether it is `flushAtClose()`'s, which a queued delivery it joins becomes. + /// - paced: whether nobody asked for it, so that it gives way to flag requests. One that is not ends the pacing of + /// the delivery running, so that it is not held behind it. + private func queueDelivery(_ completion: FlushOutcomeClosure?, closing: Bool = false, paced: Bool = false) { + if !paced { + endPacing() + } deliveryLock.lock() if queuedDelivery != nil { if let completion { queuedDelivery?.append(completion) } queuedDeliveryIsClosing = queuedDeliveryIsClosing || closing + queuedDeliveryIsPaced = queuedDeliveryIsPaced && paced deliveryLock.unlock() return } queuedDelivery = completion.map { [$0] } ?? [] queuedDeliveryIsClosing = closing + queuedDeliveryIsPaced = paced let starts = !isDelivering isDelivering = true deliveryLock.unlock() @@ -639,6 +765,9 @@ extension DirectEventProcessor { let closing = queuedDeliveryIsClosing queuedDeliveryIsClosing = false isClosingDelivery = closing + isPacedDelivery = queuedDeliveryIsPaced && givesWayToFlags && !closing + queuedDeliveryIsPaced = false + lastPacedPostAnswered = nil #if DEBUG deliveryRunCount += 1 #endif @@ -668,6 +797,7 @@ extension DirectEventProcessor { private func startQueuedDeliveryOrStop() { let next = deliveryLock.withLock { () -> Bool in isClosingDelivery = false + isPacedDelivery = false isDelivering = queuedDelivery != nil return isDelivering } @@ -885,7 +1015,11 @@ extension DirectEventProcessor { /// /// `lostAnything` carries whether a batch already taken in this pass was lost for good, so that the caller hears /// about it even where the batches after it were accepted. `retried` whether this pass has had its retry. - private func deliver(_ batches: [EventBatch], lostAnything: Bool, retried: Bool, _ completion: @escaping (PassOutcome) -> Void) { + /// + /// A paced delivery holds each batch first: until a gap has passed since the last one was answered, and then while + /// a flag request is in flight, up to a limit. A backlog this sends unasked otherwise takes the network for as long + /// as it lasts, an `identify`'s flag request included. `hadTurn` is whether the first batch has been held already. + private func deliver(_ batches: [EventBatch], lostAnything: Bool, retried: Bool, hadTurn: Bool = false, _ completion: @escaping (PassOutcome) -> Void) { var remaining = batches guard !remaining.isEmpty else { @@ -893,6 +1027,18 @@ extension DirectEventProcessor { return } + if !hadTurn, let gapEnds = pacedTurn() { + let isStopped = { [weak self] () -> Bool in + guard let self + else { return true } + return self.deliveryLock.withLock { !self.isPacedDelivery } + } + flagTraffic.awaitTurn(gapEnds: gapEnds, flagWait: flagTimings.flagRequestWait, isStopped: isStopped) { + self.deliver(batches, lostAnything: lostAnything, retried: retried, hadTurn: true, completion) + } + return + } + let wait = deliveryLock.withLock { isClosingDelivery } ? 0 : retryState.remainingWait guard wait <= 0 else { @@ -1045,7 +1191,7 @@ extension DirectEventProcessor { guard let self else { return } self.deliveryLock.withLock { self.isRetryScheduled = false } - self.queueScheduledDelivery() + self.queueDelivery(nil) } } @@ -1056,6 +1202,11 @@ extension DirectEventProcessor { store.startedSending(batch) // The store is settled before observers are told, so that one looking at it on hearing the outcome sees it. sender.send(payload.body, payloadId: batch.payloadId, eventCount: payload.eventCount) { outcome in + self.deliveryLock.withLock { + if self.isPacedDelivery { + self.lastPacedPostAnswered = .now() + } + } switch outcome { case .accepted(let serverTime): self.retryState.recordSuccess() diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 60645d815..d7b11d062 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -77,7 +77,8 @@ class DirectEventProcessorSpec: QuickSpec { self.lastEventResponseDate = lastEventResponseDate.adjustedForHttpUrlHeaderUse self.store = store ?? EventStore.temporary(capacity: config.eventCapacity, persistEvents: persistence != .disabled) eventProcessor = DirectEventProcessor(service: serviceMock, onSyncComplete: onSyncComplete, - store: self.store, commitQueue: commitQueue, retryState: retryState) + store: self.store, commitQueue: commitQueue, retryState: retryState, + flagTimings: .noWaits) (0.. DirectEventProcessor { + for _ in 0.. [Date] { + var times: [Date] = [] + let deadline = Date().addingTimeInterval(10) + while times.count < count && Date() < deadline { + let posted = testContext.serviceMock.publishEventDataCallCount + while times.count < posted { + times.append(Date()) + } + Thread.sleep(forTimeInterval: 0.005) + } + return times + } + + itPersisting("sends what a previous run left only once the floor has passed after going online") { + let reporter = nextRunLeft(1, timings: FlagTraffic.Timings(launchFloor: 0.6, launchCap: 5, gap: 0, flagRequestWait: 0)) + + reporter.isOnline = true + + Thread.sleep(forTimeInterval: 0.4) + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + } + + itPersisting("holds what a previous run left while the first flag request is in flight") { + let reporter = nextRunLeft(1, timings: FlagTraffic.Timings(launchFloor: 0.1, launchCap: 5, gap: 0, flagRequestWait: 0)) + reporter.flagRequestStarted() + reporter.isOnline = true + Thread.sleep(forTimeInterval: 0.8) + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + + reporter.flagRequestSettled(.loaded) + + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: .seconds(2)) + } + + itPersisting("holds what a previous run left for a flag request only until the cap") { + let reporter = nextRunLeft(1, timings: FlagTraffic.Timings(launchFloor: 0.1, launchCap: 0.8, gap: 0, flagRequestWait: 0)) + reporter.flagRequestStarted() + let wentOnline = Date() + + reporter.isOnline = true + + let times = postTimes(1) + expect(times.count) == 1 + expect(times.first?.timeIntervalSince(wentOnline)) >= 0.7 + } + + itPersisting("sends what a previous run left on a flush, without waiting for the launch gate") { + let reporter = nextRunLeft(1, timings: FlagTraffic.Timings(launchFloor: 5, launchCap: 10, gap: 0, flagRequestWait: 0)) + reporter.flagRequestStarted() + reporter.isOnline = true + + waitUntil(timeout: .seconds(3)) { done in + reporter.flush(completion: done) + } + + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + } + + itPersisting("sends what a previous run left a batch at a time, with a gap after each answer") { + let reporter = nextRunLeft(3, timings: FlagTraffic.Timings(launchFloor: 0, launchCap: 0, gap: 0.4, flagRequestWait: 0)) + + reporter.isOnline = true + + let times = postTimes(3) + expect(times.count) == 3 + for (earlier, later) in zip(times, times.dropFirst()) { + expect(later.timeIntervalSince(earlier)) >= 0.3 + } + } + + itPersisting("sends the rest at once when a flush is asked for while pacing") { + let reporter = nextRunLeft(3, timings: FlagTraffic.Timings(launchFloor: 0, launchCap: 0, gap: 10, flagRequestWait: 0)) + reporter.isOnline = true + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + + waitUntil(timeout: .seconds(3)) { done in + reporter.flush(completion: done) + } + + expect(testContext.serviceMock.publishEventDataCallCount) == 3 + } + + itPersisting("holds a paced batch while a flag request is in flight") { + let reporter = nextRunLeft(2, timings: FlagTraffic.Timings(launchFloor: 0, launchCap: 0, gap: 0.3, flagRequestWait: 5)) + reporter.isOnline = true + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + reporter.flagRequestStarted() + Thread.sleep(forTimeInterval: 1) + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + + reporter.flagRequestSettled(.loaded) + + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(2), timeout: .seconds(2)) + } + + it("holds what waits for the launch gate only where events persist") { + let reporter = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: testContext.store, + flagTimings: FlagTraffic.Timings(launchFloor: 0.1, launchCap: 5, gap: 0, flagRequestWait: 0)) + nextRun = reporter + let opened = UnfairLock() + var hasOpened = false + reporter.whenLaunchGateOpens { opened.withLock { hasOpened = true } } + reporter.flagRequestStarted() + reporter.isOnline = true + + if self.persists { + Thread.sleep(forTimeInterval: 0.6) + expect(opened.withLock { hasOpened }) == false + reporter.flagRequestSettled(.failed) + } + + expect(opened.withLock { hasOpened }).toEventually(beTrue(), timeout: .seconds(2)) + } + } + } + /// An example whose subject is the disk, left out where the spec runs without one. private func itPersisting(_ description: String, file: FileString = #file, line: UInt = #line, closure: @escaping () throws -> Void) { guard persists @@ -361,7 +507,7 @@ class DirectEventProcessorSpec: QuickSpec { var testContext: TestContext! beforeEach { testContext = TestContext() - testContext.eventProcessor = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: { _ in }, store: testContext.store) + testContext.eventProcessor = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: { _ in }, store: testContext.store, flagTimings: .noWaits) } it("starts offline without reporting events") { expect(testContext.eventProcessor.service) === testContext.serviceMock @@ -504,7 +650,7 @@ class DirectEventProcessorSpec: QuickSpec { serviceMock.config = config serviceMock.stubEventResponse(success: true) store = EventStore.temporary(capacity: config.eventCapacity) - return DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: encoding) + return DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: encoding, flagTimings: .noWaits) } afterEach { @@ -666,7 +812,7 @@ class DirectEventProcessorSpec: QuickSpec { serviceMock.config = config serviceMock.stubEventResponse(success: true) store = EventStore.temporary(capacity: config.eventCapacity) - reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: encoding) + reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: encoding, flagTimings: .noWaits) reporter.isOnline = true return reporter } @@ -1269,7 +1415,7 @@ class DirectEventProcessorSpec: QuickSpec { /// A reporter writing to a store of its own, and that store, so a test can read back what was recorded. func makeReporter() -> (DirectEventProcessor, EventStore) { let store = EventStore.temporary() - return (DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store), store) + return (DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, flagTimings: .noWaits), store) } describe("recordFlagEvaluationEvents") { @@ -1282,7 +1428,7 @@ class DirectEventProcessorSpec: QuickSpec { service.diagnosticCache = diagnosticCache let store = EventStore.temporary(capacity: config.eventCapacity) defer { store.deleteEverything() } - let reporter = DirectEventProcessor(service: service, onSyncComplete: nil, store: store) + let reporter = DirectEventProcessor(service: service, onSyncComplete: nil, store: store, flagTimings: .noWaits) let untracked = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: false) let tracked = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: true) @@ -1647,7 +1793,7 @@ extension DirectEventProcessorSpec { // Nothing delivered them and nothing closed the log, as would be the case had the process died here. let recovered = testContext.store.nextRun() - let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) + let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered, flagTimings: .noWaits) nextRun.isOnline = true waitUntil(timeout: Constants.deliveryTimeout) { done in @@ -1669,7 +1815,7 @@ extension DirectEventProcessorSpec { testContext.eventProcessor.commitRecordedEvents() let recovered = testContext.store.nextRun() - let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) + let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered, flagTimings: .noWaits) // Nothing asks for a flush, and the report interval is several times longer than this waits, so the // only thing that can deliver these is the reporter coming online. @@ -1691,7 +1837,7 @@ extension DirectEventProcessorSpec { try FileManager.default.setAttributes([.modificationDate: longAgo], ofItemAtPath: file.path) } - let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: EventStore.sharing(directory)) + let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: EventStore.sharing(directory), flagTimings: .noWaits) nextRun.isOnline = true var delivered: Bool? waitUntil(timeout: Constants.deliveryTimeout) { done in @@ -1865,7 +2011,7 @@ extension DirectEventProcessorSpec { // after launch reads every batch a previous run left behind. let store = SlowListingStore(testContext.store) defer { store.release.signal() } - let reporter = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: store) + let reporter = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: store, flagTimings: .noWaits) reporter.isOnline = true reporter.record(CustomEvent(key: "tracked", context: testContext.context)) @@ -1966,7 +2112,7 @@ extension DirectEventProcessorSpec { testContext.eventProcessor.record(CustomEvent(key: "kept", context: testContext.context)) let recovered = testContext.store.nextRun() - let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) + let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered, flagTimings: .noWaits) nextRun.isOnline = true defer { nextRun.isOnline = false } @@ -1991,7 +2137,7 @@ extension DirectEventProcessorSpec { // The same directory, as the application's next launch would find it. testContext.serviceMock.stubEventResponse(success: true) let recovered = EventStore.sharing(testContext.store.directory!) - let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered) + let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered, flagTimings: .noWaits) nextRun.isOnline = true defer { nextRun.isOnline = false } @@ -2108,7 +2254,7 @@ extension DirectEventProcessorSpec { let commits = DispatchQueue(label: "com.launchdarkly.tests.suspended") commits.suspend() defer { commits.resume() } - let reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: .codable, commitQueue: commits) + let reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: .codable, commitQueue: commits, flagTimings: .noWaits) let slow = EncodingGate(key: "slow", context: LDContext.stub()) reporter.record(slow) @@ -2142,7 +2288,7 @@ extension DirectEventProcessorSpec { let commits = DispatchQueue(label: "com.launchdarkly.tests.suspended") commits.suspend() defer { commits.resume() } - let reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: .codable, commitQueue: commits) + let reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: .codable, commitQueue: commits, flagTimings: .noWaits) let slow = EncodingGate(key: "slow", context: LDContext.stub()) let competingCommit = DispatchSemaphore(value: 0) @@ -2184,7 +2330,7 @@ extension DirectEventProcessorSpec { let commits = DispatchQueue(label: "com.launchdarkly.tests.suspended") commits.suspend() defer { commits.resume() } - let reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: .codable, commitQueue: commits) + let reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: .codable, commitQueue: commits, flagTimings: .noWaits) let slow = EncodingGate(key: "slow", context: LDContext.stub()) reporter.record(CustomEvent(key: "first", context: LDContext.stub())) From 8b6e2d671a126b801ad2b1731c2b960d2a5e9495 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 27 Oct 2026 20:49:38 -0700 Subject: [PATCH 110/158] feat: share the byte capacity equally across the configured environments eventCapacityBytes was each environment's, so an application configured for several environments could keep that many times the bytes it set. It is now for all of them together, each environment's store getting an equal share. eventCapacity is still each environment's in full. Co-authored-by: Cursor --- .../LaunchDarkly/Models/LDConfig.swift | 13 +++++----- .../ObjectiveC/ObjcLDConfig.swift | 2 +- .../ServiceObjects/DirectEventProcessor.swift | 10 +++++++- .../DirectEventProcessorSpec.swift | 25 +++++++++++++++++++ 4 files changed, 42 insertions(+), 8 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift index 20a726cee..54051005a 100644 --- a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift @@ -235,7 +235,7 @@ public struct LDConfig { /// The default maximum number of events the LDClient can store static let eventCapacity = 2000 - /// The default maximum number of bytes of events the LDClient can persist per environment. (20 MiB) + /// The default maximum number of bytes of events the SDK can persist, for all environments together. (20 MiB) static let eventCapacityBytes = 20 * 1024 * 1024 /// The default maximum age of persisted events. (7 days) @@ -383,11 +383,12 @@ public struct LDConfig { /// any backwards compatibility guarantees or semantic versioning. (Default: `.disabled`) public var eventPersistence: EventPersistence = Defaults.eventPersistence - /// The maximum number of bytes of events the SDK keeps for each environment where `eventPersistence` keeps events across launches, - /// on disk and in memory together. When it is reached, the SDK discards new events until it successfully reports the stored ones, - /// as it does when `eventCapacity` is reached; it never deletes stored events to make room. A fifth of it is kept for the summaries - /// of flag evaluations, so that evaluations are still counted after full events start being discarded. A value of zero or less - /// means the default. (Default: 20 MiB) + /// The maximum number of bytes of events the SDK keeps where `eventPersistence` keeps events across launches, on disk and in + /// memory together. It is for every environment the application configures together: with `secondaryMobileKeys`, each environment + /// gets an equal share of it, unlike `eventCapacity`, which each environment has in full. When an environment reaches its share, + /// the SDK discards new events until it successfully reports the stored ones, as it does when `eventCapacity` is reached; it never + /// deletes stored events to make room. A fifth of each share is kept for the summaries of flag evaluations, so that evaluations + /// are still counted after full events start being discarded. A value of zero or less means the default. (Default: 20 MiB) /// /// Where `eventPersistence` is `.disabled`, full events are bounded by `eventCapacity` alone, and this bounds only the summaries, /// which `eventCapacity` never refuses: a delivery that keeps failing would otherwise leave them growing for the life of the process. diff --git a/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift b/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift index 77536aab5..c0593e584 100644 --- a/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift @@ -53,7 +53,7 @@ public final class ObjcLDConfig: NSObject { set { config.eventPersistence = newValue } } - /// The maximum number of bytes of events the SDK keeps for each environment where `eventPersistence` keeps events across launches. When it is reached, the SDK discards new events until it reports the stored ones; a fifth of it is kept for the summaries of flag evaluations. No effect where `eventPersistence` is `.disabled`. (Default: 20 MiB) + /// The maximum number of bytes of events the SDK keeps where `eventPersistence` keeps events across launches, for every environment together: each environment in `secondaryMobileKeys` gets an equal share of it, unlike `eventCapacity`. When an environment reaches its share, the SDK discards new events until it reports the stored ones; a fifth of each share is kept for the summaries of flag evaluations. No effect where `eventPersistence` is `.disabled`. (Default: 20 MiB) @objc public var eventCapacityBytes: Int { get { config.eventCapacityBytes } set { config.eventCapacityBytes = newValue } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 3c5f98c34..b12010856 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -267,12 +267,20 @@ class DirectEventProcessor: EventProcessor, FlagRequestListening { let mobileKey = config.mobileKey return EventStore(locatingDirectory: { EventDirectory.defaultUrl(mobileKey: mobileKey) }, capacity: config.eventCapacity, - byteCapacity: config.eventCapacityBytes, + byteCapacity: shareOfCapacityBytes(config: config), maxAge: config.eventMaxAge, persistEvents: config.eventPersistence != .disabled, logger: config.logger) } + /// This environment's share of the byte capacity, which is for every environment the application configured together, + /// where the event capacity is for each one. + static func shareOfCapacityBytes(config: LDConfig) -> Int { + let total = config.eventCapacityBytes > 0 ? config.eventCapacityBytes : LDConfig.Defaults.eventCapacityBytes + // At least one byte, since the store reads zero as the default. + return max(1, total / (1 + config.getSecondaryMobileKeys().count)) + } + // MARK: Recording func record(_ event: Event) { diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index d7b11d062..7f209f1b2 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -516,6 +516,31 @@ class DirectEventProcessorSpec: QuickSpec { expect(testContext.serviceMock.publishEventDataCallCount) == 0 } } + describe("the byte capacity") { + var config: LDConfig! + beforeEach { + config = LDConfig(mobileKey: LDConfig.Constants.mockMobileKey, autoEnvAttributes: .disabled) + } + it("is the whole of it for a single environment") { + config.eventCapacityBytes = 2_400 + expect(DirectEventProcessor.shareOfCapacityBytes(config: config)) == 2_400 + } + it("is shared equally by the configured environments") { + config.eventCapacityBytes = 2_400 + try? config.setSecondaryMobileKeys(["second": "mob-second", "third": "mob-third", "fourth": "mob-fourth"]) + expect(DirectEventProcessor.shareOfCapacityBytes(config: config)) == 600 + } + it("shares the default where none is set") { + config.eventCapacityBytes = 0 + try? config.setSecondaryMobileKeys(["second": "mob-second"]) + expect(DirectEventProcessor.shareOfCapacityBytes(config: config)) == LDConfig.Defaults.eventCapacityBytes / 2 + } + it("leaves each environment at least a byte") { + config.eventCapacityBytes = 2 + try? config.setSecondaryMobileKeys(["second": "mob-second", "third": "mob-third"]) + expect(DirectEventProcessor.shareOfCapacityBytes(config: config)) == 1 + } + } } private func isOnlineSpec() { From 07ed8c5273b4a4aaf0b5ea7f71479413f1c0e692 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 27 Oct 2026 21:31:44 -0700 Subject: [PATCH 111/158] fix: count the wait for a flag request from when it started, so the launch cap is not waited twice Co-authored-by: Cursor --- .../ServiceObjects/DirectEventProcessorSpec.swift | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 7f209f1b2..45f29551b 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -400,6 +400,20 @@ class DirectEventProcessorSpec: QuickSpec { expect(times.first?.timeIntervalSince(wentOnline)) >= 0.7 } + itPersisting("holds no batch any longer for a flag request past the cap") { + let reporter = nextRunLeft(2, timings: FlagTraffic.Timings(launchFloor: 0.1, launchCap: 2, gap: 0.3, flagRequestWait: 2)) + reporter.flagRequestStarted() + let wentOnline = Date() + + reporter.isOnline = true + + // Waiting on the request again would add the two seconds' wait to each batch. + let times = postTimes(2) + expect(times.count) == 2 + expect(times.first?.timeIntervalSince(wentOnline)) < 3 + expect(times.last?.timeIntervalSince(times.first ?? wentOnline)) < 1.5 + } + itPersisting("sends what a previous run left on a flush, without waiting for the launch gate") { let reporter = nextRunLeft(1, timings: FlagTraffic.Timings(launchFloor: 5, launchCap: 10, gap: 0, flagRequestWait: 0)) reporter.flagRequestStarted() From e6514b90cc63ab37e3cd65230a099502a0145535 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 27 Oct 2026 23:19:50 -0700 Subject: [PATCH 112/158] fix: count the wait for a flag request from when it started, so no paced batch waits on it again Co-authored-by: Cursor --- .../ServiceObjects/FlagTraffic.swift | 31 ++++++++++++------- .../DirectEventProcessorSpec.swift | 12 +++++++ 2 files changed, 31 insertions(+), 12 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/FlagTraffic.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/FlagTraffic.swift index f710dd72a..c5ce4bac0 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/FlagTraffic.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/FlagTraffic.swift @@ -34,7 +34,9 @@ final class FlagTraffic { let launchCap: TimeInterval /// The least time between one paced batch's answer and the next batch's request. let gap: TimeInterval - /// The longest a paced batch waits for a flag request in flight, so that a hung one cannot stop delivery. + /// How long after a flag request starts that paced batches stop waiting for it, so that a hung one cannot stop + /// delivery. It counts from the request rather than from each batch, so the launch gate's cap and the batches + /// after it never wait on the same request again. let flagRequestWait: TimeInterval } @@ -44,8 +46,8 @@ final class FlagTraffic { let flagWait: TimeInterval let isStopped: () -> Bool let proceed: () -> Void - /// When it stops waiting on the flag request, set once the gap has passed with one in flight. Only to be used - /// while holding the lock. + /// When it stops waiting on the flag request in flight, as last scheduled for a check. Only to be used while + /// holding the lock. var flagWaitEnds: DispatchTime? init(gapEnds: DispatchTime, flagWait: TimeInterval, isStopped: @escaping () -> Bool, proceed: @escaping () -> Void) { @@ -59,12 +61,17 @@ final class FlagTraffic { private let lock = UnfairLock() // Only to be used while holding `lock`. private var inFlight = false + /// When the request in flight started. + private var startedAt = DispatchTime.now() /// How the last flag request ended, or nil if none has. private var lastOutcome: FlagRequestOutcome? private var turns: [Turn] = [] func started() { - lock.withLock { inFlight = true } + lock.withLock { + inFlight = true + startedAt = DispatchTime.now() + } } func settled(_ outcome: FlagRequestOutcome) { @@ -100,7 +107,7 @@ final class FlagTraffic { } /// Calls `proceed`, on a global queue, once `gapEnds` has passed and then no flag request is in flight, waiting on - /// one for no longer than `flagWait`; or at once when `isStopped` says so. `isStopped` is asked outside this + /// one until `flagWait` after it started; or at once when `isStopped` says so. `isStopped` is asked outside this /// object's lock, so it may take a lock of its own. func awaitTurn(gapEnds: DispatchTime, flagWait: TimeInterval, isStopped: @escaping () -> Bool, then proceed: @escaping () -> Void) { let turn = Turn(gapEnds: gapEnds, flagWait: flagWait, isStopped: isStopped, proceed: proceed) @@ -126,16 +133,16 @@ final class FlagTraffic { due.append(turn) continue } - if turn.flagWaitEnds == nil { - let ends = now + turn.flagWait + let ends = startedAt + turn.flagWait + if now >= ends { + due.append(turn) + continue + } + if turn.flagWaitEnds != ends { turn.flagWaitEnds = ends deadlines.append(ends) } - if let ends = turn.flagWaitEnds, now >= ends { - due.append(turn) - } else { - waiting.append(turn) - } + waiting.append(turn) } } due += waiting.filter { $0.isStopped() } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 45f29551b..d541a8496 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -463,6 +463,18 @@ class DirectEventProcessorSpec: QuickSpec { expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(2), timeout: .seconds(2)) } + itPersisting("holds paced batches for a hung flag request only until the wait after it started") { + let reporter = nextRunLeft(3, timings: FlagTraffic.Timings(launchFloor: 0, launchCap: 0, gap: 0.3, flagRequestWait: 1)) + reporter.flagRequestStarted() + + reporter.isOnline = true + + // Waiting on the request again would add a second to each batch. + let times = postTimes(3) + expect(times.count) == 3 + expect(times.last?.timeIntervalSince(times.first ?? Date())) < 1.5 + } + it("holds what waits for the launch gate only where events persist") { let reporter = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: testContext.store, flagTimings: FlagTraffic.Timings(launchFloor: 0.1, launchCap: 5, gap: 0, flagRequestWait: 0)) From bc29c3d6f669425b939b164aa1e0718ffa077273 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Tue, 27 Oct 2026 23:19:50 -0700 Subject: [PATCH 113/158] fix: going offline stops a delivery before its next post, its retry's included Co-authored-by: Cursor --- .../ServiceObjects/DirectEventProcessor.swift | 26 +++++++++++++--- .../DirectEventProcessorSpec.swift | 30 +++++++++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index b12010856..0b2912933 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -171,8 +171,8 @@ class DirectEventProcessor: EventProcessor, FlagRequestListening { private var isClosingDelivery = false /// Whether the delivery queued was asked for only by the report timer or by the launch gate, and so is paced. private var queuedDeliveryIsPaced = false - /// Whether the delivery running is paced: until a delivery somebody asked for is queued, or the reporter goes - /// offline, after which it sends the rest back to back. + /// Whether the delivery running is paced: until a delivery somebody asked for is queued, after which it sends the + /// rest back to back, or the reporter goes offline, after which it sends nothing more. private var isPacedDelivery = false /// When the paced delivery running last had an answer for a post, or nil before its first. private var lastPacedPostAnswered: DispatchTime? @@ -702,7 +702,7 @@ extension DirectEventProcessor { queueDelivery(nil, paced: true) } - /// Ends the pacing of a delivery under way, which then sends the rest back to back. + /// Ends the pacing of a delivery under way, which then sends the rest back to back, or, offline, stops. private func endPacing() { let wasPaced = deliveryLock.withLock { () -> Bool in defer { isPacedDelivery = false } @@ -1047,6 +1047,12 @@ extension DirectEventProcessor { return } + guard mayPost() + else { + completion(.blocked) + return + } + let wait = deliveryLock.withLock { isClosingDelivery } ? 0 : retryState.remainingWait guard wait <= 0 else { @@ -1104,6 +1110,12 @@ extension DirectEventProcessor { post(batch, payload, leftOut: leftOut, remaining: remaining, lostAnything: lostAnything, retried: retried, completion) } + /// Whether the delivery running may post: online, or `flushAtClose()`'s. Going offline stops a delivery before its + /// next post, its retry's included, and leaves what it has not sent for a later one. + private func mayPost() -> Bool { + deliveryLock.withLock { isClosingDelivery } || isOnline + } + /// Posts a batch already read, then goes on to the rest. Requires the backoff to allow a post. // swiftlint:disable:next function_parameter_count private func post(_ batch: EventBatch, @@ -1146,7 +1158,13 @@ extension DirectEventProcessor { } os_log("%s Retrying event post after %.1fs", log: self.service.config.logger, type: .debug, self.typeName(and: #function), wait) DispatchQueue.global().asyncAfter(deadline: .now() + wait) { - // Another delivery's failure may have set a longer wait meanwhile. + // Another delivery's failure may have set a longer wait meanwhile, or the reporter gone offline. + guard self.mayPost() + else { + self.reportSyncComplete(failure.error) + completion(.blocked) + return + } guard self.retryState.remainingWait <= 0 else { self.reportSyncComplete(failure.error) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index d541a8496..3bdb48386 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -475,6 +475,22 @@ class DirectEventProcessorSpec: QuickSpec { expect(times.last?.timeIntervalSince(times.first ?? Date())) < 1.5 } + itPersisting("stops a paced delivery on going offline, and keeps the rest") { + let reporter = nextRunLeft(3, timings: FlagTraffic.Timings(launchFloor: 0, launchCap: 0, gap: 0.5, flagRequestWait: 0)) + reporter.isOnline = true + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + + reporter.isOnline = false + Thread.sleep(forTimeInterval: 1.5) + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + + reporter.isOnline = true + waitUntil(timeout: .seconds(3)) { done in + reporter.flush(completion: done) + } + expect(testContext.serviceMock.publishEventDataCallCount) == 3 + } + it("holds what waits for the launch gate only where events persist") { let reporter = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: testContext.store, flagTimings: FlagTraffic.Timings(launchFloor: 0.1, launchCap: 5, gap: 0, flagRequestWait: 0)) @@ -3093,6 +3109,20 @@ extension DirectEventProcessorSpec { } } + it("does not retry a failed post once the reporter has gone offline") { + testContext = TestContext() + testContext.serviceMock.respondToEvents(with: 503) + testContext.eventProcessor.record(CustomEvent(key: "refused", context: testContext.context)) + testContext.eventProcessor.isOnline = true + testContext.eventProcessor.flush(completion: nil) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + + testContext.eventProcessor.isOnline = false + + waitLettingTimersFire(2) + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + } + for status in [400, 413, 415, 422] { it("drops a batch refused with HTTP \(status) twice in a row, and no longer holds the ones behind it") { testContext = TestContext(retryState: DirectEventProcessorSpec.withoutBackoff()) From 2887cd3b0139f01fc86def0c5cfeab53596155ca Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Wed, 28 Oct 2026 09:48:41 -0700 Subject: [PATCH 114/158] fix: leave flag requests untracked and FlagTraffic unbuilt where events do not persist Co-authored-by: Cursor --- LaunchDarkly/LaunchDarkly/LDClient.swift | 10 ++++++--- .../ServiceObjects/DirectEventProcessor.swift | 22 +++++++++++++------ .../ServiceObjects/FlagSynchronizer.swift | 7 +++++- .../ServiceObjects/FlagTraffic.swift | 4 +++- .../DirectEventProcessorSpec.swift | 18 +++++++++++++++ .../ServiceObjects/FlagSynchronizerSpec.swift | 1 + 6 files changed, 50 insertions(+), 12 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/LDClient.swift b/LaunchDarkly/LaunchDarkly/LDClient.swift index e5bf1d14e..608c3d5eb 100644 --- a/LaunchDarkly/LaunchDarkly/LDClient.swift +++ b/LaunchDarkly/LaunchDarkly/LDClient.swift @@ -1106,15 +1106,19 @@ extension LDClient: TypeIdentifying { } extension LDClient { /// Has the flag synchronizer tell the event processor about its flag requests, so that events can give way to them. + /// Only where they do: otherwise the synchronizer tells nothing. fileprivate func reportFlagRequests() { - (flagSynchronizer as? FlagSynchronizer)?.flagRequests = eventProcessor as? FlagRequestListening + guard let listener = eventProcessor as? FlagRequestListening, listener.givesWayToFlags + else { return } + (flagSynchronizer as? FlagSynchronizer)?.flagRequests = listener } /// Has events give way to flag requests: the synchronizer reports its requests to the event processor, and the - /// diagnostic init event waits for the processor's launch gate as a previous run's events do. + /// diagnostic init event waits for the processor's launch gate as a previous run's events do. Where events do not + /// persist neither happens, and the init event goes out as it did before persistence. fileprivate func giveFlagRequestsTheNetworkFirst() { reportFlagRequests() - if let eventProcessor = eventProcessor as? DirectEventProcessor { + if let eventProcessor = eventProcessor as? DirectEventProcessor, eventProcessor.givesWayToFlags { (diagnosticReporter as? DiagnosticReporter)?.sendsInitAfter = eventProcessor.whenLaunchGateOpens } } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 0b2912933..aaedd83dc 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -67,11 +67,13 @@ class DirectEventProcessor: EventProcessor, FlagRequestListening { /// Its summarizer is guarded by `recordLock`, and its encoders are reached only under `commitLock`. let eventBuffer: OutboundEventBuffer - private let flagTraffic = FlagTraffic() private let flagTimings: FlagTraffic.Timings /// Whether events give way to flag values: only where they persist, like the rest of what persistence brings. - /// Without it there is no previous run to hold back. - private let givesWayToFlags: Bool + /// Without it there is no previous run to hold back, and delivery is what it was before persistence: no launch + /// gate, no pacing, and nothing heard about flag requests. + let givesWayToFlags: Bool + /// Only where events give way to flags; nil otherwise, so that nothing of it runs. + private let flagTraffic: FlagTraffic? // MARK: Guarded by commitLock @@ -248,11 +250,15 @@ class DirectEventProcessor: EventProcessor, FlagRequestListening { self.store = eventStore self.flagTimings = flagTimings self.givesWayToFlags = eventStore.isPersisting + self.flagTraffic = eventStore.isPersisting ? FlagTraffic() : nil self.launchGateOpen = !eventStore.isPersisting // A log left open by a previous run has to be closed before it can be delivered, but nothing waits on that: // doing it on the caller's thread would put file I/O in the way of the client starting up. Below the queue's // own priority, and so below the flag requests, which matter more at launch than what a previous run left. + // Only where events persist, since memory holds nothing from a previous run. + guard eventStore.isPersisting + else { return } let store = eventStore deliveryQueue.async(qos: .utility, flags: .enforceQoS) { [weak self] in store.preparePayloadsFromInterruptedLogs() @@ -588,10 +594,12 @@ extension DirectEventProcessor { } func flagRequestStarted() { - flagTraffic.started() + flagTraffic?.started() } func flagRequestSettled(_ outcome: FlagRequestOutcome) { + guard let flagTraffic + else { return } flagTraffic.settled(outcome) deliveryQueue.async { [weak self] in self?.checkLaunchGate() } } @@ -615,7 +623,7 @@ extension DirectEventProcessor { /// Opens the launch gate if it may open now, and otherwise looks again when it next could: at the floor, then at /// the cap, or sooner once the flag request settles. On `deliveryQueue`. private func checkLaunchGate() { - guard !launchGateOpen, let wentOnlineAt + guard !launchGateOpen, let wentOnlineAt, let flagTraffic else { return } let sinceOnline = TimeInterval(DispatchTime.now().uptimeNanoseconds - wentOnlineAt.uptimeNanoseconds) / 1_000_000_000 guard sinceOnline >= flagTimings.launchFloor @@ -709,7 +717,7 @@ extension DirectEventProcessor { return isPacedDelivery } if wasPaced { - flagTraffic.wake() + flagTraffic?.wake() } } @@ -1035,7 +1043,7 @@ extension DirectEventProcessor { return } - if !hadTurn, let gapEnds = pacedTurn() { + if !hadTurn, let flagTraffic, let gapEnds = pacedTurn() { let isStopped = { [weak self] () -> Bool in guard let self else { return true } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/FlagSynchronizer.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/FlagSynchronizer.swift index a9e58a361..a738db9fa 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/FlagSynchronizer.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/FlagSynchronizer.swift @@ -154,13 +154,18 @@ class FlagSynchronizer: LDFlagSynchronizing, EventHandler { } } + /// Without a listener, which is where events do not give way to flags, nothing is tracked. private func flagRequestStarted() { + guard let flagRequests + else { return } flagRequestLock.withLock { flagRequestInFlight = true } - flagRequests?.flagRequestStarted() + flagRequests.flagRequestStarted() } /// Tells the listener how the first request ended, once; any answer after it, and every later poll, says nothing. private func flagRequestSettled(_ outcome: FlagRequestOutcome) { + guard flagRequests != nil + else { return } let wasInFlight = flagRequestLock.withLock { () -> Bool in defer { flagRequestInFlight = false } return flagRequestInFlight diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/FlagTraffic.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/FlagTraffic.swift index c5ce4bac0..619bbe30b 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/FlagTraffic.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/FlagTraffic.swift @@ -1,8 +1,10 @@ import Foundation /// Hears when the flag synchronizer's first request after going online starts and how it ended, so that events can -/// give way to it. +/// give way to it. The client connects the two only where `givesWayToFlags`; otherwise the synchronizer tells nothing. protocol FlagRequestListening: AnyObject { + /// Whether events give way to flag values, and so want to hear about flag requests. Fixed for its life. + var givesWayToFlags: Bool { get } func flagRequestStarted() func flagRequestSettled(_ outcome: FlagRequestOutcome) } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 3bdb48386..d8bc9548f 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -509,6 +509,24 @@ class DirectEventProcessorSpec: QuickSpec { expect(opened.withLock { hasOpened }).toEventually(beTrue(), timeout: .seconds(2)) } + + it("holds a periodic delivery for a flag request in flight only where events persist") { + testContext = TestContext(eventFlushInterval: 0.2) + let reporter = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: testContext.store, + flagTimings: FlagTraffic.Timings(launchFloor: 0, launchCap: 0, gap: 0, flagRequestWait: 10)) + nextRun = reporter + expect(reporter.givesWayToFlags) == self.persists + reporter.flagRequestStarted() + reporter.isOnline = true + reporter.record(CustomEvent(key: "an-event", context: testContext.context)) + + if self.persists { + Thread.sleep(forTimeInterval: 1) + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + } else { + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: .seconds(2)) + } + } } } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/FlagSynchronizerSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/FlagSynchronizerSpec.swift index be7f40833..d3d519718 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/FlagSynchronizerSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/FlagSynchronizerSpec.swift @@ -46,6 +46,7 @@ final class FlagSynchronizerSpec: QuickSpec { /// Records what a synchronizer tells the event processor about its flag requests. final class FlagRequestRecorder: FlagRequestListening { + let givesWayToFlags = true private let lock = NSLock() private var _reports: [String] = [] var reports: [String] { From b1a7ed1b0da4c02f1be315e4575739f3e843ee08 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Wed, 28 Oct 2026 10:39:05 -0700 Subject: [PATCH 115/158] feat: version batch names, leaving a batch another version named to expiry Batches are now named ready-v1-., as on Android, so that a later version may change how the rest of the name reads without this one misreading the count. A ready- batch not named v1- is another version's: it is never listed, counted against capacity or sent. Expiry still deletes it once it is past the maximum age, without reading or counting it, both in this environment and in abandoned ones, so it cannot stay on the device for good. Co-authored-by: Cursor --- .../ServiceObjects/EventDirectory.swift | 54 +++++++++--- .../DirectEventProcessorSpec.swift | 8 +- .../ServiceObjects/EventStoreSpec.swift | 84 +++++++++++++++---- 3 files changed, 115 insertions(+), 31 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index e2aedd133..ccd043d84 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -30,9 +30,10 @@ struct EventTally: Equatable { /// The lock is handed down so that each call can check it is held, rather than each level taking a lock of its own /// that would have to be ordered against the others. final class EventDirectory { - /// Names a log that has been closed off for delivery. The rest of the name is the batch's payload ID and how many - /// events are in it, `ready-.`. Anything more a batch needs to say belongs in a frame type readers - /// skip, rather than in the name. + /// Names a log that has been closed off for delivery, whichever version named it. What follows is the version of the + /// rest of the name, `batchNameVersion`, then the batch's payload ID and how many events are in it: + /// `ready-v1-.`. Anything more a batch needs to say belongs in a frame type readers skip, rather + /// than in the name. /// /// The count is in the name so that listing a batch this process did not close costs a directory entry rather than /// a read of the whole file. That is what every batch is at startup: the ones a previous run left behind, and the @@ -41,6 +42,13 @@ final class EventDirectory { /// /// A batch is never appended to once it is closed, so the count is as good as the file is. private static let batchPrefix = "ready-" + /// The version of how the rest of a batch's name reads, so that a later version may change it without this one + /// misreading the count. A batch another version named is never listed, counted or sent: reading it is that + /// version's to do. Expiry still removes it, as it does any batch, so that a file this version does not read cannot + /// stay on the device for good. A payload ID is a UUID, which cannot start with a `v`. + private static let batchNameVersion = "v1-" + /// Names a batch this version named, and so reads. + private static let ownBatchPrefix = batchPrefix + batchNameVersion private static let nameFieldSeparator: Character = "." /// Names a log whose events are not yet in a batch: one its owner is appending to, or one left by an owner that is /// gone, for recovery to close off. The rest of the name is the UUID it was opened under. @@ -238,18 +246,33 @@ final class EventDirectory { /// Requires `ioLock`. private func newUrl(of payloadId: String, holding events: Int) -> URL { disk.directory.appendingPathComponent( - "\(EventDirectory.batchPrefix)\(payloadId)\(EventDirectory.nameFieldSeparator)\(events)") + "\(EventDirectory.ownBatchPrefix)\(payloadId)\(EventDirectory.nameFieldSeparator)\(events)") } - /// A batch as its name, `ready-.`, gives it: each field by its position. + /// A name for a batch of events not yet counted, which the first listing to read it counts. Requires `ioLock`. + private func newUncountedUrl() -> URL { + disk.directory.appendingPathComponent("\(EventDirectory.ownBatchPrefix)\(UUID().uuidString)") + } + + /// Whether a file is a batch this version named, and so reads. + private static func isOwnBatch(_ name: String) -> Bool { + name.hasPrefix(ownBatchPrefix) + } + + /// Whether a file is a batch some other version named, which this one leaves to expiry. + private static func isAnotherVersionsBatch(_ name: String) -> Bool { + name.hasPrefix(batchPrefix) && !name.hasPrefix(ownBatchPrefix) + } + + /// A batch this version named, `ready-v1-.`, as its name gives it: each field by its position. /// /// Fields are split at `nameFieldSeparator`, which a payload ID never contains, so nothing about the count's own - /// characters is needed to tell it apart. Fields after it, which a later version might add, are ignored. + /// characters is needed to tell it apart. Fields after it are ignored. /// /// - Returns: the payload ID, and the count, which is nil where the name has none or it does not parse -- a batch a /// recovery could not read, which is read once to count it. private static func parse(batchName name: String) -> (payloadId: String, events: Int?) { - let fields = name.dropFirst(batchPrefix.count) + let fields = name.dropFirst(ownBatchPrefix.count) .split(separator: nameFieldSeparator, omittingEmptySubsequences: false) let payloadId = String(fields[0]) guard fields.count > 1, !fields[1].isEmpty, fields[1].allSatisfy({ $0.isASCII && $0.isNumber }), @@ -376,7 +399,7 @@ final class EventDirectory { } let batches = contents - .filter { $0.lastPathComponent.hasPrefix(EventDirectory.batchPrefix) } + .filter { EventDirectory.isOwnBatch($0.lastPathComponent) } .compactMap { file -> (EventBatch, Date)? in let (payloadId, counted) = EventDirectory.parse(batchName: file.lastPathComponent) guard !isDeliveredButUndeleted(payloadId, at: file) @@ -523,7 +546,7 @@ final class EventDirectory { let contents = (try? FileManager.default.contentsOfDirectory(at: disk.directory, includingPropertiesForKeys: nil, options: [.skipsHiddenFiles])) ?? [] - var logs = contents.filter { $0.lastPathComponent.hasPrefix(EventDirectory.batchPrefix) }.sorted { $0.path < $1.path } + var logs = contents.filter { EventDirectory.isOwnBatch($0.lastPathComponent) }.sorted { $0.path < $1.path } if disk.log.exists { logs.append(disk.log.url) } @@ -677,7 +700,7 @@ final class EventDirectory { // to. // Named without a count, which is what a batch whose events have never been counted is: the listing that // first manages to read it is what counts it. - let uncounted = disk.directory.appendingPathComponent("\(EventDirectory.batchPrefix)\(UUID().uuidString)") + let uncounted = newUncountedUrl() if (try? FileManager.default.moveItem(at: leftOpen, to: uncounted)) != nil { os_log("%s could not read events from a previous run, will try again at the next delivery: %s", log: logger, type: .info, typeName(and: #function), String(describing: error)) } else { @@ -730,7 +753,8 @@ extension EventDirectory { /// event. /// /// A batch whose request is out is left alone however old it is, whichever store or process sent it; see - /// `BatchLocks`. + /// `BatchLocks`. A batch another version named is deleted without being read or counted: it was never counted + /// against capacity, and its events were never this version's to lose. /// /// - Parameter sending: batches whose request is out from this store. /// - Returns: how many events were discarded, and how much this process had counted of the batches that left the @@ -744,7 +768,9 @@ extension EventDirectory { var counted = EventTally() for file in contents where EventDirectory.listed(file).modified.map({ $0 < cutoff }) ?? false { let name = file.lastPathComponent - if name.hasPrefix(EventDirectory.batchPrefix) { + if EventDirectory.isAnotherVersionsBatch(name) { + _ = BatchLocks.deleteUnlessBeingSent(file) + } else if EventDirectory.isOwnBatch(name) { let (payloadId, named) = EventDirectory.parse(batchName: name) guard !sending.contains(payloadId), !deliveredButUndeleted.contains(payloadId), !BatchLocks.isSentInThisProcess(file) @@ -807,7 +833,9 @@ extension EventDirectory { options: [])) ?? [] for file in contents where listed(file).modified.map({ $0 < cutoff }) ?? false { let name = file.lastPathComponent - if name.hasPrefix(batchPrefix) { + if isAnotherVersionsBatch(name) { + _ = BatchLocks.deleteUnlessBeingSent(file) + } else if isOwnBatch(name) { guard !BatchLocks.isSentInThisProcess(file) else { continue } let held = parse(batchName: name).events ?? eventsReading(file) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index d8bc9548f..86f4d40fe 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -2558,7 +2558,7 @@ extension DirectEventProcessorSpec { testContext.eventProcessor.commitRecordedEvents() let damaged = store.closeBatch()! // Its name still counts it, so it is listed without being read; the damage is only found on sending. - let damagedUrl = store.directory!.appendingPathComponent("ready-\(damaged.payloadId).\(damaged.eventCount)") + let damagedUrl = store.directory!.appendingPathComponent("ready-v1-\(damaged.payloadId).\(damaged.eventCount)") try Data("not an event log".utf8).write(to: damagedUrl) testContext.recordEvents(1) @@ -2585,7 +2585,7 @@ extension DirectEventProcessorSpec { testContext.recordEvents(3) testContext.eventProcessor.commitRecordedEvents() let damaged = store.closeBatch()! - let damagedUrl = store.directory!.appendingPathComponent("ready-\(damaged.payloadId).\(damaged.eventCount)") + let damagedUrl = store.directory!.appendingPathComponent("ready-v1-\(damaged.payloadId).\(damaged.eventCount)") var bytes = try Data(contentsOf: damagedUrl) bytes[bytes.count - 1] ^= 0x01 try bytes.write(to: damagedUrl) @@ -2618,7 +2618,7 @@ extension DirectEventProcessorSpec { testContext.eventProcessor.commitRecordedEvents() let gone = store.closeBatch()! try FileManager.default.removeItem( - at: store.directory!.appendingPathComponent("ready-\(gone.payloadId).\(gone.eventCount)")) + at: store.directory!.appendingPathComponent("ready-v1-\(gone.payloadId).\(gone.eventCount)")) testContext.recordEvents(1) var delivered: Bool? @@ -2682,7 +2682,7 @@ extension DirectEventProcessorSpec { testContext.recordEvents(1) testContext.eventProcessor.commitRecordedEvents() let damaged = store.closeBatch()! - let damagedUrl = store.directory!.appendingPathComponent("ready-\(damaged.payloadId).\(damaged.eventCount)") + let damagedUrl = store.directory!.appendingPathComponent("ready-v1-\(damaged.payloadId).\(damaged.eventCount)") try Data("not an event log".utf8).write(to: damagedUrl) testContext.eventProcessor.flush(completion: nil) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 5d177efda..9e968b206 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -323,6 +323,15 @@ final class EventStoreSpec: QuickSpec { return renamed } + /// Renames a batch to a name a later version might give it, holding events this version would otherwise read. + @discardableResult + private static func asAnotherVersionsBatch(_ batch: URL) throws -> URL { + // Fields this version would take for a payload ID and a count, were the name its own. + let renamed = batch.deletingLastPathComponent().appendingPathComponent("ready-v2-7.\(UUID().uuidString)") + try FileManager.default.moveItem(at: batch, to: renamed) + return renamed + } + /// What a process out of file descriptors gets for a file that is perfectly intact. private static func readFailing(while failing: @escaping () -> Bool) -> (URL) throws -> Data { { url in @@ -770,7 +779,7 @@ final class EventStoreSpec: QuickSpec { } it("is listed by the count in its name, and unreadable") { - try EventStoreSpec.tooLargeFile("ready-\(UUID().uuidString).3", in: store) + try EventStoreSpec.tooLargeFile("ready-v1-\(UUID().uuidString).3", in: store) let batches = store.pendingBatches() expect(batches.map { $0.eventCount }) == [3] @@ -778,7 +787,7 @@ final class EventStoreSpec: QuickSpec { } it("is listed as one event, the fewest it can hold, where its name has no count, and unreadable") { - try EventStoreSpec.tooLargeFile("ready-\(UUID().uuidString)", in: store) + try EventStoreSpec.tooLargeFile("ready-v1-\(UUID().uuidString)", in: store) let batches = store.pendingBatches() expect(batches.map { $0.eventCount }) == [1] @@ -786,7 +795,7 @@ final class EventStoreSpec: QuickSpec { } it("is discarded as one event once it expires, where nothing counts it") { - let uncounted = try EventStoreSpec.tooLargeFile("ready-\(UUID().uuidString)", in: store) + let uncounted = try EventStoreSpec.tooLargeFile("ready-v1-\(UUID().uuidString)", in: store) let log = try EventStoreSpec.tooLargeFile(EventDirectory.logPrefix + UUID().uuidString, in: store) try EventStoreSpec.age(uncounted, byDays: 8) try EventStoreSpec.age(log, byDays: 8) @@ -875,7 +884,7 @@ final class EventStoreSpec: QuickSpec { // Nothing appends to a batch once it is closed, so the count its name carries still holds and // listing it has no reason to go to the disk. Damaging the file is how the test tells the two apart: // a listing that read would refuse this batch. - let batchUrl = store.directory!.appendingPathComponent("ready-\(batch.payloadId).2") + let batchUrl = store.directory!.appendingPathComponent("ready-v1-\(batch.payloadId).2") try Data("not an event log".utf8).write(to: batchUrl) // The store that closed it and one that did not both count it from the name alone. @@ -893,8 +902,8 @@ final class EventStoreSpec: QuickSpec { // A name without a count, which is what a recovery that could not read a log leaves behind for a // listing to count. - let counted = store.directory!.appendingPathComponent("ready-\(batch.payloadId).1") - let uncounted = store.directory!.appendingPathComponent("ready-\(batch.payloadId)") + let counted = store.directory!.appendingPathComponent("ready-v1-\(batch.payloadId).1") + let uncounted = store.directory!.appendingPathComponent("ready-v1-\(batch.payloadId)") try FileManager.default.moveItem(at: counted, to: uncounted) let reader = EventStoreSpec.reader(sharing: store) @@ -911,8 +920,8 @@ final class EventStoreSpec: QuickSpec { // Its last group fits an Int32 once the leading zeros are dropped. let payloadId = "6F9619FF-8B86-D011-B42D-000012345678" - let counted = store.directory!.appendingPathComponent("ready-\(batch.payloadId).1") - let uncounted = store.directory!.appendingPathComponent("ready-\(payloadId)") + let counted = store.directory!.appendingPathComponent("ready-v1-\(batch.payloadId).1") + let uncounted = store.directory!.appendingPathComponent("ready-v1-\(payloadId)") try FileManager.default.moveItem(at: counted, to: uncounted) expect(EventStoreSpec.reader(sharing: store).pendingBatches()) == [EventBatch(payloadId: payloadId, eventCount: 1)] @@ -924,8 +933,8 @@ final class EventStoreSpec: QuickSpec { let batch = store.closeBatch()! // What a later version might add, which this one has no use for. - let counted = store.directory!.appendingPathComponent("ready-\(batch.payloadId).2") - let extended = store.directory!.appendingPathComponent("ready-\(batch.payloadId).2.v2-something") + let counted = store.directory!.appendingPathComponent("ready-v1-\(batch.payloadId).2") + let extended = store.directory!.appendingPathComponent("ready-v1-\(batch.payloadId).2.v2-something") try FileManager.default.moveItem(at: counted, to: extended) expect(EventStoreSpec.reader(sharing: store).pendingBatches()) == [batch] @@ -935,13 +944,34 @@ final class EventStoreSpec: QuickSpec { _ = store.stage(EventStoreSpec.payload("only")) let batch = store.closeBatch()! - let counted = store.directory!.appendingPathComponent("ready-\(batch.payloadId).1") - let unparseable = store.directory!.appendingPathComponent("ready-\(batch.payloadId).not-a-count") + let counted = store.directory!.appendingPathComponent("ready-v1-\(batch.payloadId).1") + let unparseable = store.directory!.appendingPathComponent("ready-v1-\(batch.payloadId).not-a-count") try FileManager.default.moveItem(at: counted, to: unparseable) expect(EventStoreSpec.reader(sharing: store).pendingBatches()) == [batch] } + it("names a batch with the version of how its name reads") { + _ = store.stage(EventStoreSpec.payload("only")) + let batch = store.closeBatch()! + + let named = store.directory!.appendingPathComponent("ready-v1-\(batch.payloadId).1") + expect(FileManager.default.fileExists(atPath: named.path)) == true + } + + it("neither counts nor sends a batch another version named") { + _ = store.stage(EventStoreSpec.payload("first")) + _ = store.stage(EventStoreSpec.payload("second")) + _ = store.closeBatch() + let batch = try EventStoreSpec.asAnotherVersionsBatch(try EventStoreSpec.onlyBatchFile(in: store)) + + let reader = EventStoreSpec.reader(sharing: store) + expect(reader.pendingBatches()).to(beEmpty()) + expect(reader.pendingEventCount) == 0 + expect(reader.pendingEventPayloads()).to(beEmpty()) + expect(FileManager.default.fileExists(atPath: batch.path)) == true + } + it("gives a batch its own payload id, so a retry is not read as a new delivery") { _ = store.stage(EventStoreSpec.payload("first")) let first = store.closeBatch()! @@ -1368,7 +1398,7 @@ final class EventStoreSpec: QuickSpec { // The listing that can delete it does, rather than leaving it for a run that would send it again. _ = store.pendingBatches() let left = try FileManager.default.contentsOfDirectory(at: directory, includingPropertiesForKeys: nil) - expect(left.filter { $0.lastPathComponent.hasPrefix("ready-\(first.payloadId)") }).to(beEmpty()) + expect(left.filter { $0.lastPathComponent.hasPrefix("ready-v1-\(first.payloadId)") }).to(beEmpty()) } it("stops counting a batch once when another process delivered it and a listing came between") { @@ -1417,7 +1447,7 @@ final class EventStoreSpec: QuickSpec { let delivered = store.closeBatch()! try FileManager.default.removeItem( - at: store.directory!.appendingPathComponent("ready-\(deleted.payloadId).\(deleted.eventCount)")) + at: store.directory!.appendingPathComponent("ready-v1-\(deleted.payloadId).\(deleted.eventCount)")) EventStoreSpec.reader(sharing: store).remove(delivered) expect(store.listPendingBatches().vanishedEvents) == 2 @@ -1833,6 +1863,21 @@ final class EventStoreSpec: QuickSpec { expect(store.stage(EventStoreSpec.payload("new"))) == true } + it("discards an expired batch another version named without counting it") { + _ = store.stage(EventStoreSpec.payload("first")) + _ = store.stage(EventStoreSpec.payload("second")) + _ = store.closeBatch() + let batch = try EventStoreSpec.asAnotherVersionsBatch(try EventStoreSpec.onlyBatchFile(in: store)) + let reader = EventStoreSpec.reader(sharing: store) + + expect(reader.discardExpired()) == 0 + expect(FileManager.default.fileExists(atPath: batch.path)) == true + + try EventStoreSpec.age(batch, byDays: 8) + expect(reader.discardExpired()) == 0 + expect(FileManager.default.fileExists(atPath: batch.path)) == false + } + it("keeps a batch younger than the maximum age") { _ = store.stage(EventStoreSpec.payload("recent")) _ = store.closeBatch() @@ -1963,6 +2008,17 @@ final class EventStoreSpec: QuickSpec { expect(try FileManager.default.contentsOfDirectory(atPath: otherApplication.path)).toNot(beEmpty()) } + it("empties an abandoned environment of batches another version named, without counting them") { + let abandoned = try EventStoreSpec.environment("app.abandoned", in: root, events: 2) + let batch = try FileManager.default.contentsOfDirectory(at: abandoned, includingPropertiesForKeys: nil) + .first { $0.lastPathComponent.hasPrefix("ready-") }! + try EventStoreSpec.asAnotherVersionsBatch(batch) + try EventStoreSpec.age(contentsOf: abandoned, byDays: 8) + + expect(store.discardExpired()) == 0 + expect(FileManager.default.fileExists(atPath: abandoned.path)) == false + } + it("does nothing where the application did not ask for persistence") { let abandoned = try EventStoreSpec.environment("app.abandoned", in: root, events: 1) try EventStoreSpec.age(contentsOf: abandoned, byDays: 8) From a015c89347c15f000fdb7c439e4c5383626003e1 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Wed, 28 Oct 2026 11:37:09 -0700 Subject: [PATCH 116/158] refactor: give way to flag values by time alone, without tracking flag requests Flag request tracking is removed: FlagTraffic, FlagRequestListening and the reporting from FlagSynchronizer. It covered only the request the synchronizer starts with, and streaming and polling each settle differently, so there were more cases it did not see than it did. What stays is timing. Where events persist, a previous run's events and the diagnostic init event wait 2 seconds after the reporter first goes online, and the deliveries the SDK starts on its own leave at least 1 second between one batch's answer and the next batch's request. A flush, backgrounding, going offline and close end the pacing as before. At disabled nothing waits. Co-authored-by: Cursor --- LaunchDarkly.xcodeproj/project.pbxproj | 10 -- LaunchDarkly/LaunchDarkly/LDClient.swift | 20 +-- .../ServiceObjects/DirectEventProcessor.swift | 145 ++++++++------- .../ServiceObjects/FlagSynchronizer.swift | 58 +----- .../ServiceObjects/FlagTraffic.swift | 167 ------------------ .../DirectEventProcessorSpec.swift | 143 +++++---------- .../ServiceObjects/FlagSynchronizerSpec.swift | 82 --------- 7 files changed, 139 insertions(+), 486 deletions(-) delete mode 100644 LaunchDarkly/LaunchDarkly/ServiceObjects/FlagTraffic.swift diff --git a/LaunchDarkly.xcodeproj/project.pbxproj b/LaunchDarkly.xcodeproj/project.pbxproj index 5ca0b5b49..82a32c4d2 100644 --- a/LaunchDarkly.xcodeproj/project.pbxproj +++ b/LaunchDarkly.xcodeproj/project.pbxproj @@ -21,7 +21,6 @@ A1B2C3D4E5F60718293A4B01 /* EventDirectory.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D4E5F60718293A4B5C /* EventDirectory.swift */; }; 3C55384C9E3485B493E1F86A /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; C3F7E502BD916C40AE238F64 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; - 713C1C0A71192DA31CD28E1F /* FlagTraffic.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34ED898C31FA1DC1E35C8266 /* FlagTraffic.swift */; }; AB4827D11084D58661E24B05 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; 3D2406142E0D90E000F91253 /* Plugin.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3D2406102E0D90E000F91253 /* Plugin.swift */; }; 3D2406152E0D90E000F91253 /* PluginMetadata.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3D2406112E0D90E000F91253 /* PluginMetadata.swift */; }; @@ -59,7 +58,6 @@ A1B2C3D4E5F60718293A4B02 /* EventDirectory.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D4E5F60718293A4B5C /* EventDirectory.swift */; }; DDA788E3056F8AC3CA203933 /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; D4080613CEA27D51BF349075 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; - 45B28E8795FFF24BDA23A37C /* FlagTraffic.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34ED898C31FA1DC1E35C8266 /* FlagTraffic.swift */; }; ECBE45806B470ACC5187B7D6 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; 7185A3C30E7FFC4F725769F1 /* BackgroundActivity.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9A771B316E19ED598CD4FB0B /* BackgroundActivity.swift */; }; 830BF933202D188E006DF9B1 /* HTTPURLRequest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 830BF932202D188E006DF9B1 /* HTTPURLRequest.swift */; }; @@ -414,7 +412,6 @@ A1B2C3D4E5F60718293A4B03 /* EventDirectory.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D4E5F60718293A4B5C /* EventDirectory.swift */; }; 44E0BE449714952A68628CEB /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; E5191724DFB38E62C0451086 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; - 8C5CDD837AA4B41BF2D94268 /* FlagTraffic.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34ED898C31FA1DC1E35C8266 /* FlagTraffic.swift */; }; CA2ADB3D20A229D70CA34F73 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; B40B419C249ADA6B00CD0726 /* DiagnosticCacheSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = B40B419B249ADA6B00CD0726 /* DiagnosticCacheSpec.swift */; }; B4265EB124E7390C001CFD2C /* TestUtil.swift in Sources */ = {isa = PBXBuildFile; fileRef = B4265EB024E7390C001CFD2C /* TestUtil.swift */; }; @@ -471,7 +468,6 @@ A1B2C3D4E5F60718293A4B04 /* EventDirectory.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D4E5F60718293A4B5C /* EventDirectory.swift */; }; F0F504E5EC32223905356098 /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; B2E6D4F1AC805B3F9D127E53 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; - CF5053295713576276009C0F /* FlagTraffic.swift in Sources */ = {isa = PBXBuildFile; fileRef = 34ED898C31FA1DC1E35C8266 /* FlagTraffic.swift */; }; 625ADAC0CCFE3BE324C45752 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; D7A500FCA04E3412F931003F /* ContextEncodingCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = D93B7F5F6CF5C3DE79FB4286 /* ContextEncodingCache.swift */; }; DAF5695D58E28B8CF9221213 /* LDContextJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88DAB8D1864C14D9C7E06F11 /* LDContextJSONWriter.swift */; }; @@ -707,7 +703,6 @@ A1B2C3D4E5F60718293A4B5C /* EventDirectory.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventDirectory.swift; sourceTree = ""; }; 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventLogFile.swift; sourceTree = ""; }; A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = DeliveryClaims.swift; sourceTree = ""; }; - 34ED898C31FA1DC1E35C8266 /* FlagTraffic.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = FlagTraffic.swift; sourceTree = ""; }; B58551BB165F28784C785720 /* EventLogFormat.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventLogFormat.swift; sourceTree = ""; }; C408884623033B3600420721 /* ConnectionInformationStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConnectionInformationStore.swift; sourceTree = ""; }; C408884823033B7500420721 /* ConnectionInformation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConnectionInformation.swift; sourceTree = ""; }; @@ -1103,7 +1098,6 @@ A1B2C3D4E5F60718293A4B5C /* EventDirectory.swift */, 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */, A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */, - 34ED898C31FA1DC1E35C8266 /* FlagTraffic.swift */, B58551BB165F28784C785720 /* EventLogFormat.swift */, ); path = ServiceObjects; @@ -1641,7 +1635,6 @@ A1B2C3D4E5F60718293A4B04 /* EventDirectory.swift in Sources */, F0F504E5EC32223905356098 /* EventLogFile.swift in Sources */, B2E6D4F1AC805B3F9D127E53 /* DeliveryClaims.swift in Sources */, - CF5053295713576276009C0F /* FlagTraffic.swift in Sources */, 625ADAC0CCFE3BE324C45752 /* EventLogFormat.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; @@ -1747,7 +1740,6 @@ A1B2C3D4E5F60718293A4B01 /* EventDirectory.swift in Sources */, 3C55384C9E3485B493E1F86A /* EventLogFile.swift in Sources */, C3F7E502BD916C40AE238F64 /* DeliveryClaims.swift in Sources */, - 713C1C0A71192DA31CD28E1F /* FlagTraffic.swift in Sources */, AB4827D11084D58661E24B05 /* EventLogFormat.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; @@ -1852,7 +1844,6 @@ A1B2C3D4E5F60718293A4B02 /* EventDirectory.swift in Sources */, DDA788E3056F8AC3CA203933 /* EventLogFile.swift in Sources */, D4080613CEA27D51BF349075 /* DeliveryClaims.swift in Sources */, - 45B28E8795FFF24BDA23A37C /* FlagTraffic.swift in Sources */, ECBE45806B470ACC5187B7D6 /* EventLogFormat.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; @@ -2024,7 +2015,6 @@ A1B2C3D4E5F60718293A4B03 /* EventDirectory.swift in Sources */, 44E0BE449714952A68628CEB /* EventLogFile.swift in Sources */, E5191724DFB38E62C0451086 /* DeliveryClaims.swift in Sources */, - 8C5CDD837AA4B41BF2D94268 /* FlagTraffic.swift in Sources */, CA2ADB3D20A229D70CA34F73 /* EventLogFormat.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; diff --git a/LaunchDarkly/LaunchDarkly/LDClient.swift b/LaunchDarkly/LaunchDarkly/LDClient.swift index 608c3d5eb..504d57371 100644 --- a/LaunchDarkly/LaunchDarkly/LDClient.swift +++ b/LaunchDarkly/LaunchDarkly/LDClient.swift @@ -982,7 +982,7 @@ public class LDClient { let networkReporter: NetworkReporting private(set) var flagCache: FeatureFlagCaching - private(set) var flagSynchronizer: LDFlagSynchronizing { didSet { reportFlagRequests() } } + private(set) var flagSynchronizer: LDFlagSynchronizing var flagChangeNotifier: FlagChangeNotifying private(set) var eventProcessor: EventProcessor private(set) var environmentReporter: EnvironmentReporting @@ -1083,7 +1083,7 @@ public class LDClient { service.resetFlagResponseCache(etag: cachedData.etag) // Reports to this client, so it could not be given at construction: `self` is only usable from here. flagSynchronizer.onSyncComplete = onFlagSyncComplete - giveFlagRequestsTheNetworkFirst() + giveFlagValuesTheNetworkFirst() if let cachedFlags = cachedData.items, !cachedFlags.isEmpty { flagStore.replaceStore(newStoredItems: cachedFlags) @@ -1105,19 +1105,9 @@ public class LDClient { extension LDClient: TypeIdentifying { } extension LDClient { - /// Has the flag synchronizer tell the event processor about its flag requests, so that events can give way to them. - /// Only where they do: otherwise the synchronizer tells nothing. - fileprivate func reportFlagRequests() { - guard let listener = eventProcessor as? FlagRequestListening, listener.givesWayToFlags - else { return } - (flagSynchronizer as? FlagSynchronizer)?.flagRequests = listener - } - - /// Has events give way to flag requests: the synchronizer reports its requests to the event processor, and the - /// diagnostic init event waits for the processor's launch gate as a previous run's events do. Where events do not - /// persist neither happens, and the init event goes out as it did before persistence. - fileprivate func giveFlagRequestsTheNetworkFirst() { - reportFlagRequests() + /// Has the diagnostic init event wait for the event processor's launch gate, as a previous run's events do. Where + /// events do not persist there is no gate, and the init event goes out as it did before persistence. + fileprivate func giveFlagValuesTheNetworkFirst() { if let eventProcessor = eventProcessor as? DirectEventProcessor, eventProcessor.givesWayToFlags { (diagnosticReporter as? DiagnosticReporter)?.sendsInitAfter = eventProcessor.whenLaunchGateOpens } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index aaedd83dc..52f65dab5 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -22,11 +22,34 @@ import OSLog /// `LDConfig.eventPersistence`. Only at `.immediate` can `track` promise the event is on disk by the time it returns. /// At `.disabled` nothing commits early: the held run waits for the delivery, which encodes it. /// -/// Flag values come first. Where events persist, what a previous run left waits at launch for the first flag request, -/// and the deliveries this reporter starts on its own send one batch at a time, giving way to any flag request in -/// flight. The flag synchronizer says what it is doing through `FlagRequestListening`. A delivery the application asks -/// for is never held. -class DirectEventProcessor: EventProcessor, FlagRequestListening { +/// Flag values come first. Where events persist, what a previous run left waits at launch until a fixed time after the +/// reporter first goes online, and the deliveries this reporter starts on its own send one batch at a time, a gap apart, +/// leaving room for flag requests. Both go by time alone: nothing here hears what the flag synchronizer is doing. A +/// delivery the application asks for is never held. +class DirectEventProcessor: EventProcessor { + /// How long the deliveries this reporter starts on its own wait, so that flag values come first. + struct Pacing { + static let specified = Pacing(launchDelay: 2, gap: 1) + /// No waits at all, for tests about something else. + static let noWaits = Pacing(launchDelay: 0, gap: 0) + + /// How long after first going online that a previous run's events, and the diagnostic init event, wait. It + /// gives the application's startup traffic, the first flag request among it, the network first, and is short + /// enough for an application that dies a few seconds after every launch to still send them. + let launchDelay: TimeInterval + /// The least time between one paced batch's answer and the next batch's request. + let gap: TimeInterval + } + + /// A paced batch waiting out its gap. + private final class PacedTurn { + let proceed: () -> Void + + init(_ proceed: @escaping () -> Void) { + self.proceed = proceed + } + } + /// How many full events may be held unencoded before one of them pays for a commit. /// /// It sets two things at once. The first is how many evaluations a termination can take -- not everything a crash @@ -67,13 +90,11 @@ class DirectEventProcessor: EventProcessor, FlagRequestListening { /// Its summarizer is guarded by `recordLock`, and its encoders are reached only under `commitLock`. let eventBuffer: OutboundEventBuffer - private let flagTimings: FlagTraffic.Timings + private let pacing: Pacing /// Whether events give way to flag values: only where they persist, like the rest of what persistence brings. /// Without it there is no previous run to hold back, and delivery is what it was before persistence: no launch - /// gate, no pacing, and nothing heard about flag requests. + /// gate and no pacing. let givesWayToFlags: Bool - /// Only where events give way to flags; nil otherwise, so that nothing of it runs. - private let flagTraffic: FlagTraffic? // MARK: Guarded by commitLock @@ -178,6 +199,8 @@ class DirectEventProcessor: EventProcessor, FlagRequestListening { private var isPacedDelivery = false /// When the paced delivery running last had an answer for a post, or nil before its first. private var lastPacedPostAnswered: DispatchTime? + /// The batch of the paced delivery running that waits out its gap, which ending the pacing lets go at once. + private var waitingTurn: PacedTurn? #if DEBUG /// How many deliveries have started, which is fewer than the flushes asked for where they were joined. @@ -222,9 +245,9 @@ class DirectEventProcessor: EventProcessor, FlagRequestListening { /// question and the delivery that acts on it are both enqueued there, in that order. private var hasEventsFromPreviousRun = false - /// Whether the launch gate has opened. It opens once the floor has passed since first going online, and the first - /// flag request has settled, or there was none, or the cap has passed too. Until then neither a previous run's events - /// nor the diagnostic init event are sent. Open from the start where events do not give way. + /// Whether the launch gate has opened. It opens once the launch delay has passed since first going online. Until then + /// neither a previous run's events nor the diagnostic init event are sent. Open from the start where events do not + /// give way. private var launchGateOpen: Bool /// When the reporter first went online, which the gate counts from. private var wentOnlineAt: DispatchTime? @@ -237,7 +260,7 @@ class DirectEventProcessor: EventProcessor, FlagRequestListening { encoding: OutboundEventBuffer.Encoding = .handWritten, commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.DirectEventProcessor.commitQueue", qos: .userInitiated), retryState: DeliveryRetryState? = nil, - flagTimings: FlagTraffic.Timings = .specified) { + pacing: Pacing = .specified) { let eventStore = store ?? DirectEventProcessor.makeStore(config: service.config) self.service = service self.onSyncComplete = onSyncComplete @@ -248,9 +271,8 @@ class DirectEventProcessor: EventProcessor, FlagRequestListening { self.commitQueue = commitQueue self.eventBuffer = OutboundEventBuffer(config: service.config, encoding: encoding) self.store = eventStore - self.flagTimings = flagTimings + self.pacing = pacing self.givesWayToFlags = eventStore.isPersisting - self.flagTraffic = eventStore.isPersisting ? FlagTraffic() : nil self.launchGateOpen = !eventStore.isPersisting // A log left open by a previous run has to be closed before it can be delivered, but nothing waits on that: @@ -577,7 +599,7 @@ extension DirectEventProcessor { if self.wentOnlineAt == nil { self.wentOnlineAt = .now() if !self.launchGateOpen { - self.checkLaunchGate(after: self.flagTimings.launchFloor) + self.checkLaunchGate(after: self.pacing.launchDelay) } } guard self.launchGateOpen, self.hasEventsFromPreviousRun @@ -593,17 +615,6 @@ extension DirectEventProcessor { endPacing() } - func flagRequestStarted() { - flagTraffic?.started() - } - - func flagRequestSettled(_ outcome: FlagRequestOutcome) { - guard let flagTraffic - else { return } - flagTraffic.settled(outcome) - deliveryQueue.async { [weak self] in self?.checkLaunchGate() } - } - /// Runs `block`, on a global queue, once the launch gate is open. func whenLaunchGateOpens(_ block: @escaping () -> Void) { deliveryQueue.async { [weak self] in @@ -620,20 +631,15 @@ extension DirectEventProcessor { deliveryQueue.asyncAfter(deadline: .now() + max(0, delay)) { [weak self] in self?.checkLaunchGate() } } - /// Opens the launch gate if it may open now, and otherwise looks again when it next could: at the floor, then at - /// the cap, or sooner once the flag request settles. On `deliveryQueue`. + /// Opens the launch gate once the launch delay has passed since first going online, and otherwise looks again then. + /// On `deliveryQueue`. private func checkLaunchGate() { - guard !launchGateOpen, let wentOnlineAt, let flagTraffic + guard !launchGateOpen, let wentOnlineAt else { return } let sinceOnline = TimeInterval(DispatchTime.now().uptimeNanoseconds - wentOnlineAt.uptimeNanoseconds) / 1_000_000_000 - guard sinceOnline >= flagTimings.launchFloor - else { - checkLaunchGate(after: flagTimings.launchFloor - sinceOnline) - return - } - guard let why = flagTraffic.launchGateReason(capReached: sinceOnline >= flagTimings.launchCap, cap: flagTimings.launchCap) + guard sinceOnline >= pacing.launchDelay else { - checkLaunchGate(after: flagTimings.launchCap - sinceOnline) + checkLaunchGate(after: pacing.launchDelay - sinceOnline) return } launchGateOpen = true @@ -642,7 +648,7 @@ extension DirectEventProcessor { waiters.forEach { DispatchQueue.global().async(execute: $0) } guard hasEventsFromPreviousRun else { return } - os_log("%s Sending events from a previous run after %s", log: service.config.logger, type: .debug, typeName(and: #function), why) + os_log("%s Sending events from a previous run, %dms after going online", log: service.config.logger, type: .debug, typeName(and: #function), Int(pacing.launchDelay * 1000)) // Offline, they wait for the reporter to go online, which finds the gate open. guard isOnline else { return } @@ -712,22 +718,47 @@ extension DirectEventProcessor { /// Ends the pacing of a delivery under way, which then sends the rest back to back, or, offline, stops. private func endPacing() { - let wasPaced = deliveryLock.withLock { () -> Bool in - defer { isPacedDelivery = false } - return isPacedDelivery + let turn = deliveryLock.withLock { () -> PacedTurn? in + defer { + isPacedDelivery = false + waitingTurn = nil + } + return waitingTurn } - if wasPaced { - flagTraffic?.wake() + if let turn { + DispatchQueue.global().async(execute: turn.proceed) } } - /// When the next batch of the delivery running may be posted, or nil where it is not paced. - private func pacedTurn() -> DispatchTime? { - deliveryLock.withLock { - guard isPacedDelivery + /// Holds the next batch of the delivery running until the gap has passed since the last one was answered, where the + /// delivery is paced and has posted before, and then calls `proceed` on a global queue; or sooner, once the pacing + /// ends. + /// + /// - Returns: whether it holds the batch; false where the batch may go now, and `proceed` is not called. + private func awaitPacedTurn(then proceed: @escaping () -> Void) -> Bool { + let turn = PacedTurn(proceed) + // Under the same lock that ends the pacing, so that the pacing cannot end between the check and the wait. + guard let gapEnds = deliveryLock.withLock({ () -> DispatchTime? in + guard isPacedDelivery, let answered = lastPacedPostAnswered else { return nil } - return lastPacedPostAnswered.map { $0 + flagTimings.gap } ?? .now() + waitingTurn = turn + return answered + pacing.gap + }) + else { return false } + DispatchQueue.global().asyncAfter(deadline: gapEnds) { [weak self] in + guard let self + else { return } + let taken = self.deliveryLock.withLock { () -> Bool in + guard self.waitingTurn === turn + else { return false } + self.waitingTurn = nil + return true + } + if taken { + turn.proceed() + } } + return true } /// Queues a delivery, or joins one that is queued and has not started. @@ -1032,9 +1063,9 @@ extension DirectEventProcessor { /// `lostAnything` carries whether a batch already taken in this pass was lost for good, so that the caller hears /// about it even where the batches after it were accepted. `retried` whether this pass has had its retry. /// - /// A paced delivery holds each batch first: until a gap has passed since the last one was answered, and then while - /// a flag request is in flight, up to a limit. A backlog this sends unasked otherwise takes the network for as long - /// as it lasts, an `identify`'s flag request included. `hadTurn` is whether the first batch has been held already. + /// A paced delivery holds each batch after its first until a gap has passed since the last one was answered. A + /// backlog this sends unasked otherwise takes the network for as long as it lasts, an `identify`'s flag request + /// included. `hadTurn` is whether the first batch has been held already. private func deliver(_ batches: [EventBatch], lostAnything: Bool, retried: Bool, hadTurn: Bool = false, _ completion: @escaping (PassOutcome) -> Void) { var remaining = batches guard !remaining.isEmpty @@ -1043,15 +1074,9 @@ extension DirectEventProcessor { return } - if !hadTurn, let flagTraffic, let gapEnds = pacedTurn() { - let isStopped = { [weak self] () -> Bool in - guard let self - else { return true } - return self.deliveryLock.withLock { !self.isPacedDelivery } - } - flagTraffic.awaitTurn(gapEnds: gapEnds, flagWait: flagTimings.flagRequestWait, isStopped: isStopped) { - self.deliver(batches, lostAnything: lostAnything, retried: retried, hadTurn: true, completion) - } + if !hadTurn, awaitPacedTurn(then: { + self.deliver(batches, lostAnything: lostAnything, retried: retried, hadTurn: true, completion) + }) { return } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/FlagSynchronizer.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/FlagSynchronizer.swift index a738db9fa..8fb10d36a 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/FlagSynchronizer.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/FlagSynchronizer.swift @@ -99,14 +99,6 @@ class FlagSynchronizer: LDFlagSynchronizing, EventHandler { private var lastCachedRequestedTime: Date? private var syncQueue = DispatchQueue(label: Constants.queueName, qos: .utility) - - /// Told when the first flag request after going online starts and how it ends, so that events can give way to it. - /// Set once the client exists, as `onSyncComplete` is. - weak var flagRequests: FlagRequestListening? - private let flagRequestLock = UnfairLock() - /// Whether that request has started and not yet been told settled. Only to be used while holding `flagRequestLock`. - private var flagRequestInFlight = false - private var eventSourceStarted: Date? // Only accessed on the event source callback queue. private var connectedAt: Date? @@ -135,43 +127,14 @@ class FlagSynchronizer: LDFlagSynchronizing, EventHandler { switch streamingMode { case .streaming: stopPolling() - if eventSource == nil { - flagRequestStarted() - } startEventSource() case .polling: stopEventSource() - // A poll a fresh cache puts off makes no request now, and events do not wait for the later one. - if flagRequestTimer == nil && firstPollDelay == 0 { - flagRequestStarted() - } startPolling() } } else { stopEventSource() stopPolling() - flagRequestSettled(.stopped) - } - } - - /// Without a listener, which is where events do not give way to flags, nothing is tracked. - private func flagRequestStarted() { - guard let flagRequests - else { return } - flagRequestLock.withLock { flagRequestInFlight = true } - flagRequests.flagRequestStarted() - } - - /// Tells the listener how the first request ended, once; any answer after it, and every later poll, says nothing. - private func flagRequestSettled(_ outcome: FlagRequestOutcome) { - guard flagRequests != nil - else { return } - let wasInFlight = flagRequestLock.withLock { () -> Bool in - defer { flagRequestInFlight = false } - return flagRequestInFlight - } - if wasInFlight { - flagRequests?.flagRequestSettled(outcome) } } @@ -229,8 +192,10 @@ class FlagSynchronizer: LDFlagSynchronizing, EventHandler { return } - let initialDelay = firstPollDelay - if self.lastCachedRequestedTime != nil { + var initialDelay: TimeInterval = 0 + if let lastTime = self.lastCachedRequestedTime { + // The cache is still fresh, so delay the first poll until the cached flags would go stale. + initialDelay = max(0, lastTime.addingTimeInterval(pollingInterval).timeIntervalSinceNow) // We loaded cached flags, so report completion immediately rather than blocking on the first poll. syncQueue.async { [self] in guard isOnline @@ -242,13 +207,6 @@ class FlagSynchronizer: LDFlagSynchronizing, EventHandler { os_log("%s", log: service.config.logger, type: .debug, typeName(and: #function)) } - /// How long the first poll waits: while the cache is still fresh, until the cached flags would go stale. - private var firstPollDelay: TimeInterval { - guard let lastTime = lastCachedRequestedTime - else { return 0 } - return max(0, lastTime.addingTimeInterval(pollingInterval).timeIntervalSinceNow) - } - private func stopPolling() { guard flagRequestTimer != nil else { @@ -328,10 +286,7 @@ class FlagSynchronizer: LDFlagSynchronizing, EventHandler { } var failed = false var unexpected = false - defer { - flagRequestSettled(failed ? .failed : .loaded) - pollDidComplete(failed: failed, unexpected: unexpected) - } + defer { pollDidComplete(failed: failed, unexpected: unexpected) } if let serviceResponseError = serviceResponse.error { os_log("%s error: %s", log: service.config.logger, type: .debug, typeName(and: #function), String(describing: serviceResponseError)) @@ -399,7 +354,6 @@ class FlagSynchronizer: LDFlagSynchronizing, EventHandler { streamingRetry.recordFailure(unexpected: SynchronizingError.streamError(error).isTerminal) let delay = streamingRetry.nextDelay() os_log("%s stream error; reconnecting in %.3fs. error: %s", log: service.config.logger, type: .debug, typeName(and: #function), delay, String(describing: error)) - flagRequestSettled(.failed) reportSyncComplete(.error(.streamError(error))) // Tear down the failed connection now, then reconnect after the backoff. @@ -462,12 +416,10 @@ class FlagSynchronizer: LDFlagSynchronizing, EventHandler { guard let data = messageEvent.data.data(using: .utf8), let flagCollection = try? JSONDecoder().decode(FeatureFlagCollection.self, from: data) else { - flagRequestSettled(.failed) reportDataError(messageEvent.data.data(using: .utf8)) return } - flagRequestSettled(.loaded) // NOTE: If you are adding e-tag support through the streaming // connection, make sure you read the documentation on the // FeatureFlagCaching.saveCachedData method. diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/FlagTraffic.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/FlagTraffic.swift deleted file mode 100644 index 619bbe30b..000000000 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/FlagTraffic.swift +++ /dev/null @@ -1,167 +0,0 @@ -import Foundation - -/// Hears when the flag synchronizer's first request after going online starts and how it ended, so that events can -/// give way to it. The client connects the two only where `givesWayToFlags`; otherwise the synchronizer tells nothing. -protocol FlagRequestListening: AnyObject { - /// Whether events give way to flag values, and so want to hear about flag requests. Fixed for its life. - var givesWayToFlags: Bool { get } - func flagRequestStarted() - func flagRequestSettled(_ outcome: FlagRequestOutcome) -} - -enum FlagRequestOutcome { - case loaded - case failed - /// The synchronizer went offline, or was replaced, before an answer came. - case stopped -} - -/// Whether a flag request is in flight, and the waits events make for it. -/// -/// Flag values decide what the application shows, and events only report on it afterwards. So at launch, and whenever -/// the SDK sends a backlog on its own, events wait for the flag request rather than compete with it for the network. -final class FlagTraffic { - /// How long events give way. - struct Timings { - static let specified = Timings(launchFloor: 2, launchCap: 10, gap: 1, flagRequestWait: 10) - /// No waits at all, for tests about something else. - static let noWaits = Timings(launchFloor: 0, launchCap: 0, gap: 0, flagRequestWait: 0) - - /// The soonest after first going online that a previous run's events are sent, whatever the flag request is - /// doing. It gives the application's own startup traffic the network first, and is short enough for an - /// application that dies a few seconds after every launch to still send them. - let launchFloor: TimeInterval - /// The longest after first going online that a previous run's events wait for the first flag request: the - /// default connection timeout, by which a request that has not answered has most likely failed. - let launchCap: TimeInterval - /// The least time between one paced batch's answer and the next batch's request. - let gap: TimeInterval - /// How long after a flag request starts that paced batches stop waiting for it, so that a hung one cannot stop - /// delivery. It counts from the request rather than from each batch, so the launch gate's cap and the batches - /// after it never wait on the same request again. - let flagRequestWait: TimeInterval - } - - /// One batch waiting for its turn. - private final class Turn { - let gapEnds: DispatchTime - let flagWait: TimeInterval - let isStopped: () -> Bool - let proceed: () -> Void - /// When it stops waiting on the flag request in flight, as last scheduled for a check. Only to be used while - /// holding the lock. - var flagWaitEnds: DispatchTime? - - init(gapEnds: DispatchTime, flagWait: TimeInterval, isStopped: @escaping () -> Bool, proceed: @escaping () -> Void) { - self.gapEnds = gapEnds - self.flagWait = flagWait - self.isStopped = isStopped - self.proceed = proceed - } - } - - private let lock = UnfairLock() - // Only to be used while holding `lock`. - private var inFlight = false - /// When the request in flight started. - private var startedAt = DispatchTime.now() - /// How the last flag request ended, or nil if none has. - private var lastOutcome: FlagRequestOutcome? - private var turns: [Turn] = [] - - func started() { - lock.withLock { - inFlight = true - startedAt = DispatchTime.now() - } - } - - func settled(_ outcome: FlagRequestOutcome) { - lock.withLock { - inFlight = false - lastOutcome = outcome - } - checkTurns() - } - - /// Has the batches waiting look at whether they were stopped again. - func wake() { - checkTurns() - } - - /// Why the launch gate may open, once its floor has passed, or nil while it waits on the flag request. - /// - /// - Parameters: - /// - capReached: whether the cap has passed, after which it waits on nothing - /// - cap: the cap, for saying so - func launchGateReason(capReached: Bool, cap: TimeInterval) -> String? { - lock.withLock { - guard inFlight - else { - switch lastOutcome { - case .loaded: return "flags loaded" - case .failed: return "the flag request failed" - case .stopped, nil: return "no flag request at launch" - } - } - return capReached ? "waiting \(Int(cap * 1000))ms for flags" : nil - } - } - - /// Calls `proceed`, on a global queue, once `gapEnds` has passed and then no flag request is in flight, waiting on - /// one until `flagWait` after it started; or at once when `isStopped` says so. `isStopped` is asked outside this - /// object's lock, so it may take a lock of its own. - func awaitTurn(gapEnds: DispatchTime, flagWait: TimeInterval, isStopped: @escaping () -> Bool, then proceed: @escaping () -> Void) { - let turn = Turn(gapEnds: gapEnds, flagWait: flagWait, isStopped: isStopped, proceed: proceed) - lock.withLock { turns.append(turn) } - DispatchQueue.global().asyncAfter(deadline: gapEnds) { [weak self] in self?.checkTurns() } - checkTurns() - } - - private func checkTurns() { - let now = DispatchTime.now() - var due: [Turn] = [] - var waiting: [Turn] = [] - var deadlines: [DispatchTime] = [] - lock.withLock { - for turn in turns { - guard now >= turn.gapEnds - else { - waiting.append(turn) - continue - } - guard inFlight - else { - due.append(turn) - continue - } - let ends = startedAt + turn.flagWait - if now >= ends { - due.append(turn) - continue - } - if turn.flagWaitEnds != ends { - turn.flagWaitEnds = ends - deadlines.append(ends) - } - waiting.append(turn) - } - } - due += waiting.filter { $0.isStopped() } - for deadline in deadlines { - DispatchQueue.global().asyncAfter(deadline: deadline) { [weak self] in self?.checkTurns() } - } - for turn in due { - // Two checks can find the same turn due; only the one that takes it proceeds. - let taken = lock.withLock { () -> Bool in - guard let index = turns.firstIndex(where: { $0 === turn }) - else { return false } - turns.remove(at: index) - return true - } - if taken { - DispatchQueue.global().async(execute: turn.proceed) - } - } - } -} diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 86f4d40fe..09598f53b 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -78,7 +78,7 @@ class DirectEventProcessorSpec: QuickSpec { self.store = store ?? EventStore.temporary(capacity: config.eventCapacity, persistEvents: persistence != .disabled) eventProcessor = DirectEventProcessor(service: serviceMock, onSyncComplete: onSyncComplete, store: self.store, commitQueue: commitQueue, retryState: retryState, - flagTimings: .noWaits) + pacing: .noWaits) (0.. DirectEventProcessor { + func nextRunLeft(_ batches: Int, pacing: DirectEventProcessor.Pacing) -> DirectEventProcessor { for _ in 0..= 0.7 + expect(times.first?.timeIntervalSince(wentOnline)) >= 0.75 } - itPersisting("holds no batch any longer for a flag request past the cap") { - let reporter = nextRunLeft(2, timings: FlagTraffic.Timings(launchFloor: 0.1, launchCap: 2, gap: 0.3, flagRequestWait: 2)) - reporter.flagRequestStarted() - let wentOnline = Date() + itPersisting("counts the launch delay from going online rather than from the reporter starting") { + let reporter = nextRunLeft(1, pacing: DirectEventProcessor.Pacing(launchDelay: 0.6, gap: 0)) + Thread.sleep(forTimeInterval: 0.8) reporter.isOnline = true - // Waiting on the request again would add the two seconds' wait to each batch. - let times = postTimes(2) - expect(times.count) == 2 - expect(times.first?.timeIntervalSince(wentOnline)) < 3 - expect(times.last?.timeIntervalSince(times.first ?? wentOnline)) < 1.5 + Thread.sleep(forTimeInterval: 0.3) + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) } itPersisting("sends what a previous run left on a flush, without waiting for the launch gate") { - let reporter = nextRunLeft(1, timings: FlagTraffic.Timings(launchFloor: 5, launchCap: 10, gap: 0, flagRequestWait: 0)) - reporter.flagRequestStarted() + let reporter = nextRunLeft(1, pacing: DirectEventProcessor.Pacing(launchDelay: 5, gap: 0)) reporter.isOnline = true waitUntil(timeout: .seconds(3)) { done in @@ -427,7 +402,7 @@ class DirectEventProcessorSpec: QuickSpec { } itPersisting("sends what a previous run left a batch at a time, with a gap after each answer") { - let reporter = nextRunLeft(3, timings: FlagTraffic.Timings(launchFloor: 0, launchCap: 0, gap: 0.4, flagRequestWait: 0)) + let reporter = nextRunLeft(3, pacing: DirectEventProcessor.Pacing(launchDelay: 0, gap: 0.4)) reporter.isOnline = true @@ -439,7 +414,7 @@ class DirectEventProcessorSpec: QuickSpec { } itPersisting("sends the rest at once when a flush is asked for while pacing") { - let reporter = nextRunLeft(3, timings: FlagTraffic.Timings(launchFloor: 0, launchCap: 0, gap: 10, flagRequestWait: 0)) + let reporter = nextRunLeft(3, pacing: DirectEventProcessor.Pacing(launchDelay: 0, gap: 10)) reporter.isOnline = true expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) @@ -450,33 +425,8 @@ class DirectEventProcessorSpec: QuickSpec { expect(testContext.serviceMock.publishEventDataCallCount) == 3 } - itPersisting("holds a paced batch while a flag request is in flight") { - let reporter = nextRunLeft(2, timings: FlagTraffic.Timings(launchFloor: 0, launchCap: 0, gap: 0.3, flagRequestWait: 5)) - reporter.isOnline = true - expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) - reporter.flagRequestStarted() - Thread.sleep(forTimeInterval: 1) - expect(testContext.serviceMock.publishEventDataCallCount) == 1 - - reporter.flagRequestSettled(.loaded) - - expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(2), timeout: .seconds(2)) - } - - itPersisting("holds paced batches for a hung flag request only until the wait after it started") { - let reporter = nextRunLeft(3, timings: FlagTraffic.Timings(launchFloor: 0, launchCap: 0, gap: 0.3, flagRequestWait: 1)) - reporter.flagRequestStarted() - - reporter.isOnline = true - - // Waiting on the request again would add a second to each batch. - let times = postTimes(3) - expect(times.count) == 3 - expect(times.last?.timeIntervalSince(times.first ?? Date())) < 1.5 - } - itPersisting("stops a paced delivery on going offline, and keeps the rest") { - let reporter = nextRunLeft(3, timings: FlagTraffic.Timings(launchFloor: 0, launchCap: 0, gap: 0.5, flagRequestWait: 0)) + let reporter = nextRunLeft(3, pacing: DirectEventProcessor.Pacing(launchDelay: 0, gap: 0.5)) reporter.isOnline = true expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) @@ -493,39 +443,34 @@ class DirectEventProcessorSpec: QuickSpec { it("holds what waits for the launch gate only where events persist") { let reporter = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: testContext.store, - flagTimings: FlagTraffic.Timings(launchFloor: 0.1, launchCap: 5, gap: 0, flagRequestWait: 0)) + pacing: DirectEventProcessor.Pacing(launchDelay: 1, gap: 0)) nextRun = reporter + expect(reporter.givesWayToFlags) == self.persists let opened = UnfairLock() var hasOpened = false reporter.whenLaunchGateOpens { opened.withLock { hasOpened = true } } - reporter.flagRequestStarted() reporter.isOnline = true if self.persists { Thread.sleep(forTimeInterval: 0.6) expect(opened.withLock { hasOpened }) == false - reporter.flagRequestSettled(.failed) } expect(opened.withLock { hasOpened }).toEventually(beTrue(), timeout: .seconds(2)) } - it("holds a periodic delivery for a flag request in flight only where events persist") { + itPersisting("paces a periodic delivery of a backlog too") { testContext = TestContext(eventFlushInterval: 0.2) - let reporter = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: testContext.store, - flagTimings: FlagTraffic.Timings(launchFloor: 0, launchCap: 0, gap: 0, flagRequestWait: 10)) - nextRun = reporter - expect(reporter.givesWayToFlags) == self.persists - reporter.flagRequestStarted() + // A launch delay longer than the example, so that the report timer rather than the gate starts it. + let reporter = nextRunLeft(2, pacing: DirectEventProcessor.Pacing(launchDelay: 60, gap: 1)) + reporter.isOnline = true - reporter.record(CustomEvent(key: "an-event", context: testContext.context)) - if self.persists { - Thread.sleep(forTimeInterval: 1) - expect(testContext.serviceMock.publishEventDataCallCount) == 0 - } else { - expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: .seconds(2)) - } + // Waited for on the run loop, which the report timer fires on. + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + let first = Date() + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(2), timeout: .seconds(3)) + expect(Date().timeIntervalSince(first)) >= 0.8 } } } @@ -567,7 +512,7 @@ class DirectEventProcessorSpec: QuickSpec { var testContext: TestContext! beforeEach { testContext = TestContext() - testContext.eventProcessor = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: { _ in }, store: testContext.store, flagTimings: .noWaits) + testContext.eventProcessor = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: { _ in }, store: testContext.store, pacing: .noWaits) } it("starts offline without reporting events") { expect(testContext.eventProcessor.service) === testContext.serviceMock @@ -735,7 +680,7 @@ class DirectEventProcessorSpec: QuickSpec { serviceMock.config = config serviceMock.stubEventResponse(success: true) store = EventStore.temporary(capacity: config.eventCapacity) - return DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: encoding, flagTimings: .noWaits) + return DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: encoding, pacing: .noWaits) } afterEach { @@ -897,7 +842,7 @@ class DirectEventProcessorSpec: QuickSpec { serviceMock.config = config serviceMock.stubEventResponse(success: true) store = EventStore.temporary(capacity: config.eventCapacity) - reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: encoding, flagTimings: .noWaits) + reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: encoding, pacing: .noWaits) reporter.isOnline = true return reporter } @@ -1500,7 +1445,7 @@ class DirectEventProcessorSpec: QuickSpec { /// A reporter writing to a store of its own, and that store, so a test can read back what was recorded. func makeReporter() -> (DirectEventProcessor, EventStore) { let store = EventStore.temporary() - return (DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, flagTimings: .noWaits), store) + return (DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, pacing: .noWaits), store) } describe("recordFlagEvaluationEvents") { @@ -1513,7 +1458,7 @@ class DirectEventProcessorSpec: QuickSpec { service.diagnosticCache = diagnosticCache let store = EventStore.temporary(capacity: config.eventCapacity) defer { store.deleteEverything() } - let reporter = DirectEventProcessor(service: service, onSyncComplete: nil, store: store, flagTimings: .noWaits) + let reporter = DirectEventProcessor(service: service, onSyncComplete: nil, store: store, pacing: .noWaits) let untracked = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: false) let tracked = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: true) @@ -1878,7 +1823,7 @@ extension DirectEventProcessorSpec { // Nothing delivered them and nothing closed the log, as would be the case had the process died here. let recovered = testContext.store.nextRun() - let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered, flagTimings: .noWaits) + let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered, pacing: .noWaits) nextRun.isOnline = true waitUntil(timeout: Constants.deliveryTimeout) { done in @@ -1900,7 +1845,7 @@ extension DirectEventProcessorSpec { testContext.eventProcessor.commitRecordedEvents() let recovered = testContext.store.nextRun() - let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered, flagTimings: .noWaits) + let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered, pacing: .noWaits) // Nothing asks for a flush, and the report interval is several times longer than this waits, so the // only thing that can deliver these is the reporter coming online. @@ -1922,7 +1867,7 @@ extension DirectEventProcessorSpec { try FileManager.default.setAttributes([.modificationDate: longAgo], ofItemAtPath: file.path) } - let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: EventStore.sharing(directory), flagTimings: .noWaits) + let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: EventStore.sharing(directory), pacing: .noWaits) nextRun.isOnline = true var delivered: Bool? waitUntil(timeout: Constants.deliveryTimeout) { done in @@ -2096,7 +2041,7 @@ extension DirectEventProcessorSpec { // after launch reads every batch a previous run left behind. let store = SlowListingStore(testContext.store) defer { store.release.signal() } - let reporter = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: store, flagTimings: .noWaits) + let reporter = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: store, pacing: .noWaits) reporter.isOnline = true reporter.record(CustomEvent(key: "tracked", context: testContext.context)) @@ -2197,7 +2142,7 @@ extension DirectEventProcessorSpec { testContext.eventProcessor.record(CustomEvent(key: "kept", context: testContext.context)) let recovered = testContext.store.nextRun() - let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered, flagTimings: .noWaits) + let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered, pacing: .noWaits) nextRun.isOnline = true defer { nextRun.isOnline = false } @@ -2222,7 +2167,7 @@ extension DirectEventProcessorSpec { // The same directory, as the application's next launch would find it. testContext.serviceMock.stubEventResponse(success: true) let recovered = EventStore.sharing(testContext.store.directory!) - let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered, flagTimings: .noWaits) + let nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: recovered, pacing: .noWaits) nextRun.isOnline = true defer { nextRun.isOnline = false } @@ -2339,7 +2284,7 @@ extension DirectEventProcessorSpec { let commits = DispatchQueue(label: "com.launchdarkly.tests.suspended") commits.suspend() defer { commits.resume() } - let reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: .codable, commitQueue: commits, flagTimings: .noWaits) + let reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: .codable, commitQueue: commits, pacing: .noWaits) let slow = EncodingGate(key: "slow", context: LDContext.stub()) reporter.record(slow) @@ -2373,7 +2318,7 @@ extension DirectEventProcessorSpec { let commits = DispatchQueue(label: "com.launchdarkly.tests.suspended") commits.suspend() defer { commits.resume() } - let reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: .codable, commitQueue: commits, flagTimings: .noWaits) + let reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: .codable, commitQueue: commits, pacing: .noWaits) let slow = EncodingGate(key: "slow", context: LDContext.stub()) let competingCommit = DispatchSemaphore(value: 0) @@ -2415,7 +2360,7 @@ extension DirectEventProcessorSpec { let commits = DispatchQueue(label: "com.launchdarkly.tests.suspended") commits.suspend() defer { commits.resume() } - let reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: .codable, commitQueue: commits, flagTimings: .noWaits) + let reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: .codable, commitQueue: commits, pacing: .noWaits) let slow = EncodingGate(key: "slow", context: LDContext.stub()) reporter.record(CustomEvent(key: "first", context: LDContext.stub())) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/FlagSynchronizerSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/FlagSynchronizerSpec.swift index d3d519718..3534f20f8 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/FlagSynchronizerSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/FlagSynchronizerSpec.swift @@ -41,88 +41,6 @@ final class FlagSynchronizerSpec: QuickSpec { pollingTimerFiresSpec() pollingBackoffSpec() flagRequestSpec() - flagRequestReportingSpec() - } - - /// Records what a synchronizer tells the event processor about its flag requests. - final class FlagRequestRecorder: FlagRequestListening { - let givesWayToFlags = true - private let lock = NSLock() - private var _reports: [String] = [] - var reports: [String] { - lock.lock() - defer { lock.unlock() } - return _reports - } - - func flagRequestStarted() { - record("started") - } - - func flagRequestSettled(_ outcome: FlagRequestOutcome) { - record("settled \(outcome)") - } - - private func record(_ report: String) { - lock.lock() - defer { lock.unlock() } - _reports.append(report) - } - } - - func flagRequestReportingSpec() { - describe("reporting flag requests") { - it("reports the stream starting, and that its first payload loaded flags, once") { - let testContext = TestContext(streamingMode: .streaming, useReport: false) - let recorder = FlagRequestRecorder() - testContext.flagSynchronizer.flagRequests = recorder - - testContext.flagSynchronizer.isOnline = true - expect(recorder.reports) == ["started"] - testContext.flagSynchronizer.testStreamOnMessage(event: "put", messageEvent: MessageEvent(data: "{\"flag1\": {}}")) - testContext.flagSynchronizer.testStreamOnMessage(event: "put", messageEvent: MessageEvent(data: "{\"flag1\": {}}")) - - expect(recorder.reports) == ["started", "settled loaded"] - testContext.flagSynchronizer.isOnline = false - expect(recorder.reports) == ["started", "settled loaded"] - } - - it("reports a request going offline before it answered as stopped") { - let testContext = TestContext(streamingMode: .streaming, useReport: false) - let recorder = FlagRequestRecorder() - testContext.flagSynchronizer.flagRequests = recorder - - testContext.flagSynchronizer.isOnline = true - testContext.flagSynchronizer.isOnline = false - - expect(recorder.reports) == ["started", "settled stopped"] - } - - it("reports the first poll and that it failed") { - let testContext = TestContext(streamingMode: .polling, useReport: false) - testContext.serviceMock.stubFlagResponse(statusCode: HTTPURLResponse.StatusCodes.internalServerError, responseOnly: true) - let recorder = FlagRequestRecorder() - testContext.flagSynchronizer.flagRequests = recorder - - testContext.flagSynchronizer.isOnline = true - - expect(recorder.reports).toEventually(equal(["started", "settled failed"])) - testContext.flagSynchronizer.isOnline = false - } - - it("reports nothing for a poll a fresh cache puts off") { - let serviceMock = DarklyServiceMock() - let flagSynchronizer = FlagSynchronizer(streamingMode: .polling, pollingInterval: 60, useReport: false, - lastUpdated: Date(), service: serviceMock, onSyncComplete: nil) - let recorder = FlagRequestRecorder() - flagSynchronizer.flagRequests = recorder - - flagSynchronizer.isOnline = true - flagSynchronizer.isOnline = false - - expect(recorder.reports) == [] - } - } } func initSpec() { From cedff6af259002cd23c66bd4ddaa95a8ef9b7304 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Wed, 28 Oct 2026 12:28:46 -0700 Subject: [PATCH 117/158] refactor: move recording and commits out of DirectEventProcessor into EventCommitter Co-authored-by: Cursor --- LaunchDarkly.xcodeproj/project.pbxproj | 10 + .../ServiceObjects/DirectEventProcessor.swift | 394 ++------------- .../ServiceObjects/EventCommitter.swift | 455 ++++++++++++++++++ 3 files changed, 496 insertions(+), 363 deletions(-) create mode 100644 LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift diff --git a/LaunchDarkly.xcodeproj/project.pbxproj b/LaunchDarkly.xcodeproj/project.pbxproj index 82a32c4d2..fd0f3d49a 100644 --- a/LaunchDarkly.xcodeproj/project.pbxproj +++ b/LaunchDarkly.xcodeproj/project.pbxproj @@ -80,6 +80,7 @@ 831188562113AE0800D77CB5 /* FlagSynchronizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */; }; 831188572113AE0B00D77CB5 /* FlagChangeNotifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8358F25F1F476AD800ECE1AF /* FlagChangeNotifier.swift */; }; 831188582113AE0F00D77CB5 /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; + E7C0A11E2F00000000000002 /* EventCommitter.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000001 /* EventCommitter.swift */; }; B66ECB2014E0FC95C23DA0A2 /* AnalyticsEventSender.swift in Sources */ = {isa = PBXBuildFile; fileRef = 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */; }; 629D5A9FE152DBC3E20D6DFC /* OutboundEventBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */; }; 014901E06BE1F2B0FAAD6AA6 /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; @@ -125,6 +126,7 @@ 831EF35520655E730001C643 /* FlagSynchronizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */; }; 831EF35620655E730001C643 /* FlagChangeNotifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8358F25F1F476AD800ECE1AF /* FlagChangeNotifier.swift */; }; 831EF35720655E730001C643 /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; + E7C0A11E2F00000000000003 /* EventCommitter.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000001 /* EventCommitter.swift */; }; 367D374B5CD1B2A8596C0738 /* AnalyticsEventSender.swift in Sources */ = {isa = PBXBuildFile; fileRef = 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */; }; 5A89648200D2566D3F8409BA /* OutboundEventBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */; }; CFF357D68BBF2DC8395930F2 /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; @@ -211,6 +213,7 @@ 83D9EC872062DEAB004D7FA6 /* FlagSynchronizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */; }; 83D9EC882062DEAB004D7FA6 /* FlagChangeNotifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8358F25F1F476AD800ECE1AF /* FlagChangeNotifier.swift */; }; 83D9EC892062DEAB004D7FA6 /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; + E7C0A11E2F00000000000004 /* EventCommitter.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000001 /* EventCommitter.swift */; }; 321659EE1824B83BD0D0F22D /* AnalyticsEventSender.swift in Sources */ = {isa = PBXBuildFile; fileRef = 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */; }; 73E23C0FA009141BB4CBC24C /* OutboundEventBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */; }; 54745AEF1A43D165F5A6E08F /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; @@ -239,6 +242,7 @@ 83F0A5641FB5F33800550A95 /* LDConfigSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83F0A5631FB5F33800550A95 /* LDConfigSpec.swift */; }; 83FEF8DD1F266742001CF12C /* FlagSynchronizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */; }; 83FEF8DF1F2667E4001CF12C /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; + E7C0A11E2F00000000000005 /* EventCommitter.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000001 /* EventCommitter.swift */; }; E91F5DCAEEE5FC4AC408BA67 /* AnalyticsEventSender.swift in Sources */ = {isa = PBXBuildFile; fileRef = 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */; }; 7D37AD3ECAA9D8E2927228C5 /* OutboundEventBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */; }; 9FA0EE3A87678F58F4406B24 /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; @@ -629,6 +633,7 @@ 83F0A5631FB5F33800550A95 /* LDConfigSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LDConfigSpec.swift; sourceTree = ""; }; 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FlagSynchronizer.swift; sourceTree = ""; }; 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DirectEventProcessor.swift; sourceTree = ""; }; + E7C0A11E2F00000000000001 /* EventCommitter.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = EventCommitter.swift; sourceTree = ""; }; 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AnalyticsEventSender.swift; sourceTree = ""; }; 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OutboundEventBuffer.swift; sourceTree = ""; }; BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NullEventProcessor.swift; sourceTree = ""; }; @@ -1078,6 +1083,7 @@ B4C9D4372489E20A004A9B03 /* DiagnosticReporter.swift */, 831425B0206B030100F2EF36 /* EnvironmentReporter.swift */, 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */, + E7C0A11E2F00000000000001 /* EventCommitter.swift */, 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */, 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */, BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */, @@ -1561,6 +1567,7 @@ 3D3AB9462A4F16FE003AECF1 /* ReportingConsts.swift in Sources */, 831188522113ADF700D77CB5 /* KeyedValueCache.swift in Sources */, 831188582113AE0F00D77CB5 /* DirectEventProcessor.swift in Sources */, + E7C0A11E2F00000000000002 /* EventCommitter.swift in Sources */, B66ECB2014E0FC95C23DA0A2 /* AnalyticsEventSender.swift in Sources */, 629D5A9FE152DBC3E20D6DFC /* OutboundEventBuffer.swift in Sources */, 014901E06BE1F2B0FAAD6AA6 /* NullEventProcessor.swift in Sources */, @@ -1687,6 +1694,7 @@ 831EF35620655E730001C643 /* FlagChangeNotifier.swift in Sources */, A358D6D92A4DE6A500270C60 /* ApplicationInfoEnvironmentReporter.swift in Sources */, 831EF35720655E730001C643 /* DirectEventProcessor.swift in Sources */, + E7C0A11E2F00000000000003 /* EventCommitter.swift in Sources */, 367D374B5CD1B2A8596C0738 /* AnalyticsEventSender.swift in Sources */, 5A89648200D2566D3F8409BA /* OutboundEventBuffer.swift in Sources */, CFF357D68BBF2DC8395930F2 /* NullEventProcessor.swift in Sources */, @@ -1799,6 +1807,7 @@ A3599E882A4B4AD400DB5C67 /* Modifier.swift in Sources */, 8358F2601F476AD800ECE1AF /* FlagChangeNotifier.swift in Sources */, 83FEF8DF1F2667E4001CF12C /* DirectEventProcessor.swift in Sources */, + E7C0A11E2F00000000000005 /* EventCommitter.swift in Sources */, E91F5DCAEEE5FC4AC408BA67 /* AnalyticsEventSender.swift in Sources */, 7D37AD3ECAA9D8E2927228C5 /* OutboundEventBuffer.swift in Sources */, 9FA0EE3A87678F58F4406B24 /* NullEventProcessor.swift in Sources */, @@ -1966,6 +1975,7 @@ 83D9EC882062DEAB004D7FA6 /* FlagChangeNotifier.swift in Sources */, C443A40B2315AA4D00145710 /* NetworkReporter.swift in Sources */, 83D9EC892062DEAB004D7FA6 /* DirectEventProcessor.swift in Sources */, + E7C0A11E2F00000000000004 /* EventCommitter.swift in Sources */, 321659EE1824B83BD0D0F22D /* AnalyticsEventSender.swift in Sources */, 73E23C0FA009141BB4CBC24C /* OutboundEventBuffer.swift in Sources */, 54745AEF1A43D165F5A6E08F /* NullEventProcessor.swift in Sources */, diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 52f65dab5..b8397a9dd 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -7,20 +7,9 @@ import OSLog /// That is what lets an application that dies moments after recording an event still report it: the events are on disk /// before the process is gone, and the next run of the application delivers them. /// -/// Recording an event summarizes it and, if it has to be delivered in full, holds it as an `Event`. Nothing is encoded -/// on the thread that recorded it; a commit turns the whole held run into bytes at once. So an evaluation of an -/// untracked flag costs a counter update, and an evaluation of a tracked one costs that plus an array append. -/// -/// Where the line falls is between the events an application asked for by name and the ones it did not. Recording a -/// custom or identify event is a *commit point*, because an application reporting something it chose to report is -/// saying this matters more than the microseconds it costs, and the crash it describes may be moments away. A commit -/// takes the whole held run with it, so the exposures leading up to the error go down alongside it. Evaluations get no -/// such promise, and are committed once `pendingCommitThreshold` of them have accumulated, when a delivery starts, or -/// on `flush`. -/// -/// Whether a commit point runs on the caller's thread is the application's choice, through -/// `LDConfig.eventPersistence`. Only at `.immediate` can `track` promise the event is on disk by the time it returns. -/// At `.disabled` nothing commits early: the held run waits for the delivery, which encodes it. +/// What the application records is accepted, held and committed to the store by `EventCommitter`, which says which +/// calls are commit points and which thread each commit runs on. This class delivers what is committed, and runs the +/// reporter's life: flushes, going online and offline, and the flush at close. /// /// Flag values come first. Where events persist, what a previous run left waits at launch until a fixed time after the /// reporter first goes online, and the deliveries this reporter starts on its own send one batch at a time, a gap apart, @@ -50,35 +39,12 @@ class DirectEventProcessor: EventProcessor { } } - /// How many full events may be held unencoded before one of them pays for a commit. - /// - /// It sets two things at once. The first is how many evaluations a termination can take -- not everything a crash - /// could take, since an application that flushes on its way out commits the whole run, but the window for the - /// terminations that run nothing on the way out, such as the system reclaiming a backgrounded process. Recording a - /// custom or identify event closes it too, because that commits. - /// - /// The second is what a commit point usually costs, since it encodes whatever is held before returning. The commit - /// queue keeps the run drained, so the common case is far below this; raising it trades that cost against the - /// number of writes. - /// - /// Neither is a ceiling. Reaching this only schedules a commit, and recording does not wait for it, so where events - /// arrive faster than the commit queue encodes them, or the queue is starved, the run grows up to the event - /// capacity, and the next commit point encodes all of it. - /// - /// It is the only thing that writes events no caller asked to have written. The store writes only when told to, so - /// every staged byte reaches the disk through a commit that started here, at a commit point, or at a delivery. - private static let pendingCommitThreshold = 32 - // MARK: Set in init and never changed let service: DarklyServiceProvider let store: EventStoring - - /// How many events the SDK will hold in total, across `pending` and the store. - private let eventCapacity: Int - - /// Where a commit runs when no caller is waiting on it. - private let commitQueue: DispatchQueue + let committer: EventCommitter + private let lostEvents: LostEventCount private let onSyncComplete: EventSyncCompleteClosure? @@ -87,73 +53,20 @@ class DirectEventProcessor: EventProcessor { /// When a post is allowed after a failure. Nothing is posted while it says to wait. private let retryState: DeliveryRetryState - /// Its summarizer is guarded by `recordLock`, and its encoders are reached only under `commitLock`. - let eventBuffer: OutboundEventBuffer - private let pacing: Pacing /// Whether events give way to flag values: only where they persist, like the rest of what persistence brings. /// Without it there is no previous run to hold back, and delivery is what it was before persistence: no launch /// gate and no pacing. let givesWayToFlags: Bool - // MARK: Guarded by commitLock - - /// Held for the whole of a commit, so that only one runs at a time. - /// - /// This is what a commit point's guarantee rests on. Without it a commit already in flight could take the caller's - /// event out of `pending` before the caller got there, leaving the caller nothing to write and returning while - /// those bytes were still being produced somewhere else. Waiting here instead means that when the call returns the - /// event is on disk, whichever commit put it there. It also keeps two encoders from staging their runs in - /// whichever order they happened to finish. - /// - /// Taken before `recordLock` and before anything the store locks, never after. - private let commitLock = UnfairLock() - - // MARK: Guarded by recordLock - - /// Guards everything one recording writes: the held events and the summarizer. - /// - /// One evaluation can produce a counter, a full event and a debug event, and the three have to land together. Taken - /// separately, a commit landing between them splits one evaluation across two payloads. - /// - /// Held for the appends and the handover of the run, never across the encoder or a syscall. It is a lock rather - /// than a queue because it is taken once per evaluation, and a `DispatchQueue.sync` costs enough at that rate to be - /// worth avoiding: the same measurement that made `EvaluationExposureDeduper` a lock applies here. - private let recordLock = UnfairLock() - - /// Full events recorded but not yet encoded. Only to be used while holding `recordLock`. - /// - /// Held rather than encoded because encoding early would not make them durable: the store stages bytes into memory - /// too, and only a commit reaches the file. Both forms are equally lost to a crash, so the encode may as well - /// happen where it is cheapest. - private var pending: [Event] = [] - - /// Whether a commit is already queued, so a run of recordings past the threshold asks for one write rather than - /// one each. Only to be used while holding `recordLock`. - private var isCommitScheduled = false - - // MARK: Guarded by stateLock - var lastEventResponseDate: Date { - stateLock.withLock { responseDate } + committer.lastEventResponseDate } - /// Guards the last response date, the record of what has been lost, and the count of refused payloads. - private let stateLock = UnfairLock() - /// Only to be used while holding `stateLock`. - private var responseDate: Date + // MARK: Guarded by stateLock - /// How many events the SDK accepted and then dropped for good. Only to be used while holding `stateLock`. - /// - /// A batch LaunchDarkly refuses is kept and retried, so it is not a loss, unless the refusal is a second refusal of - /// the payload in a row. Otherwise what is lost is what the store never received: an event or a summary that could not be serialized - /// when it was committed, or that serialized too large to store. A delivery that finds nothing to send cannot tell - /// those apart from events that arrived, so without this it would report success for them. - /// - /// A count rather than a flag. Each delivery takes the count as it closes its batch, and answers for the losses up - /// to that mark, so a loss is told to the caller whose delivery it happened before rather than to whichever - /// delivery answers next. - private var lostEvents: UInt64 = 0 + /// Guards which losses have been told to a caller, and the count of refused payloads. + private let stateLock = UnfairLock() /// The mark up to which losses have already been reported to a caller, so that none is told twice. Only to be /// used while holding `stateLock`. @@ -262,14 +175,15 @@ class DirectEventProcessor: EventProcessor { retryState: DeliveryRetryState? = nil, pacing: Pacing = .specified) { let eventStore = store ?? DirectEventProcessor.makeStore(config: service.config) + let lostEvents = LostEventCount() self.service = service self.onSyncComplete = onSyncComplete self.sender = AnalyticsEventSender(service: service) self.retryState = retryState ?? DeliveryRetryState(flushInterval: service.config.eventFlushInterval) - self.responseDate = Date() - self.eventCapacity = service.config.eventCapacity - self.commitQueue = commitQueue - self.eventBuffer = OutboundEventBuffer(config: service.config, encoding: encoding) + self.lostEvents = lostEvents + self.committer = EventCommitter(config: service.config, store: eventStore, encoding: encoding, + commitQueue: commitQueue, diagnosticCache: service.diagnosticCache, + lostEvents: lostEvents) self.store = eventStore self.pacing = pacing self.givesWayToFlags = eventStore.isPersisting @@ -312,18 +226,7 @@ class DirectEventProcessor: EventProcessor { // MARK: Recording func record(_ event: Event) { - let (dropped, needsCommit) = recordLock.withLock { - (!admitToPending(event), pending.count >= DirectEventProcessor.pendingCommitThreshold) - } - - if dropped { - reportDropped() - } - if DirectEventProcessor.isCommitPoint(event.kind) { - commitAtCommitPoint() - } else if needsCommit { - scheduleCommitWherePersisting() - } + committer.record(event) } // swiftlint:disable:next function_parameter_count @@ -333,58 +236,16 @@ class DirectEventProcessor: EventProcessor { featureFlag: FeatureFlag?, context: LDContext, includeReason: Bool) { - let recordingFeatureEvent = featureFlag?.trackEvents == true - let recordingDebugEvent = featureFlag?.shouldCreateDebugEvents(lastEventReportResponseTime: lastEventResponseDate) ?? false - // Built before the lock is taken, so that the critical section is only the writes. - let featureEvent = recordingFeatureEvent ? FeatureEvent(key: flagKey, context: context, value: value, defaultValue: defaultValue, featureFlag: featureFlag, includeReason: includeReason, isDebug: false) : nil - let debugEvent = recordingDebugEvent ? FeatureEvent(key: flagKey, context: context, value: value, defaultValue: defaultValue, featureFlag: featureFlag, includeReason: includeReason, isDebug: true) : nil - - var dropped = 0 - let (counted, needsCommit) = recordLock.withLock { () -> (Bool, Bool) in - let counted = eventBuffer.summarize(flagKey: flagKey, reportedValue: value, featureFlag: featureFlag, defaultValue: defaultValue, context: context) - if let featureEvent, !admitToPending(featureEvent) { dropped += 1 } - if let debugEvent, !admitToPending(debugEvent) { dropped += 1 } - return (counted, pending.count >= DirectEventProcessor.pendingCommitThreshold) - } - - // A refused evaluation is a loss like a refused event, since nothing later reconstructs its counter. Its full - // event is still decided by the event capacity alone. - if !counted { - service.diagnosticCache?.incrementDroppedEventCount() - } - for _ in 0.. Bool { - recordLock.assertOwned() - guard pending.count + store.pendingEventCount < eventCapacity, store.hasRoomForEventBytes - else { return false } - - pending.append(event) - return true + func commitRecordedEvents() { + committer.commitRecordedEvents() } - private func reportDropped() { - os_log("%s aborted. Event store is full", log: service.config.logger, type: .debug, typeName(and: #function)) - service.diagnosticCache?.incrementDroppedEventCount() + func commitWherePersisting() { + committer.commitWherePersisting() } /// Records stored events discarded for being older than the maximum age. @@ -410,14 +271,7 @@ class DirectEventProcessor: EventProcessor { /// `reportBatchLost(_:)` for part of a batch. private func reportStoredEventsLost(_ events: Int) { - stateLock.withLock { lostEvents += UInt64(events) } - } - - /// Records the loss of an event the SDK accepted and then could not keep, which no later attempt recovers. - private func reportLost() { - os_log("%s dropping an event the store would not take", log: service.config.logger, type: .error, typeName(and: #function)) - service.diagnosticCache?.incrementDroppedEventCount() - stateLock.withLock { lostEvents += 1 } + lostEvents.add(events) } /// Whether anything was lost up to `mark` that no caller has been told about, which this caller is now the one to @@ -434,151 +288,6 @@ class DirectEventProcessor: EventProcessor { } } - /// Commits at a commit point, on the caller's thread or off it as the application asked. - /// - /// Committing on the caller's thread is what lets `track` promise its event is on disk by the time it returns. - /// Scheduling it instead keeps the encode and the write off that thread, and the event is durable a moment later - /// rather than immediately. - private func commitAtCommitPoint() { - if service.config.eventPersistence == .immediate { - commitRecordedEvents() - } else { - scheduleCommitWherePersisting() - } - } - - /// Queues a commit where there is a disk for it to reach. - /// - /// Without persistence a commit makes nothing durable. All it would do is move the encode into the middle of the - /// application's evaluations, where it competes with them for the CPU and splits their counters across summaries; - /// left alone, the run waits for the delivery, which encodes it anyway, and capacity still bounds how much is held. - private func scheduleCommitWherePersisting() { - if store.isPersisting { - scheduleCommit() - } - } - - /// Queues a commit unless one is already queued, so a run of recordings asks for one write rather than one each. - private func scheduleCommit() { - let alreadyScheduled = recordLock.withLock { () -> Bool in - defer { isCommitScheduled = true } - return isCommitScheduled - } - guard !alreadyScheduled - else { return } - - commitQueue.async { [weak self] in - guard let self - else { return } - // Cleared before the commit rather than after it, so that an event held while this one is in flight can ask - // for another commit instead of finding one apparently already on its way. - self.recordLock.withLock { self.isCommitScheduled = false } - - self.commitRecordedEvents() - } - } - - /// Encodes everything recorded since the last commit, stages it, and commits it. - /// - /// The run and the counters are taken in one critical section, so that an evaluation's counter and its full event - /// are staged by the same commit, and encoded outside it, so recording does not wait on the encoder. - func commitRecordedEvents() { - commitLock.withLock { encodeAndCommitPending() } - } - - /// Commits what is held and closes it into a batch, as one step with respect to other commits. - /// - /// Closing takes everything staged so far, and a commit stages its run one event at a time, so a commit running - /// between the two would be closed partway: an evaluation's event in this batch and its summary in the next. - /// - /// - Returns: the loss count as of the close, which is the mark this delivery answers for, and whether the close - /// took every committed event into a batch. Both taken under the same lock, so they cover the closed batch and - /// nothing a later commit does. - private func commitAndCloseBatch() -> (lossMark: UInt64, closedEverything: Bool) { - commitLock.withLock { - encodeAndCommitPending() - _ = store.closeBatch() - reportVanished(store.takeVanishedLogEvents()) - return (stateLock.withLock { lostEvents }, !store.holdsUnclosedEvents) - } - } - - /// Requires `commitLock`. - private func encodeAndCommitPending() { - commitLock.assertOwned() - let (run, summaries) = recordLock.withLock { () -> ([Event], [(tracker: FlagRequestTracker, context: LDContext)]) in - let run = pending - pending = [] - let summaries = eventBuffer.takeSummaries() - // Reserved in the same critical section the run leaves `pending` in, so capacity counts it in one place or - // the other throughout the encode. - store.reserve(run.count) - return (run, summaries) - } - - stage(run) - store.releaseReservations() - stageSummaries(summaries) - store.commit() - } - - /// Encodes the held events as one run and stages the bytes. - /// - /// Staging uses the reservation taken with the run rather than checking capacity, because the decision to keep - /// these events was made when they were held, and refusing them here would drop events the SDK has already counted - /// as accepted. - /// - /// Requires `commitLock`: two threads draining separate runs would stage them in whichever order they finished - /// encoding, which is not the order they were recorded in. - private func stage(_ run: [Event]) { - commitLock.assertOwned() - for event in run { - guard let encoded = eventBuffer.encode(event) - else { - reportLost() - continue - } - // The event capacity is bypassed here, so what is left is an event too large for a frame or for the bytes - // left in the store, which were not known until now. Either way it was accepted and is not kept. - if !store.stageReserved(encoded) { - reportLost() - } - } - } - - /// Whether recording this kind of event should encode and write the held run before it returns. - /// - /// Custom and identify events are recorded because the application did something it chose to report, which is both - /// rare enough to afford a write and the moment it is least acceptable to lose. Evaluations are neither. - private static func isCommitPoint(_ kind: Event.Kind) -> Bool { - switch kind { - case .custom, .identify: return true - case .feature, .debug, .summary: return false - } - } - - /// Turns the counters taken with the run into summary events and stages them. - /// - /// Counters live only in memory, so a crash takes whatever has not been summarized with it. Summarizing at each - /// commit point rather than only at a delivery is what bounds that loss, and it costs nothing in accuracy: - /// LaunchDarkly sums the counters of every summary it receives, so a session that produced several summaries is - /// counted the same as one that produced a single summary covering the same evaluations. - private func stageSummaries(_ summaries: [(tracker: FlagRequestTracker, context: LDContext)]) { - for summary in summaries { - guard let encoded = eventBuffer.encodeSummary(summary) - else { - reportLost() - continue - } - // Summaries are an aggregate of evaluations that were already counted, so refusing one for capacity would - // lose evaluations the SDK promised to report rather than shed new load. Only the byte capacity refuses - // one, once even the room held back for summaries is used up, since nothing else bounds how many are - // staged while delivery keeps failing. - if !store.stage(encoded, bypassingCapacity: true) { - reportLost() - } - } - } } // MARK: Reporting @@ -662,7 +371,7 @@ extension DirectEventProcessor { } func flushReportingOutcome(completion: @escaping FlushOutcomeClosure) { - queueCommitWherePersisting() + committer.queueCommitWherePersisting() queueDelivery(completion) } @@ -684,33 +393,6 @@ extension DirectEventProcessor { queueDelivery(completion) } - /// Writes on the caller's thread at `.deferred` as well as at `.immediate`, unlike `track` and `identify`. - /// - /// A flush is what an application calls when it is about to end the process on purpose, and entering the background - /// is when the system is most likely to end it; a write only queued is one the process may end before. Deferring is - /// for the recordings an application makes all the time, which neither is. Where nothing persists, a commit makes - /// nothing durable, and is left to the delivery. - func commitWherePersisting() { - if service.config.eventPersistence == .immediate || store.isPersisting { - commitRecordedEvents() - } - } - - /// `commitWherePersisting()` queued on `commitQueue` rather than written on the caller's thread, for a flush whose - /// caller has a deadline. - /// - /// `flushAndWait` can only hold to its timeout once every environment has handed back, and each step of a commit - /// can block for as long as the device takes: on `commitLock` behind a commit stalled on the disk, on the store's - /// I/O lock, and on the write itself. Queued rather than left to the delivery, because `deliveryQueue` can be held - /// by a request and its retry long after the caller stopped waiting, while `commitQueue` never waits on the - /// network. The answer still covers the write: the delivery commits again before it sends, so `true` means the - /// events were written first. A commit still running when the caller stops waiting is left to finish. - private func queueCommitWherePersisting() { - if service.config.eventPersistence == .immediate || store.isPersisting { - scheduleCommit() - } - } - /// Queues the delivery the report timer and the launch gate start, which nobody asked for now and which is paced. private func queueScheduledDelivery() { queueDelivery(nil, paced: true) @@ -885,13 +567,13 @@ extension DirectEventProcessor { else { // Committed all the same, since a delivery is a commit point. Nothing was closed, so the caller's events // did not go, and the mark is as of this commit, the last point anything the caller recorded was staged. - settled = (self.commitLeavingTheLogOpen(), false) + settled = (self.committer.commitLeavingTheLogOpen(), false) os_log("%s will report failure: an earlier batch could not be delivered", log: self.service.config.logger, type: .debug, self.typeName(and: #function)) answer.report(.blocked) return } - let (lossMark, closedEverything) = self.commitAndCloseBatch() + let (lossMark, closedEverything) = self.committer.commitAndCloseBatch() let new = self.claimDeliverableBatches(discardingExpired: false) // Events left in a log that would not close, or in a batch a listing that failed could not see, are still on // the disk for a later delivery. This one delivers what it has, and cannot say the caller's events got out. @@ -936,18 +618,6 @@ extension DirectEventProcessor { } } - /// `commitAndCloseBatch()` for a pass that found an earlier batch blocked, and so closes nothing. - /// - /// - Returns: the loss count as of the commit. - private func commitLeavingTheLogOpen() -> UInt64 { - commitLock.withLock { - encodeAndCommitPending() - // A commit that fills the log closes it, and can find it gone. - reportVanished(store.takeVanishedLogEvents()) - return stateLock.withLock { lostEvents } - } - } - /// What `claimDeliverableBatches(discardingExpired:)` took. private struct ClaimedBatches { let deliverable: [EventBatch] @@ -1269,10 +939,8 @@ extension DirectEventProcessor { switch outcome { case .accepted(let serverTime): self.retryState.recordSuccess() - self.stateLock.withLock { - if let serverTime = serverTime, serverTime > self.responseDate { - self.responseDate = serverTime - } + if let serverTime { + self.committer.recordResponseDate(serverTime) } self.store.remove(batch) self.store.stoppedSending(batch) @@ -1305,11 +973,11 @@ extension DirectEventProcessor: TypeIdentifying { } extension DirectEventProcessor { /// Full events accepted but not yet handed to a commit. var pendingEventsForTesting: [Event] { - recordLock.withLock { pending } + committer.pendingEventsForTesting } func setLastEventResponseDate(_ date: Date) { - stateLock.withLock { responseDate = date } + committer.setLastEventResponseDate(date) } /// Occupies the queue a delivery runs on, so a test can see what one that has not started yet collects. @@ -1331,9 +999,9 @@ extension DirectEventProcessor: TypeIdentifying { } return deliveryRunCount } - /// Reads the summarizer under `recordLock`, which a scheduled commit may be holding to take its counters. + /// As `EventCommitter.withContextSummarizer(_:)`. func withContextSummarizer(_ read: (ContextSummarizer) -> T) -> T { - recordLock.withLock { read(eventBuffer.contextSummarizer) } + committer.withContextSummarizer(read) } } #endif diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift new file mode 100644 index 000000000..fe4154255 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift @@ -0,0 +1,455 @@ +import Foundation +import OSLog + +/// How many events the SDK accepted and then lost for good, which `DirectEventProcessor` and its `EventCommitter` both +/// add to. +/// +/// A batch LaunchDarkly refuses is kept and retried, so it is not a loss, unless the refusal is a second refusal of the +/// payload in a row. Otherwise what is lost is what the store never received: an event or a summary that could not be +/// serialized when it was committed, or that serialized too large to store. A delivery that finds nothing to send cannot +/// tell those apart from events that arrived, so without this it would report success for them. +/// +/// A count rather than a flag. Each delivery takes the count as it closes its batch, and answers for the losses up to that +/// mark, so a loss is told to the caller whose delivery it happened before rather than to whichever delivery answers next. +final class LostEventCount { + private let lock = UnfairLock() + /// Only to be used while holding `lock`. + private var count: UInt64 = 0 + + var value: UInt64 { lock.withLock { count } } + + func add(_ events: Int) { + lock.withLock { count += UInt64(events) } + } +} + +/// Where `DirectEventProcessor` accepts events, holds them and commits them to the store. +/// +/// Recording an event summarizes it and, if it has to be delivered in full, holds it as an `Event`. Nothing is encoded +/// on the thread that recorded it; a commit turns the whole held run into bytes at once. So an evaluation of an +/// untracked flag costs a counter update, and an evaluation of a tracked one costs that plus an array append. +/// +/// Where the line falls is between the events an application asked for by name and the ones it did not. Recording a +/// custom or identify event is a *commit point*, because an application reporting something it chose to report is +/// saying this matters more than the microseconds it costs, and the crash it describes may be moments away. A commit +/// takes the whole held run with it, so the exposures leading up to the error go down alongside it. Evaluations get no +/// such promise, and are committed once `pendingCommitThreshold` of them have accumulated, when a delivery starts, or +/// on `flush`. +/// +/// Whether a commit point runs on the caller's thread is the application's choice, through +/// `LDConfig.eventPersistence`. Only at `.immediate` can `track` promise the event is on disk by the time it returns. +/// At `.disabled` nothing commits early: the held run waits for the delivery, which encodes it. +/// +/// Threads. Recording runs on whichever thread evaluated or tracked. Commits run on `commitQueue`, or on the caller's +/// thread at a commit point, one at a time under `commitLock`. A delivery commits too, before it closes a batch. +final class EventCommitter { + /// How many full events may be held unencoded before one of them pays for a commit. + /// + /// It sets two things at once. The first is how many evaluations a termination can take -- not everything a crash + /// could take, since an application that flushes on its way out commits the whole run, but the window for the + /// terminations that run nothing on the way out, such as the system reclaiming a backgrounded process. Recording a + /// custom or identify event closes it too, because that commits. + /// + /// The second is what a commit point usually costs, since it encodes whatever is held before returning. The commit + /// queue keeps the run drained, so the common case is far below this; raising it trades that cost against the + /// number of writes. + /// + /// Neither is a ceiling. Reaching this only schedules a commit, and recording does not wait for it, so where events + /// arrive faster than the commit queue encodes them, or the queue is starved, the run grows up to the event + /// capacity, and the next commit point encodes all of it. + /// + /// It is the only thing that writes events no caller asked to have written. The store writes only when told to, so + /// every staged byte reaches the disk through a commit that started here, at a commit point, or at a delivery. + private static let pendingCommitThreshold = 32 + + // MARK: Set in init and never changed + + private let store: EventStoring + private let persistence: EventPersistence + private let logger: OSLog + private let diagnosticCache: DiagnosticCaching? + private let lostEvents: LostEventCount + + /// How many events the SDK will hold in total, across `pending` and the store. + private let eventCapacity: Int + + /// Where a commit runs when no caller is waiting on it. + private let commitQueue: DispatchQueue + + /// Its summarizer is guarded by `recordLock`, and its encoders are reached only under `commitLock`. + private let eventBuffer: OutboundEventBuffer + + // MARK: Guarded by commitLock + + /// Held for the whole of a commit, so that only one runs at a time. + /// + /// This is what a commit point's guarantee rests on. Without it a commit already in flight could take the caller's + /// event out of `pending` before the caller got there, leaving the caller nothing to write and returning while + /// those bytes were still being produced somewhere else. Waiting here instead means that when the call returns the + /// event is on disk, whichever commit put it there. It also keeps two encoders from staging their runs in + /// whichever order they happened to finish. + /// + /// Taken before `recordLock` and before anything the store locks, never after. + private let commitLock = UnfairLock() + + // MARK: Guarded by recordLock + + /// Guards everything one recording writes: the held events and the summarizer. + /// + /// One evaluation can produce a counter, a full event and a debug event, and the three have to land together. Taken + /// separately, a commit landing between them splits one evaluation across two payloads. + /// + /// Held for the appends and the handover of the run, never across the encoder or a syscall. It is a lock rather + /// than a queue because it is taken once per evaluation, and a `DispatchQueue.sync` costs enough at that rate to be + /// worth avoiding: the same measurement that made `EvaluationExposureDeduper` a lock applies here. + private let recordLock = UnfairLock() + + /// Full events recorded but not yet encoded. Only to be used while holding `recordLock`. + /// + /// Held rather than encoded because encoding early would not make them durable: the store stages bytes into memory + /// too, and only a commit reaches the file. Both forms are equally lost to a crash, so the encode may as well + /// happen where it is cheapest. + private var pending: [Event] = [] + + /// Whether a commit is already queued, so a run of recordings past the threshold asks for one write rather than + /// one each. Only to be used while holding `recordLock`. + private var isCommitScheduled = false + + // MARK: Guarded by responseDateLock + + /// When LaunchDarkly last answered a post, by its own clock, which is how long debug events go on for. + var lastEventResponseDate: Date { + responseDateLock.withLock { responseDate } + } + + /// Its own lock rather than `recordLock`, since an evaluation reads it before it takes that one. + private let responseDateLock = UnfairLock() + /// Only to be used while holding `responseDateLock`. + private var responseDate = Date() + + init(config: LDConfig, + store: EventStoring, + encoding: OutboundEventBuffer.Encoding, + commitQueue: DispatchQueue, + diagnosticCache: DiagnosticCaching?, + lostEvents: LostEventCount) { + self.store = store + self.persistence = config.eventPersistence + self.logger = config.logger + self.diagnosticCache = diagnosticCache + self.lostEvents = lostEvents + self.eventCapacity = config.eventCapacity + self.commitQueue = commitQueue + self.eventBuffer = OutboundEventBuffer(config: config, encoding: encoding) + } + + // MARK: Recording + + func record(_ event: Event) { + let (dropped, needsCommit) = recordLock.withLock { + (!admitToPending(event), pending.count >= EventCommitter.pendingCommitThreshold) + } + + if dropped { + reportDropped() + } + if EventCommitter.isCommitPoint(event.kind) { + commitAtCommitPoint() + } else if needsCommit { + scheduleCommitWherePersisting() + } + } + + // swiftlint:disable:next function_parameter_count + func recordFlagEvaluationEvents(flagKey: LDFlagKey, + value: LDValue, + defaultValue: LDValue, + featureFlag: FeatureFlag?, + context: LDContext, + includeReason: Bool) { + let recordingFeatureEvent = featureFlag?.trackEvents == true + let recordingDebugEvent = featureFlag?.shouldCreateDebugEvents(lastEventReportResponseTime: lastEventResponseDate) ?? false + // Built before the lock is taken, so that the critical section is only the writes. + let featureEvent = recordingFeatureEvent ? FeatureEvent(key: flagKey, context: context, value: value, defaultValue: defaultValue, featureFlag: featureFlag, includeReason: includeReason, isDebug: false) : nil + let debugEvent = recordingDebugEvent ? FeatureEvent(key: flagKey, context: context, value: value, defaultValue: defaultValue, featureFlag: featureFlag, includeReason: includeReason, isDebug: true) : nil + + var dropped = 0 + let (counted, needsCommit) = recordLock.withLock { () -> (Bool, Bool) in + let counted = eventBuffer.summarize(flagKey: flagKey, reportedValue: value, featureFlag: featureFlag, defaultValue: defaultValue, context: context) + if let featureEvent, !admitToPending(featureEvent) { dropped += 1 } + if let debugEvent, !admitToPending(debugEvent) { dropped += 1 } + return (counted, pending.count >= EventCommitter.pendingCommitThreshold) + } + + // A refused evaluation is a loss like a refused event, since nothing later reconstructs its counter. Its full + // event is still decided by the event capacity alone. + if !counted { + diagnosticCache?.incrementDroppedEventCount() + } + for _ in 0.. responseDate { + responseDate = serverTime + } + } + } + + /// Adds an event to `pending` for the next commit to encode, returning false if the SDK is already full. Requires + /// `recordLock`. + /// + /// Capacity is consulted before anything else, so an event that will not be kept is never encoded. That ordering is + /// what bounds an application re-evaluating a tracked flag in a render loop: once the limit is reached an + /// evaluation costs no more than its summary counter, however fast the loop runs. + /// + /// The byte capacity can only be consulted as far as whether any room is left, since the event has no size until + /// it is encoded; whether it fits is decided when it is staged. + private func admitToPending(_ event: Event) -> Bool { + recordLock.assertOwned() + guard pending.count + store.pendingEventCount < eventCapacity, store.hasRoomForEventBytes + else { return false } + + pending.append(event) + return true + } + + private func reportDropped() { + os_log("%s aborted. Event store is full", log: logger, type: .debug, typeName(and: #function)) + diagnosticCache?.incrementDroppedEventCount() + } + + /// Records the loss of an event the SDK accepted and then could not keep, which no later attempt recovers. + private func reportLost() { + os_log("%s dropping an event the store would not take", log: logger, type: .error, typeName(and: #function)) + diagnosticCache?.incrementDroppedEventCount() + lostEvents.add(1) + } + + /// Counts as lost the events of an open log a close found gone. + private func reportVanished(_ events: Int) { + guard events > 0 + else { return } + os_log("%s %d stored event(s) were deleted before they could be sent", log: logger, type: .error, typeName(and: #function), events) + lostEvents.add(events) + } + + // MARK: Committing + + /// Commits at a commit point, on the caller's thread or off it as the application asked. + /// + /// Committing on the caller's thread is what lets `track` promise its event is on disk by the time it returns. + /// Scheduling it instead keeps the encode and the write off that thread, and the event is durable a moment later + /// rather than immediately. + private func commitAtCommitPoint() { + if persistence == .immediate { + commitRecordedEvents() + } else { + scheduleCommitWherePersisting() + } + } + + /// Queues a commit where there is a disk for it to reach. + /// + /// Without persistence a commit makes nothing durable. All it would do is move the encode into the middle of the + /// application's evaluations, where it competes with them for the CPU and splits their counters across summaries; + /// left alone, the run waits for the delivery, which encodes it anyway, and capacity still bounds how much is held. + private func scheduleCommitWherePersisting() { + if store.isPersisting { + scheduleCommit() + } + } + + /// Queues a commit unless one is already queued, so a run of recordings asks for one write rather than one each. + private func scheduleCommit() { + let alreadyScheduled = recordLock.withLock { () -> Bool in + defer { isCommitScheduled = true } + return isCommitScheduled + } + guard !alreadyScheduled + else { return } + + commitQueue.async { [weak self] in + guard let self + else { return } + // Cleared before the commit rather than after it, so that an event held while this one is in flight can ask + // for another commit instead of finding one apparently already on its way. + self.recordLock.withLock { self.isCommitScheduled = false } + + self.commitRecordedEvents() + } + } + + /// Writes on the caller's thread at `.deferred` as well as at `.immediate`, unlike `track` and `identify`. + /// + /// A flush is what an application calls when it is about to end the process on purpose, and entering the background + /// is when the system is most likely to end it; a write only queued is one the process may end before. Deferring is + /// for the recordings an application makes all the time, which neither is. Where nothing persists, a commit makes + /// nothing durable, and is left to the delivery. + func commitWherePersisting() { + if persistence == .immediate || store.isPersisting { + commitRecordedEvents() + } + } + + /// `commitWherePersisting()` queued on `commitQueue` rather than written on the caller's thread, for a flush whose + /// caller has a deadline. + /// + /// `flushAndWait` can only hold to its timeout once every environment has handed back, and each step of a commit + /// can block for as long as the device takes: on `commitLock` behind a commit stalled on the disk, on the store's + /// I/O lock, and on the write itself. Queued rather than left to the delivery, because the delivery queue can be + /// held by a request and its retry long after the caller stopped waiting, while `commitQueue` never waits on the + /// network. The answer still covers the write: the delivery commits again before it sends, so `true` means the + /// events were written first. A commit still running when the caller stops waiting is left to finish. + func queueCommitWherePersisting() { + if persistence == .immediate || store.isPersisting { + scheduleCommit() + } + } + + /// Encodes everything recorded since the last commit, stages it, and commits it. + /// + /// The run and the counters are taken in one critical section, so that an evaluation's counter and its full event + /// are staged by the same commit, and encoded outside it, so recording does not wait on the encoder. + func commitRecordedEvents() { + commitLock.withLock { encodeAndCommitPending() } + } + + /// Commits what is held and closes it into a batch, as one step with respect to other commits. + /// + /// Closing takes everything staged so far, and a commit stages its run one event at a time, so a commit running + /// between the two would be closed partway: an evaluation's event in this batch and its summary in the next. + /// + /// - Returns: the loss count as of the close, which is the mark this delivery answers for, and whether the close + /// took every committed event into a batch. Both taken under the same lock, so they cover the closed batch and + /// nothing a later commit does. + func commitAndCloseBatch() -> (lossMark: UInt64, closedEverything: Bool) { + commitLock.withLock { + encodeAndCommitPending() + _ = store.closeBatch() + reportVanished(store.takeVanishedLogEvents()) + return (lostEvents.value, !store.holdsUnclosedEvents) + } + } + + /// `commitAndCloseBatch()` for a pass that found an earlier batch blocked, and so closes nothing. + /// + /// - Returns: the loss count as of the commit. + func commitLeavingTheLogOpen() -> UInt64 { + commitLock.withLock { + encodeAndCommitPending() + // A commit that fills the log closes it, and can find it gone. + reportVanished(store.takeVanishedLogEvents()) + return lostEvents.value + } + } + + /// Requires `commitLock`. + private func encodeAndCommitPending() { + commitLock.assertOwned() + let (run, summaries) = recordLock.withLock { () -> ([Event], [(tracker: FlagRequestTracker, context: LDContext)]) in + let run = pending + pending = [] + let summaries = eventBuffer.takeSummaries() + // Reserved in the same critical section the run leaves `pending` in, so capacity counts it in one place or + // the other throughout the encode. + store.reserve(run.count) + return (run, summaries) + } + + stage(run) + store.releaseReservations() + stageSummaries(summaries) + store.commit() + } + + /// Encodes the held events as one run and stages the bytes. + /// + /// Staging uses the reservation taken with the run rather than checking capacity, because the decision to keep + /// these events was made when they were held, and refusing them here would drop events the SDK has already counted + /// as accepted. + /// + /// Requires `commitLock`: two threads draining separate runs would stage them in whichever order they finished + /// encoding, which is not the order they were recorded in. + private func stage(_ run: [Event]) { + commitLock.assertOwned() + for event in run { + guard let encoded = eventBuffer.encode(event) + else { + reportLost() + continue + } + // The event capacity is bypassed here, so what is left is an event too large for a frame or for the bytes + // left in the store, which were not known until now. Either way it was accepted and is not kept. + if !store.stageReserved(encoded) { + reportLost() + } + } + } + + /// Whether recording this kind of event should encode and write the held run before it returns. + /// + /// Custom and identify events are recorded because the application did something it chose to report, which is both + /// rare enough to afford a write and the moment it is least acceptable to lose. Evaluations are neither. + private static func isCommitPoint(_ kind: Event.Kind) -> Bool { + switch kind { + case .custom, .identify: return true + case .feature, .debug, .summary: return false + } + } + + /// Turns the counters taken with the run into summary events and stages them. + /// + /// Counters live only in memory, so a crash takes whatever has not been summarized with it. Summarizing at each + /// commit point rather than only at a delivery is what bounds that loss, and it costs nothing in accuracy: + /// LaunchDarkly sums the counters of every summary it receives, so a session that produced several summaries is + /// counted the same as one that produced a single summary covering the same evaluations. + private func stageSummaries(_ summaries: [(tracker: FlagRequestTracker, context: LDContext)]) { + for summary in summaries { + guard let encoded = eventBuffer.encodeSummary(summary) + else { + reportLost() + continue + } + // Summaries are an aggregate of evaluations that were already counted, so refusing one for capacity would + // lose evaluations the SDK promised to report rather than shed new load. Only the byte capacity refuses + // one, once even the room held back for summaries is used up, since nothing else bounds how many are + // staged while delivery keeps failing. + if !store.stage(encoded, bypassingCapacity: true) { + reportLost() + } + } + } +} + +extension EventCommitter: TypeIdentifying { } + +#if DEBUG + extension EventCommitter { + /// Full events accepted but not yet handed to a commit. + var pendingEventsForTesting: [Event] { + recordLock.withLock { pending } + } + + func setLastEventResponseDate(_ date: Date) { + responseDateLock.withLock { responseDate = date } + } + + /// Reads the summarizer under `recordLock`, which a scheduled commit may be holding to take its counters. + func withContextSummarizer(_ read: (ContextSummarizer) -> T) -> T { + recordLock.withLock { read(eventBuffer.contextSummarizer) } + } + } +#endif From 53eb6878340fc30b45d04a1275a012ee23154624 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Wed, 28 Oct 2026 12:47:18 -0700 Subject: [PATCH 118/158] Move delivery queueing and pacing out of DirectEventProcessor DeliveryScheduler owns the queued request, the one-delivery-at-a-time rule and the retry timer claim. DeliveryRun carries what a single delivery needs: whether it is closing, and its pacing state and gap wait. The processor passes the run through deliver, post and publish instead of reading shared fields under deliveryLock. Co-authored-by: Cursor --- LaunchDarkly.xcodeproj/project.pbxproj | 10 + .../ServiceObjects/DeliveryScheduler.swift | 238 +++++++++++++++++ .../ServiceObjects/DirectEventProcessor.swift | 239 +++--------------- 3 files changed, 290 insertions(+), 197 deletions(-) create mode 100644 LaunchDarkly/LaunchDarkly/ServiceObjects/DeliveryScheduler.swift diff --git a/LaunchDarkly.xcodeproj/project.pbxproj b/LaunchDarkly.xcodeproj/project.pbxproj index fd0f3d49a..2cc05ecf3 100644 --- a/LaunchDarkly.xcodeproj/project.pbxproj +++ b/LaunchDarkly.xcodeproj/project.pbxproj @@ -81,6 +81,7 @@ 831188572113AE0B00D77CB5 /* FlagChangeNotifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8358F25F1F476AD800ECE1AF /* FlagChangeNotifier.swift */; }; 831188582113AE0F00D77CB5 /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; E7C0A11E2F00000000000002 /* EventCommitter.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000001 /* EventCommitter.swift */; }; + E7C0A11E2F00000000000007 /* DeliveryScheduler.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000006 /* DeliveryScheduler.swift */; }; B66ECB2014E0FC95C23DA0A2 /* AnalyticsEventSender.swift in Sources */ = {isa = PBXBuildFile; fileRef = 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */; }; 629D5A9FE152DBC3E20D6DFC /* OutboundEventBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */; }; 014901E06BE1F2B0FAAD6AA6 /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; @@ -127,6 +128,7 @@ 831EF35620655E730001C643 /* FlagChangeNotifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8358F25F1F476AD800ECE1AF /* FlagChangeNotifier.swift */; }; 831EF35720655E730001C643 /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; E7C0A11E2F00000000000003 /* EventCommitter.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000001 /* EventCommitter.swift */; }; + E7C0A11E2F00000000000008 /* DeliveryScheduler.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000006 /* DeliveryScheduler.swift */; }; 367D374B5CD1B2A8596C0738 /* AnalyticsEventSender.swift in Sources */ = {isa = PBXBuildFile; fileRef = 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */; }; 5A89648200D2566D3F8409BA /* OutboundEventBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */; }; CFF357D68BBF2DC8395930F2 /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; @@ -214,6 +216,7 @@ 83D9EC882062DEAB004D7FA6 /* FlagChangeNotifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8358F25F1F476AD800ECE1AF /* FlagChangeNotifier.swift */; }; 83D9EC892062DEAB004D7FA6 /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; E7C0A11E2F00000000000004 /* EventCommitter.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000001 /* EventCommitter.swift */; }; + E7C0A11E2F00000000000009 /* DeliveryScheduler.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000006 /* DeliveryScheduler.swift */; }; 321659EE1824B83BD0D0F22D /* AnalyticsEventSender.swift in Sources */ = {isa = PBXBuildFile; fileRef = 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */; }; 73E23C0FA009141BB4CBC24C /* OutboundEventBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */; }; 54745AEF1A43D165F5A6E08F /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; @@ -243,6 +246,7 @@ 83FEF8DD1F266742001CF12C /* FlagSynchronizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */; }; 83FEF8DF1F2667E4001CF12C /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; E7C0A11E2F00000000000005 /* EventCommitter.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000001 /* EventCommitter.swift */; }; + E7C0A11E2F0000000000000A /* DeliveryScheduler.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000006 /* DeliveryScheduler.swift */; }; E91F5DCAEEE5FC4AC408BA67 /* AnalyticsEventSender.swift in Sources */ = {isa = PBXBuildFile; fileRef = 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */; }; 7D37AD3ECAA9D8E2927228C5 /* OutboundEventBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */; }; 9FA0EE3A87678F58F4406B24 /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; @@ -634,6 +638,7 @@ 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FlagSynchronizer.swift; sourceTree = ""; }; 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DirectEventProcessor.swift; sourceTree = ""; }; E7C0A11E2F00000000000001 /* EventCommitter.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = EventCommitter.swift; sourceTree = ""; }; + E7C0A11E2F00000000000006 /* DeliveryScheduler.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DeliveryScheduler.swift; sourceTree = ""; }; 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AnalyticsEventSender.swift; sourceTree = ""; }; 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OutboundEventBuffer.swift; sourceTree = ""; }; BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NullEventProcessor.swift; sourceTree = ""; }; @@ -1084,6 +1089,7 @@ 831425B0206B030100F2EF36 /* EnvironmentReporter.swift */, 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */, E7C0A11E2F00000000000001 /* EventCommitter.swift */, + E7C0A11E2F00000000000006 /* DeliveryScheduler.swift */, 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */, 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */, BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */, @@ -1568,6 +1574,7 @@ 831188522113ADF700D77CB5 /* KeyedValueCache.swift in Sources */, 831188582113AE0F00D77CB5 /* DirectEventProcessor.swift in Sources */, E7C0A11E2F00000000000002 /* EventCommitter.swift in Sources */, + E7C0A11E2F00000000000007 /* DeliveryScheduler.swift in Sources */, B66ECB2014E0FC95C23DA0A2 /* AnalyticsEventSender.swift in Sources */, 629D5A9FE152DBC3E20D6DFC /* OutboundEventBuffer.swift in Sources */, 014901E06BE1F2B0FAAD6AA6 /* NullEventProcessor.swift in Sources */, @@ -1695,6 +1702,7 @@ A358D6D92A4DE6A500270C60 /* ApplicationInfoEnvironmentReporter.swift in Sources */, 831EF35720655E730001C643 /* DirectEventProcessor.swift in Sources */, E7C0A11E2F00000000000003 /* EventCommitter.swift in Sources */, + E7C0A11E2F00000000000008 /* DeliveryScheduler.swift in Sources */, 367D374B5CD1B2A8596C0738 /* AnalyticsEventSender.swift in Sources */, 5A89648200D2566D3F8409BA /* OutboundEventBuffer.swift in Sources */, CFF357D68BBF2DC8395930F2 /* NullEventProcessor.swift in Sources */, @@ -1808,6 +1816,7 @@ 8358F2601F476AD800ECE1AF /* FlagChangeNotifier.swift in Sources */, 83FEF8DF1F2667E4001CF12C /* DirectEventProcessor.swift in Sources */, E7C0A11E2F00000000000005 /* EventCommitter.swift in Sources */, + E7C0A11E2F0000000000000A /* DeliveryScheduler.swift in Sources */, E91F5DCAEEE5FC4AC408BA67 /* AnalyticsEventSender.swift in Sources */, 7D37AD3ECAA9D8E2927228C5 /* OutboundEventBuffer.swift in Sources */, 9FA0EE3A87678F58F4406B24 /* NullEventProcessor.swift in Sources */, @@ -1976,6 +1985,7 @@ C443A40B2315AA4D00145710 /* NetworkReporter.swift in Sources */, 83D9EC892062DEAB004D7FA6 /* DirectEventProcessor.swift in Sources */, E7C0A11E2F00000000000004 /* EventCommitter.swift in Sources */, + E7C0A11E2F00000000000009 /* DeliveryScheduler.swift in Sources */, 321659EE1824B83BD0D0F22D /* AnalyticsEventSender.swift in Sources */, 73E23C0FA009141BB4CBC24C /* OutboundEventBuffer.swift in Sources */, 54745AEF1A43D165F5A6E08F /* NullEventProcessor.swift in Sources */, diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DeliveryScheduler.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DeliveryScheduler.swift new file mode 100644 index 000000000..ebcba7940 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DeliveryScheduler.swift @@ -0,0 +1,238 @@ +import Foundation + +/// One delivery, from taking its batches until it has an answer for everything it sent, its retry included. +/// +/// Made by `DeliveryScheduler.begin()` and passed through the delivery, so that what is true of the delivery running is +/// held by it, and starts afresh with each one. +final class DeliveryRun { + /// A paced batch waiting out its gap. + private final class PacedTurn { + let proceed: () -> Void + + init(_ proceed: @escaping () -> Void) { + self.proceed = proceed + } + } + + /// Whether it is `flushAtClose()`'s, which posts whatever wait the backoff is in and runs although closing has + /// turned `isOnline` off since. The delivery one of its failures sets for the end of the wait is another one, held + /// by the backoff, and finds the reporter offline as any other. + let isClosing: Bool + + /// The least time between one paced batch's answer and the next batch's request. + private let gap: TimeInterval + + /// Guards the pacing, which a delivery somebody asks for, or going offline, ends from another thread. + private let lock = UnfairLock() + /// Whether it is paced: until a delivery somebody asked for is queued, after which it sends the rest back to back, + /// or the reporter goes offline, after which it sends nothing more. Only to be used while holding `lock`. + private var isPaced: Bool + /// When it last had an answer for a post, or nil before its first. Only to be used while holding `lock`. + private var lastPostAnswered: DispatchTime? + /// The batch waiting out its gap, which ending the pacing lets go at once. Only to be used while holding `lock`. + private var waitingTurn: PacedTurn? + + init(isClosing: Bool, isPaced: Bool, gap: TimeInterval) { + self.isClosing = isClosing + self.isPaced = isPaced + self.gap = gap + } + + /// Notes that a post had its answer, which the gap before the next one counts from. + func postAnswered() { + lock.withLock { + if isPaced { + lastPostAnswered = .now() + } + } + } + + /// Ends the pacing, so that the rest go back to back, or, offline, stop. + func endPacing() { + let turn = lock.withLock { () -> PacedTurn? in + defer { + isPaced = false + waitingTurn = nil + } + return waitingTurn + } + if let turn { + DispatchQueue.global().async(execute: turn.proceed) + } + } + + /// Holds the next batch until the gap has passed since the last one was answered, where the delivery is paced and + /// has posted before, and then calls `proceed` on a global queue; or sooner, once the pacing ends. + /// + /// - Returns: whether it holds the batch; false where the batch may go now, and `proceed` is not called. + func awaitTurn(then proceed: @escaping () -> Void) -> Bool { + let turn = PacedTurn(proceed) + // Under the same lock that ends the pacing, so that the pacing cannot end between the check and the wait. + guard let gapEnds = lock.withLock({ () -> DispatchTime? in + guard isPaced, let answered = lastPostAnswered + else { return nil } + waitingTurn = turn + return answered + gap + }) + else { return false } + DispatchQueue.global().asyncAfter(deadline: gapEnds) { + let taken = self.lock.withLock { () -> Bool in + guard self.waitingTurn === turn + else { return false } + self.waitingTurn = nil + return true + } + if taken { + turn.proceed() + } + } + return true + } +} + +/// Decides which delivery runs next, one at a time: queues one, joins a caller to one that is queued and has not +/// started, starts the queued one once the one running has its answer, and sets one for when the backoff's wait ends. +/// +/// Its lock is taken on a caller's thread, on the delivery queue and where a delivery's posts are answered, and never +/// held across any of them. +final class DeliveryScheduler { + /// A delivery queued that has not taken its batches yet. + private struct Queued { + /// The callers waiting on it. + var waiters: [FlushOutcomeClosure] + /// Whether it is `flushAtClose()`'s, or has been joined by it. + var isClosing: Bool + /// Whether it was asked for only by the report timer or by the launch gate, and so is paced. + var isPaced: Bool + } + + /// Whether deliveries nobody asked for are paced: only where events give way to flag values. + private let pacesDeliveries: Bool + private let gap: TimeInterval + + private let lock = UnfairLock() + /// Only to be used while holding `lock`. + private var queued: Queued? + /// Whether a delivery has started, or is about to, and not yet had an answer for everything it sent, its retry + /// included. Only to be used while holding `lock`. + /// + /// A delivery returns from the delivery queue as soon as its first request is out, so the queue alone does not keep + /// two apart. Two at once would each leave the batch it is sending out of both capacities, and a run of flushes, + /// each held up by a slow response, could take the store that many batches past them. + private var isDelivering = false + /// The delivery running, whose pacing a delivery somebody asks for ends. Only to be used while holding `lock`. + private var running: DeliveryRun? + /// Whether a delivery is already set to run when the backoff's wait ends, so a run of passes inside the wait sets + /// one rather than one each. Only to be used while holding `lock`. + private var isRetryScheduled = false + + #if DEBUG + /// How many deliveries have started, which is fewer than the flushes asked for where they were joined. + private var deliveryRunCount = 0 + #endif + + init(pacesDeliveries: Bool, gap: TimeInterval) { + self.pacesDeliveries = pacesDeliveries + self.gap = gap + } + + /// Queues a delivery, or joins one that is queued and has not started. + /// + /// A delivery that has not started yet will send everything recorded before it starts, which includes this + /// caller's events, so joining it is as good as queueing another. Without this, flushes arriving faster than a + /// post completes each queue their own, and the one that matters -- the flush at shutdown -- waits behind all of + /// them. + /// + /// A delivery that has already started is not joined. It has closed and taken its batches, so it cannot speak for + /// anything recorded since. Nor does another start beside it: the one queued waits until the running one has an + /// answer, then sends what that one left behind as well as what was recorded since, so its caller hears `true` + /// only if both got out. + /// + /// - Parameters: + /// - closing: whether it is `flushAtClose()`'s, which a queued delivery it joins becomes. + /// - paced: whether nobody asked for it, so that it gives way to flag requests. One that is not ends the pacing of + /// the delivery running, so that it is not held behind it. + /// - Returns: whether the caller is to start it, which is where no delivery is running. + func queue(_ completion: FlushOutcomeClosure?, closing: Bool, paced: Bool) -> Bool { + let (starts, unpaced) = lock.withLock { () -> (Bool, DeliveryRun?) in + let unpaced = paced ? nil : running + if var joined = queued { + if let completion { + joined.waiters.append(completion) + } + joined.isClosing = joined.isClosing || closing + joined.isPaced = joined.isPaced && paced + queued = joined + return (false, unpaced) + } + queued = Queued(waiters: completion.map { [$0] } ?? [], isClosing: closing, isPaced: paced) + let starts = !isDelivering + isDelivering = true + return (starts, unpaced) + } + unpaced?.endPacing() + return starts + } + + /// Starts the delivery queued. Requests arriving from here on need a delivery of their own: this one is about to + /// close and take its batches, and what it takes is all it can speak for. + /// + /// - Returns: the delivery, and the callers waiting on it. + func begin() -> (run: DeliveryRun, waiters: [FlushOutcomeClosure]) { + lock.withLock { + let next = queued ?? Queued(waiters: [], isClosing: false, isPaced: false) + queued = nil + let run = DeliveryRun(isClosing: next.isClosing, + isPaced: next.isPaced && pacesDeliveries && !next.isClosing, + gap: gap) + running = run + #if DEBUG + deliveryRunCount += 1 + #endif + return (run, next.waiters) + } + } + + /// Ends the delivery running, which has had its answer. + /// + /// - Returns: whether one queued behind it is to start now. + func finish() -> Bool { + lock.withLock { + running = nil + isDelivering = queued != nil + return isDelivering + } + } + + /// Ends the pacing of the delivery running, which then sends the rest back to back, or, offline, stops. + func endPacing() { + lock.withLock { running }?.endPacing() + } + + /// Takes the one delivery set for when the backoff's wait ends. + /// + /// - Returns: false where one is set already. + func claimRetry() -> Bool { + lock.withLock { + defer { isRetryScheduled = true } + return !isRetryScheduled + } + } + + /// Notes that the wait has ended and the delivery set for it is being queued, so a later failure can set another. + func retryDue() { + lock.withLock { isRetryScheduled = false } + } + + #if DEBUG + /// Callers waiting on the queued delivery. + var queuedWaiterCount: Int { + lock.withLock { queued?.waiters.count ?? 0 } + } + + /// Deliveries started, counting one for a run of flushes that joined each other. + var startedCount: Int { + lock.withLock { deliveryRunCount } + } + #endif +} diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index b8397a9dd..f85b7dd9a 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -30,21 +30,13 @@ class DirectEventProcessor: EventProcessor { let gap: TimeInterval } - /// A paced batch waiting out its gap. - private final class PacedTurn { - let proceed: () -> Void - - init(_ proceed: @escaping () -> Void) { - self.proceed = proceed - } - } - // MARK: Set in init and never changed let service: DarklyServiceProvider let store: EventStoring let committer: EventCommitter private let lostEvents: LostEventCount + private let deliveryScheduler: DeliveryScheduler private let onSyncComplete: EventSyncCompleteClosure? @@ -83,43 +75,6 @@ class DirectEventProcessor: EventProcessor { /// the application counts again. Only to be used while holding `stateLock`. private var badRequests: (payloadId: String, inARow: Int)? - // MARK: Guarded by deliveryLock - - /// Guards `queuedDelivery`, `isDelivering`, `isRetryScheduled` and whether a delivery is close's. Taken on a caller's - /// thread, on `deliveryQueue` and where a delivery's posts are answered, and never held across any of them. - private let deliveryLock = UnfairLock() - /// Callers waiting on a delivery that is queued and has not taken the batches yet, or nil where none is queued. - private var queuedDelivery: [FlushOutcomeClosure]? - /// Whether a delivery has started and not yet had an answer for everything it sent, its retry included. - /// - /// A delivery returns from `deliveryQueue` as soon as its first request is out, so the queue alone does not keep two - /// apart. Two at once would each leave the batch it is sending out of both capacities, and a run of flushes, each - /// held up by a slow response, could take the store that many batches past them. - private var isDelivering = false - /// Whether a delivery is already set to run when the backoff's wait ends, so a run of passes inside the wait sets - /// one rather than one each. - private var isRetryScheduled = false - /// Whether the delivery queued is `flushAtClose()`'s, or has been joined by it. - private var queuedDeliveryIsClosing = false - /// Whether the delivery running is `flushAtClose()`'s, which posts whatever wait the backoff is in and runs although - /// closing has turned `isOnline` off since. For its length only: the delivery one of its failures sets for the end - /// of the wait is held by the backoff, and finds the reporter offline, as any other. - private var isClosingDelivery = false - /// Whether the delivery queued was asked for only by the report timer or by the launch gate, and so is paced. - private var queuedDeliveryIsPaced = false - /// Whether the delivery running is paced: until a delivery somebody asked for is queued, after which it sends the - /// rest back to back, or the reporter goes offline, after which it sends nothing more. - private var isPacedDelivery = false - /// When the paced delivery running last had an answer for a post, or nil before its first. - private var lastPacedPostAnswered: DispatchTime? - /// The batch of the paced delivery running that waits out its gap, which ending the pacing lets go at once. - private var waitingTurn: PacedTurn? - - #if DEBUG - /// How many deliveries have started, which is fewer than the flushes asked for where they were joined. - private var deliveryRunCount = 0 - #endif - // MARK: Guarded by claimLock /// Guards `batchesInFlight`. @@ -131,8 +86,8 @@ class DirectEventProcessor: EventProcessor { /// The batches a delivery has taken and not yet had an answer for, by payload ID. Only to be used while holding /// `claimLock`. /// - /// A batch stays on disk until its response arrives. `isDelivering` keeps a second delivery from listing it inside - /// that window; this is what keeps it from being sent twice, under the same payload ID, should two ever overlap. + /// A batch stays on disk until its response arrives. `DeliveryScheduler` keeps a second delivery from listing it + /// inside that window; this is what keeps it from being sent twice, under the same payload ID, should two ever overlap. /// A delivery that finds a batch claimed waits on that claim's outcome rather than sending it. private var batchesInFlight: [String: Claim] = [:] @@ -187,6 +142,7 @@ class DirectEventProcessor: EventProcessor { self.store = eventStore self.pacing = pacing self.givesWayToFlags = eventStore.isPersisting + self.deliveryScheduler = DeliveryScheduler(pacesDeliveries: eventStore.isPersisting, gap: pacing.gap) self.launchGateOpen = !eventStore.isPersisting // A log left open by a previous run has to be closed before it can be delivered, but nothing waits on that: @@ -321,7 +277,7 @@ extension DirectEventProcessor { private func stopReporting() { eventReportTimer?.cancel() eventReportTimer = nil - endPacing() + deliveryScheduler.endPacing() } /// Runs `block`, on a global queue, once the launch gate is open. @@ -398,111 +354,16 @@ extension DirectEventProcessor { queueDelivery(nil, paced: true) } - /// Ends the pacing of a delivery under way, which then sends the rest back to back, or, offline, stops. - private func endPacing() { - let turn = deliveryLock.withLock { () -> PacedTurn? in - defer { - isPacedDelivery = false - waitingTurn = nil - } - return waitingTurn - } - if let turn { - DispatchQueue.global().async(execute: turn.proceed) - } - } - - /// Holds the next batch of the delivery running until the gap has passed since the last one was answered, where the - /// delivery is paced and has posted before, and then calls `proceed` on a global queue; or sooner, once the pacing - /// ends. - /// - /// - Returns: whether it holds the batch; false where the batch may go now, and `proceed` is not called. - private func awaitPacedTurn(then proceed: @escaping () -> Void) -> Bool { - let turn = PacedTurn(proceed) - // Under the same lock that ends the pacing, so that the pacing cannot end between the check and the wait. - guard let gapEnds = deliveryLock.withLock({ () -> DispatchTime? in - guard isPacedDelivery, let answered = lastPacedPostAnswered - else { return nil } - waitingTurn = turn - return answered + pacing.gap - }) - else { return false } - DispatchQueue.global().asyncAfter(deadline: gapEnds) { [weak self] in - guard let self - else { return } - let taken = self.deliveryLock.withLock { () -> Bool in - guard self.waitingTurn === turn - else { return false } - self.waitingTurn = nil - return true - } - if taken { - turn.proceed() - } - } - return true - } - - /// Queues a delivery, or joins one that is queued and has not started. - /// - /// A delivery that has not started yet will send everything recorded before it starts, which includes this - /// caller's events, so joining it is as good as queueing another. Without this, flushes arriving faster than a - /// post completes each queue their own, and the one that matters -- the flush at shutdown -- waits behind all of - /// them. - /// - /// A delivery that has already started is not joined. It has closed and taken its batches, so it cannot speak for - /// anything recorded since. Nor does another start beside it: the one queued waits until the running one has an - /// answer, then sends what that one left behind as well as what was recorded since, so its caller hears `true` - /// only if both got out. - /// - /// - Parameters: - /// - closing: whether it is `flushAtClose()`'s, which a queued delivery it joins becomes. - /// - paced: whether nobody asked for it, so that it gives way to flag requests. One that is not ends the pacing of - /// the delivery running, so that it is not held behind it. + /// As `DeliveryScheduler.queue(_:closing:paced:)`, starting the delivery where nothing is running. private func queueDelivery(_ completion: FlushOutcomeClosure?, closing: Bool = false, paced: Bool = false) { - if !paced { - endPacing() - } - deliveryLock.lock() - if queuedDelivery != nil { - if let completion { - queuedDelivery?.append(completion) - } - queuedDeliveryIsClosing = queuedDeliveryIsClosing || closing - queuedDeliveryIsPaced = queuedDeliveryIsPaced && paced - deliveryLock.unlock() - return - } - queuedDelivery = completion.map { [$0] } ?? [] - queuedDeliveryIsClosing = closing - queuedDeliveryIsPaced = paced - let starts = !isDelivering - isDelivering = true - deliveryLock.unlock() - - if starts { + if deliveryScheduler.queue(completion, closing: closing, paced: paced) { deliveryQueue.async { self.runDelivery() } } } private func runDelivery() { - // Requests arriving from here on need a delivery of their own: this one is about to close and take its - // batches, and what it takes is all it can speak for. - deliveryLock.lock() - let waiters = queuedDelivery ?? [] - queuedDelivery = nil - let closing = queuedDeliveryIsClosing - queuedDeliveryIsClosing = false - isClosingDelivery = closing - isPacedDelivery = queuedDeliveryIsPaced && givesWayToFlags && !closing - queuedDeliveryIsPaced = false - lastPacedPostAnswered = nil - #if DEBUG - deliveryRunCount += 1 - #endif - deliveryLock.unlock() - - reportEvents(closing: closing) { delivered, lossMark in + let (run, waiters) = deliveryScheduler.begin() + reportEvents(run) { delivered, lossMark in // Taken once for the delivery rather than once per waiter, so that every caller joined to it hears about // a loss instead of only whichever happened to be answered first. Taken before the next delivery starts, // which could otherwise answer first and take this one's loss with its own. @@ -524,20 +385,14 @@ extension DirectEventProcessor { /// Ends the delivery that just had its answer, starting the one queued behind it if there is one. private func startQueuedDeliveryOrStop() { - let next = deliveryLock.withLock { () -> Bool in - isClosingDelivery = false - isPacedDelivery = false - isDelivering = queuedDelivery != nil - return isDelivering - } - if next { + if deliveryScheduler.finish() { deliveryQueue.async { self.runDelivery() } } } - /// - Parameter closing: whether it is `flushAtClose()`'s, which was online when it was asked for. - private func reportEvents(closing: Bool = false, completion: ((_ delivered: Bool, _ lossMark: UInt64) -> Void)?) { - guard closing || isOnline + /// - Parameter run: the delivery; a closing one was online when it was asked for. + private func reportEvents(_ run: DeliveryRun, completion: ((_ delivered: Bool, _ lossMark: UInt64) -> Void)?) { + guard run.isClosing || isOnline else { os_log("%s aborted. DirectEventProcessor is offline", log: service.config.logger, type: .debug, typeName(and: #function)) reportSyncComplete(.isOffline) @@ -562,7 +417,7 @@ extension DirectEventProcessor { // Waited on for the answer, but not before closing. waitOn(waiting.waitingOn, answer) - deliverOwn(waiting.deliverable, lostAnything: waiting.lostAnything) { waitingOutcome in + deliverOwn(waiting.deliverable, run, lostAnything: waiting.lostAnything) { waitingOutcome in guard waitingOutcome != .blocked else { // Committed all the same, since a delivery is a commit point. Nothing was closed, so the caller's events @@ -591,21 +446,21 @@ extension DirectEventProcessor { answer.report(max(waitingOutcome, newOutcome)) } self.waitOn(new.waitingOn, newAnswer) - self.deliverOwn(new.deliverable, lostAnything: new.lostAnything, newAnswer.report) + self.deliverOwn(new.deliverable, run, lostAnything: new.lostAnything, newAnswer.report) } } /// Delivers the batches this delivery claimed, releasing the claims once it has an outcome. /// /// - Parameter lostAnything: whether the listing that found them found other batches deleted unsent. - private func deliverOwn(_ batches: [EventBatch], lostAnything: Bool, _ completion: @escaping (PassOutcome) -> Void) { + private func deliverOwn(_ batches: [EventBatch], _ run: DeliveryRun, lostAnything: Bool, _ completion: @escaping (PassOutcome) -> Void) { guard !batches.isEmpty else { completion(lostAnything ? .lost : .delivered) return } os_log("%s starting", log: service.config.logger, type: .debug, typeName(and: #function)) - deliver(batches, lostAnything: lostAnything, retried: false) { outcome in + deliver(batches, run, lostAnything: lostAnything, retried: false) { outcome in self.releaseClaim(on: batches, delivered: outcome == .delivered) completion(outcome) } @@ -736,7 +591,7 @@ extension DirectEventProcessor { /// A paced delivery holds each batch after its first until a gap has passed since the last one was answered. A /// backlog this sends unasked otherwise takes the network for as long as it lasts, an `identify`'s flag request /// included. `hadTurn` is whether the first batch has been held already. - private func deliver(_ batches: [EventBatch], lostAnything: Bool, retried: Bool, hadTurn: Bool = false, _ completion: @escaping (PassOutcome) -> Void) { + private func deliver(_ batches: [EventBatch], _ run: DeliveryRun, lostAnything: Bool, retried: Bool, hadTurn: Bool = false, _ completion: @escaping (PassOutcome) -> Void) { var remaining = batches guard !remaining.isEmpty else { @@ -744,26 +599,26 @@ extension DirectEventProcessor { return } - if !hadTurn, awaitPacedTurn(then: { - self.deliver(batches, lostAnything: lostAnything, retried: retried, hadTurn: true, completion) + if !hadTurn, run.awaitTurn(then: { + self.deliver(batches, run, lostAnything: lostAnything, retried: retried, hadTurn: true, completion) }) { return } - guard mayPost() + guard mayPost(run) else { completion(.blocked) return } - let wait = deliveryLock.withLock { isClosingDelivery } ? 0 : retryState.remainingWait + let wait = run.isClosing ? 0 : retryState.remainingWait guard wait <= 0 else { // Not an attempt, so it moves nothing in the backoff. A wait no longer than a retry's is sat out, as this // pass's retry; a longer one ends the pass. if !retried && wait <= retryState.initialDelay { DispatchQueue.global().asyncAfter(deadline: .now() + wait) { - self.deliver(batches, lostAnything: lostAnything, retried: true, completion) + self.deliver(batches, run, lostAnything: lostAnything, retried: true, completion) } } else { scheduleDeliveryWhenWaitEnds() @@ -782,13 +637,13 @@ extension DirectEventProcessor { os_log("%s dropping %d stored event(s) this version cannot read", log: service.config.logger, type: .error, typeName(and: #function), batch.eventCount) store.remove(batch) reportBatchLost(batch) - deliver(remaining, lostAnything: true, retried: retried, completion) + deliver(remaining, run, lostAnything: true, retried: retried, completion) return } catch is VanishedBatch { os_log("%s %d stored event(s) were deleted before they could be sent", log: service.config.logger, type: .error, typeName(and: #function), batch.eventCount) store.remove(batch) reportBatchLost(batch) - deliver(remaining, lostAnything: true, retried: retried, completion) + deliver(remaining, run, lostAnything: true, retried: retried, completion) return } catch { // Kept: the batch is still there, and a read that failed may not fail next time. @@ -802,7 +657,7 @@ extension DirectEventProcessor { // Delivered by another process or store, or holding no events. Either way nothing is lost and there is // nothing to keep. store.remove(batch) - deliver(remaining, lostAnything: lostAnything, retried: retried, completion) + deliver(remaining, run, lostAnything: lostAnything, retried: retried, completion) return } @@ -810,36 +665,37 @@ extension DirectEventProcessor { // the batch is removed rather than now, since one kept for a retry is read, and found short, again. let leftOut = max(0, batch.eventCount - payload.eventCount) service.diagnosticCache?.recordEventsInLastBatch(eventsInLastBatch: payload.eventCount) - post(batch, payload, leftOut: leftOut, remaining: remaining, lostAnything: lostAnything, retried: retried, completion) + post(batch, payload, run, leftOut: leftOut, remaining: remaining, lostAnything: lostAnything, retried: retried, completion) } /// Whether the delivery running may post: online, or `flushAtClose()`'s. Going offline stops a delivery before its /// next post, its retry's included, and leaves what it has not sent for a later one. - private func mayPost() -> Bool { - deliveryLock.withLock { isClosingDelivery } || isOnline + private func mayPost(_ run: DeliveryRun) -> Bool { + run.isClosing || isOnline } /// Posts a batch already read, then goes on to the rest. Requires the backoff to allow a post. // swiftlint:disable:next function_parameter_count private func post(_ batch: EventBatch, _ payload: BatchPayload, + _ run: DeliveryRun, leftOut: Int, remaining: [EventBatch], lostAnything: Bool, retried: Bool, _ completion: @escaping (PassOutcome) -> Void) { - publish(batch, payload) { outcome in + publish(batch, payload, run) { outcome in switch outcome { case .accepted: _ = self.countBadRequest(batch, statusCode: nil) guard leftOut > 0 else { - self.deliver(remaining, lostAnything: lostAnything, retried: retried, completion) + self.deliver(remaining, run, lostAnything: lostAnything, retried: retried, completion) return } os_log("%s dropped %d damaged stored event(s) from a batch it delivered", log: self.service.config.logger, type: .error, self.typeName(and: #function), leftOut) self.reportStoredEventsLost(leftOut) - self.deliver(remaining, lostAnything: true, retried: retried, completion) + self.deliver(remaining, run, lostAnything: true, retried: retried, completion) case .failed(let failure): let wait = self.retryState.recordFailure(unexpected: failure.unexpected, retryAfter: failure.retryAfter) if self.countBadRequest(batch, statusCode: failure.statusCode) >= DirectEventProcessor.badRequestsBeforeDropping { @@ -848,7 +704,7 @@ extension DirectEventProcessor { self.store.remove(batch) self.reportBatchLost(batch) self.reportSyncComplete(failure.error) - self.deliver(remaining, lostAnything: true, retried: retried, completion) + self.deliver(remaining, run, lostAnything: true, retried: retried, completion) return } guard !retried, wait <= self.retryState.initialDelay @@ -862,7 +718,7 @@ extension DirectEventProcessor { os_log("%s Retrying event post after %.1fs", log: self.service.config.logger, type: .debug, self.typeName(and: #function), wait) DispatchQueue.global().asyncAfter(deadline: .now() + wait) { // Another delivery's failure may have set a longer wait meanwhile, or the reporter gone offline. - guard self.mayPost() + guard self.mayPost(run) else { self.reportSyncComplete(failure.error) completion(.blocked) @@ -875,7 +731,7 @@ extension DirectEventProcessor { completion(.blocked) return } - self.post(batch, payload, leftOut: leftOut, remaining: remaining, lostAnything: lostAnything, retried: true, completion) + self.post(batch, payload, run, leftOut: leftOut, remaining: remaining, lostAnything: lostAnything, retried: true, completion) } } } @@ -908,18 +764,14 @@ extension DirectEventProcessor { let wait = retryState.remainingWait guard wait > 0 else { return } - let alreadyScheduled = deliveryLock.withLock { () -> Bool in - defer { isRetryScheduled = true } - return isRetryScheduled - } - guard !alreadyScheduled + guard deliveryScheduler.claimRetry() else { return } os_log("%s Will retry posting events in %.1fs after a failure", log: service.config.logger, type: .debug, typeName(and: #function), wait) DispatchQueue.global().asyncAfter(deadline: .now() + wait) { [weak self] in guard let self else { return } - self.deliveryLock.withLock { self.isRetryScheduled = false } + self.deliveryScheduler.retryDue() self.queueDelivery(nil) } } @@ -927,15 +779,12 @@ extension DirectEventProcessor { /// Sends one batch and acts on what LaunchDarkly made of it, before handing the outcome on. private func publish(_ batch: EventBatch, _ payload: BatchPayload, + _ run: DeliveryRun, _ completion: @escaping (AnalyticsEventSender.Outcome) -> Void) { store.startedSending(batch) // The store is settled before observers are told, so that one looking at it on hearing the outcome sees it. sender.send(payload.body, payloadId: batch.payloadId, eventCount: payload.eventCount) { outcome in - self.deliveryLock.withLock { - if self.isPacedDelivery { - self.lastPacedPostAnswered = .now() - } - } + run.postAnswered() switch outcome { case .accepted(let serverTime): self.retryState.recordSuccess() @@ -987,16 +836,12 @@ extension DirectEventProcessor: TypeIdentifying { } /// Callers waiting on the queued delivery. var queuedDeliveryWaiterCount: Int { - deliveryLock.lock() - defer { deliveryLock.unlock() } - return queuedDelivery?.count ?? 0 + deliveryScheduler.queuedWaiterCount } /// Deliveries started, counting one for a run of flushes that joined each other. var startedDeliveryCount: Int { - deliveryLock.lock() - defer { deliveryLock.unlock() } - return deliveryRunCount + deliveryScheduler.startedCount } /// As `EventCommitter.withContextSummarizer(_:)`. From f2b515e0ba727a11166983e1c3f1f202eb629991 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Wed, 28 Oct 2026 12:59:17 -0700 Subject: [PATCH 119/158] Rename DeliveryScheduler to EventDeliveryScheduler Co-authored-by: Cursor --- LaunchDarkly.xcodeproj/project.pbxproj | 20 +++++++++---------- .../ServiceObjects/DirectEventProcessor.swift | 8 ++++---- ...ler.swift => EventDeliveryScheduler.swift} | 4 ++-- 3 files changed, 16 insertions(+), 16 deletions(-) rename LaunchDarkly/LaunchDarkly/ServiceObjects/{DeliveryScheduler.swift => EventDeliveryScheduler.swift} (98%) diff --git a/LaunchDarkly.xcodeproj/project.pbxproj b/LaunchDarkly.xcodeproj/project.pbxproj index 2cc05ecf3..7999ec30a 100644 --- a/LaunchDarkly.xcodeproj/project.pbxproj +++ b/LaunchDarkly.xcodeproj/project.pbxproj @@ -81,7 +81,7 @@ 831188572113AE0B00D77CB5 /* FlagChangeNotifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8358F25F1F476AD800ECE1AF /* FlagChangeNotifier.swift */; }; 831188582113AE0F00D77CB5 /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; E7C0A11E2F00000000000002 /* EventCommitter.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000001 /* EventCommitter.swift */; }; - E7C0A11E2F00000000000007 /* DeliveryScheduler.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000006 /* DeliveryScheduler.swift */; }; + E7C0A11E2F00000000000007 /* EventDeliveryScheduler.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000006 /* EventDeliveryScheduler.swift */; }; B66ECB2014E0FC95C23DA0A2 /* AnalyticsEventSender.swift in Sources */ = {isa = PBXBuildFile; fileRef = 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */; }; 629D5A9FE152DBC3E20D6DFC /* OutboundEventBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */; }; 014901E06BE1F2B0FAAD6AA6 /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; @@ -128,7 +128,7 @@ 831EF35620655E730001C643 /* FlagChangeNotifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8358F25F1F476AD800ECE1AF /* FlagChangeNotifier.swift */; }; 831EF35720655E730001C643 /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; E7C0A11E2F00000000000003 /* EventCommitter.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000001 /* EventCommitter.swift */; }; - E7C0A11E2F00000000000008 /* DeliveryScheduler.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000006 /* DeliveryScheduler.swift */; }; + E7C0A11E2F00000000000008 /* EventDeliveryScheduler.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000006 /* EventDeliveryScheduler.swift */; }; 367D374B5CD1B2A8596C0738 /* AnalyticsEventSender.swift in Sources */ = {isa = PBXBuildFile; fileRef = 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */; }; 5A89648200D2566D3F8409BA /* OutboundEventBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */; }; CFF357D68BBF2DC8395930F2 /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; @@ -216,7 +216,7 @@ 83D9EC882062DEAB004D7FA6 /* FlagChangeNotifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8358F25F1F476AD800ECE1AF /* FlagChangeNotifier.swift */; }; 83D9EC892062DEAB004D7FA6 /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; E7C0A11E2F00000000000004 /* EventCommitter.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000001 /* EventCommitter.swift */; }; - E7C0A11E2F00000000000009 /* DeliveryScheduler.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000006 /* DeliveryScheduler.swift */; }; + E7C0A11E2F00000000000009 /* EventDeliveryScheduler.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000006 /* EventDeliveryScheduler.swift */; }; 321659EE1824B83BD0D0F22D /* AnalyticsEventSender.swift in Sources */ = {isa = PBXBuildFile; fileRef = 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */; }; 73E23C0FA009141BB4CBC24C /* OutboundEventBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */; }; 54745AEF1A43D165F5A6E08F /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; @@ -246,7 +246,7 @@ 83FEF8DD1F266742001CF12C /* FlagSynchronizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */; }; 83FEF8DF1F2667E4001CF12C /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; E7C0A11E2F00000000000005 /* EventCommitter.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000001 /* EventCommitter.swift */; }; - E7C0A11E2F0000000000000A /* DeliveryScheduler.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000006 /* DeliveryScheduler.swift */; }; + E7C0A11E2F0000000000000A /* EventDeliveryScheduler.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000006 /* EventDeliveryScheduler.swift */; }; E91F5DCAEEE5FC4AC408BA67 /* AnalyticsEventSender.swift in Sources */ = {isa = PBXBuildFile; fileRef = 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */; }; 7D37AD3ECAA9D8E2927228C5 /* OutboundEventBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */; }; 9FA0EE3A87678F58F4406B24 /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; @@ -638,7 +638,7 @@ 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FlagSynchronizer.swift; sourceTree = ""; }; 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DirectEventProcessor.swift; sourceTree = ""; }; E7C0A11E2F00000000000001 /* EventCommitter.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = EventCommitter.swift; sourceTree = ""; }; - E7C0A11E2F00000000000006 /* DeliveryScheduler.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DeliveryScheduler.swift; sourceTree = ""; }; + E7C0A11E2F00000000000006 /* EventDeliveryScheduler.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = EventDeliveryScheduler.swift; sourceTree = ""; }; 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AnalyticsEventSender.swift; sourceTree = ""; }; 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OutboundEventBuffer.swift; sourceTree = ""; }; BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NullEventProcessor.swift; sourceTree = ""; }; @@ -1089,7 +1089,7 @@ 831425B0206B030100F2EF36 /* EnvironmentReporter.swift */, 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */, E7C0A11E2F00000000000001 /* EventCommitter.swift */, - E7C0A11E2F00000000000006 /* DeliveryScheduler.swift */, + E7C0A11E2F00000000000006 /* EventDeliveryScheduler.swift */, 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */, 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */, BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */, @@ -1574,7 +1574,7 @@ 831188522113ADF700D77CB5 /* KeyedValueCache.swift in Sources */, 831188582113AE0F00D77CB5 /* DirectEventProcessor.swift in Sources */, E7C0A11E2F00000000000002 /* EventCommitter.swift in Sources */, - E7C0A11E2F00000000000007 /* DeliveryScheduler.swift in Sources */, + E7C0A11E2F00000000000007 /* EventDeliveryScheduler.swift in Sources */, B66ECB2014E0FC95C23DA0A2 /* AnalyticsEventSender.swift in Sources */, 629D5A9FE152DBC3E20D6DFC /* OutboundEventBuffer.swift in Sources */, 014901E06BE1F2B0FAAD6AA6 /* NullEventProcessor.swift in Sources */, @@ -1702,7 +1702,7 @@ A358D6D92A4DE6A500270C60 /* ApplicationInfoEnvironmentReporter.swift in Sources */, 831EF35720655E730001C643 /* DirectEventProcessor.swift in Sources */, E7C0A11E2F00000000000003 /* EventCommitter.swift in Sources */, - E7C0A11E2F00000000000008 /* DeliveryScheduler.swift in Sources */, + E7C0A11E2F00000000000008 /* EventDeliveryScheduler.swift in Sources */, 367D374B5CD1B2A8596C0738 /* AnalyticsEventSender.swift in Sources */, 5A89648200D2566D3F8409BA /* OutboundEventBuffer.swift in Sources */, CFF357D68BBF2DC8395930F2 /* NullEventProcessor.swift in Sources */, @@ -1816,7 +1816,7 @@ 8358F2601F476AD800ECE1AF /* FlagChangeNotifier.swift in Sources */, 83FEF8DF1F2667E4001CF12C /* DirectEventProcessor.swift in Sources */, E7C0A11E2F00000000000005 /* EventCommitter.swift in Sources */, - E7C0A11E2F0000000000000A /* DeliveryScheduler.swift in Sources */, + E7C0A11E2F0000000000000A /* EventDeliveryScheduler.swift in Sources */, E91F5DCAEEE5FC4AC408BA67 /* AnalyticsEventSender.swift in Sources */, 7D37AD3ECAA9D8E2927228C5 /* OutboundEventBuffer.swift in Sources */, 9FA0EE3A87678F58F4406B24 /* NullEventProcessor.swift in Sources */, @@ -1985,7 +1985,7 @@ C443A40B2315AA4D00145710 /* NetworkReporter.swift in Sources */, 83D9EC892062DEAB004D7FA6 /* DirectEventProcessor.swift in Sources */, E7C0A11E2F00000000000004 /* EventCommitter.swift in Sources */, - E7C0A11E2F00000000000009 /* DeliveryScheduler.swift in Sources */, + E7C0A11E2F00000000000009 /* EventDeliveryScheduler.swift in Sources */, 321659EE1824B83BD0D0F22D /* AnalyticsEventSender.swift in Sources */, 73E23C0FA009141BB4CBC24C /* OutboundEventBuffer.swift in Sources */, 54745AEF1A43D165F5A6E08F /* NullEventProcessor.swift in Sources */, diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index f85b7dd9a..2a96d4b09 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -36,7 +36,7 @@ class DirectEventProcessor: EventProcessor { let store: EventStoring let committer: EventCommitter private let lostEvents: LostEventCount - private let deliveryScheduler: DeliveryScheduler + private let deliveryScheduler: EventDeliveryScheduler private let onSyncComplete: EventSyncCompleteClosure? @@ -86,7 +86,7 @@ class DirectEventProcessor: EventProcessor { /// The batches a delivery has taken and not yet had an answer for, by payload ID. Only to be used while holding /// `claimLock`. /// - /// A batch stays on disk until its response arrives. `DeliveryScheduler` keeps a second delivery from listing it + /// A batch stays on disk until its response arrives. `EventDeliveryScheduler` keeps a second delivery from listing it /// inside that window; this is what keeps it from being sent twice, under the same payload ID, should two ever overlap. /// A delivery that finds a batch claimed waits on that claim's outcome rather than sending it. private var batchesInFlight: [String: Claim] = [:] @@ -142,7 +142,7 @@ class DirectEventProcessor: EventProcessor { self.store = eventStore self.pacing = pacing self.givesWayToFlags = eventStore.isPersisting - self.deliveryScheduler = DeliveryScheduler(pacesDeliveries: eventStore.isPersisting, gap: pacing.gap) + self.deliveryScheduler = EventDeliveryScheduler(pacesDeliveries: eventStore.isPersisting, gap: pacing.gap) self.launchGateOpen = !eventStore.isPersisting // A log left open by a previous run has to be closed before it can be delivered, but nothing waits on that: @@ -354,7 +354,7 @@ extension DirectEventProcessor { queueDelivery(nil, paced: true) } - /// As `DeliveryScheduler.queue(_:closing:paced:)`, starting the delivery where nothing is running. + /// As `EventDeliveryScheduler.queue(_:closing:paced:)`, starting the delivery where nothing is running. private func queueDelivery(_ completion: FlushOutcomeClosure?, closing: Bool = false, paced: Bool = false) { if deliveryScheduler.queue(completion, closing: closing, paced: paced) { deliveryQueue.async { self.runDelivery() } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DeliveryScheduler.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift similarity index 98% rename from LaunchDarkly/LaunchDarkly/ServiceObjects/DeliveryScheduler.swift rename to LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift index ebcba7940..bae5a28bb 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DeliveryScheduler.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift @@ -2,7 +2,7 @@ import Foundation /// One delivery, from taking its batches until it has an answer for everything it sent, its retry included. /// -/// Made by `DeliveryScheduler.begin()` and passed through the delivery, so that what is true of the delivery running is +/// Made by `EventDeliveryScheduler.begin()` and passed through the delivery, so that what is true of the delivery running is /// held by it, and starts afresh with each one. final class DeliveryRun { /// A paced batch waiting out its gap. @@ -95,7 +95,7 @@ final class DeliveryRun { /// /// Its lock is taken on a caller's thread, on the delivery queue and where a delivery's posts are answered, and never /// held across any of them. -final class DeliveryScheduler { +final class EventDeliveryScheduler { /// A delivery queued that has not taken its batches yet. private struct Queued { /// The callers waiting on it. From 9e629b688abef9ecd1ad24ad0e190876b2678d9a Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Wed, 28 Oct 2026 13:12:17 -0700 Subject: [PATCH 120/158] Rename DeliveryRun to EventDeliveryRun Co-authored-by: Cursor --- .../ServiceObjects/DirectEventProcessor.swift | 12 ++++++------ .../ServiceObjects/EventDeliveryScheduler.swift | 10 +++++----- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 2a96d4b09..c767a3901 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -391,7 +391,7 @@ extension DirectEventProcessor { } /// - Parameter run: the delivery; a closing one was online when it was asked for. - private func reportEvents(_ run: DeliveryRun, completion: ((_ delivered: Bool, _ lossMark: UInt64) -> Void)?) { + private func reportEvents(_ run: EventDeliveryRun, completion: ((_ delivered: Bool, _ lossMark: UInt64) -> Void)?) { guard run.isClosing || isOnline else { os_log("%s aborted. DirectEventProcessor is offline", log: service.config.logger, type: .debug, typeName(and: #function)) @@ -453,7 +453,7 @@ extension DirectEventProcessor { /// Delivers the batches this delivery claimed, releasing the claims once it has an outcome. /// /// - Parameter lostAnything: whether the listing that found them found other batches deleted unsent. - private func deliverOwn(_ batches: [EventBatch], _ run: DeliveryRun, lostAnything: Bool, _ completion: @escaping (PassOutcome) -> Void) { + private func deliverOwn(_ batches: [EventBatch], _ run: EventDeliveryRun, lostAnything: Bool, _ completion: @escaping (PassOutcome) -> Void) { guard !batches.isEmpty else { completion(lostAnything ? .lost : .delivered) @@ -591,7 +591,7 @@ extension DirectEventProcessor { /// A paced delivery holds each batch after its first until a gap has passed since the last one was answered. A /// backlog this sends unasked otherwise takes the network for as long as it lasts, an `identify`'s flag request /// included. `hadTurn` is whether the first batch has been held already. - private func deliver(_ batches: [EventBatch], _ run: DeliveryRun, lostAnything: Bool, retried: Bool, hadTurn: Bool = false, _ completion: @escaping (PassOutcome) -> Void) { + private func deliver(_ batches: [EventBatch], _ run: EventDeliveryRun, lostAnything: Bool, retried: Bool, hadTurn: Bool = false, _ completion: @escaping (PassOutcome) -> Void) { var remaining = batches guard !remaining.isEmpty else { @@ -670,7 +670,7 @@ extension DirectEventProcessor { /// Whether the delivery running may post: online, or `flushAtClose()`'s. Going offline stops a delivery before its /// next post, its retry's included, and leaves what it has not sent for a later one. - private func mayPost(_ run: DeliveryRun) -> Bool { + private func mayPost(_ run: EventDeliveryRun) -> Bool { run.isClosing || isOnline } @@ -678,7 +678,7 @@ extension DirectEventProcessor { // swiftlint:disable:next function_parameter_count private func post(_ batch: EventBatch, _ payload: BatchPayload, - _ run: DeliveryRun, + _ run: EventDeliveryRun, leftOut: Int, remaining: [EventBatch], lostAnything: Bool, @@ -779,7 +779,7 @@ extension DirectEventProcessor { /// Sends one batch and acts on what LaunchDarkly made of it, before handing the outcome on. private func publish(_ batch: EventBatch, _ payload: BatchPayload, - _ run: DeliveryRun, + _ run: EventDeliveryRun, _ completion: @escaping (AnalyticsEventSender.Outcome) -> Void) { store.startedSending(batch) // The store is settled before observers are told, so that one looking at it on hearing the outcome sees it. diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift index bae5a28bb..03ff19016 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift @@ -4,7 +4,7 @@ import Foundation /// /// Made by `EventDeliveryScheduler.begin()` and passed through the delivery, so that what is true of the delivery running is /// held by it, and starts afresh with each one. -final class DeliveryRun { +final class EventDeliveryRun { /// A paced batch waiting out its gap. private final class PacedTurn { let proceed: () -> Void @@ -121,7 +121,7 @@ final class EventDeliveryScheduler { /// each held up by a slow response, could take the store that many batches past them. private var isDelivering = false /// The delivery running, whose pacing a delivery somebody asks for ends. Only to be used while holding `lock`. - private var running: DeliveryRun? + private var running: EventDeliveryRun? /// Whether a delivery is already set to run when the backoff's wait ends, so a run of passes inside the wait sets /// one rather than one each. Only to be used while holding `lock`. private var isRetryScheduled = false @@ -154,7 +154,7 @@ final class EventDeliveryScheduler { /// the delivery running, so that it is not held behind it. /// - Returns: whether the caller is to start it, which is where no delivery is running. func queue(_ completion: FlushOutcomeClosure?, closing: Bool, paced: Bool) -> Bool { - let (starts, unpaced) = lock.withLock { () -> (Bool, DeliveryRun?) in + let (starts, unpaced) = lock.withLock { () -> (Bool, EventDeliveryRun?) in let unpaced = paced ? nil : running if var joined = queued { if let completion { @@ -178,11 +178,11 @@ final class EventDeliveryScheduler { /// close and take its batches, and what it takes is all it can speak for. /// /// - Returns: the delivery, and the callers waiting on it. - func begin() -> (run: DeliveryRun, waiters: [FlushOutcomeClosure]) { + func begin() -> (run: EventDeliveryRun, waiters: [FlushOutcomeClosure]) { lock.withLock { let next = queued ?? Queued(waiters: [], isClosing: false, isPaced: false) queued = nil - let run = DeliveryRun(isClosing: next.isClosing, + let run = EventDeliveryRun(isClosing: next.isClosing, isPaced: next.isPaced && pacesDeliveries && !next.isClosing, gap: gap) running = run From 99133066bca699a5efb390ca03b344f99321f165 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Wed, 28 Oct 2026 16:05:46 -0700 Subject: [PATCH 121/158] Hold the report timer's deliveries until the launch gate opens A delivery sends the oldest batches first, which are what a previous run left, so a report interval shorter than the launch delay sent them ahead of the gate. Until it opens, the timer only commits. Co-authored-by: Cursor --- .../ServiceObjects/DirectEventProcessor.swift | 16 +++++++++++++++- .../DirectEventProcessorSpec.swift | 19 ++++++++++++++++--- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index c767a3901..1f966a96a 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -252,7 +252,9 @@ extension DirectEventProcessor { private func startReporting() { guard eventReportTimer == nil else { return } - eventReportTimer = LDTimer(withTimeInterval: service.config.eventFlushInterval, fireQueue: deliveryQueue, execute: queueScheduledDelivery) + eventReportTimer = LDTimer(withTimeInterval: service.config.eventFlushInterval, fireQueue: deliveryQueue) { [weak self] in + self?.reportIntervalEnded() + } // Events a previous run left behind are already late by however long the application was gone, so they do not // wait out a report interval on top of that, only the launch gate: an application that crashed reports it as @@ -349,6 +351,18 @@ extension DirectEventProcessor { queueDelivery(completion) } + /// The report timer's delivery, on `deliveryQueue`. Until the launch gate opens it only commits: a delivery sends the + /// oldest batches first, which are what a previous run left, so one now would send them ahead of the gate. What this + /// run recorded goes with the first delivery after the gate opens. + private func reportIntervalEnded() { + guard launchGateOpen + else { + committer.queueCommitWherePersisting() + return + } + queueScheduledDelivery() + } + /// Queues the delivery the report timer and the launch gate start, which nobody asked for now and which is paced. private func queueScheduledDelivery() { queueDelivery(nil, paced: true) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 09598f53b..3010b1d62 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -459,10 +459,23 @@ class DirectEventProcessorSpec: QuickSpec { expect(opened.withLock { hasOpened }).toEventually(beTrue(), timeout: .seconds(2)) } - itPersisting("paces a periodic delivery of a backlog too") { + itPersisting("does not let a report interval shorter than the launch delay send what a previous run left") { + testContext = TestContext(eventFlushInterval: 0.1) + let reporter = nextRunLeft(1, pacing: DirectEventProcessor.Pacing(launchDelay: 0.8, gap: 0)) + let wentOnline = Date() + + reporter.isOnline = true + + // Waited for on the run loop, which the report timer fires on. + waitLettingTimersFire(0.5) + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + expect(Date().timeIntervalSince(wentOnline)) >= 0.75 + } + + itPersisting("keeps pacing a backlog through the report intervals that end during it") { testContext = TestContext(eventFlushInterval: 0.2) - // A launch delay longer than the example, so that the report timer rather than the gate starts it. - let reporter = nextRunLeft(2, pacing: DirectEventProcessor.Pacing(launchDelay: 60, gap: 1)) + let reporter = nextRunLeft(2, pacing: DirectEventProcessor.Pacing(launchDelay: 0.2, gap: 1)) reporter.isOnline = true From 8a8a670ba7641df2ac41057ad01d9ed5265d8987 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 29 Oct 2026 17:03:20 -0700 Subject: [PATCH 122/158] fix: flushAndWait posts through the backoff's wait, as close does A blocking flush is called just before the app is suspended or ends, when nothing can come after it. During a backoff it made no attempt and answered false, so the events waited for a next run. It now makes one attempt through the wait; its answer updates the retry state. flush() and flush(completion:) still keep to the wait. Also corrects the flushAndWait docs, which still said a refusal stops delivery. Co-authored-by: Cursor --- .../GeneratedCode/mocks.generated.swift | 9 ++++ LaunchDarkly/LaunchDarkly/LDClientFlush.swift | 18 ++++--- .../ObjectiveC/ObjcLDClient.swift | 14 ++++-- .../ServiceObjects/DirectEventProcessor.swift | 17 +++++-- .../EventDeliveryScheduler.swift | 20 +++++--- .../ServiceObjects/EventProcessor.swift | 5 ++ .../ServiceObjects/NullEventProcessor.swift | 4 ++ .../LaunchDarklyTests/LDClientSpec.swift | 23 ++++----- .../DirectEventProcessorSpec.swift | 49 +++++++++++++++++++ 9 files changed, 128 insertions(+), 31 deletions(-) diff --git a/LaunchDarkly/GeneratedCode/mocks.generated.swift b/LaunchDarkly/GeneratedCode/mocks.generated.swift index 61d9c7835..a982005e9 100644 --- a/LaunchDarkly/GeneratedCode/mocks.generated.swift +++ b/LaunchDarkly/GeneratedCode/mocks.generated.swift @@ -231,6 +231,15 @@ final class EventProcessorMock: EventProcessor { try! flushReportingOutcomeCallback?() } + var flushBypassingBackoffCallCount = 0 + var flushBypassingBackoffCallback: (() throws -> Void)? + var flushBypassingBackoffReceivedCompletion: (FlushOutcomeClosure)? + func flushBypassingBackoff(completion: @escaping FlushOutcomeClosure) { + flushBypassingBackoffCallCount += 1 + flushBypassingBackoffReceivedCompletion = completion + try! flushBypassingBackoffCallback?() + } + var flushAtCloseCallCount = 0 var flushAtCloseCallback: (() throws -> Void)? func flushAtClose() { diff --git a/LaunchDarkly/LaunchDarkly/LDClientFlush.swift b/LaunchDarkly/LaunchDarkly/LDClientFlush.swift index abb37317f..3f080b2f3 100644 --- a/LaunchDarkly/LaunchDarkly/LDClientFlush.swift +++ b/LaunchDarkly/LaunchDarkly/LDClientFlush.swift @@ -105,6 +105,12 @@ extension LDClient { This is not a crash-time mechanism. It is safe to call from the main thread, but keep the budget well below 15 seconds there. + It posts even while delivery is backing off after a failure, as closing the client does, since it is for a moment + after which the SDK may not get to send. After a refusal such as a 401 or 403 the SDK waits about eight and a half + minutes before posting again, doubling up to an hour, and a call that sat that out would make no attempt at all. + `flush` and `flush(completion:)` still wait it out, so a call made routinely, rather than before the process is + suspended or ends, posts once per call while LaunchDarkly keeps refusing. + This method is not stable, and not subject to any backwards compatibility guarantees or semantic versioning. It is experimental. @@ -112,11 +118,11 @@ extension LDClient { no upper limit: a delivery always ends, by its own request timeouts if nothing else, so a very long timeout waits for that. - returns: `true` if LaunchDarkly accepted the events, or there were none to send. `false` if the timeout expired - first, the client is offline or closed, or any of the events were lost: refused by LaunchDarkly, still failing - after one retry, or unable to be serialized. Lost events are not kept for a later flush, so calling this again - does not resend them. A refusal also stops event delivery until the client is next set online, and until then - this returns `false` even with nothing to send. A `false` because the timeout expired does not mean the events - were not sent: the delivery is left running, and may still arrive. + first, the client is offline or closed, or the events did not get out. Events the service refused or that + failed to post are kept, and posted again once the backoff allows, or by the next call to this method. Events + that could not be serialized, or whose payload LaunchDarkly refused twice in a row, are lost, and calling this + again does not resend them. A `false` because the timeout expired does not mean the events were not sent: the + delivery is left running, and may still arrive. */ @discardableResult public func flushAndWait(timeout: TimeInterval) -> Bool { @@ -146,7 +152,7 @@ extension LDClient { // one before starting the next would spend the caller's budget on deliveries that could have been running all // along, and leave the last environment with none of it. for client in clients { - client.eventProcessor.flushReportingOutcome { result in + client.eventProcessor.flushBypassingBackoff { result in lock.lock() delivered = delivered && result lock.unlock() diff --git a/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDClient.swift b/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDClient.swift index 0b7da3aa1..1e470051c 100644 --- a/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDClient.swift +++ b/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDClient.swift @@ -575,6 +575,12 @@ public final class ObjcLDClient: NSObject { This is not a crash-time mechanism. It is safe to call from the main thread, but keep the budget well below 15 seconds there. + It posts even while delivery is backing off after a failure, as closing the client does, since it is for a moment + after which the SDK may not get to send. After a refusal such as a 401 or 403 the SDK waits about eight and a half + minutes before posting again, doubling up to an hour, and a call that sat that out would make no attempt at all. + `flush` still waits it out, so a call made routinely, rather than before the process is suspended or ends, posts + once per call while LaunchDarkly keeps refusing. + This method is not stable, and not subject to any backwards compatibility guarantees or semantic versioning. It is experimental. @@ -582,10 +588,10 @@ public final class ObjcLDClient: NSObject { no upper limit: a delivery always ends, by its own request timeouts if nothing else, so a very long timeout waits for that. - returns: `YES` if LaunchDarkly accepted the events, or there were none to send. `NO` if the timeout expired - first, the client is offline or closed, or any of the events were lost: refused by LaunchDarkly, still failing - after one retry, or unable to be serialized. Lost events are not kept for a later flush, so calling this again - does not resend them. A refusal also stops event delivery until the client is next set online, and until then - this returns `NO` even with nothing to send. A `NO` because the timeout expired does not mean the events were + first, the client is offline or closed, or the events did not get out. Events the service refused or that + failed to post are kept, and posted again once the backoff allows, or by the next call to this method. Events + that could not be serialized, or whose payload LaunchDarkly refused twice in a row, are lost, and calling this + again does not resend them. A `NO` because the timeout expired does not mean the events were not sent: the delivery is left running, and may still arrive. */ @objc public func flushAndWait(timeout: TimeInterval) -> Bool { diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 1f966a96a..d4121dcfa 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -333,6 +333,11 @@ extension DirectEventProcessor { queueDelivery(completion) } + func flushBypassingBackoff(completion: @escaping FlushOutcomeClosure) { + committer.queueCommitWherePersisting() + queueDelivery(completion, bypassingBackoff: true) + } + func flushAtClose() { commitWherePersisting() // Offline, it is an ordinary delivery, which finds the reporter offline and sends nothing: offline is the @@ -368,9 +373,13 @@ extension DirectEventProcessor { queueDelivery(nil, paced: true) } - /// As `EventDeliveryScheduler.queue(_:closing:paced:)`, starting the delivery where nothing is running. - private func queueDelivery(_ completion: FlushOutcomeClosure?, closing: Bool = false, paced: Bool = false) { - if deliveryScheduler.queue(completion, closing: closing, paced: paced) { + /// As `EventDeliveryScheduler.queue(_:closing:bypassingBackoff:paced:)`, starting the delivery where nothing is + /// running. + private func queueDelivery(_ completion: FlushOutcomeClosure?, + closing: Bool = false, + bypassingBackoff: Bool = false, + paced: Bool = false) { + if deliveryScheduler.queue(completion, closing: closing, bypassingBackoff: bypassingBackoff, paced: paced) { deliveryQueue.async { self.runDelivery() } } } @@ -625,7 +634,7 @@ extension DirectEventProcessor { return } - let wait = run.isClosing ? 0 : retryState.remainingWait + let wait = run.bypassesBackoff ? 0 : retryState.remainingWait guard wait <= 0 else { // Not an attempt, so it moves nothing in the backoff. A wait no longer than a retry's is sat out, as this diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift index 03ff19016..07be87dd3 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift @@ -14,10 +14,11 @@ final class EventDeliveryRun { } } - /// Whether it is `flushAtClose()`'s, which posts whatever wait the backoff is in and runs although closing has - /// turned `isOnline` off since. The delivery one of its failures sets for the end of the wait is another one, held + /// Whether it is `flushAtClose()`'s, which runs although closing has turned `isOnline` off since. The delivery one of its failures sets for the end of the wait is another one, held /// by the backoff, and finds the reporter offline as any other. let isClosing: Bool + /// Whether it posts whatever wait the backoff is in: `flushAtClose()`'s, and `flushBypassingBackoff(completion:)`'s. + let bypassesBackoff: Bool /// The least time between one paced batch's answer and the next batch's request. private let gap: TimeInterval @@ -32,8 +33,9 @@ final class EventDeliveryRun { /// The batch waiting out its gap, which ending the pacing lets go at once. Only to be used while holding `lock`. private var waitingTurn: PacedTurn? - init(isClosing: Bool, isPaced: Bool, gap: TimeInterval) { + init(isClosing: Bool, bypassesBackoff: Bool, isPaced: Bool, gap: TimeInterval) { self.isClosing = isClosing + self.bypassesBackoff = bypassesBackoff self.isPaced = isPaced self.gap = gap } @@ -102,6 +104,8 @@ final class EventDeliveryScheduler { var waiters: [FlushOutcomeClosure] /// Whether it is `flushAtClose()`'s, or has been joined by it. var isClosing: Bool + /// Whether a caller bypassing the backoff asked for it, or joined it. + var bypassesBackoff: Bool /// Whether it was asked for only by the report timer or by the launch gate, and so is paced. var isPaced: Bool } @@ -150,10 +154,11 @@ final class EventDeliveryScheduler { /// /// - Parameters: /// - closing: whether it is `flushAtClose()`'s, which a queued delivery it joins becomes. + /// - bypassingBackoff: whether it posts whatever wait the backoff is in, which a queued delivery it joins does too. /// - paced: whether nobody asked for it, so that it gives way to flag requests. One that is not ends the pacing of /// the delivery running, so that it is not held behind it. /// - Returns: whether the caller is to start it, which is where no delivery is running. - func queue(_ completion: FlushOutcomeClosure?, closing: Bool, paced: Bool) -> Bool { + func queue(_ completion: FlushOutcomeClosure?, closing: Bool, bypassingBackoff: Bool, paced: Bool) -> Bool { let (starts, unpaced) = lock.withLock { () -> (Bool, EventDeliveryRun?) in let unpaced = paced ? nil : running if var joined = queued { @@ -161,11 +166,13 @@ final class EventDeliveryScheduler { joined.waiters.append(completion) } joined.isClosing = joined.isClosing || closing + joined.bypassesBackoff = joined.bypassesBackoff || bypassingBackoff joined.isPaced = joined.isPaced && paced queued = joined return (false, unpaced) } - queued = Queued(waiters: completion.map { [$0] } ?? [], isClosing: closing, isPaced: paced) + queued = Queued(waiters: completion.map { [$0] } ?? [], isClosing: closing, bypassesBackoff: bypassingBackoff, + isPaced: paced) let starts = !isDelivering isDelivering = true return (starts, unpaced) @@ -180,9 +187,10 @@ final class EventDeliveryScheduler { /// - Returns: the delivery, and the callers waiting on it. func begin() -> (run: EventDeliveryRun, waiters: [FlushOutcomeClosure]) { lock.withLock { - let next = queued ?? Queued(waiters: [], isClosing: false, isPaced: false) + let next = queued ?? Queued(waiters: [], isClosing: false, bypassesBackoff: false, isPaced: false) queued = nil let run = EventDeliveryRun(isClosing: next.isClosing, + bypassesBackoff: next.isClosing || next.bypassesBackoff, isPaced: next.isPaced && pacesDeliveries && !next.isClosing, gap: gap) running = run diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift index 75949de91..e8e951d76 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift @@ -37,6 +37,11 @@ protocol EventProcessor { /// were written, since the delivery writes them before it sends. func flushReportingOutcome(completion: @escaping FlushOutcomeClosure) + /// `flushReportingOutcome` for a caller about to lose the ability to send, `LDClient.flushAndWait`: it posts + /// whatever wait the backoff is in, as `flushAtClose()` does. After a refusal that wait is minutes to an hour, and + /// where nothing persists the events would not outlive it. + func flushBypassingBackoff(completion: @escaping FlushOutcomeClosure) + /// The last delivery before the client closes: `flush`, except that it posts whatever wait the backoff is in, which /// closing cuts short. After a refusal that wait is minutes to an hour, and where nothing persists the events would /// not outlive it. diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventProcessor.swift index 38cd660a2..aad1ca9fc 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventProcessor.swift @@ -30,6 +30,10 @@ class NullEventProcessor: EventProcessor { DispatchQueue.global().async { completion(true) } } + func flushBypassingBackoff(completion: @escaping FlushOutcomeClosure) { + flushReportingOutcome(completion: completion) + } + func flushAtClose() { } diff --git a/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift b/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift index 5b9134e2b..5512a9633 100644 --- a/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift @@ -1778,8 +1778,8 @@ final class LDClientSpec: QuickSpec { describe("flushAndWait") { /// Answers inside the call, because `flushAndWait` blocks the test thread. func answer(_ mock: EventProcessorMock, with delivered: Bool) { - mock.flushReportingOutcomeCallback = { [weak mock] in - mock?.flushReportingOutcomeReceivedCompletion?(delivered) + mock.flushBypassingBackoffCallback = { [weak mock] in + mock?.flushBypassingBackoffReceivedCompletion?(delivered) } } @@ -1789,7 +1789,8 @@ final class LDClientSpec: QuickSpec { answer(testContext.eventProcessorMock, with: true) expect(testContext.subject.flushAndWait(timeout: 1.0)) == true - expect(testContext.eventProcessorMock.flushReportingOutcomeCallCount) == 1 + expect(testContext.eventProcessorMock.flushBypassingBackoffCallCount) == 1 + expect(testContext.eventProcessorMock.flushReportingOutcomeCallCount) == 0 } it("reports false when delivery cannot finish") { @@ -1803,7 +1804,7 @@ final class LDClientSpec: QuickSpec { it("reports false when the budget expires first") { let testContext = TestContext() testContext.start() - testContext.eventProcessorMock.flushReportingOutcomeCallback = nil + testContext.eventProcessorMock.flushBypassingBackoffCallback = nil expect(testContext.subject.flushAndWait(timeout: 0.05)) == false } @@ -1816,7 +1817,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.subject.flushAndWait(timeout: 0)) == false // Still worth calling: at `.immediate` the events are on disk by the time the delivery is queued. - expect(testContext.eventProcessorMock.flushReportingOutcomeCallCount) == 1 + expect(testContext.eventProcessorMock.flushBypassingBackoffCallCount) == 1 } it("reports false once closed") { @@ -1838,7 +1839,7 @@ final class LDClientSpec: QuickSpec { // Its own events were never attempted, so the outcome of another client's delivery says nothing of them. expect(stale.subject.flushAndWait(timeout: 1.0)) == false - expect(current.eventProcessorMock.flushReportingOutcomeCallCount) == 0 + expect(current.eventProcessorMock.flushBypassingBackoffCallCount) == 0 } it("gives every environment the whole budget rather than what the one before it left") { @@ -1851,8 +1852,8 @@ final class LDClientSpec: QuickSpec { for environment in ["alternate", LDConfig.Constants.primaryEnvironmentName] { let reporter = LDClient.get(environment: environment)?.eventProcessor as? EventProcessorMock expect(reporter).toNot(beNil()) - reporter?.flushReportingOutcomeCallback = { [weak reporter] in - let completion = reporter?.flushReportingOutcomeReceivedCompletion + reporter?.flushBypassingBackoffCallback = { [weak reporter] in + let completion = reporter?.flushBypassingBackoffReceivedCompletion DispatchQueue.global().asyncAfter(deadline: .now() + 0.3) { completion?(true) } } } @@ -1868,11 +1869,11 @@ final class LDClientSpec: QuickSpec { // spent is the budget the second has left. Giving each its own copy of it would take twice as long. let answering = LDClient.get(environment: "alternate")?.eventProcessor as? EventProcessorMock expect(answering).toNot(beNil()) - answering?.flushReportingOutcomeCallback = { [weak answering] in - let completion = answering?.flushReportingOutcomeReceivedCompletion + answering?.flushBypassingBackoffCallback = { [weak answering] in + let completion = answering?.flushBypassingBackoffReceivedCompletion DispatchQueue.global().asyncAfter(deadline: .now() + 0.5) { completion?(true) } } - testContext.eventProcessorMock.flushReportingOutcomeCallback = nil + testContext.eventProcessorMock.flushBypassingBackoffCallback = nil // Answering at 0.5s into a 0.6s budget leaves the second wait 0.1s, against the 0.6s a fresh copy // would give it, so the two outcomes are 0.6s and 1.1s with room on either side of the threshold. diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 3010b1d62..89155a38c 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -138,6 +138,17 @@ class DirectEventProcessorSpec: QuickSpec { return delivered } + func flushBypassingBackoffOutcome() -> Bool? { + var delivered: Bool? + waitUntil(timeout: .seconds(10)) { done in + eventProcessor.flushBypassingBackoff { result in + delivered = result + done() + } + } + return delivered + } + func cleanUp() { eventProcessor.isOnline = false store.deleteEverything() @@ -3209,6 +3220,44 @@ extension DirectEventProcessorSpec { .toEventually(equal(["refused", "at-close"]), timeout: Constants.deliveryTimeout) } + it("posts through the backoff's wait for a flush bypassing it, and answers whether the events got out") { + testContext = TestContext(retryState: DeliveryRetryState(initialDelay: 1, normalCeiling: 1, extendedCeiling: 3600)) + testContext.serviceMock.respondToEvents(with: HTTPURLResponse.StatusCodes.unauthorized) + testContext.eventProcessor.record(CustomEvent(key: "refused", context: testContext.context)) + testContext.eventProcessor.isOnline = true + expect(testContext.flushOutcome()) == false + testContext.serviceMock.stubEventResponse(success: true) + testContext.serviceMock.eventStatusForPost = nil + testContext.eventProcessor.record(CustomEvent(key: "bypassing", context: testContext.context)) + + expect(testContext.flushBypassingBackoffOutcome()) == true + expect(testContext.serviceMock.publishedEventPayloads.dropFirst().flatMap(customKeys)) == ["refused", "bypassing"] + } + + it("leaves ordinary flushes waiting out the backoff after a flush bypassing it is refused again") { + testContext = TestContext(retryState: DeliveryRetryState(initialDelay: 1, normalCeiling: 1, extendedCeiling: 3600)) + testContext.serviceMock.respondToEvents(with: HTTPURLResponse.StatusCodes.unauthorized) + testContext.eventProcessor.record(CustomEvent(key: "refused", context: testContext.context)) + testContext.eventProcessor.isOnline = true + expect(testContext.flushOutcome()) == false + + expect(testContext.flushBypassingBackoffOutcome()) == false + expect(testContext.serviceMock.publishEventDataCallCount) == 2 + + expect(testContext.flushOutcome()) == false + testContext.eventProcessor.flush(completion: nil) + waitLettingTimersFire(0.3) + expect(testContext.serviceMock.publishEventDataCallCount) == 2 + } + + it("posts nothing for a flush bypassing the backoff while offline") { + testContext = TestContext() + testContext.eventProcessor.record(CustomEvent(key: "an-event", context: testContext.context)) + + expect(testContext.flushBypassingBackoffOutcome()) == false + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + } + it("posts nothing at close when the client was already offline") { testContext = TestContext() testContext.eventProcessor.record(CustomEvent(key: "an-event", context: testContext.context)) From bcd41ae69d8ae9b2123f8e52ce375ad7a46faf31 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 29 Oct 2026 19:19:15 -0700 Subject: [PATCH 123/158] fix: a post that gets no response leaves the run of payload refusals as it was countBadRequest ended the run for a nil status code, which is what a network or TLS failure reports, and an accepted post passed nil to end it on purpose. So a batch the service will never take, refused with failures in between, never reached two refusals in a row, and stayed the oldest, holding back every batch behind it. A failure with no response now leaves the count unchanged; an accepted post ends the run through endBadRequests(), and any other response still ends it. The service mock answers a scripted status of 0 with an error. Co-authored-by: Cursor --- .../ServiceObjects/DirectEventProcessor.swift | 26 ++++++++++++++++--- .../Mocks/DarklyServiceMock.swift | 15 ++++++----- .../DirectEventProcessorSpec.swift | 16 ++++++++++++ 3 files changed, 47 insertions(+), 10 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index d4121dcfa..9e31a8909 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -71,8 +71,9 @@ class DirectEventProcessor: EventProcessor { static let badRequestsBeforeDropping = 2 /// The batch that last had its payload refused and how many times in a row: delivery goes oldest first and stops at - /// a kept failure, so the next post is that batch's unless capacity dropped it. Held only in memory, so a new run of - /// the application counts again. Only to be used while holding `stateLock`. + /// a kept failure, so the next post is that batch's unless capacity dropped it. Any response other than a refusal + /// ends the run; a post that got no response leaves it. Held only in memory, so a new run of the application counts + /// again. Only to be used while holding `stateLock`. private var badRequests: (payloadId: String, inARow: Int)? // MARK: Guarded by claimLock @@ -710,7 +711,7 @@ extension DirectEventProcessor { publish(batch, payload, run) { outcome in switch outcome { case .accepted: - _ = self.countBadRequest(batch, statusCode: nil) + self.endBadRequests() guard leftOut > 0 else { self.deliver(remaining, run, lostAnything: lostAnything, retried: retried, completion) @@ -760,11 +761,23 @@ extension DirectEventProcessor { } } + /// - Parameter statusCode: the response's status, or nil where there was no response. /// - Returns: how many refusals of the payload in a row this batch has had, counting this response; zero for any /// other status, which ends the run, and once the batch is dropped. private func countBadRequest(_ batch: EventBatch, statusCode: Int?) -> Int { stateLock.withLock { - guard let statusCode, HTTPURLResponse.StatusCodes.isEventPayloadRefused(statusCode) + guard let statusCode + else { + // A network or TLS failure leaves the run as it was. The service did not answer, so it said nothing + // about the payload: had it ended the run, refusals with network failures between them would never + // reach two, and a batch the service will never take would stay the oldest, holding back every + // batch behind it. + if let previous = badRequests, previous.payloadId == batch.payloadId { + return previous.inARow + } + return 0 + } + guard HTTPURLResponse.StatusCodes.isEventPayloadRefused(statusCode) else { badRequests = nil return 0 @@ -778,6 +791,11 @@ extension DirectEventProcessor { } } + /// Ends the run of refusals of the payload, as an accepted post does. + private func endBadRequests() { + stateLock.withLock { badRequests = nil } + } + /// Sets a delivery for when the backoff's wait ends, unless one is set already, so a batch kept through a failure /// goes out then rather than at the next report interval after it. /// diff --git a/LaunchDarkly/LaunchDarklyTests/Mocks/DarklyServiceMock.swift b/LaunchDarkly/LaunchDarklyTests/Mocks/DarklyServiceMock.swift index c71a58326..0d4bce729 100644 --- a/LaunchDarkly/LaunchDarklyTests/Mocks/DarklyServiceMock.swift +++ b/LaunchDarkly/LaunchDarklyTests/Mocks/DarklyServiceMock.swift @@ -146,7 +146,8 @@ final class DarklyServiceMock: DarklyServiceProvider { private var _eventStatusForPost: ((Int) -> Int)? private var _publishedEventStatuses: [Int?] = [] - /// The status to answer the nth post with, counting from 1, in place of `stubbedEventResponse`. + /// The status to answer the nth post with, counting from 1, in place of `stubbedEventResponse`; 0 for a network + /// error with no response. var eventStatusForPost: ((Int) -> Int)? { get { withEventLock { _eventStatusForPost } } set { withEventLock { _eventStatusForPost = newValue } } @@ -199,11 +200,13 @@ final class DarklyServiceMock: DarklyServiceProvider { } var response = _stubbedEventResponse ?? (nil, nil, nil, nil) if let statusForPost = _eventStatusForPost { - response = (nil, - HTTPURLResponse(url: config.eventsUrl, statusCode: statusForPost(_publishEventDataCallCount), - httpVersion: Constants.httpVersion, headerFields: nil), - nil, - nil) + let status = statusForPost(_publishEventDataCallCount) + response = status == 0 + ? (nil, nil, Constants.error, nil) + : (nil, + HTTPURLResponse(url: config.eventsUrl, statusCode: status, httpVersion: Constants.httpVersion, headerFields: nil), + nil, + nil) } _publishedEventStatuses.append((response.urlResponse as? HTTPURLResponse)?.statusCode) return response diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 89155a38c..add092897 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -3145,6 +3145,22 @@ extension DirectEventProcessorSpec { expect(testContext.serviceMock.publishedEventPayloads.map(customKeys)) == [["refused"], ["refused"], ["next"]] } + it("leaves refusals of the payload in a row through a post that gets no response") { + testContext = TestContext(retryState: DirectEventProcessorSpec.withoutBackoff()) + testContext.serviceMock.eventStatusForPost = { post in [1: 400, 2: 0, 3: 400][post] ?? 202 } + testContext.eventProcessor.record(CustomEvent(key: "malformed", context: testContext.context)) + testContext.eventProcessor.isOnline = true + // A refusal, and its retry, which gets no response. + expect(testContext.flushOutcome()) == false + expect(testContext.store.pendingBatches()).toNot(beEmpty()) + + // The second refusal drops the batch, which no longer holds back the one behind it. + testContext.eventProcessor.record(CustomEvent(key: "next", context: testContext.context)) + expect(testContext.flushOutcome()) == false + expect(testContext.serviceMock.publishedEventPayloads.map(customKeys)) == [["malformed"], ["malformed"], ["malformed"], ["next"]] + expect(testContext.flushOutcome()) == true + } + // The SDK's own backoff: a refusal such as a 401 would wait eight and a half minutes before the second post. it("retries a batch refused with HTTP 413 a second later, as it does a 400") { testContext = TestContext() From 0d4e4a45a6b3f397f6063bd341325b7174abc213 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 29 Oct 2026 20:05:58 -0700 Subject: [PATCH 124/158] fix: bound full events by bytes where events are kept off the disk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit At .disabled, full events were bounded by eventCapacity alone, which this branch raises from 100 to 2000, so held in memory through an outage they could take 2000 times their size. The byte capacity now bounds them wherever they are kept, with the same fifth held back for summaries. The admission check every recorded event makes takes bufferLock once for count and bytes together, through admitsEvent, rather than once more for the bytes: on an Apple Watch, recording a tracked evaluation at .disabled went from 8.86 to 8.94 µs, within run-to-run noise. Co-authored-by: Cursor --- .../LaunchDarkly/Models/LDConfig.swift | 16 +++++----- .../ObjectiveC/ObjcLDConfig.swift | 2 +- .../ServiceObjects/EventCommitter.swift | 2 +- .../ServiceObjects/EventStore.swift | 25 +++++++++++---- .../DirectEventProcessorSpec.swift | 2 ++ .../ServiceObjects/EventStoreSpec.swift | 32 ++++++++++++------- 6 files changed, 51 insertions(+), 28 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift index 54051005a..6b3dc94db 100644 --- a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift @@ -383,15 +383,15 @@ public struct LDConfig { /// any backwards compatibility guarantees or semantic versioning. (Default: `.disabled`) public var eventPersistence: EventPersistence = Defaults.eventPersistence - /// The maximum number of bytes of events the SDK keeps where `eventPersistence` keeps events across launches, on disk and in - /// memory together. It is for every environment the application configures together: with `secondaryMobileKeys`, each environment - /// gets an equal share of it, unlike `eventCapacity`, which each environment has in full. When an environment reaches its share, - /// the SDK discards new events until it successfully reports the stored ones, as it does when `eventCapacity` is reached; it never - /// deletes stored events to make room. A fifth of each share is kept for the summaries of flag evaluations, so that evaluations - /// are still counted after full events start being discarded. A value of zero or less means the default. (Default: 20 MiB) + /// The maximum number of bytes of events the SDK keeps, on disk and in memory together, whatever `eventPersistence` is. It is for + /// every environment the application configures together: with `secondaryMobileKeys`, each environment gets an equal share of it, + /// unlike `eventCapacity`, which each environment has in full. When an environment reaches its share, the SDK discards new events + /// until it successfully reports the stored ones, as it does when `eventCapacity` is reached; it never deletes stored events to make + /// room. A fifth of each share is kept for the summaries of flag evaluations, so that evaluations are still counted after full + /// events start being discarded. A value of zero or less means the default. (Default: 20 MiB) /// - /// Where `eventPersistence` is `.disabled`, full events are bounded by `eventCapacity` alone, and this bounds only the summaries, - /// which `eventCapacity` never refuses: a delivery that keeps failing would otherwise leave them growing for the life of the process. + /// Ordinary events reach `eventCapacity` long before this. It binds where events are unusually large, or where delivery keeps + /// failing and summaries, which `eventCapacity` never refuses, accumulate. public var eventCapacityBytes: Int = Defaults.eventCapacityBytes /// How long the SDK keeps events it has persisted but not yet reported, where `eventPersistence` keeps events across launches. Events diff --git a/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift b/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift index c0593e584..da985578f 100644 --- a/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift @@ -53,7 +53,7 @@ public final class ObjcLDConfig: NSObject { set { config.eventPersistence = newValue } } - /// The maximum number of bytes of events the SDK keeps where `eventPersistence` keeps events across launches, for every environment together: each environment in `secondaryMobileKeys` gets an equal share of it, unlike `eventCapacity`. When an environment reaches its share, the SDK discards new events until it reports the stored ones; a fifth of each share is kept for the summaries of flag evaluations. No effect where `eventPersistence` is `.disabled`. (Default: 20 MiB) + /// The maximum number of bytes of events the SDK keeps, on disk and in memory together, whatever `eventPersistence` is, for every environment together: each environment in `secondaryMobileKeys` gets an equal share of it, unlike `eventCapacity`. When an environment reaches its share, the SDK discards new events until it reports the stored ones; a fifth of each share is kept for the summaries of flag evaluations. (Default: 20 MiB) @objc public var eventCapacityBytes: Int { get { config.eventCapacityBytes } set { config.eventCapacityBytes = newValue } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift index fe4154255..b1d029f59 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift @@ -219,7 +219,7 @@ final class EventCommitter { /// it is encoded; whether it fits is decided when it is staged. private func admitToPending(_ event: Event) -> Bool { recordLock.assertOwned() - guard pending.count + store.pendingEventCount < eventCapacity, store.hasRoomForEventBytes + guard store.admitsEvent(heldElsewhere: pending.count, capacity: eventCapacity) else { return false } pending.append(event) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index c8a173ead..a9bff26c1 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -46,6 +46,15 @@ protocol EventStoring { /// there is room can still be refused at `stageReserved(_:)` if it turns out not to fit. var hasRoomForEventBytes: Bool { get } + /// `pendingEventCount` and `hasRoomForEventBytes` in one acquisition of the lock, for the check every recorded event + /// makes, where the lock is contended by the commit staging events. + /// + /// - Parameters: + /// - heldElsewhere: events the caller holds that the store does not count yet. + /// - capacity: the most events to hold, counting `heldElsewhere`. + /// - Returns: whether one more full event can be accepted, by count and by bytes. + func admitsEvent(heldElsewhere: Int, capacity: Int) -> Bool + /// Whether a commit writes anything, which is false where persistence is off and once a write has failed and the /// store has given up on it. var isPersisting: Bool { get } @@ -350,8 +359,11 @@ final class EventStore: EventStoring { } var hasRoomForEventBytes: Bool { - // A store kept off the disk holds events the way the SDK did before persistence, bounded by count alone. - disk == nil || bufferLock.withLock { countedBytes < eventByteCapacity } + bufferLock.withLock { countedBytes < eventByteCapacity } + } + + func admitsEvent(heldElsewhere: Int, capacity: Int) -> Bool { + bufferLock.withLock { heldElsewhere + countedEvents < capacity && countedBytes < eventByteCapacity } } var isPersisting: Bool { @@ -388,17 +400,16 @@ final class EventStore: EventStoring { let frame = EventLogFormat.frame(for: encodedEvent) // Only a summary bypasses capacity without a reservation, and only a summary may use the bytes held back. let isSummary = bypassingCapacity && !usingReservation - // Kept off the disk, full events are bounded by count alone, as they were before persistence. A summary bypasses - // the count wherever it is kept, so there its bytes are all that bounds it: a delivery that keeps failing would - // otherwise go on holding one more for the life of the process. - let boundedByBytes = disk != nil || isSummary let staged = bufferLock.withLock { () -> Bool in // In the same critical section as the append, so the event is never counted twice or not at all. if usingReservation { reservedEvents = max(0, reservedEvents - 1) } + // Bytes bound events kept off the disk too: capacity counts events, so held in memory through an outage it + // would allow it that many times whatever size they are, and a summary, which capacity never refuses, + // would otherwise go on growing for the life of the process. guard bypassingCapacity || countedEvents < capacity, - !boundedByBytes || countedBytes + frame.count <= (isSummary ? byteCapacity : eventByteCapacity) + countedBytes + frame.count <= (isSummary ? byteCapacity : eventByteCapacity) else { return false } if isSummary { diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index add092897..dcf3b7fcf 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -3505,6 +3505,7 @@ private final class ClosingHookStore: EventStoring { var pendingEventCount: Int { inner.pendingEventCount } var hasRoomForEventBytes: Bool { inner.hasRoomForEventBytes } + func admitsEvent(heldElsewhere: Int, capacity: Int) -> Bool { inner.admitsEvent(heldElsewhere: heldElsewhere, capacity: capacity) } var isPersisting: Bool { inner.isPersisting } func stage(_ encodedEvent: Data, bypassingCapacity: Bool) -> Bool { inner.stage(encodedEvent, bypassingCapacity: bypassingCapacity) } func reserve(_ events: Int) { inner.reserve(events) } @@ -3585,6 +3586,7 @@ private final class SlowListingStore: EventStoring { var pendingEventCount: Int { inner.pendingEventCount } var hasRoomForEventBytes: Bool { inner.hasRoomForEventBytes } + func admitsEvent(heldElsewhere: Int, capacity: Int) -> Bool { inner.admitsEvent(heldElsewhere: heldElsewhere, capacity: capacity) } var isPersisting: Bool { inner.isPersisting } func stage(_ encodedEvent: Data, bypassingCapacity: Bool) -> Bool { inner.stage(encodedEvent, bypassingCapacity: bypassingCapacity) } func reserve(_ events: Int) { inner.reserve(events) } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 9e968b206..9953b14d3 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -1780,15 +1780,6 @@ final class EventStoreSpec: QuickSpec { expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "past-capacity"))) == false } - it("is not bounded by bytes where it keeps events off the disk") { - store = EventStore.temporary(capacity: 100, byteCapacity: 1000, persistEvents: false) - for index in 1...20 { - expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-\(index)"))) == true - } - expect(store.hasRoomForEventBytes) == true - expect(store.pendingEventCount) == 20 - } - it("bounds summaries by bytes even where it keeps events off the disk") { // Nothing else bounds them: capacity never refuses a summary, and a delivery that keeps failing keeps // every one it was given. @@ -1801,8 +1792,27 @@ final class EventStoreSpec: QuickSpec { expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "summary-2"), bypassingCapacity: true)) == true expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "summary-3"), bypassingCapacity: true)) == false - // Full events are still bounded by count alone. - expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-9"))) == true + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-9"))) == false + } + + it("bounds full events by bytes where it keeps events off the disk, leaving the summaries' fifth") { + // Capacity counts events, so in memory through an outage it would otherwise hold that many of any size. + store = EventStore.temporary(capacity: 100, byteCapacity: 1000, persistEvents: false) + for index in 1...8 { + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-\(index)"))) == true + } + expect(store.hasRoomForEventBytes) == false + expect(store.admitsEvent(heldElsewhere: 0, capacity: 100)) == false + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-9"))) == false + expect(store.pendingEventCount) == 8 + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "summary"), bypassingCapacity: true)) == true + } + + it("admits an event by count and by bytes in one check") { + store = EventStore.temporary(capacity: 100, byteCapacity: 1000, persistEvents: false) + _ = store.stage(EventStoreSpec.payload(framedIn: 100, key: "event")) + expect(store.admitsEvent(heldElsewhere: 98, capacity: 100)) == true + expect(store.admitsEvent(heldElsewhere: 99, capacity: 100)) == false } it("counts committed bytes as well as staged ones") { From fdaf88203f8a8a0d9136802ad793b9a50326f3cf Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 29 Oct 2026 20:56:58 -0700 Subject: [PATCH 125/158] fix: count an uncounted batch this version cannot read as lost rather than deleting it unnoticed A batch named without a count, in a format this version does not read, was deleted by the listing with no loss counted (ATR3-003 item 2). It is now listed as holding one event, the fewest it can, as one too large to read already is, so the delivery that reaches it drops it and counts the loss. Co-authored-by: Cursor --- .../LaunchDarkly/ServiceObjects/EventDirectory.swift | 7 ++++--- .../ServiceObjects/EventStoreSpec.swift | 12 +++++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index ccd043d84..373e7420a 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -620,9 +620,10 @@ final class EventDirectory { guard let events = EventLogFormat.eventCount(in: log) else { // Written by a version of the SDK whose format this one does not read, or damaged beyond what the - // torn tail recovery tolerates. Either way it can never be delivered, so it is not kept. - try? FileManager.default.removeItem(at: file) - return nil + // torn tail recovery tolerates. Either way it can never be delivered, but it held events. Listed as + // holding one, the fewest it can, so that the delivery reaching it drops it and counts the loss, as + // for one too large to read. + return EventTally(events: 1, bytes: log.count) } return EventTally(events: events, bytes: log.count) } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 9953b14d3..a09b8bf32 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -741,7 +741,7 @@ final class EventStoreSpec: QuickSpec { expect(EventStoreSpec.keys(in: next.payloadsOnDisk)) == ["from-the-previous-run", "this-run"] } - it("is discarded when a batch named without a count is in a format this version does not read") { + it("is listed as one event and unreadable when a batch named without a count is in a format this version does not read") { store = EventStore.temporary() _ = store.stage(EventStoreSpec.payload("unreadable")) store.commit() @@ -753,9 +753,15 @@ final class EventStoreSpec: QuickSpec { corrupted[5] = 0xFF try corrupted.write(to: batchUrl) + // Left for the delivery that reaches it, which removes it and counts the loss, rather than deleted + // here uncounted. let next = EventStoreSpec.reader(sharing: store) - expect(next.pendingBatches()).to(beEmpty()) - // A batch that can never be delivered is not left to occupy the device either. + let batches = next.pendingBatches() + expect(batches.map { $0.eventCount }) == [1] + guard let batch = batches.first + else { return } + expect { try next.body(of: batch) }.to(throwError(errorType: UnreadableBatch.self)) + next.remove(batch) expect(FileManager.default.fileExists(atPath: batchUrl.path)) == false } } From 8d97c4168b997a770fd06bffc7dece141d89aa6d Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 29 Oct 2026 21:07:44 -0700 Subject: [PATCH 126/158] fix: count an uncounted batch damaged from its first frame as one event lost, not none Its frames count as no events, so the delivery that dropped it as unreadable added zero to the loss count and no flush heard of it (ATR3-003 item 3). The listing now counts it as holding one event, the fewest it can. Co-authored-by: Cursor --- .../ServiceObjects/EventDirectory.swift | 3 ++- .../ServiceObjects/EventStoreSpec.swift | 18 ++++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index 373e7420a..a1f7d22c8 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -625,7 +625,8 @@ final class EventDirectory { // for one too large to read. return EventTally(events: 1, bytes: log.count) } - return EventTally(events: events, bytes: log.count) + // Damaged from its first frame, it counts as no events, and would be dropped as a loss of none. + return EventTally(events: events == 0 && EventLogFormat.isDamaged(log) ? 1 : events, bytes: log.count) } } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index a09b8bf32..83bac0181 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -636,6 +636,24 @@ final class EventStoreSpec: QuickSpec { expect { try store.body(of: batch) }.to(throwError(errorType: UnreadableBatch.self)) } + it("lists a batch named without a count whose first frame is damaged as one event, and loses it") { + store = EventStore.temporary() + _ = store.stage(EventStoreSpec.payload("damaged")) + _ = store.closeBatch() + let file = try EventStoreSpec.asUncountedName(EventStoreSpec.onlyBatchFile(in: store)) + var damaged = try Data(contentsOf: file) + damaged[EventLogFormat.fileHeaderSize + EventLogFormat.frameHeaderSize] ^= 0x01 + try damaged.write(to: file) + + // Counted as no events, it would be dropped as a loss of none, which no flush hears about. + let next = EventStoreSpec.reader(sharing: store) + let batches = next.pendingBatches() + expect(batches.map { $0.eventCount }) == [1] + guard let batch = batches.first + else { return } + expect { try next.body(of: batch) }.to(throwError(errorType: UnreadableBatch.self)) + } + it("says how many events made it into the body of a batch damaged after its first frame") { store = EventStore.temporary() ["first", "second", "third"].forEach { _ = store.stage(EventStoreSpec.payload($0)) } From af90a9c566b278f16f87d94bcaf160882e6d1939 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 29 Oct 2026 21:16:54 -0700 Subject: [PATCH 127/158] fix: pass by a stored batch whose read fails, and drop it after three in a row A read that failed kept the batch and stopped the pass at it, so every batch behind it, the open log's included, waited for as long as the read kept failing: for good where expiry cannot open the file either (N1). The pass now goes on past it, reading it once a delivery, and a third failed read in a row drops it and counts its events as lost, as on Android. Co-authored-by: Cursor --- .../ServiceObjects/DirectEventProcessor.swift | 77 +++++++++++-- .../EventDeliveryScheduler.swift | 22 +++- .../DirectEventProcessorSpec.swift | 108 ++++++++++++++++++ 3 files changed, 196 insertions(+), 11 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 9e31a8909..71ea530a0 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -57,7 +57,7 @@ class DirectEventProcessor: EventProcessor { // MARK: Guarded by stateLock - /// Guards which losses have been told to a caller, and the count of refused payloads. + /// Guards which losses have been told to a caller, and the counts of refused payloads and failed reads. private let stateLock = UnfairLock() /// The mark up to which losses have already been reported to a caller, so that none is told twice. Only to be @@ -76,6 +76,18 @@ class DirectEventProcessor: EventProcessor { /// again. Only to be used while holding `stateLock`. private var badRequests: (payloadId: String, inARow: Int)? + /// How many failed reads in a row drop a batch. A read that fails is mostly one that will go on failing, a + /// permission taken away or a directory named like a batch, and expiry cannot delete a file it cannot open, so the + /// batch would be read, and fail, at every delivery for good. More than one, so a passing failure such as running out + /// of file descriptors costs nothing; a failed read sets no delivery of its own, so the reads are usually deliveries + /// apart. + static let readFailuresBeforeDropping = 3 + + /// Failed reads in a row, by payload ID, of the batches whose last read failed. Unlike refusals, these are counted + /// for several batches at once, since a batch whose read failed is passed by. Held only in memory, so a new run of + /// the application counts again. Only to be used while holding `stateLock`. + private var readFailures: [String: Int] = [:] + // MARK: Guarded by claimLock /// Guards `batchesInFlight`. @@ -592,8 +604,11 @@ extension DirectEventProcessor { /// Every batch was attempted, and at least one was lost for good: unreadable, damaged, deleted unsent, or had its /// payload refused `badRequestsBeforeDropping` times in a row. case lost - /// A batch is still there to be sent again: its request failed, the backoff did not allow it to be posted yet, - /// or it could not be read this time. The batches after it were not attempted. + /// Every batch was attempted or passed by, and at least one was passed by: its read failed, and it is kept for a + /// later delivery to read again, while the batches after it were still attempted. + case passedBy + /// A batch is still there to be sent again: its request failed, or the backoff did not allow it to be posted + /// yet. The batches after it were not attempted. case blocked static func < (lhs: PassOutcome, rhs: PassOutcome) -> Bool { @@ -601,10 +616,12 @@ extension DirectEventProcessor { } } - /// Delivers batches oldest first, stopping at the first one that failed. + /// Delivers batches oldest first, stopping at the first one whose post failed. One whose read failed is passed by, + /// and is not read again in this delivery. /// /// Stopping matters: the batches that are left keep their place in the log, and a later delivery attempts them - /// again rather than the SDK spending the rest of the session's requests on a service that is refusing them. + /// again rather than the SDK spending the rest of the session's requests on a service that is refusing them. A read + /// says nothing about the service, so it stops nothing. /// /// A failure whose wait is no longer than the initial delay is retried here, once a pass, so a caller waiting on /// the pass hears from the retry; a longer wait ends the pass at once, with a delivery set for when the wait ends. @@ -617,9 +634,14 @@ extension DirectEventProcessor { /// included. `hadTurn` is whether the first batch has been held already. private func deliver(_ batches: [EventBatch], _ run: EventDeliveryRun, lostAnything: Bool, retried: Bool, hadTurn: Bool = false, _ completion: @escaping (PassOutcome) -> Void) { var remaining = batches - guard !remaining.isEmpty + guard let next = remaining.first else { - completion(lostAnything ? .lost : .delivered) + completion(run.passedAnyBy ? .passedBy : lostAnything ? .lost : .delivered) + return + } + guard !run.passedBy(next.payloadId) + else { + deliver(Array(remaining.dropFirst()), run, lostAnything: lostAnything, retried: retried, hadTurn: hadTurn, completion) return } @@ -670,11 +692,11 @@ extension DirectEventProcessor { deliver(remaining, run, lostAnything: true, retried: retried, completion) return } catch { - // Kept: the batch is still there, and a read that failed may not fail next time. - os_log("%s could not read stored events: %s", log: service.config.logger, type: .debug, typeName(and: #function), String(describing: error)) - completion(.blocked) + let dropped = passByOrDrop(batch, readFailedWith: error, run) + deliver(remaining, run, lostAnything: lostAnything || dropped, retried: retried, completion) return } + endReadFailures(batch) guard let payload = read else { @@ -692,6 +714,26 @@ extension DirectEventProcessor { post(batch, payload, run, leftOut: leftOut, remaining: remaining, lostAnything: lostAnything, retried: retried, completion) } + /// Passes by a batch whose read failed, or drops it as lost once that is `readFailuresBeforeDropping` in a row. + /// + /// Kept, since a read that failed may not fail next time, but passed by rather than stopped at. Stopped at, it held + /// back every batch behind it, the open log's included, on every pass for as long as the read kept failing: for good + /// where expiry cannot open the file to delete it either. + /// + /// - Returns: whether it dropped the batch. + private func passByOrDrop(_ batch: EventBatch, readFailedWith error: Error, _ run: EventDeliveryRun) -> Bool { + guard countReadFailure(batch) < DirectEventProcessor.readFailuresBeforeDropping + else { + os_log("%s dropping %d stored event(s) that could not be read %d times in a row: %s", log: service.config.logger, type: .error, typeName(and: #function), batch.eventCount, DirectEventProcessor.readFailuresBeforeDropping, String(describing: error)) + store.remove(batch) + reportBatchLost(batch) + return true + } + os_log("%s could not read stored events, will try again at the next delivery: %s", log: service.config.logger, type: .info, typeName(and: #function), String(describing: error)) + run.passBy(batch.payloadId) + return false + } + /// Whether the delivery running may post: online, or `flushAtClose()`'s. Going offline stops a delivery before its /// next post, its retry's included, and leaves what it has not sent for a later one. private func mayPost(_ run: EventDeliveryRun) -> Bool { @@ -796,6 +838,21 @@ extension DirectEventProcessor { stateLock.withLock { badRequests = nil } } + /// - Returns: how many reads in a row of this batch have failed, counting this one; once that reaches + /// `readFailuresBeforeDropping` the count is ended, since the batch is dropped. + private func countReadFailure(_ batch: EventBatch) -> Int { + stateLock.withLock { + let inARow = (readFailures[batch.payloadId] ?? 0) + 1 + readFailures[batch.payloadId] = inARow >= DirectEventProcessor.readFailuresBeforeDropping ? nil : inARow + return inARow + } + } + + /// Ends a batch's run of failed reads, at a read that worked. + private func endReadFailures(_ batch: EventBatch) { + stateLock.withLock { _ = readFailures.removeValue(forKey: batch.payloadId) } + } + /// Sets a delivery for when the backoff's wait ends, unless one is set already, so a batch kept through a failure /// goes out then rather than at the next report interval after it. /// diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift index 07be87dd3..dabe01aad 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift @@ -23,7 +23,8 @@ final class EventDeliveryRun { /// The least time between one paced batch's answer and the next batch's request. private let gap: TimeInterval - /// Guards the pacing, which a delivery somebody asks for, or going offline, ends from another thread. + /// Guards the pacing, which a delivery somebody asks for, or going offline, ends from another thread, and the + /// batches passed by, which the delivery notes from whichever thread it goes on from. private let lock = UnfairLock() /// Whether it is paced: until a delivery somebody asked for is queued, after which it sends the rest back to back, /// or the reporter goes offline, after which it sends nothing more. Only to be used while holding `lock`. @@ -32,6 +33,9 @@ final class EventDeliveryRun { private var lastPostAnswered: DispatchTime? /// The batch waiting out its gap, which ending the pacing lets go at once. Only to be used while holding `lock`. private var waitingTurn: PacedTurn? + /// The batches whose read failed in this delivery, which it passes by rather than reading again. Only to be used + /// while holding `lock`. + private var passedBy: Set = [] init(isClosing: Bool, bypassesBackoff: Bool, isPaced: Bool, gap: TimeInterval) { self.isClosing = isClosing @@ -49,6 +53,22 @@ final class EventDeliveryRun { } } + /// Notes that a batch's read failed, so this delivery passes it by from here on: read again moments later, it would + /// most likely fail as it did, and count as two failures in a row. + func passBy(_ payloadId: String) { + lock.withLock { _ = passedBy.insert(payloadId) } + } + + /// Whether this delivery passed a batch by after its read failed. + func passedBy(_ payloadId: String) -> Bool { + lock.withLock { passedBy.contains(payloadId) } + } + + /// Whether this delivery passed any batch by after its read failed, and so left one unsent. + var passedAnyBy: Bool { + lock.withLock { !passedBy.isEmpty } + } + /// Ends the pacing, so that the rest go back to back, or, offline, stop. func endPacing() { let turn = lock.withLock { () -> PacedTurn? in diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index dcf3b7fcf..25eb9be70 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -2512,6 +2512,83 @@ extension DirectEventProcessorSpec { expect(store.pendingEventCount) == 0 } + it("passes by a stored batch it could not read, reading it once a flush, and still delivers the batches behind it") { + let reads = FailingReads() + let store = EventStore.temporary(readFile: { try reads.read($0) }) + testContext = TestContext(store: store) + // Online, and past the check for what a previous run left, before there is a batch for that check to find. + testContext.eventProcessor.isOnline = true + waitUntil(timeout: Constants.deliveryTimeout) { done in + testContext.eventProcessor.flushReportingOutcome { _ in done() } + } + testContext.recordEvents(1) + testContext.eventProcessor.commitRecordedEvents() + let unreadable = store.closeBatch()! + reads.fail(unreadable.payloadId) + testContext.recordEvents(1) + + var delivered: Bool? + waitUntil(timeout: Constants.deliveryTimeout) { done in + testContext.eventProcessor.flushReportingOutcome { result in + delivered = result + done() + } + } + expect(delivered) == false + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + expect(testContext.serviceMock.publishedPayloadIds).toNot(contain(unreadable.payloadId)) + expect(reads.failures) == 1 + expect(store.pendingBatches().map { $0.payloadId }) == [unreadable.payloadId] + + reads.fail(nil) + waitUntil(timeout: Constants.deliveryTimeout) { done in + testContext.eventProcessor.flushReportingOutcome { result in + delivered = result + done() + } + } + expect(delivered) == true + expect(testContext.serviceMock.publishedPayloadIds.last) == unreadable.payloadId + } + + it("drops a stored batch it could not read at three flushes in a row, and reports the loss once") { + let reads = FailingReads() + let store = EventStore.temporary(readFile: { try reads.read($0) }) + testContext = TestContext(store: store) + testContext.eventProcessor.isOnline = true + waitUntil(timeout: Constants.deliveryTimeout) { done in + testContext.eventProcessor.flushReportingOutcome { _ in done() } + } + testContext.recordEvents(1) + testContext.eventProcessor.commitRecordedEvents() + reads.fail(store.closeBatch()!.payloadId) + + var answers: [Bool] = [] + for _ in 0.. Data { + let fails = lock.withLock { () -> Bool in + guard let payloadId, url.lastPathComponent.contains(payloadId) + else { return false } + failed += 1 + return true + } + if fails { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(EACCES)) + } + return try Data(contentsOf: url) + } +} + /// What a run of flushes were told, written from whichever threads answered them and read from the test's. private final class Answers { private let lock = UnfairLock() From f0596534cb99eaf8ed97b823ee92a9f18dfd3316 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 29 Oct 2026 22:18:09 -0700 Subject: [PATCH 128/158] fix: give each event directory its own BatchLocks lock Expiry and sending in one environment no longer wait on those of another. The locks are process-wide by directory, so stores sharing a directory still share one. Co-authored-by: Cursor --- .../ServiceObjects/EventDirectory.swift | 74 ++++++++++++++----- .../ServiceObjects/EventStoreSpec.swift | 69 +++++++++++++++++ 2 files changed, 123 insertions(+), 20 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index a1f7d22c8..8090602ac 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -900,24 +900,48 @@ enum BatchLocks { var senders = 0 } - /// The batches this process has a request out for, by `key(_:)`. Guarded by `lock`, which covers the probes and - /// deletions below as well, so that a store in this process cannot start sending a batch another is expiring. - private static var heldInThisProcess: [String: Held] = [:] - private static let lock = UnfairLock() + /// The batches of one directory this process has a request out for, by name. Guarded by `lock`, which covers the + /// probes and deletions below as well, so that a store in this process cannot start sending a batch another is + /// expiring. One per directory rather than one for all, since a lock taken or dropped on one file leaves every other + /// file's alone, so stores of different environments have nothing to wait on each other for. + private final class Directory { + var heldInThisProcess: [String: Held] = [:] + let lock = UnfairLock() + } + + /// Every directory this process has sent from or expired in, by `key(_:)`. Process-wide, because two stores in one + /// process can share a directory and must share its lock. Never shrinks: a directory dropped while a call still held + /// it would let the next call take a second lock for the same files. There is one entry per environment directory + /// the process has touched, which stays a handful. + private static var directories: [String: Directory] = [:] + private static let directoriesLock = UnfairLock() - /// One name for a batch however its URL was come by, from a listing or put together from the directory: only the - /// directory is resolved, since the batch itself may be gone by the time its send stops. + /// One name for a batch's directory however its URL was come by, from a listing or put together from the directory: + /// only the directory is resolved, since the batch itself may be gone by the time its send stops. private static func key(_ batch: URL) -> String { - batch.deletingLastPathComponent().resolvingSymlinksInPath().appendingPathComponent(batch.lastPathComponent).path + batch.deletingLastPathComponent().resolvingSymlinksInPath().path + } + + private static func directory(of batch: URL) -> Directory { + let path = key(batch) + return directoriesLock.withLock { + if let known = directories[path] { + return known + } + let created = Directory() + directories[path] = created + return created + } } /// Locks a batch for as long as a request for it is out. Each call is ended by one `stoppedSending(_:)`; a batch /// two deliveries send at once stays locked until both have stopped. static func startedSending(_ batch: URL) { - let path = key(batch) - lock.withLock { - let held = heldInThisProcess[path] ?? Held() - heldInThisProcess[path] = held + let directory = directory(of: batch) + let name = batch.lastPathComponent + directory.lock.withLock { + let held = directory.heldInThisProcess[name] ?? Held() + directory.heldInThisProcess[name] = held held.senders += 1 #if os(macOS) || targetEnvironment(macCatalyst) // Only fails to open where the batch is gone already, leaving nothing for expiry to take. Still remembered @@ -936,14 +960,15 @@ enum BatchLocks { /// Ends one `startedSending(_:)`, releasing the lock once the last sender has stopped. static func stoppedSending(_ batch: URL) { - let path = key(batch) - lock.withLock { - guard let held = heldInThisProcess[path] + let directory = directory(of: batch) + let name = batch.lastPathComponent + directory.lock.withLock { + guard let held = directory.heldInThisProcess[name] else { return } held.senders -= 1 guard held.senders <= 0 else { return } - heldInThisProcess[path] = nil + directory.heldInThisProcess[name] = nil // Releases the lock with it. held.file?.close() } @@ -951,8 +976,9 @@ enum BatchLocks { /// Whether a store in this process is sending a batch, answered without opening the file. static func isSentInThisProcess(_ batch: URL) -> Bool { - let path = key(batch) - return lock.withLock { heldInThisProcess[path] != nil } + let directory = directory(of: batch) + let name = batch.lastPathComponent + return directory.lock.withLock { directory.heldInThisProcess[name] != nil } } /// Deletes a batch for expiry, unless some live process is sending it. @@ -960,9 +986,10 @@ enum BatchLocks { /// - Returns: whether this call deleted it; false for one being sent, one that could not be looked at, and one /// already gone, whose events are someone else's to count. static func deleteUnlessBeingSent(_ batch: URL) -> Bool { - let path = key(batch) - return lock.withLock { - guard heldInThisProcess[path] == nil + let directory = directory(of: batch) + let name = batch.lastPathComponent + return directory.lock.withLock { + guard directory.heldInThisProcess[name] == nil else { return false } #if os(macOS) || targetEnvironment(macCatalyst) // Opened without being created: a batch delivered since it was listed has to stay gone, since a delivery @@ -977,6 +1004,13 @@ enum BatchLocks { return (try? FileManager.default.removeItem(at: batch)) != nil } } + + #if DEBUG + /// The lock every call on a batch takes: its directory's, whichever store names the batch. + static func lockForTesting(_ batch: URL) -> UnfairLock { + directory(of: batch).lock + } + #endif } extension EventDirectory { diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 83bac0181..ecaf0ecfe 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -2089,6 +2089,75 @@ final class EventStoreSpec: QuickSpec { /// Losing the disk should cost durability and nothing else. These are the events an application would otherwise /// have reported before any of this existed, so the store keeps delivering them from memory. private func fallbackSpec() { + describe("batch locks") { + var root: URL! + beforeEach { + root = EventStore.temporaryDirectory() + } + afterEach { + try? FileManager.default.removeItem(at: root) + } + func batch(in directory: String, named name: String) throws -> URL { + let directory = root.appendingPathComponent(directory, isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + let batch = directory.appendingPathComponent(name) + try Data([1, 2, 3]).write(to: batch) + return batch + } + + it("gives the batches of one directory one lock, however their URLs were come by, and another directory its own") { + let first = try batch(in: "one", named: "ready-a-1") + let second = URL(fileURLWithPath: first.deletingLastPathComponent().path + "/ready-b-1") + let elsewhere = try batch(in: "other", named: "ready-a-1") + + expect(BatchLocks.lockForTesting(first) === BatchLocks.lockForTesting(second)) == true + expect(BatchLocks.lockForTesting(first) === BatchLocks.lockForTesting(elsewhere)) == false + } + + it("expires in one directory while another directory's lock is held, and waits on its own") { + let held = try batch(in: "held", named: "ready-held-1") + let besideIt = try batch(in: "held", named: "ready-beside-1") + let elsewhere = try batch(in: "elsewhere", named: "ready-elsewhere-1") + let elsewhereDone = DispatchSemaphore(value: 0) + let besideItDone = DispatchSemaphore(value: 0) + + let lock = BatchLocks.lockForTesting(held) + lock.lock() + DispatchQueue.global().async { + _ = BatchLocks.deleteUnlessBeingSent(elsewhere) + elsewhereDone.signal() + } + let elsewhereWaited = elsewhereDone.wait(timeout: .now() + 5) == .timedOut + DispatchQueue.global().async { + _ = BatchLocks.deleteUnlessBeingSent(besideIt) + besideItDone.signal() + } + let besideItWaited = besideItDone.wait(timeout: .now() + 0.2) == .timedOut + lock.unlock() + + expect(elsewhereWaited) == false + expect(besideItWaited) == true + expect(besideItDone.wait(timeout: .now() + 5)) == .success + expect(FileManager.default.fileExists(atPath: elsewhere.path)) == false + expect(FileManager.default.fileExists(atPath: besideIt.path)) == false + } + + it("leaves a batch of the same name in another directory to expiry while one is sent") { + let sent = try batch(in: "sending", named: "ready-same-1") + let namesake = try batch(in: "expiring", named: "ready-same-1") + + BatchLocks.startedSending(sent) + expect(BatchLocks.isSentInThisProcess(sent)) == true + expect(BatchLocks.isSentInThisProcess(namesake)) == false + expect(BatchLocks.deleteUnlessBeingSent(sent)) == false + expect(BatchLocks.deleteUnlessBeingSent(namesake)) == true + BatchLocks.stoppedSending(sent) + + expect(BatchLocks.isSentInThisProcess(sent)) == false + expect(BatchLocks.deleteUnlessBeingSent(sent)) == true + } + } + describe("a store the filesystem will not take") { var store: EventStore! beforeEach { From 6877ac1911c13ade057cada88440af523c3aaf26 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 29 Oct 2026 23:53:23 -0700 Subject: [PATCH 129/158] fix: split a summary too large for a log frame rather than dropping it A context's summary over a frame is halved by flag, again as needed, into summaries of the same context and period. LaunchDarkly sums them, so nothing is lost; only one flag's counters too large for a frame are left whole for the store to refuse. Co-authored-by: Cursor --- .../FeatureFlag/FlagRequestTracker.swift | 27 +++++++++++++- .../ServiceObjects/EventCommitter.swift | 4 +- .../ServiceObjects/OutboundEventBuffer.swift | 18 ++++++++- .../FlagRequestTrackerSpec.swift | 26 +++++++++++++ .../DirectEventProcessorSpec.swift | 37 +++++++++++++++++++ 5 files changed, 106 insertions(+), 6 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/Models/FeatureFlag/FlagRequestTracker.swift b/LaunchDarkly/LaunchDarkly/Models/FeatureFlag/FlagRequestTracker.swift index 9480aa32c..08b041910 100644 --- a/LaunchDarkly/LaunchDarkly/Models/FeatureFlag/FlagRequestTracker.swift +++ b/LaunchDarkly/LaunchDarkly/Models/FeatureFlag/FlagRequestTracker.swift @@ -2,12 +2,35 @@ import Foundation import OSLog struct FlagRequestTracker { - let startDate = Date() + let startDate: Date var flagCounters: [LDFlagKey: FlagCounter] = [:] let logger: OSLog - init(logger: OSLog) { + init(logger: OSLog, startDate: Date = Date()) { self.logger = logger + self.startDate = startDate + } + + /// The flags split between two trackers over the same period. LaunchDarkly sums the counters of every summary it + /// receives, so the two count exactly what the one did. Nothing may count into this tracker any more: the halves + /// share its counters. + /// + /// - Returns: the two halves, or nil for a tracker of one flag, which cannot be split. + func halved() -> (FlagRequestTracker, FlagRequestTracker)? { + guard flagCounters.count > 1 + else { return nil } + let flagKeys = flagCounters.keys.sorted() + let middle = flagKeys.count / 2 + var first = FlagRequestTracker(logger: logger, startDate: startDate) + var second = FlagRequestTracker(logger: logger, startDate: startDate) + for (index, flagKey) in flagKeys.enumerated() { + if index < middle { + first.flagCounters[flagKey] = flagCounters[flagKey] + } else { + second.flagCounters[flagKey] = flagCounters[flagKey] + } + } + return (first, second) } mutating func trackRequest(flagKey: LDFlagKey, reportedValue: LDValue, featureFlag: FeatureFlag?, defaultValue: LDValue, context: LDContext) { diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift index b1d029f59..0f22538c1 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift @@ -417,8 +417,8 @@ final class EventCommitter { /// LaunchDarkly sums the counters of every summary it receives, so a session that produced several summaries is /// counted the same as one that produced a single summary covering the same evaluations. private func stageSummaries(_ summaries: [(tracker: FlagRequestTracker, context: LDContext)]) { - for summary in summaries { - guard let encoded = eventBuffer.encodeSummary(summary) + for encoded in summaries.flatMap({ eventBuffer.encodeSummary($0) }) { + guard let encoded = encoded else { reportLost() continue diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift index aa007270d..0e07979eb 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift @@ -74,8 +74,22 @@ final class OutboundEventBuffer { } /// Encodes the summary of one context's counters. Requires the processor's `commitLock`. - func encodeSummary(_ summary: (tracker: FlagRequestTracker, context: LDContext)) -> Data? { - encode(SummaryEvent(flagRequestTracker: summary.tracker, context: summary.context)) + /// + /// A summary larger than one log frame holds is split between summaries of the same context and period, each with + /// some of its flags, until each fits. Whole, it would be refused by the store and every evaluation in it lost. Only + /// a single flag's counters too large for a frame are left whole, for the store to refuse. + /// + /// - Returns: at least one summary, nil for each part that could not be encoded. + func encodeSummary(_ summary: (tracker: FlagRequestTracker, context: LDContext)) -> [Data?] { + encodeSummary(summary.tracker, context: summary.context, endDate: Date()) + } + + private func encodeSummary(_ tracker: FlagRequestTracker, context: LDContext, endDate: Date) -> [Data?] { + let encoded = encode(SummaryEvent(flagRequestTracker: tracker, context: context, endDate: endDate)) + guard let size = encoded?.count, size > EventLogFormat.maxFrameSize, let halves = tracker.halved() + else { return [encoded] } + return encodeSummary(halves.0, context: context, endDate: endDate) + + encodeSummary(halves.1, context: context, endDate: endDate) } private static func makeEncoder(config: LDConfig) -> JSONEncoder { diff --git a/LaunchDarkly/LaunchDarklyTests/Models/FeatureFlag/FlagRequestTracking/FlagRequestTrackerSpec.swift b/LaunchDarkly/LaunchDarklyTests/Models/FeatureFlag/FlagRequestTracking/FlagRequestTrackerSpec.swift index 9a67bb87c..7c2edf951 100644 --- a/LaunchDarkly/LaunchDarklyTests/Models/FeatureFlag/FlagRequestTracking/FlagRequestTrackerSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/Models/FeatureFlag/FlagRequestTracking/FlagRequestTrackerSpec.swift @@ -63,6 +63,32 @@ final class FlagRequestTrackerSpec: XCTestCase { XCTAssertEqual(flagRequestTracker.flagCounters["alt-flag"], counter2) } + func testHalvedLeavesATrackerOfOneFlagWhole() { + var tracker = FlagRequestTracker(logger: .disabled) + tracker.trackRequest(flagKey: "only", reportedValue: true, featureFlag: FeatureFlag(flagKey: "only"), defaultValue: false, context: LDContext.stub()) + tracker.trackRequest(flagKey: "only", reportedValue: true, featureFlag: FeatureFlag(flagKey: "only"), defaultValue: false, context: LDContext.stub()) + XCTAssertNil(tracker.halved()) + } + + func testHalvedSharesTheFlagsBetweenTwoTrackersOverTheSamePeriod() { + var tracker = FlagRequestTracker(logger: .disabled) + for index in 0..<5 { + tracker.trackRequest(flagKey: "flag-\(index)", reportedValue: true, featureFlag: FeatureFlag(flagKey: "flag-\(index)"), defaultValue: false, context: LDContext.stub()) + } + guard let halves = tracker.halved() + else { return XCTFail("a tracker of five flags was not split") } + let (first, second) = halves + XCTAssertEqual(first.flagCounters.count, 2) + XCTAssertEqual(second.flagCounters.count, 3) + XCTAssertEqual(first.startDate, tracker.startDate) + XCTAssertEqual(second.startDate, tracker.startDate) + let rejoined = first.flagCounters.merging(second.flagCounters) { kept, _ in kept } + XCTAssertEqual(Set(rejoined.keys), Set(tracker.flagCounters.keys)) + for (flagKey, counter) in rejoined { + XCTAssertTrue(counter === tracker.flagCounters[flagKey]) + } + } + func testHasLoggedRequests() { var flagRequestTracker = FlagRequestTracker(logger: OSLog(subsystem: "com.launchdarkly", category: "tests")) flagRequestTracker.trackRequest(flagKey: "test-key", reportedValue: nil, featureFlag: FeatureFlag(flagKey: "test-key"), defaultValue: nil, context: LDContext.stub()) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 25eb9be70..8363b69cc 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -2857,6 +2857,43 @@ extension DirectEventProcessorSpec { expect(bodies.flatMap(customKeys).sorted()) == (1...eventCount).map { "event-\($0)" }.sorted() } + it("delivers a summary larger than a frame in pieces that count every flag once") { + // Long keys, so that a few thousand flags take one context's summary past what a frame holds. + let keyPrefix = String(repeating: "k", count: 1000) + let flagCount = EventLogFormat.maxFrameSize / 1000 * 3 / 2 + testContext = TestContext() + let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: false) + let context = LDContext.stub() + for index in 0.. [LDValue] in + guard case .array(let events)? = try? JSONDecoder().decode(LDValue.self, from: body) + else { return [] } + return events.filter { $0.kindField == "summary" } + } + let flagKeys = summaries.flatMap { summary -> [String] in + guard case .object(let fields) = summary, case .object(let features)? = fields["features"] + else { return [] } + return Array(features.keys) + } + expect(summaries.count) > 1 + expect(flagKeys.count) == flagCount + expect(Set(flagKeys).count) == flagCount + } + it("reports false when a retryable failure leaves the batch on disk") { testContext = TestContext(stubResponseSuccess: false) testContext.recordEvents(1) From 2a1d1a88b8dc6fa7301cf51155756ba4b2f64115 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Fri, 30 Oct 2026 11:47:58 -0700 Subject: [PATCH 130/158] Stop .deferred track and identify from scheduling a commit each (ATR3-010) At .deferred a custom or identify event now writes nothing itself, as durability 13.3 says. Its events are committed with the rest of what is held at the pending threshold, a flush, a delivery or entering the background. A commit per call cost an application that tracks every few seconds one encode and write per event, each contending with flag evaluations for the record lock and with delivery for the store's locks. .immediate still commits on the caller's thread. Co-authored-by: Cursor --- .../LaunchDarkly/Models/LDConfig.swift | 7 ++- .../ServiceObjects/EventCommitter.swift | 51 ++++++++----------- .../DirectEventProcessorSpec.swift | 31 ++++++----- 3 files changed, 44 insertions(+), 45 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift index 6b3dc94db..e5665d4de 100644 --- a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift @@ -36,10 +36,9 @@ public enum LDStreamingMode { case disabled /// Events are written to disk, off the thread that recorded them. /// - /// Recording stays as cheap as it is without persistence. An event from `track` or `identify` becomes durable a - /// moment after it was recorded rather than immediately, and a process that dies inside that window still loses - /// it. Evaluating a flag does not schedule a write of its own: its events wait in memory until enough of them have - /// built up, or until the next `track`, `identify`, flush or delivery writes them. + /// Recording stays as cheap as it is without persistence. Neither `track`, `identify` nor evaluating a flag + /// schedules a write of its own: events wait in memory until enough of them have built up, or until the next + /// flush, delivery or move to the background writes them. A process that dies before then loses them. /// /// A flush and entering the background are the exceptions. Both write on the calling thread, as `.immediate` does: /// a flush so that an application ending the process on purpose can rely on it, and entering the background because diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift index 0f22538c1..b9b35e323 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift @@ -29,16 +29,16 @@ final class LostEventCount { /// on the thread that recorded it; a commit turns the whole held run into bytes at once. So an evaluation of an /// untracked flag costs a counter update, and an evaluation of a tracked one costs that plus an array append. /// -/// Where the line falls is between the events an application asked for by name and the ones it did not. Recording a -/// custom or identify event is a *commit point*, because an application reporting something it chose to report is -/// saying this matters more than the microseconds it costs, and the crash it describes may be moments away. A commit -/// takes the whole held run with it, so the exposures leading up to the error go down alongside it. Evaluations get no -/// such promise, and are committed once `pendingCommitThreshold` of them have accumulated, when a delivery starts, or -/// on `flush`. +/// Where the line falls is between the events an application asked for by name and the ones it did not. At `.immediate` +/// recording a custom or identify event is a *commit point*, because an application reporting something it chose to +/// report is saying this matters more than the microseconds it costs, and the crash it describes may be moments away. A +/// commit takes the whole held run with it, so the exposures leading up to the error go down alongside it. Everything +/// else is committed once `pendingCommitThreshold` events have accumulated, when a delivery starts, on entering the +/// background, or on `flush`. /// -/// Whether a commit point runs on the caller's thread is the application's choice, through -/// `LDConfig.eventPersistence`. Only at `.immediate` can `track` promise the event is on disk by the time it returns. -/// At `.disabled` nothing commits early: the held run waits for the delivery, which encodes it. +/// Which of the two `track` gets is the application's choice, through `LDConfig.eventPersistence`. Only at `.immediate` +/// can `track` promise the event is on disk by the time it returns; at `.deferred` it costs what an evaluation does. At +/// `.disabled` nothing commits early: the held run waits for the delivery, which encodes it. /// /// Threads. Recording runs on whichever thread evaluated or tracked. Commits run on `commitQueue`, or on the caller's /// thread at a commit point, one at a time under `commitLock`. A delivery commits too, before it closes a batch. @@ -48,7 +48,7 @@ final class EventCommitter { /// It sets two things at once. The first is how many evaluations a termination can take -- not everything a crash /// could take, since an application that flushes on its way out commits the whole run, but the window for the /// terminations that run nothing on the way out, such as the system reclaiming a backgrounded process. Recording a - /// custom or identify event closes it too, because that commits. + /// custom or identify event at `.immediate` closes it too, because that commits. /// /// The second is what a commit point usually costs, since it encodes whatever is held before returning. The commit /// queue keeps the run drained, so the common case is far below this; raising it trades that cost against the @@ -60,7 +60,7 @@ final class EventCommitter { /// /// It is the only thing that writes events no caller asked to have written. The store writes only when told to, so /// every staged byte reaches the disk through a commit that started here, at a commit point, or at a delivery. - private static let pendingCommitThreshold = 32 + static let pendingCommitThreshold = 32 // MARK: Set in init and never changed @@ -145,6 +145,12 @@ final class EventCommitter { // MARK: Recording + /// Records an event, and at `.immediate` commits before returning if it is a commit point. + /// + /// At `.deferred` a custom or identify event writes nothing itself (durability §13.3), and is committed with what + /// is held at the pending threshold, a flush, a delivery or entering the background. A commit per call would cost + /// an application that tracks every few seconds one encode and write per event, each taking `recordLock` from the + /// threads evaluating flags and the store's locks from delivery. func record(_ event: Event) { let (dropped, needsCommit) = recordLock.withLock { (!admitToPending(event), pending.count >= EventCommitter.pendingCommitThreshold) @@ -153,8 +159,8 @@ final class EventCommitter { if dropped { reportDropped() } - if EventCommitter.isCommitPoint(event.kind) { - commitAtCommitPoint() + if persistence == .immediate && EventCommitter.isCommitPoint(event.kind) { + commitRecordedEvents() } else if needsCommit { scheduleCommitWherePersisting() } @@ -191,8 +197,8 @@ final class EventCommitter { } // Deliberately no commit point. An evaluation is expected to cost what bookkeeping costs, and it is usually the // main thread doing it; where events are persisted, the run is encoded and written on the commit queue once - // enough of them have piled up, and the next event recorded at a commit point makes them durable along with - // itself. + // enough of them have piled up, and at `.immediate` the next event recorded at a commit point makes them + // durable along with itself. if needsCommit { scheduleCommitWherePersisting() } @@ -248,19 +254,6 @@ final class EventCommitter { // MARK: Committing - /// Commits at a commit point, on the caller's thread or off it as the application asked. - /// - /// Committing on the caller's thread is what lets `track` promise its event is on disk by the time it returns. - /// Scheduling it instead keeps the encode and the write off that thread, and the event is durable a moment later - /// rather than immediately. - private func commitAtCommitPoint() { - if persistence == .immediate { - commitRecordedEvents() - } else { - scheduleCommitWherePersisting() - } - } - /// Queues a commit where there is a disk for it to reach. /// /// Without persistence a commit makes nothing durable. All it would do is move the encode into the middle of the @@ -399,7 +392,7 @@ final class EventCommitter { } } - /// Whether recording this kind of event should encode and write the held run before it returns. + /// Whether recording this kind of event at `.immediate` should encode and write the held run before it returns. /// /// Custom and identify events are recorded because the application did something it chose to report, which is both /// rare enough to afford a write and the moment it is least acceptable to lose. Evaluations are neither. diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 8363b69cc..53f62a400 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -1722,14 +1722,19 @@ extension DirectEventProcessorSpec { expect(onDisk.first?.kindField) == "custom" } - itPersisting("still writes a tracked event when the write is not on the caller's thread") { - testContext = TestContext(eventPersistence: .deferred) + itPersisting("writes nothing for a tracked event at .deferred, and writes it with the rest at the pending threshold") { + let commits = DispatchQueue(label: "com.launchdarkly.tests.commits") + testContext = TestContext(eventCapacity: 100, eventPersistence: .deferred, commitQueue: commits) testContext.eventProcessor.record(CustomEvent(key: "fatal-error", context: testContext.context, data: ["message": "boom"])) + // Behind any write the track queued, so that write has run by now. + commits.sync { } + expect(eventsLeftOnDisk()).to(beEmpty()) - // The promise is weaker by exactly one scheduling hop: the event reaches the disk, just not before - // recording it returned. - expect(eventsLeftOnDisk().compactMap { $0.kindField }).toEventually(equal(["custom"]), timeout: Constants.deliveryTimeout) + for index in 1.. Date: Fri, 30 Oct 2026 12:43:34 -0700 Subject: [PATCH 131/158] Group concurrent .immediate commit points into one commit, as Android does Number every recording, and remember the last one a finished commit covered. A track, identify or flush whose recording a commit already made durable returns without committing again; the next caller still uncovered commits everything that arrived meanwhile. Previously every caller waiting on the commit lock committed in turn. Co-authored-by: Cursor --- .../ServiceObjects/EventCommitter.swift | 37 ++++++++++-- .../DirectEventProcessorSpec.swift | 56 ++++++++++++------- 2 files changed, 68 insertions(+), 25 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift index b9b35e323..2205cff85 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift @@ -92,6 +92,14 @@ final class EventCommitter { /// Taken before `recordLock` and before anything the store locks, never after. private let commitLock = UnfairLock() + /// The last recording a finished commit covered. Only to be used while holding `commitLock`. + /// + /// This is what makes concurrent `.immediate` commit points a group commit. The first caller's commit takes every + /// recording made before it took the run, including those of callers now waiting behind it. They find their + /// recordings covered and return, and the next caller still uncovered commits everything that arrived in the + /// meantime. Without it each waiter committed again in turn. + private var committedSequence: UInt64 = 0 + // MARK: Guarded by recordLock /// Guards everything one recording writes: the held events and the summarizer. @@ -111,6 +119,10 @@ final class EventCommitter { /// happen where it is cheapest. private var pending: [Event] = [] + /// Numbers every recording, so a caller at a commit point can tell whether a commit has already covered it. Only to + /// be used while holding `recordLock`. + private var recordedSequence: UInt64 = 0 + /// Whether a commit is already queued, so a run of recordings past the threshold asks for one write rather than /// one each. Only to be used while holding `recordLock`. private var isCommitScheduled = false @@ -152,15 +164,18 @@ final class EventCommitter { /// an application that tracks every few seconds one encode and write per event, each taking `recordLock` from the /// threads evaluating flags and the store's locks from delivery. func record(_ event: Event) { - let (dropped, needsCommit) = recordLock.withLock { - (!admitToPending(event), pending.count >= EventCommitter.pendingCommitThreshold) + var sequence: UInt64 = 0 + let (dropped, needsCommit) = recordLock.withLock { () -> (Bool, Bool) in + recordedSequence += 1 + sequence = recordedSequence + return (!admitToPending(event), pending.count >= EventCommitter.pendingCommitThreshold) } if dropped { reportDropped() } if persistence == .immediate && EventCommitter.isCommitPoint(event.kind) { - commitRecordedEvents() + commitThrough(sequence) } else if needsCommit { scheduleCommitWherePersisting() } @@ -181,6 +196,7 @@ final class EventCommitter { var dropped = 0 let (counted, needsCommit) = recordLock.withLock { () -> (Bool, Bool) in + recordedSequence += 1 let counted = eventBuffer.summarize(flagKey: flagKey, reportedValue: value, featureFlag: featureFlag, defaultValue: defaultValue, context: context) if let featureEvent, !admitToPending(featureEvent) { dropped += 1 } if let debugEvent, !admitToPending(debugEvent) { dropped += 1 } @@ -293,7 +309,7 @@ final class EventCommitter { /// nothing durable, and is left to the delivery. func commitWherePersisting() { if persistence == .immediate || store.isPersisting { - commitRecordedEvents() + commitThrough(recordLock.withLock { recordedSequence }) } } @@ -320,6 +336,16 @@ final class EventCommitter { commitLock.withLock { encodeAndCommitPending() } } + /// Commits unless a finished commit already covers recording `sequence`, checked once the lock is held, since the + /// commit a caller waited behind is the one most likely to have covered it. + private func commitThrough(_ sequence: UInt64) { + commitLock.withLock { + guard committedSequence < sequence + else { return } + encodeAndCommitPending() + } + } + /// Commits what is held and closes it into a batch, as one step with respect to other commits. /// /// Closing takes everything staged so far, and a commit stages its run one event at a time, so a commit running @@ -352,6 +378,7 @@ final class EventCommitter { /// Requires `commitLock`. private func encodeAndCommitPending() { commitLock.assertOwned() + var through: UInt64 = 0 let (run, summaries) = recordLock.withLock { () -> ([Event], [(tracker: FlagRequestTracker, context: LDContext)]) in let run = pending pending = [] @@ -359,6 +386,7 @@ final class EventCommitter { // Reserved in the same critical section the run leaves `pending` in, so capacity counts it in one place or // the other throughout the encode. store.reserve(run.count) + through = recordedSequence return (run, summaries) } @@ -366,6 +394,7 @@ final class EventCommitter { store.releaseReservations() stageSummaries(summaries) store.commit() + committedSequence = through } /// Encodes the held events as one run and stages the bytes. diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 53f62a400..9548086f5 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -1703,7 +1703,8 @@ extension DirectEventProcessorSpec { describe("surviving the process") { var testContext: TestContext! afterEach { - testContext.cleanUp() + testContext?.cleanUp() + testContext = nil } /// The events the next run of the application would find. @@ -2120,21 +2121,30 @@ extension DirectEventProcessorSpec { expect(eventsLeftOnDisk().compactMap { $0.kindField }) == ["feature", "summary"] } - itPersisting("shares writes between tracks recorded at once, and has each on disk when recording it returns") { + itPersisting("shares commits between tracks recorded at once, and has each on disk when recording it returns") { let trackers = 8 - let writesLock = NSLock() - var writes = 0 - // A slow disk, so that the other tracks arrive while a write is under way. - let store = EventStore.temporary(writeLog: { file, bytes in - writesLock.lock() - writes += 1 - writesLock.unlock() + var config = LDConfig.stub + config.eventCapacity = 100 + config.eventPersistence = .immediate + let serviceMock = DarklyServiceMock() + serviceMock.config = config + let inner = EventStore.temporary(capacity: config.eventCapacity) + defer { inner.deleteEverything() } + let store = ClosingHookStore(inner) + let commitsLock = NSLock() + var commits = 0 + // A slow disk, so that the other tracks arrive while a commit is under way. + store.beforeCommitting = { + commitsLock.lock() + commits += 1 + commitsLock.unlock() Thread.sleep(forTimeInterval: 0.05) - return file.writeAll(bytes) - }) - testContext = TestContext(store: store, eventCapacity: 100) - let reporter = testContext.eventProcessor! - let context = testContext.context! + } + let reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: .codable, commitQueue: DispatchQueue(label: "com.launchdarkly.tests.commits"), pacing: .noWaits) + let context = LDContext.stub() + func keysOnDisk() -> [String] { + inner.payloadsOnDisk.compactMap { try? JSONDecoder().decode(LDValue.self, from: $0).keyField } + } let start = DispatchSemaphore(value: 0) let finished = DispatchGroup() @@ -2146,7 +2156,7 @@ extension DirectEventProcessorSpec { Thread { start.wait() reporter.record(CustomEvent(key: key, context: context)) - if !eventsLeftOnDisk().compactMap({ $0.keyField }).contains(key) { + if !keysOnDisk().contains(key) { missingLock.lock() missingOnReturn.append(key) missingLock.unlock() @@ -2160,11 +2170,10 @@ extension DirectEventProcessorSpec { expect(finished.wait(timeout: .now() + 10)) == .success expect(missingOnReturn).to(beEmpty()) - // One write is the log's header. A write per track would be one more for each of them. - writesLock.lock() - let totalWrites = writes - writesLock.unlock() - expect(totalWrites) < trackers + 1 + commitsLock.lock() + let totalCommits = commits + commitsLock.unlock() + expect(totalCommits) < trackers } itPersisting("tries a batch a previous run left once at start, with its retry, and then waits out the backoff") { @@ -3618,6 +3627,8 @@ private final class ClosingHookStore: EventStoring { let inner: EventStoring var beforeClosing: (() -> Void)? var afterClosing: (() -> Void)? + /// Runs at the start of every commit. + var beforeCommitting: (() -> Void)? private(set) var closedKeys: [[String]] = [] init(_ inner: EventStoring) { @@ -3632,7 +3643,10 @@ private final class ClosingHookStore: EventStoring { func reserve(_ events: Int) { inner.reserve(events) } func stageReserved(_ encodedEvent: Data) -> Bool { inner.stageReserved(encodedEvent) } func releaseReservations() { inner.releaseReservations() } - func commit() { inner.commit() } + func commit() { + beforeCommitting?() + inner.commit() + } var holdsUnclosedEvents: Bool { inner.holdsUnclosedEvents } func takeVanishedLogEvents() -> Int { inner.takeVanishedLogEvents() } func listPendingBatches() -> BatchListing { inner.listPendingBatches() } From 9bda454b8f805c27b469d93b74506eb864ef8634 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Fri, 30 Oct 2026 13:00:34 -0700 Subject: [PATCH 132/158] Commit at close off the caller's thread within a budget, as Android does flushAtClose committed on the caller's thread, usually the main one, with no bound, so a stalled disk write held close for as long as the device took. It now queues the commit on the commit queue and waits for it at most two seconds. A commit that outlasts that is left to finish, and the final delivery waits for it on the delivery queue. Co-authored-by: Cursor --- .../ServiceObjects/DirectEventProcessor.swift | 7 ++++- .../ServiceObjects/EventCommitter.swift | 21 ++++++++++++++ .../DirectEventProcessorSpec.swift | 29 +++++++++++++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 71ea530a0..03fe5e3c8 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -30,6 +30,11 @@ class DirectEventProcessor: EventProcessor { let gap: TimeInterval } + /// How long `flushAtClose()` waits for its commit, on a thread that for an application closing the client is + /// usually the main one. The same two seconds Android allows its whole `close()`, well inside what the system + /// tolerates from a main thread that has stopped answering. + static let closeCommitBudget: TimeInterval = 2 + // MARK: Set in init and never changed let service: DarklyServiceProvider @@ -352,7 +357,7 @@ extension DirectEventProcessor { } func flushAtClose() { - commitWherePersisting() + committer.commitWithinBudget(DirectEventProcessor.closeCommitBudget) // Offline, it is an ordinary delivery, which finds the reporter offline and sends nothing: offline is the // application keeping the SDK off the network, and closing does not revoke that. queueDelivery(nil, closing: isOnline) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift index 2205cff85..fb43b90d3 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift @@ -313,6 +313,27 @@ final class EventCommitter { } } + /// Commits everything recorded so far for `flushAtClose()`, waiting for it no longer than `budget`. + /// + /// On `commitQueue` rather than the caller's thread, which for an application closing the client is usually the + /// main one. Each step of a commit can block for as long as the device takes: on `commitLock` behind a commit + /// stalled on the disk, on the store's I/O lock, and on the write itself. A commit that outlasts the budget is left + /// to finish, and the delivery queued after it waits for it on the delivery queue, not on the caller's thread. + /// Where nothing persists there is nothing to write: the final delivery encodes what it sends. + func commitWithinBudget(_ budget: TimeInterval) { + guard store.isPersisting + else { return } + let through = recordLock.withLock { recordedSequence } + let committed = DispatchSemaphore(value: 0) + commitQueue.async { + self.commitThrough(through) + committed.signal() + } + if committed.wait(timeout: .now() + budget) == .timedOut { + os_log("%s gave up waiting for the final event commit after %dms; it continues in the background", log: logger, type: .info, typeName(and: #function), Int(budget * 1000)) + } + } + /// `commitWherePersisting()` queued on `commitQueue` rather than written on the caller's thread, for a flush whose /// caller has a deadline. /// diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 9548086f5..bb301e7b7 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -2176,6 +2176,35 @@ extension DirectEventProcessorSpec { expect(totalCommits) < trackers } + itPersisting("returns from closing within its budget while the final commit is stalled on the disk, and still writes it") { + var config = LDConfig.stub + config.eventPersistence = .deferred + let serviceMock = DarklyServiceMock() + serviceMock.config = config + let inner = EventStore.temporary(capacity: config.eventCapacity) + defer { inner.deleteEverything() } + let store = ClosingHookStore(inner) + let diskAnswers = DispatchGroup() + diskAnswers.enter() + let stalled = DispatchSemaphore(value: 0) + store.beforeCommitting = { + stalled.signal() + _ = diskAnswers.wait(timeout: .now() + 10) + } + let reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: .codable, commitQueue: DispatchQueue(label: "com.launchdarkly.tests.commits"), pacing: .noWaits) + reporter.record(CustomEvent(key: "before-close", context: LDContext.stub())) + + let started = Date() + reporter.flushAtClose() + let took = Date().timeIntervalSince(started) + diskAnswers.leave() + + expect(stalled.wait(timeout: .now())) == .success + expect(took) < DirectEventProcessor.closeCommitBudget + 1 + let keysOnDisk = { inner.payloadsOnDisk.compactMap { try? JSONDecoder().decode(LDValue.self, from: $0).keyField } } + expect(keysOnDisk()).toEventually(contain("before-close"), timeout: Constants.deliveryTimeout) + } + itPersisting("tries a batch a previous run left once at start, with its retry, and then waits out the backoff") { testContext = TestContext() testContext.serviceMock.respondToEvents(with: 503) From 18e73a694a3d2f12a2eb0ff6ea93597e24e69246 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Fri, 30 Oct 2026 13:36:39 -0700 Subject: [PATCH 133/158] A failed post through the backoff leaves the wait as it was The blocking flushAndWait and close post through the backoff. A failure there used to step it: four refused calls during a 401 took the wait from about 8.5 minutes to an hour, holding flush, the periodic delivery and the retry with it. Now such a failure keeps the attempt count and the ceiling, and a Retry-After can only lengthen the wait. flushAndWait's docs say it is for the last moments only. Matches Android. Co-authored-by: Cursor --- LaunchDarkly/LaunchDarkly/LDClientFlush.swift | 7 ++-- .../ObjectiveC/ObjcLDClient.swift | 6 ++-- .../ServiceObjects/DirectEventProcessor.swift | 5 ++- .../ServiceObjects/RetryState.swift | 24 ++++++++++++++ .../DirectEventProcessorSpec.swift | 19 +++++++++++ .../ServiceObjects/RetryStateSpec.swift | 33 +++++++++++++++++++ 6 files changed, 89 insertions(+), 5 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/LDClientFlush.swift b/LaunchDarkly/LaunchDarkly/LDClientFlush.swift index 3f080b2f3..871b21162 100644 --- a/LaunchDarkly/LaunchDarkly/LDClientFlush.swift +++ b/LaunchDarkly/LaunchDarkly/LDClientFlush.swift @@ -108,8 +108,11 @@ extension LDClient { It posts even while delivery is backing off after a failure, as closing the client does, since it is for a moment after which the SDK may not get to send. After a refusal such as a 401 or 403 the SDK waits about eight and a half minutes before posting again, doubling up to an hour, and a call that sat that out would make no attempt at all. - `flush` and `flush(completion:)` still wait it out, so a call made routinely, rather than before the process is - suspended or ends, posts once per call while LaunchDarkly keeps refusing. + A post that fails then leaves the wait as it was, so `flush`, `flush(completion:)` and the periodic delivery go on + waiting out the same one rather than a longer one. That also means nothing slows these posts down: a call made + routinely, rather than just before the process is suspended or ends, posts once per call for as long as + LaunchDarkly keeps refusing. Keep it for those last moments; the SDK already flushes on every move to the + background. This method is not stable, and not subject to any backwards compatibility guarantees or semantic versioning. It is experimental. diff --git a/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDClient.swift b/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDClient.swift index 1e470051c..1f8259735 100644 --- a/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDClient.swift +++ b/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDClient.swift @@ -578,8 +578,10 @@ public final class ObjcLDClient: NSObject { It posts even while delivery is backing off after a failure, as closing the client does, since it is for a moment after which the SDK may not get to send. After a refusal such as a 401 or 403 the SDK waits about eight and a half minutes before posting again, doubling up to an hour, and a call that sat that out would make no attempt at all. - `flush` still waits it out, so a call made routinely, rather than before the process is suspended or ends, posts - once per call while LaunchDarkly keeps refusing. + A post that fails then leaves the wait as it was, so `flush` and the periodic delivery go on waiting out the same + one rather than a longer one. That also means nothing slows these posts down: a call made routinely, rather than + just before the process is suspended or ends, posts once per call for as long as LaunchDarkly keeps refusing. + Keep it for those last moments; the SDK already flushes on every move to the background. This method is not stable, and not subject to any backwards compatibility guarantees or semantic versioning. It is experimental. diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 03fe5e3c8..b4e07e5bf 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -768,7 +768,10 @@ extension DirectEventProcessor { self.reportStoredEventsLost(leftOut) self.deliver(remaining, run, lostAnything: true, retried: retried, completion) case .failed(let failure): - let wait = self.retryState.recordFailure(unexpected: failure.unexpected, retryAfter: failure.retryAfter) + // Only a run bypassing the backoff posts during its wait. + let wait = run.bypassesBackoff + ? self.retryState.recordFailureDuringWait(unexpected: failure.unexpected, retryAfter: failure.retryAfter) + : self.retryState.recordFailure(unexpected: failure.unexpected, retryAfter: failure.retryAfter) if self.countBadRequest(batch, statusCode: failure.statusCode) >= DirectEventProcessor.badRequestsBeforeDropping { // The rest are still attempted, once the backoff allows. os_log("%s dropping %d event(s) whose payload the service refused %d times in a row, last with HTTP %d", log: self.service.config.logger, type: .error, self.typeName(and: #function), batch.eventCount, DirectEventProcessor.badRequestsBeforeDropping, failure.statusCode ?? 0) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/RetryState.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/RetryState.swift index a03e98118..956a64de0 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/RetryState.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/RetryState.swift @@ -184,6 +184,30 @@ final class DeliveryRetryState { } } + /// A failure of a post that went through a wait, as `flushAtClose()` and the blocking `flushAndWait` may: an extra + /// try, not a step of the backoff. It leaves the attempt count and the ceiling as they were, so a caller flushing + /// through a refusal several times does not push flush, the periodic delivery and the retry out to the hour. A + /// `Retry-After` can still make the wait longer, never shorter. Once the wait has run out it is an ordinary failure. + /// + /// - Parameters: as `recordFailure(unexpected:retryAfter:)`. + /// - Returns: how long nothing may be posted. + @discardableResult + func recordFailureDuringWait(unexpected: Bool, retryAfter: TimeInterval?) -> TimeInterval { + let stillWaiting: TimeInterval? = lock.withLock { + let now = DispatchTime.now() + guard let waitingUntil = nextAttempt, now < waitingUntil + else { return nil } + var until = waitingUntil + if let retryAfter, retryAfter >= 0 { + let directed = min(retryAfter, Constants.maxServerDirectedWait) + until = max(until, now + (directed - random() * (directed / 2))) + } + nextAttempt = until + return TimeInterval(until.uptimeNanoseconds - now.uptimeNanoseconds) / 1_000_000_000 + } + return stillWaiting ?? recordFailure(unexpected: unexpected, retryAfter: retryAfter) + } + /// How long until a post is allowed; zero once it is. var remainingWait: TimeInterval { lock.withLock { diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index bb301e7b7..269aaabb5 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -3425,6 +3425,25 @@ extension DirectEventProcessorSpec { expect(testContext.serviceMock.publishEventDataCallCount) == 2 } + it("leaves the backoff's wait as it was when flushes bypassing it are refused") { + let retryState = DeliveryRetryState(flushInterval: 30) + testContext = TestContext(retryState: retryState) + testContext.serviceMock.respondToEvents(with: HTTPURLResponse.StatusCodes.unauthorized) + testContext.eventProcessor.record(CustomEvent(key: "refused", context: testContext.context)) + testContext.eventProcessor.isOnline = true + expect(testContext.flushOutcome()) == false + let waitAfterTheRefusal = retryState.remainingWait + expect(waitAfterTheRefusal) > 0 + + // An application flushing on every move to the background during a refusal. + for _ in 0..<4 { + expect(testContext.flushBypassingBackoffOutcome()) == false + } + + expect(testContext.serviceMock.publishEventDataCallCount) == 5 + expect(retryState.remainingWait) <= waitAfterTheRefusal + } + it("posts nothing for a flush bypassing the backoff while offline") { testContext = TestContext() testContext.eventProcessor.record(CustomEvent(key: "an-event", context: testContext.context)) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/RetryStateSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/RetryStateSpec.swift index d3ed8ed3b..288de3a1c 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/RetryStateSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/RetryStateSpec.swift @@ -161,6 +161,39 @@ final class RetryStateSpec: XCTestCase { XCTAssertEqual(retry.recordFailure(unexpected: false, retryAfter: nil), 4) } + func testDeliveryFailureDuringTheWaitLeavesTheBackoffWhereItWas() { + let retry = DeliveryRetryState(flushInterval: 30) + retry.recordFailure(unexpected: true, retryAfter: nil) + let waiting = retry.remainingWait + for _ in 0..<5 { + XCTAssertLessThanOrEqual(retry.recordFailureDuringWait(unexpected: true, retryAfter: nil), waiting) + } + + XCTAssertLessThanOrEqual(retry.remainingWait, waiting) + // The next step is the one after the first refusal, not five steps on. + XCTAssertEqual(retry.recordFailure(unexpected: true, retryAfter: nil), 1024) + } + + func testDeliveryRetryAfterDuringTheWaitCanOnlyLengthenIt() { + let retry = DeliveryRetryState(initialDelay: 1, normalCeiling: 30, extendedCeiling: 3600, random: { 0 }) + retry.recordFailure(unexpected: true, retryAfter: nil) + let waiting = retry.remainingWait + + retry.recordFailureDuringWait(unexpected: false, retryAfter: 10) + XCTAssertLessThanOrEqual(retry.remainingWait, waiting) + + XCTAssertGreaterThan(retry.recordFailureDuringWait(unexpected: false, retryAfter: 20 * 60), 19 * 60) + XCTAssertGreaterThan(retry.remainingWait, 19 * 60) + } + + func testDeliveryFailureDuringAWaitThatHasRunOutIsAnOrdinaryOne() { + let retry = DeliveryRetryState(initialDelay: 0.02, normalCeiling: 1, extendedCeiling: 1) + retry.recordFailure(unexpected: false, retryAfter: nil) + Thread.sleep(forTimeInterval: 0.04) + + XCTAssertEqual(retry.recordFailureDuringWait(unexpected: false, retryAfter: nil), 0.04, accuracy: 0.0001) + } + func testDeliveryWaitRunsOut() { let retry = DeliveryRetryState(initialDelay: 0.02, normalCeiling: 0.02, extendedCeiling: 0.02) retry.recordFailure(unexpected: false, retryAfter: nil) From 7117c2f51b5707cb24e1d836000a86b5090f0689 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Fri, 30 Oct 2026 13:56:58 -0700 Subject: [PATCH 134/158] Space the read failures that drop a batch a flush interval apart A batch was dropped at its third failed read in a row, counted per pass. Passes can come milliseconds apart -- flushes in a row, or the blocking flushAndWait posting through the backoff -- so a failure lasting a moment deleted an intact batch. Now a failed read counts only at least a flush interval after the last counted one. Matches Android. Co-authored-by: Cursor --- .../ServiceObjects/DirectEventProcessor.swift | 47 +++++++++++----- .../DirectEventProcessorSpec.swift | 54 +++++++++++++++++++ 2 files changed, 87 insertions(+), 14 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index b4e07e5bf..3eacd6ee7 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -81,17 +81,29 @@ class DirectEventProcessor: EventProcessor { /// again. Only to be used while holding `stateLock`. private var badRequests: (payloadId: String, inARow: Int)? - /// How many failed reads in a row drop a batch. A read that fails is mostly one that will go on failing, a + /// How many counted failed reads in a row drop a batch. A read that fails is mostly one that will go on failing, a /// permission taken away or a directory named like a batch, and expiry cannot delete a file it cannot open, so the - /// batch would be read, and fail, at every delivery for good. More than one, so a passing failure such as running out - /// of file descriptors costs nothing; a failed read sets no delivery of its own, so the reads are usually deliveries - /// apart. + /// batch would be read, and fail, at every delivery for good. More than one, and counted at most once per + /// `readFailureSpacing`, so a passing failure such as running out of file descriptors costs nothing. static let readFailuresBeforeDropping = 3 - /// Failed reads in a row, by payload ID, of the batches whose last read failed. Unlike refusals, these are counted - /// for several batches at once, since a batch whose read failed is passed by. Held only in memory, so a new run of - /// the application counts again. Only to be used while holding `stateLock`. - private var readFailures: [String: Int] = [:] + /// The least time between two failed reads of a batch that both count toward dropping it: the flush interval, so a + /// drop takes failures across at least two of them. Every pass that reaches the batch reads it, and passes can come + /// milliseconds apart, from flushes called in a row or the blocking `flushAndWait` posting through the backoff; + /// counted by pass, a failure lasting a moment would drop an intact batch. A failed read closer than this to the + /// last counted one passes the batch by without counting. Settable only for tests. + var readFailureSpacing: TimeInterval { + get { stateLock.withLock { readFailureSpacingValue } } + set { stateLock.withLock { readFailureSpacingValue = newValue } } + } + // Only to be used while holding `stateLock`. + private var readFailureSpacingValue: TimeInterval + + /// Counted failed reads in a row, by payload ID, of the batches whose last read failed, and when the last of them + /// was. Unlike refusals, these are counted for several batches at once, since a batch whose read failed is passed + /// by. Held only in memory, so a new run of the application counts again. Only to be used while holding + /// `stateLock`. + private var readFailures: [String: (counted: Int, lastCounted: DispatchTime)] = [:] // MARK: Guarded by claimLock @@ -153,6 +165,7 @@ class DirectEventProcessor: EventProcessor { self.onSyncComplete = onSyncComplete self.sender = AnalyticsEventSender(service: service) self.retryState = retryState ?? DeliveryRetryState(flushInterval: service.config.eventFlushInterval) + self.readFailureSpacingValue = service.config.eventFlushInterval self.lostEvents = lostEvents self.committer = EventCommitter(config: service.config, store: eventStore, encoding: encoding, commitQueue: commitQueue, diagnosticCache: service.diagnosticCache, @@ -846,13 +859,19 @@ extension DirectEventProcessor { stateLock.withLock { badRequests = nil } } - /// - Returns: how many reads in a row of this batch have failed, counting this one; once that reaches - /// `readFailuresBeforeDropping` the count is ended, since the batch is dropped. + /// - Returns: how many counted reads in a row of this batch have failed, this one included if it is at least + /// `readFailureSpacing` after the last counted one; once that reaches `readFailuresBeforeDropping` the count is + /// ended, since the batch is dropped. private func countReadFailure(_ batch: EventBatch) -> Int { - stateLock.withLock { - let inARow = (readFailures[batch.payloadId] ?? 0) + 1 - readFailures[batch.payloadId] = inARow >= DirectEventProcessor.readFailuresBeforeDropping ? nil : inARow - return inARow + let now = DispatchTime.now() + return stateLock.withLock { + if let last = readFailures[batch.payloadId], now < last.lastCounted + readFailureSpacingValue { + return last.counted + } + let counted = (readFailures[batch.payloadId]?.counted ?? 0) + 1 + readFailures[batch.payloadId] = counted >= DirectEventProcessor.readFailuresBeforeDropping + ? nil : (counted, now) + return counted } } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 269aaabb5..91b069702 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -2607,6 +2607,7 @@ extension DirectEventProcessorSpec { testContext.recordEvents(1) testContext.eventProcessor.commitRecordedEvents() reads.fail(store.closeBatch()!.payloadId) + testContext.eventProcessor.readFailureSpacing = 0 var answers: [Bool] = [] for _ in 0.. Date: Fri, 30 Oct 2026 14:37:16 -0700 Subject: [PATCH 135/158] Stop splitting a summary that is mostly context Every part of a split summary carries the whole context, so with a context near a frame's size the halving went on down to one flag per part: one loss per flag where there was one, each holding the context. A half that kept more than three quarters of what it was split from is now left whole. Matches Android. Co-authored-by: Cursor --- .../ServiceObjects/OutboundEventBuffer.swift | 18 ++++++++----- .../DirectEventProcessorSpec.swift | 26 +++++++++++++++++++ 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift index 0e07979eb..0cb64588d 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift @@ -77,19 +77,25 @@ final class OutboundEventBuffer { /// /// A summary larger than one log frame holds is split between summaries of the same context and period, each with /// some of its flags, until each fits. Whole, it would be refused by the store and every evaluation in it lost. Only - /// a single flag's counters too large for a frame are left whole, for the store to refuse. + /// a single flag's counters too large for a frame, and a part that is mostly context, which no split makes fit, are + /// left whole, for the store to refuse. /// /// - Returns: at least one summary, nil for each part that could not be encoded. func encodeSummary(_ summary: (tracker: FlagRequestTracker, context: LDContext)) -> [Data?] { - encodeSummary(summary.tracker, context: summary.context, endDate: Date()) + encodeSummary(summary.tracker, context: summary.context, endDate: Date(), splitFrom: .max) } - private func encodeSummary(_ tracker: FlagRequestTracker, context: LDContext, endDate: Date) -> [Data?] { + /// - Parameter splitFrom: the encoded size of the summary this is half of, or `.max` for one that is not. + private func encodeSummary(_ tracker: FlagRequestTracker, context: LDContext, endDate: Date, splitFrom: Int) -> [Data?] { let encoded = encode(SummaryEvent(flagRequestTracker: tracker, context: context, endDate: endDate)) - guard let size = encoded?.count, size > EventLogFormat.maxFrameSize, let halves = tracker.halved() + // Every part carries the whole context. A half that kept more than three quarters of what it was split from is + // mostly context, and halving it again would only turn its one loss into one per flag, each a frame's worth of + // context held at once. + guard let size = encoded?.count, size > EventLogFormat.maxFrameSize, size <= splitFrom / 4 * 3, + let halves = tracker.halved() else { return [encoded] } - return encodeSummary(halves.0, context: context, endDate: endDate) - + encodeSummary(halves.1, context: context, endDate: endDate) + return encodeSummary(halves.0, context: context, endDate: endDate, splitFrom: size) + + encodeSummary(halves.1, context: context, endDate: endDate, splitFrom: size) } private static func makeEncoder(config: LDConfig) -> JSONEncoder { diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 91b069702..879aad637 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -2993,6 +2993,32 @@ extension DirectEventProcessorSpec { expect(Set(flagKeys).count) == flagCount } + it("does not split a summary large for its context down to one flag per part") { + var builder = LDContextBuilder(key: "huge") + _ = builder.trySetValue("data", .string(String(repeating: "x", count: EventLogFormat.maxFrameSize))) + let context = try! builder.build().get() + let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: false) + let flagCount = 16 + var tracker = FlagRequestTracker(logger: .disabled) + for index in 0.. [String] in + guard case .object(let fields)? = try? JSONDecoder().decode(LDValue.self, from: part), + case .object(let features)? = fields["features"] + else { return [] } + return Array(features.keys) + } + expect(Set(flagKeys).count) == flagCount + expect(flagKeys.count) == flagCount + } + it("reports false when a retryable failure leaves the batch on disk") { testContext = TestContext(stubResponseSuccess: false) testContext.recordEvents(1) From 8e6abfa1d4c1b24cb64ffda7d8337e6ea0f1e90b Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Fri, 30 Oct 2026 17:06:31 -0700 Subject: [PATCH 136/158] Pack an oversized summary's flags into parts that fit beside the context The three-quarter guard on halving stopped at any half that was mostly context, even where one more split would have fit, so those evaluations were refused. Now each flag's size is measured with a small stand-in context, and the flags are packed in key order into as few parts as fit a frame beside the context. A context that alone takes a frame still leaves the summary whole, one loss; halving remains only for a part that comes out larger than measured. Matches Android. Co-authored-by: Cursor --- .../FeatureFlag/FlagRequestTracker.swift | 10 ++ .../ServiceObjects/OutboundEventBuffer.swift | 91 ++++++++++++++++--- .../DirectEventProcessorSpec.swift | 58 ++++++++++++ 3 files changed, 145 insertions(+), 14 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/Models/FeatureFlag/FlagRequestTracker.swift b/LaunchDarkly/LaunchDarkly/Models/FeatureFlag/FlagRequestTracker.swift index 08b041910..2e39a8b40 100644 --- a/LaunchDarkly/LaunchDarkly/Models/FeatureFlag/FlagRequestTracker.swift +++ b/LaunchDarkly/LaunchDarkly/Models/FeatureFlag/FlagRequestTracker.swift @@ -33,6 +33,16 @@ struct FlagRequestTracker { return (first, second) } + /// Only these flags, over the same period. Nothing may count into this tracker any more: the part shares its + /// counters. + func withFlags(_ flagKeys: Keys) -> FlagRequestTracker where Keys.Element == LDFlagKey { + var part = FlagRequestTracker(logger: logger, startDate: startDate) + for flagKey in flagKeys { + part.flagCounters[flagKey] = flagCounters[flagKey] + } + return part + } + mutating func trackRequest(flagKey: LDFlagKey, reportedValue: LDValue, featureFlag: FeatureFlag?, defaultValue: LDValue, context: LDContext) { if flagCounters[flagKey] == nil { flagCounters[flagKey] = FlagCounter(defaultValue: defaultValue) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift index 0cb64588d..40e526e7b 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift @@ -76,26 +76,89 @@ final class OutboundEventBuffer { /// Encodes the summary of one context's counters. Requires the processor's `commitLock`. /// /// A summary larger than one log frame holds is split between summaries of the same context and period, each with - /// some of its flags, until each fits. Whole, it would be refused by the store and every evaluation in it lost. Only - /// a single flag's counters too large for a frame, and a part that is mostly context, which no split makes fit, are - /// left whole, for the store to refuse. + /// as many of its flags as fit beside the context. Whole, it would be refused by the store and every evaluation in + /// it lost. Only a single flag too large to fit beside the context, and a summary whose context alone takes a frame, + /// are left whole, for the store to refuse. /// /// - Returns: at least one summary, nil for each part that could not be encoded. func encodeSummary(_ summary: (tracker: FlagRequestTracker, context: LDContext)) -> [Data?] { - encodeSummary(summary.tracker, context: summary.context, endDate: Date(), splitFrom: .max) + let tracker = summary.tracker + let context = summary.context + let endDate = Date() + let encoded = encode(SummaryEvent(flagRequestTracker: tracker, context: context, endDate: endDate)) + guard let size = encoded?.count, size > EventLogFormat.maxFrameSize, tracker.flagCounters.count > 1 + else { return [encoded] } + let parts = packIntoFrames(tracker, flagKeys: tracker.flagCounters.keys.sorted(), endDate: endDate, size: size) + guard parts.count > 1 + else { return [encoded] } + return parts.flatMap { flagKeys -> [Data?] in + let part = tracker.withFlags(flagKeys) + return halveUntilFits(part, encode(SummaryEvent(flagRequestTracker: part, context: context, endDate: endDate)), + context: context, endDate: endDate) + } } - /// - Parameter splitFrom: the encoded size of the summary this is half of, or `.max` for one that is not. - private func encodeSummary(_ tracker: FlagRequestTracker, context: LDContext, endDate: Date, splitFrom: Int) -> [Data?] { - let encoded = encode(SummaryEvent(flagRequestTracker: tracker, context: context, endDate: endDate)) - // Every part carries the whole context. A half that kept more than three quarters of what it was split from is - // mostly context, and halving it again would only turn its one loss into one per flag, each a frame's worth of - // context held at once. - guard let size = encoded?.count, size > EventLogFormat.maxFrameSize, size <= splitFrom / 4 * 3, - let halves = tracker.halved() + private static let standInContext = try? LDContextBuilder(key: "k").build().get() + + /// Splits an oversized summary's flags into runs, in key order, that each fit a frame beside the context. Every + /// part carries the whole context, so how much room a part has for flags is a frame less the context. What each + /// flag takes is measured with a small stand-in for the context, which costs one small encoding per flag rather than + /// one of the context per part, and what is left of the whole summary is what every part repeats. Where the context + /// alone takes a frame, nothing fits, so the summary is one part, refused whole as one loss rather than one per flag. + /// + /// - Parameters: + /// - flagKeys: the summary's flags, sorted, at least two. + /// - size: the whole summary's encoded size. + /// - Returns: the flags of each part. + private func packIntoFrames(_ tracker: FlagRequestTracker, flagKeys: [LDFlagKey], endDate: Date, size: Int) -> [ArraySlice] { + let whole = [flagKeys[...]] + guard let standIn = OutboundEventBuffer.standInContext + else { return whole } + func standInSize(_ keys: ArraySlice) -> Int? { + encode(SummaryEvent(flagRequestTracker: tracker.withFlags(keys), context: standIn, endDate: endDate))?.count + } + var flagSizes: [Int] = [] + flagSizes.reserveCapacity(flagKeys.count) + for index in flagKeys.indices { + guard let single = standInSize(flagKeys[index...index]) + else { return whole } + flagSizes.append(single) + } + // Two flags measured alone and together give what a part of the stand-in repeats, less the separator a flag + // adds beside another, so that what a part takes is that plus the sum of its flags' sizes. + guard let pair = standInSize(flagKeys[0...1]) + else { return whole } + let standInRepeated = flagSizes[0] + flagSizes[1] - pair + flagSizes = flagSizes.map { $0 - standInRepeated } + let room = EventLogFormat.maxFrameSize - (size - flagSizes.reduce(0, +)) + guard room > 0 + else { return whole } + + var parts: [ArraySlice] = [] + var start = 0 + var used = 0 + for (index, flagSize) in flagSizes.enumerated() { + if index > start && used + flagSize > room { + parts.append(flagKeys[start.. [Data?] { + guard let size = encoded?.count, size > EventLogFormat.maxFrameSize, let halves = tracker.halved() else { return [encoded] } - return encodeSummary(halves.0, context: context, endDate: endDate, splitFrom: size) - + encodeSummary(halves.1, context: context, endDate: endDate, splitFrom: size) + return [halves.0, halves.1].flatMap { half -> [Data?] in + halveUntilFits(half, encode(SummaryEvent(flagRequestTracker: half, context: context, endDate: endDate)), + context: context, endDate: endDate) + } } private static func makeEncoder(config: LDConfig) -> JSONEncoder { diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 879aad637..66e156e25 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -3019,6 +3019,45 @@ extension DirectEventProcessorSpec { expect(flagKeys.count) == flagCount } + it("splits a summary mostly context into parts that fit where the context does") { + var builder = LDContextBuilder(key: "large") + _ = builder.trySetValue("data", .string(String(repeating: "x", count: 1_700_000))) + let context = try! builder.build().get() + let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: false) + let flagCount = 16 + var tracker = FlagRequestTracker(logger: .disabled) + for index in 0.. Date: Fri, 30 Oct 2026 23:31:34 -0700 Subject: [PATCH 137/158] Measure a summary's flags with a writer of their own, so its parts reuse the context's encoding Co-authored-by: Cursor --- .../ServiceObjects/ContextEncodingCache.swift | 4 ++++ .../ServiceObjects/EventJSONWriter.swift | 3 +++ .../ServiceObjects/OutboundEventBuffer.swift | 14 ++++++++++++- .../DirectEventProcessorSpec.swift | 21 ++++++++++++++++++- 4 files changed, 40 insertions(+), 2 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/ContextEncodingCache.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/ContextEncodingCache.swift index 3d6798383..bc79012e1 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/ContextEncodingCache.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/ContextEncodingCache.swift @@ -26,6 +26,9 @@ final class ContextEncodingCache { private var entries: [Entry?] = [nil, nil] + /// How many encodings have been stored, which is how many times a context was encoded rather than served. + private(set) var storeCount = 0 + /// The encoded bytes for `context` under `redactAnonymous`, or nil if that entry holds a different context. func encodedContext(for context: LDContext, redactAnonymous: Bool) -> [UInt8]? { let slot = ContextEncodingCache.slot(redactAnonymous) @@ -38,6 +41,7 @@ final class ContextEncodingCache { func store(_ context: LDContext, redactAnonymous: Bool, encoded: [UInt8]) { entries[ContextEncodingCache.slot(redactAnonymous)] = Entry(context: context, encoded: encoded) + storeCount += 1 } private static func slot(_ redactAnonymous: Bool) -> Int { diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventJSONWriter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventJSONWriter.swift index 8718520d7..fb6e8740f 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventJSONWriter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventJSONWriter.swift @@ -22,6 +22,9 @@ struct EventJSONWriter { globalPrivateAttributes: config.privateContextAttributes) } + /// How many times a context was encoded rather than served from the cache. + var contextEncodingCount: Int { contextCache.storeCount } + /// Nil if the event cannot be encoded, including when it holds a NaN or infinite number, which `JSONEncoder` also /// rejects. func encode(_ event: Event) -> Data? { diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift index 40e526e7b..7db739ba9 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift @@ -32,8 +32,12 @@ final class OutboundEventBuffer { /// `commitLock`. private let eventJSONWriter: EventJSONWriter + /// For building the writer an oversized summary is measured with. + private let config: LDConfig + init(config: LDConfig, encoding: Encoding) { self.encoding = encoding + self.config = config self.logger = config.logger self.contextSummarizer = ContextSummarizer(logger: config.logger, maxContexts: config.eventCapacity) self.encoder = OutboundEventBuffer.makeEncoder(config: config) @@ -62,6 +66,9 @@ final class OutboundEventBuffer { return summaries } + /// How many times the writer events go through encoded a context rather than reusing its last encoding. + var contextEncodingCount: Int { eventJSONWriter.contextEncodingCount } + /// Requires the processor's `commitLock`. func encode(_ event: Event) -> Data? { let encoded: Data? = encoding == .codable ? try? encoder.encode(event) : eventJSONWriter.encode(event) @@ -106,6 +113,9 @@ final class OutboundEventBuffer { /// one of the context per part, and what is left of the whole summary is what every part repeats. Where the context /// alone takes a frame, nothing fits, so the summary is one part, refused whole as one loss rather than one per flag. /// + /// The stand-in goes through a writer of its own, which leaves the context's encoding in the cache of the writer + /// the parts go through, so that no part encodes the context again. + /// /// - Parameters: /// - flagKeys: the summary's flags, sorted, at least two. /// - size: the whole summary's encoded size. @@ -114,8 +124,10 @@ final class OutboundEventBuffer { let whole = [flagKeys[...]] guard let standIn = OutboundEventBuffer.standInContext else { return whole } + let standInWriter = encoding == .handWritten ? EventJSONWriter(config: config) : nil func standInSize(_ keys: ArraySlice) -> Int? { - encode(SummaryEvent(flagRequestTracker: tracker.withFlags(keys), context: standIn, endDate: endDate))?.count + let event = SummaryEvent(flagRequestTracker: tracker.withFlags(keys), context: standIn, endDate: endDate) + return (standInWriter.map { $0.encode(event) } ?? encode(event))?.count } var flagSizes: [Int] = [] flagSizes.reserveCapacity(flagKeys.count) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 66e156e25..c03871810 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -2475,7 +2475,8 @@ extension DirectEventProcessorSpec { describe("flush reporting outcome") { var testContext: TestContext! afterEach { - testContext.cleanUp() + testContext?.cleanUp() + testContext = nil } it("reports true when there is nothing to deliver") { @@ -3040,6 +3041,24 @@ extension DirectEventProcessorSpec { expectEveryPartFitsAndEveryFlagIsCountedOnce(parts, flagCount: flagCount) } + it("encodes the context of a summary it splits only once") { + var builder = LDContextBuilder(key: "large") + _ = builder.trySetValue("data", .string(String(repeating: "x", count: 1_700_000))) + let context = try! builder.build().get() + let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: false) + var tracker = FlagRequestTracker(logger: .disabled) + for index in 0..<16 { + tracker.trackRequest(flagKey: "flag-\(index)", reportedValue: .string(String(repeating: "v", count: 64 * 1024)), + featureFlag: flag, defaultValue: "b", context: context) + } + let buffer = OutboundEventBuffer(config: LDConfig.stub, encoding: .handWritten) + + let parts = buffer.encodeSummary((tracker: tracker, context: context)) + + expect(parts.count) == 3 + expect(buffer.contextEncodingCount) == 1 + } + it("packs a large flag so the flags beside it still fit") { let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: false) let context = LDContext.stub() From 93f6ed541b1d231f9506991364a9d958d2c6484a Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Sat, 31 Oct 2026 08:33:07 -0700 Subject: [PATCH 138/158] Split an oversized summary on the commit queue rather than the caller's thread Co-authored-by: Cursor --- .../ServiceObjects/EventCommitter.swift | 69 ++++++++++++++----- .../ServiceObjects/OutboundEventBuffer.swift | 9 +++ .../DirectEventProcessorSpec.swift | 24 +++++++ 3 files changed, 83 insertions(+), 19 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift index fb43b90d3..b1164f83b 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift @@ -41,7 +41,9 @@ final class LostEventCount { /// `.disabled` nothing commits early: the held run waits for the delivery, which encodes it. /// /// Threads. Recording runs on whichever thread evaluated or tracked. Commits run on `commitQueue`, or on the caller's -/// thread at a commit point, one at a time under `commitLock`. A delivery commits too, before it closes a batch. +/// thread at a commit point, one at a time under `commitLock`. A delivery commits too, before it closes a batch. A +/// summary too large for one frame is split off the caller's thread even then: a commit there leaves it to +/// `commitQueue`, so it is durable a moment after the call returns rather than before. final class EventCommitter { /// How many full events may be held unencoded before one of them pays for a commit. /// @@ -100,6 +102,12 @@ final class EventCommitter { /// meantime. Without it each waiter committed again in turn. private var committedSequence: UInt64 = 0 + /// Summaries too large for one frame that a commit on the application's thread left for `commitQueue` to split, + /// since splitting encodes the context once per part. Every commit off that thread stages them, so they reach the + /// store before the next delivery closes a batch. Until then they are only in memory, as the counters they were + /// taken from were. Only to be used while holding `commitLock`. + private var summariesToSplit: [(tracker: FlagRequestTracker, context: LDContext)] = [] + // MARK: Guarded by recordLock /// Guards everything one recording writes: the held events and the summarizer. @@ -175,7 +183,7 @@ final class EventCommitter { reportDropped() } if persistence == .immediate && EventCommitter.isCommitPoint(event.kind) { - commitThrough(sequence) + commitThrough(sequence, onCallerThread: true) } else if needsCommit { scheduleCommitWherePersisting() } @@ -309,7 +317,7 @@ final class EventCommitter { /// nothing durable, and is left to the delivery. func commitWherePersisting() { if persistence == .immediate || store.isPersisting { - commitThrough(recordLock.withLock { recordedSequence }) + commitThrough(recordLock.withLock { recordedSequence }, onCallerThread: true) } } @@ -326,7 +334,7 @@ final class EventCommitter { let through = recordLock.withLock { recordedSequence } let committed = DispatchSemaphore(value: 0) commitQueue.async { - self.commitThrough(through) + self.commitThrough(through, onCallerThread: false) committed.signal() } if committed.wait(timeout: .now() + budget) == .timedOut { @@ -359,11 +367,11 @@ final class EventCommitter { /// Commits unless a finished commit already covers recording `sequence`, checked once the lock is held, since the /// commit a caller waited behind is the one most likely to have covered it. - private func commitThrough(_ sequence: UInt64) { + private func commitThrough(_ sequence: UInt64, onCallerThread: Bool) { commitLock.withLock { guard committedSequence < sequence else { return } - encodeAndCommitPending() + encodeAndCommitPending(onCallerThread: onCallerThread) } } @@ -397,10 +405,13 @@ final class EventCommitter { } /// Requires `commitLock`. - private func encodeAndCommitPending() { + /// + /// - Parameter onCallerThread: whether this runs on the application's thread, which leaves a summary too large for + /// one frame to `summariesToSplit` rather than splitting it here. + private func encodeAndCommitPending(onCallerThread: Bool = false) { commitLock.assertOwned() var through: UInt64 = 0 - let (run, summaries) = recordLock.withLock { () -> ([Event], [(tracker: FlagRequestTracker, context: LDContext)]) in + let (run, taken) = recordLock.withLock { () -> ([Event], [(tracker: FlagRequestTracker, context: LDContext)]) in let run = pending pending = [] let summaries = eventBuffer.takeSummaries() @@ -410,10 +421,15 @@ final class EventCommitter { through = recordedSequence return (run, summaries) } + var summaries = taken + if !onCallerThread && !summariesToSplit.isEmpty { + summaries = summariesToSplit + taken + summariesToSplit = [] + } stage(run) store.releaseReservations() - stageSummaries(summaries) + stageSummaries(summaries, onCallerThread: onCallerThread) store.commit() committedSequence = through } @@ -459,21 +475,36 @@ final class EventCommitter { /// commit point rather than only at a delivery is what bounds that loss, and it costs nothing in accuracy: /// LaunchDarkly sums the counters of every summary it receives, so a session that produced several summaries is /// counted the same as one that produced a single summary covering the same evaluations. - private func stageSummaries(_ summaries: [(tracker: FlagRequestTracker, context: LDContext)]) { - for encoded in summaries.flatMap({ eventBuffer.encodeSummary($0) }) { - guard let encoded = encoded + /// + /// On the application's thread, a summary that would have to be split is held for a commit on `commitQueue` + /// instead. Requires `commitLock`. + private func stageSummaries(_ summaries: [(tracker: FlagRequestTracker, context: LDContext)], onCallerThread: Bool) { + var toSplit: [(tracker: FlagRequestTracker, context: LDContext)] = [] + for summary in summaries { + guard let parts = eventBuffer.encodeSummary(summary, splitting: !onCallerThread) else { - reportLost() + toSplit.append(summary) continue } - // Summaries are an aggregate of evaluations that were already counted, so refusing one for capacity would - // lose evaluations the SDK promised to report rather than shed new load. Only the byte capacity refuses - // one, once even the room held back for summaries is used up, since nothing else bounds how many are - // staged while delivery keeps failing. - if !store.stage(encoded, bypassingCapacity: true) { - reportLost() + for encoded in parts { + guard let encoded = encoded + else { + reportLost() + continue + } + // Summaries are an aggregate of evaluations that were already counted, so refusing one for capacity + // would lose evaluations the SDK promised to report rather than shed new load. Only the byte capacity + // refuses one, once even the room held back for summaries is used up, since nothing else bounds how + // many are staged while delivery keeps failing. + if !store.stage(encoded, bypassingCapacity: true) { + reportLost() + } } } + if !toSplit.isEmpty { + summariesToSplit += toSplit + scheduleCommit() + } } } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift index 7db739ba9..3f9c761b7 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift @@ -89,12 +89,21 @@ final class OutboundEventBuffer { /// /// - Returns: at least one summary, nil for each part that could not be encoded. func encodeSummary(_ summary: (tracker: FlagRequestTracker, context: LDContext)) -> [Data?] { + encodeSummary(summary, splitting: true) ?? [] + } + + /// `encodeSummary(_:)`, or nil for a summary that would be split where `splitting` is false, leaving the split to + /// the caller. Splitting encodes the context once per part, which is more than a commit on the application's thread + /// should spend. + func encodeSummary(_ summary: (tracker: FlagRequestTracker, context: LDContext), splitting: Bool) -> [Data?]? { let tracker = summary.tracker let context = summary.context let endDate = Date() let encoded = encode(SummaryEvent(flagRequestTracker: tracker, context: context, endDate: endDate)) guard let size = encoded?.count, size > EventLogFormat.maxFrameSize, tracker.flagCounters.count > 1 else { return [encoded] } + guard splitting + else { return nil } let parts = packIntoFrames(tracker, flagKeys: tracker.flagCounters.keys.sorted(), endDate: endDate, size: size) guard parts.count > 1 else { return [encoded] } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index c03871810..206a1cfc1 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -1780,6 +1780,30 @@ extension DirectEventProcessorSpec { expect(eventsLeftOnDisk().compactMap { $0.kindField }) == ["custom"] } + itPersisting("leaves splitting an oversized summary to the commit queue when a flush writes on this thread") { + // Splitting a summary encodes its context once per part, which is more than the application's thread + // should spend. Suspended, so the flush's write is the only one until the queue resumes. + let commits = DispatchQueue(label: "com.launchdarkly.tests.suspended") + commits.suspend() + testContext = TestContext(lastEventResponseDate: Date(), eventPersistence: .deferred, commitQueue: commits) + let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: false) + let large = LDValue.string(String(repeating: "v", count: 64 * 1024)) + // About 3 MiB of values, so two parts. + for index in 0..<48 { + testContext.eventProcessor.recordFlagEvaluationEvents(flagKey: "flag-\(index)", value: large, defaultValue: "b", featureFlag: flag, context: testContext.context, includeReason: false) + } + testContext.eventProcessor.record(CustomEvent(key: "before-exit", context: testContext.context, data: nil)) + + testContext.eventProcessor.flush(completion: nil) + + expect(eventsLeftOnDisk().compactMap { $0.kindField }) == ["custom"] + + commits.resume() + commits.sync { } + + expect(eventsLeftOnDisk().compactMap { $0.kindField }) == ["custom", "summary", "summary"] + } + it("does not write on the main thread when a flag is evaluated there") { // Suspended for the length of the evaluations, so that a write this thread was going to cause is a // write that cannot happen: an evaluation is expected to be a memory operation, and the thread From 4ae92926906714ab9a680fcdec86ff49870a09f2 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Sun, 1 Nov 2026 12:46:41 -0800 Subject: [PATCH 139/158] Split an oversized summary on the flushing thread again, so a flush writes everything before it returns Reverts 93f6ed54: a flush, or entering the background, returned while the split of an oversized summary was only queued, leaving those evaluations in memory. Co-authored-by: Cursor --- .../ServiceObjects/EventCommitter.swift | 69 +++++-------------- .../ServiceObjects/OutboundEventBuffer.swift | 9 --- .../DirectEventProcessorSpec.swift | 15 ++-- 3 files changed, 24 insertions(+), 69 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift index b1164f83b..fb43b90d3 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift @@ -41,9 +41,7 @@ final class LostEventCount { /// `.disabled` nothing commits early: the held run waits for the delivery, which encodes it. /// /// Threads. Recording runs on whichever thread evaluated or tracked. Commits run on `commitQueue`, or on the caller's -/// thread at a commit point, one at a time under `commitLock`. A delivery commits too, before it closes a batch. A -/// summary too large for one frame is split off the caller's thread even then: a commit there leaves it to -/// `commitQueue`, so it is durable a moment after the call returns rather than before. +/// thread at a commit point, one at a time under `commitLock`. A delivery commits too, before it closes a batch. final class EventCommitter { /// How many full events may be held unencoded before one of them pays for a commit. /// @@ -102,12 +100,6 @@ final class EventCommitter { /// meantime. Without it each waiter committed again in turn. private var committedSequence: UInt64 = 0 - /// Summaries too large for one frame that a commit on the application's thread left for `commitQueue` to split, - /// since splitting encodes the context once per part. Every commit off that thread stages them, so they reach the - /// store before the next delivery closes a batch. Until then they are only in memory, as the counters they were - /// taken from were. Only to be used while holding `commitLock`. - private var summariesToSplit: [(tracker: FlagRequestTracker, context: LDContext)] = [] - // MARK: Guarded by recordLock /// Guards everything one recording writes: the held events and the summarizer. @@ -183,7 +175,7 @@ final class EventCommitter { reportDropped() } if persistence == .immediate && EventCommitter.isCommitPoint(event.kind) { - commitThrough(sequence, onCallerThread: true) + commitThrough(sequence) } else if needsCommit { scheduleCommitWherePersisting() } @@ -317,7 +309,7 @@ final class EventCommitter { /// nothing durable, and is left to the delivery. func commitWherePersisting() { if persistence == .immediate || store.isPersisting { - commitThrough(recordLock.withLock { recordedSequence }, onCallerThread: true) + commitThrough(recordLock.withLock { recordedSequence }) } } @@ -334,7 +326,7 @@ final class EventCommitter { let through = recordLock.withLock { recordedSequence } let committed = DispatchSemaphore(value: 0) commitQueue.async { - self.commitThrough(through, onCallerThread: false) + self.commitThrough(through) committed.signal() } if committed.wait(timeout: .now() + budget) == .timedOut { @@ -367,11 +359,11 @@ final class EventCommitter { /// Commits unless a finished commit already covers recording `sequence`, checked once the lock is held, since the /// commit a caller waited behind is the one most likely to have covered it. - private func commitThrough(_ sequence: UInt64, onCallerThread: Bool) { + private func commitThrough(_ sequence: UInt64) { commitLock.withLock { guard committedSequence < sequence else { return } - encodeAndCommitPending(onCallerThread: onCallerThread) + encodeAndCommitPending() } } @@ -405,13 +397,10 @@ final class EventCommitter { } /// Requires `commitLock`. - /// - /// - Parameter onCallerThread: whether this runs on the application's thread, which leaves a summary too large for - /// one frame to `summariesToSplit` rather than splitting it here. - private func encodeAndCommitPending(onCallerThread: Bool = false) { + private func encodeAndCommitPending() { commitLock.assertOwned() var through: UInt64 = 0 - let (run, taken) = recordLock.withLock { () -> ([Event], [(tracker: FlagRequestTracker, context: LDContext)]) in + let (run, summaries) = recordLock.withLock { () -> ([Event], [(tracker: FlagRequestTracker, context: LDContext)]) in let run = pending pending = [] let summaries = eventBuffer.takeSummaries() @@ -421,15 +410,10 @@ final class EventCommitter { through = recordedSequence return (run, summaries) } - var summaries = taken - if !onCallerThread && !summariesToSplit.isEmpty { - summaries = summariesToSplit + taken - summariesToSplit = [] - } stage(run) store.releaseReservations() - stageSummaries(summaries, onCallerThread: onCallerThread) + stageSummaries(summaries) store.commit() committedSequence = through } @@ -475,36 +459,21 @@ final class EventCommitter { /// commit point rather than only at a delivery is what bounds that loss, and it costs nothing in accuracy: /// LaunchDarkly sums the counters of every summary it receives, so a session that produced several summaries is /// counted the same as one that produced a single summary covering the same evaluations. - /// - /// On the application's thread, a summary that would have to be split is held for a commit on `commitQueue` - /// instead. Requires `commitLock`. - private func stageSummaries(_ summaries: [(tracker: FlagRequestTracker, context: LDContext)], onCallerThread: Bool) { - var toSplit: [(tracker: FlagRequestTracker, context: LDContext)] = [] - for summary in summaries { - guard let parts = eventBuffer.encodeSummary(summary, splitting: !onCallerThread) + private func stageSummaries(_ summaries: [(tracker: FlagRequestTracker, context: LDContext)]) { + for encoded in summaries.flatMap({ eventBuffer.encodeSummary($0) }) { + guard let encoded = encoded else { - toSplit.append(summary) + reportLost() continue } - for encoded in parts { - guard let encoded = encoded - else { - reportLost() - continue - } - // Summaries are an aggregate of evaluations that were already counted, so refusing one for capacity - // would lose evaluations the SDK promised to report rather than shed new load. Only the byte capacity - // refuses one, once even the room held back for summaries is used up, since nothing else bounds how - // many are staged while delivery keeps failing. - if !store.stage(encoded, bypassingCapacity: true) { - reportLost() - } + // Summaries are an aggregate of evaluations that were already counted, so refusing one for capacity would + // lose evaluations the SDK promised to report rather than shed new load. Only the byte capacity refuses + // one, once even the room held back for summaries is used up, since nothing else bounds how many are + // staged while delivery keeps failing. + if !store.stage(encoded, bypassingCapacity: true) { + reportLost() } } - if !toSplit.isEmpty { - summariesToSplit += toSplit - scheduleCommit() - } } } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift index 3f9c761b7..7db739ba9 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift @@ -89,21 +89,12 @@ final class OutboundEventBuffer { /// /// - Returns: at least one summary, nil for each part that could not be encoded. func encodeSummary(_ summary: (tracker: FlagRequestTracker, context: LDContext)) -> [Data?] { - encodeSummary(summary, splitting: true) ?? [] - } - - /// `encodeSummary(_:)`, or nil for a summary that would be split where `splitting` is false, leaving the split to - /// the caller. Splitting encodes the context once per part, which is more than a commit on the application's thread - /// should spend. - func encodeSummary(_ summary: (tracker: FlagRequestTracker, context: LDContext), splitting: Bool) -> [Data?]? { let tracker = summary.tracker let context = summary.context let endDate = Date() let encoded = encode(SummaryEvent(flagRequestTracker: tracker, context: context, endDate: endDate)) guard let size = encoded?.count, size > EventLogFormat.maxFrameSize, tracker.flagCounters.count > 1 else { return [encoded] } - guard splitting - else { return nil } let parts = packIntoFrames(tracker, flagKeys: tracker.flagCounters.keys.sorted(), endDate: endDate, size: size) guard parts.count > 1 else { return [encoded] } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 206a1cfc1..7fccc93db 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -1780,11 +1780,12 @@ extension DirectEventProcessorSpec { expect(eventsLeftOnDisk().compactMap { $0.kindField }) == ["custom"] } - itPersisting("leaves splitting an oversized summary to the commit queue when a flush writes on this thread") { - // Splitting a summary encodes its context once per part, which is more than the application's thread - // should spend. Suspended, so the flush's write is the only one until the queue resumes. + itPersisting("has written the split of an oversized summary by the time a flush returns") { + // A flush promises everything recorded is written before it returns, the evaluations a summary too + // large for one frame counts included. Suspended, so the flush's own write is the only one there can be. let commits = DispatchQueue(label: "com.launchdarkly.tests.suspended") commits.suspend() + defer { commits.resume() } testContext = TestContext(lastEventResponseDate: Date(), eventPersistence: .deferred, commitQueue: commits) let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: false) let large = LDValue.string(String(repeating: "v", count: 64 * 1024)) @@ -1792,16 +1793,10 @@ extension DirectEventProcessorSpec { for index in 0..<48 { testContext.eventProcessor.recordFlagEvaluationEvents(flagKey: "flag-\(index)", value: large, defaultValue: "b", featureFlag: flag, context: testContext.context, includeReason: false) } - testContext.eventProcessor.record(CustomEvent(key: "before-exit", context: testContext.context, data: nil)) testContext.eventProcessor.flush(completion: nil) - expect(eventsLeftOnDisk().compactMap { $0.kindField }) == ["custom"] - - commits.resume() - commits.sync { } - - expect(eventsLeftOnDisk().compactMap { $0.kindField }) == ["custom", "summary", "summary"] + expect(eventsLeftOnDisk().compactMap { $0.kindField }) == ["summary", "summary"] } it("does not write on the main thread when a flag is evaluated there") { From 5c612588211dfa0c50f60075b7ef0b21d1bcbd28 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Sun, 1 Nov 2026 13:05:52 -0800 Subject: [PATCH 140/158] Write only its own event for a track or identify at immediate persistence The caller's thread encodes and writes the one event, through encoders of its own and without waiting for a commit in progress; the held evaluations and the summaries are left to a commit queued on the commit queue in the same call. The summarizer's counters are still taken at each track or identify, so its contexts are cleared as before. Co-authored-by: Cursor --- .../LaunchDarkly/Models/LDConfig.swift | 11 +- .../ServiceObjects/EventCommitter.swift | 89 +++++++++---- .../ServiceObjects/OutboundEventBuffer.swift | 27 +++- .../DirectEventProcessorSpec.swift | 120 ++++++++++++------ 4 files changed, 174 insertions(+), 73 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift index e5665d4de..9da641023 100644 --- a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift @@ -48,12 +48,11 @@ public enum LDStreamingMode { case deferred /// Events are written to disk before `track` and `identify` return. /// - /// The write happens on the calling thread, costing tens of microseconds depending on how many evaluations are - /// waiting to be encoded alongside it. Evaluations are usually written in the background before many build up. A - /// burst of them faster than that can leave up to `eventCapacity` waiting, and the `track` or `identify` that - /// follows encodes them all, which can take milliseconds. In exchange there is no window: a process that dies the - /// instant after `track` returns still reports that event on the next launch. Evaluating a flag stays in memory - /// regardless. + /// The write happens on the calling thread and covers that one event, costing tens to hundreds of microseconds + /// depending on the device, however many evaluations are waiting. Those, and the summaries of them, are written in + /// the background by a write the call sets going, so they follow moments later. In exchange there is no window: a + /// process that dies the instant after `track` returns still reports that event on the next launch. Evaluating a + /// flag stays in memory regardless. /// /// This level is experimental; see `EventPersistence`. case immediate diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift index fb43b90d3..7378ca5e7 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift @@ -31,8 +31,9 @@ final class LostEventCount { /// /// Where the line falls is between the events an application asked for by name and the ones it did not. At `.immediate` /// recording a custom or identify event is a *commit point*, because an application reporting something it chose to -/// report is saying this matters more than the microseconds it costs, and the crash it describes may be moments away. A -/// commit takes the whole held run with it, so the exposures leading up to the error go down alongside it. Everything +/// report is saying this matters more than the microseconds it costs, and the crash it describes may be moments away. It +/// writes itself before returning, and only itself: what is held, and the summaries of the evaluations so far, are left +/// to a commit queued in the same call, so the exposures leading up to the error follow it moments later. Everything /// else is committed once `pendingCommitThreshold` events have accumulated, when a delivery starts, on entering the /// background, or on `flush`. /// @@ -41,22 +42,23 @@ final class LostEventCount { /// `.disabled` nothing commits early: the held run waits for the delivery, which encodes it. /// /// Threads. Recording runs on whichever thread evaluated or tracked. Commits run on `commitQueue`, or on the caller's -/// thread at a commit point, one at a time under `commitLock`. A delivery commits too, before it closes a batch. +/// thread for a flush or entering the background, one at a time under `commitLock`. A delivery commits too, before it +/// closes a batch. A commit point at `.immediate` writes its event without `commitLock`. final class EventCommitter { /// How many full events may be held unencoded before one of them pays for a commit. /// /// It sets two things at once. The first is how many evaluations a termination can take -- not everything a crash /// could take, since an application that flushes on its way out commits the whole run, but the window for the /// terminations that run nothing on the way out, such as the system reclaiming a backgrounded process. Recording a - /// custom or identify event at `.immediate` closes it too, because that commits. + /// custom or identify event at `.immediate` narrows it too, because that queues a commit. /// - /// The second is what a commit point usually costs, since it encodes whatever is held before returning. The commit - /// queue keeps the run drained, so the common case is far below this; raising it trades that cost against the - /// number of writes. + /// The second is what a flush usually costs, since it encodes whatever is held before returning. The commit queue + /// keeps the run drained, so the common case is far below this; raising it trades that cost against the number of + /// writes. /// /// Neither is a ceiling. Reaching this only schedules a commit, and recording does not wait for it, so where events /// arrive faster than the commit queue encodes them, or the queue is starved, the run grows up to the event - /// capacity, and the next commit point encodes all of it. + /// capacity, and the next flush encodes all of it. /// /// It is the only thing that writes events no caller asked to have written. The store writes only when told to, so /// every staged byte reaches the disk through a commit that started here, at a commit point, or at a delivery. @@ -83,21 +85,21 @@ final class EventCommitter { /// Held for the whole of a commit, so that only one runs at a time. /// - /// This is what a commit point's guarantee rests on. Without it a commit already in flight could take the caller's - /// event out of `pending` before the caller got there, leaving the caller nothing to write and returning while - /// those bytes were still being produced somewhere else. Waiting here instead means that when the call returns the - /// event is on disk, whichever commit put it there. It also keeps two encoders from staging their runs in - /// whichever order they happened to finish. + /// This is what a flush's guarantee rests on. Without it a commit already in flight could take the caller's events + /// out of `pending` before the caller got there, leaving the caller nothing to write and returning while those + /// bytes were still being produced somewhere else. Waiting here instead means that when the call returns the events + /// are on disk, whichever commit put them there. It also keeps two encoders from staging their runs in whichever + /// order they happened to finish. /// /// Taken before `recordLock` and before anything the store locks, never after. private let commitLock = UnfairLock() /// The last recording a finished commit covered. Only to be used while holding `commitLock`. /// - /// This is what makes concurrent `.immediate` commit points a group commit. The first caller's commit takes every - /// recording made before it took the run, including those of callers now waiting behind it. They find their - /// recordings covered and return, and the next caller still uncovered commits everything that arrived in the - /// meantime. Without it each waiter committed again in turn. + /// This is what makes concurrent flushes a group commit. The first caller's commit takes every recording made before + /// it took the run, including those of callers now waiting behind it. They find their recordings covered and + /// return, and the next caller still uncovered commits everything that arrived in the meantime. Without it each + /// waiter committed again in turn. private var committedSequence: UInt64 = 0 // MARK: Guarded by recordLock @@ -119,6 +121,15 @@ final class EventCommitter { /// happen where it is cheapest. private var pending: [Event] = [] + /// Counters a commit point at `.immediate` took from the summarizer for the next commit to encode. Only to be used + /// while holding `recordLock`. + /// + /// Taken there rather than left, so the summarizer's contexts are cleared at each of them as they were when they + /// committed the counters themselves: an application identifying its way through more contexts than capacity faster + /// than the commit queue keeps up would otherwise find the summarizer refusing evaluations. Every commit takes them + /// along with `pending`, so an evaluation's counter and its full event are still staged together. + private var takenSummaries: [(tracker: FlagRequestTracker, context: LDContext)] = [] + /// Numbers every recording, so a caller at a commit point can tell whether a commit has already covered it. Only to /// be used while holding `recordLock`. private var recordedSequence: UInt64 = 0 @@ -157,13 +168,18 @@ final class EventCommitter { // MARK: Recording - /// Records an event, and at `.immediate` commits before returning if it is a commit point. + /// Records an event, and at `.immediate` writes it before returning if it is a commit point. /// /// At `.deferred` a custom or identify event writes nothing itself (durability §13.3), and is committed with what /// is held at the pending threshold, a flush, a delivery or entering the background. A commit per call would cost /// an application that tracks every few seconds one encode and write per event, each taking `recordLock` from the /// threads evaluating flags and the store's locks from delivery. func record(_ event: Event) { + let isCommitPoint = persistence == .immediate && EventCommitter.isCommitPoint(event.kind) + if isCommitPoint && store.isPersisting { + writeAlone(event) + return + } var sequence: UInt64 = 0 let (dropped, needsCommit) = recordLock.withLock { () -> (Bool, Bool) in recordedSequence += 1 @@ -174,13 +190,41 @@ final class EventCommitter { if dropped { reportDropped() } - if persistence == .immediate && EventCommitter.isCommitPoint(event.kind) { + if isCommitPoint { commitThrough(sequence) } else if needsCommit { scheduleCommitWherePersisting() } } + /// Encodes and writes one event on the caller's thread, and queues a commit of everything else. + /// + /// Only the event itself, so the caller pays for one small encode and one write however much is held: a commit + /// encodes the held run and every summary, and a summary with a large context or many flags takes megabytes, which + /// is a dropped frame on the main thread. They follow on `commitQueue`, queued here. The event is staged ahead of + /// the run held before it, which LaunchDarkly does not mind: every event carries its own creation date. + /// + /// Not under `commitLock`, which a commit holds for the whole of its encoding, so this never waits for one. The + /// store takes its own locks, and the event goes through encoders of its own; the one wait left is for the store's + /// I/O lock, held by a write or by a delivery reading a batch back. + private func writeAlone(_ event: Event) { + let admitted = recordLock.withLock { () -> Bool in + takenSummaries.append(contentsOf: eventBuffer.takeSummaries()) + return store.admitsEvent(heldElsewhere: pending.count, capacity: eventCapacity) + } + if admitted { + // Checked against capacity again, since it is counted nowhere between being admitted and being staged. + if let encoded = eventBuffer.encodeAlone(event), store.stage(encoded, bypassingCapacity: false) { + store.commit() + } else { + reportLost() + } + } else { + reportDropped() + } + scheduleCommit() + } + // swiftlint:disable:next function_parameter_count func recordFlagEvaluationEvents(flagKey: LDFlagKey, value: LDValue, @@ -213,8 +257,8 @@ final class EventCommitter { } // Deliberately no commit point. An evaluation is expected to cost what bookkeeping costs, and it is usually the // main thread doing it; where events are persisted, the run is encoded and written on the commit queue once - // enough of them have piled up, and at `.immediate` the next event recorded at a commit point makes them - // durable along with itself. + // enough of them have piled up, and at `.immediate` the next event recorded at a commit point queues that + // write as well. if needsCommit { scheduleCommitWherePersisting() } @@ -403,7 +447,8 @@ final class EventCommitter { let (run, summaries) = recordLock.withLock { () -> ([Event], [(tracker: FlagRequestTracker, context: LDContext)]) in let run = pending pending = [] - let summaries = eventBuffer.takeSummaries() + let summaries = takenSummaries + eventBuffer.takeSummaries() + takenSummaries = [] // Reserved in the same critical section the run leaves `pending` in, so capacity counts it in one place or // the other throughout the encode. store.reserve(run.count) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift index 7db739ba9..bb3db6a8c 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift @@ -4,8 +4,8 @@ import OSLog /// Turns what `DirectEventProcessor` records into what it stages: evaluations into summary counters, and events into the /// JSON the events endpoint accepts. Android's `OutboundEventBuffer` plays the same part. /// -/// It has no lock of its own. The summarizer is only to be used while holding the processor's `recordLock`, and the -/// encoders are only reached under its `commitLock`. +/// The summarizer is only to be used while holding the processor's `recordLock`, and the encoders are only reached under +/// its `commitLock`, except the pair `encodeAlone(_:)` uses, which have a lock of their own. final class OutboundEventBuffer { /// Which encoder recorded events go through. /// @@ -32,6 +32,14 @@ final class OutboundEventBuffer { /// `commitLock`. private let eventJSONWriter: EventJSONWriter + /// Guards the encoders a `track` or `identify` writing itself goes through, which are separate from the commit's so + /// that it never waits for a commit to finish encoding. + private let aloneLock = UnfairLock() + /// Only to be used while holding `aloneLock`. + private let aloneEncoder: JSONEncoder + /// Only to be used while holding `aloneLock`. + private let aloneJSONWriter: EventJSONWriter + /// For building the writer an oversized summary is measured with. private let config: LDConfig @@ -42,6 +50,8 @@ final class OutboundEventBuffer { self.contextSummarizer = ContextSummarizer(logger: config.logger, maxContexts: config.eventCapacity) self.encoder = OutboundEventBuffer.makeEncoder(config: config) self.eventJSONWriter = EventJSONWriter(config: config) + self.aloneEncoder = OutboundEventBuffer.makeEncoder(config: config) + self.aloneJSONWriter = EventJSONWriter(config: config) } /// Counts an evaluation into its context's summary. Requires the processor's `recordLock`. @@ -71,7 +81,18 @@ final class OutboundEventBuffer { /// Requires the processor's `commitLock`. func encode(_ event: Event) -> Data? { - let encoded: Data? = encoding == .codable ? try? encoder.encode(event) : eventJSONWriter.encode(event) + logged(encoding == .codable ? try? encoder.encode(event) : eventJSONWriter.encode(event), event) + } + + /// `encode(_:)` for an event encoded outside a commit, on any thread. + func encodeAlone(_ event: Event) -> Data? { + let encoded = aloneLock.withLock { + encoding == .codable ? try? aloneEncoder.encode(event) : aloneJSONWriter.encode(event) + } + return logged(encoded, event) + } + + private func logged(_ encoded: Data?, _ event: Event) -> Data? { guard let encoded = encoded else { os_log("%s Failed to serialize event for publication: %s", log: logger, type: .error, typeName(and: #function), String(describing: event)) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 7fccc93db..c7194ce25 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -106,6 +106,11 @@ class DirectEventProcessorSpec: QuickSpec { events.compactMap { encodeToLDValue($0) } } + /// `recordedEventsAsJSON()` through `unordered(_:)`. + func recordedEventsUnordered() -> [String] { + unordered(recordedEventsAsJSON()) + } + /// Flushes, and answers whether a flush made from the completion got anywhere. /// /// A completion is free to flush again, which only works where it is not running on the queue deliveries go @@ -642,7 +647,7 @@ class DirectEventProcessorSpec: QuickSpec { expect(testContext.eventProcessor.isOnline) == false expect(testContext.eventProcessor.isReportingActive) == false expect(testContext.serviceMock.publishEventDataCallCount) == 0 - expect(testContext.committedEvents()) == testContext.recordedEventsAsJSON() + expect(unordered(testContext.committedEvents())) == testContext.recordedEventsUnordered() } } } @@ -660,7 +665,7 @@ class DirectEventProcessorSpec: QuickSpec { expect(testContext.eventProcessor.isOnline) == false expect(testContext.eventProcessor.isReportingActive) == false expect(testContext.serviceMock.publishEventDataCallCount) == 0 - expect(testContext.committedEvents()) == testContext.recordedEventsAsJSON() + expect(unordered(testContext.committedEvents())) == testContext.recordedEventsUnordered() } it("does not record a dropped event to diagnosticCache") { expect(testContext.diagnosticCache.incrementDroppedEventCountCallCount) == 0 @@ -678,7 +683,7 @@ class DirectEventProcessorSpec: QuickSpec { expect(testContext.eventProcessor.isOnline) == false expect(testContext.eventProcessor.isReportingActive) == false expect(testContext.serviceMock.publishEventDataCallCount) == 0 - expect(testContext.committedEvents()) == testContext.recordedEventsAsJSON() + expect(unordered(testContext.committedEvents())) == testContext.recordedEventsUnordered() } it("records a dropped event to diagnosticCache") { expect(testContext.diagnosticCache.incrementDroppedEventCountCallCount) == 1 @@ -1090,10 +1095,8 @@ class DirectEventProcessorSpec: QuickSpec { let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) valueIsArray(published) { valueArray in expect(valueArray.count) == testContext.events.count + 1 - expect(Array(valueArray.prefix(testContext.events.count))) == testContext.events.map { encodeToLDValue($0) } - valueIsObject(valueArray[testContext.events.count]) { summaryObject in - expect(summaryObject["kind"]) == "summary" - } + expect(unordered(valueArray.filter { $0.kindField != "summary" })) == testContext.recordedEventsUnordered() + expect(valueArray.filter { $0.kindField == "summary" }.count) == 1 } expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 @@ -1119,7 +1122,7 @@ class DirectEventProcessorSpec: QuickSpec { erOnline() expect(testContext.serviceMock.publishEventDataCallCount) == 1 let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) - expect(published) == encodeToLDValue(testContext.events) + valueIsArray(published) { expect(unordered($0)) == testContext.recordedEventsUnordered() } expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == testContext.events.count expect(testContext.store.pendingEventCount) == 0 @@ -1270,10 +1273,8 @@ class DirectEventProcessorSpec: QuickSpec { let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) valueIsArray(published) { valueArray in expect(valueArray.count) == testContext.events.count + 1 - expect(Array(valueArray.prefix(testContext.events.count))) == testContext.events.map { encodeToLDValue($0) } - valueIsObject(valueArray[testContext.events.count]) { summaryObject in - expect(summaryObject["kind"]) == "summary" - } + expect(unordered(valueArray.filter { $0.kindField != "summary" })) == testContext.recordedEventsUnordered() + expect(valueArray.filter { $0.kindField == "summary" }.count) == 1 } expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 @@ -1317,10 +1318,8 @@ class DirectEventProcessorSpec: QuickSpec { let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) valueIsArray(published) { valueArray in expect(valueArray.count) == testContext.events.count + 1 - expect(Array(valueArray.prefix(testContext.events.count))) == testContext.events.map { encodeToLDValue($0) } - valueIsObject(valueArray[testContext.events.count]) { summaryObject in - expect(summaryObject["kind"]) == "summary" - } + expect(unordered(valueArray.filter { $0.kindField != "summary" })) == testContext.recordedEventsUnordered() + expect(valueArray.filter { $0.kindField == "summary" }.count) == 1 } expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 @@ -1367,10 +1366,8 @@ class DirectEventProcessorSpec: QuickSpec { let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) valueIsArray(published) { valueArray in expect(valueArray.count) == testContext.events.count + 1 - expect(Array(valueArray.prefix(testContext.events.count))) == testContext.events.map { encodeToLDValue($0) } - valueIsObject(valueArray[testContext.events.count]) { summaryObject in - expect(summaryObject["kind"]) == "summary" - } + expect(unordered(valueArray.filter { $0.kindField != "summary" })) == testContext.recordedEventsUnordered() + expect(valueArray.filter { $0.kindField == "summary" }.count) == 1 } expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 @@ -1449,8 +1446,8 @@ class DirectEventProcessorSpec: QuickSpec { // Being offline delays delivery, not durability: a flush is a commit point, so the recorded events and // the evaluations counted so far are on disk even though nothing can be sent. let pending = testContext.committedEvents() - expect(pending.dropLast().map { $0 }) == testContext.recordedEventsAsJSON() - expect(pending.last?.kindField) == "summary" + expect(unordered(pending.filter { $0.kindField != "summary" })) == testContext.recordedEventsUnordered() + expect(pending.filter { $0.kindField == "summary" }.count) == 1 expect(testContext.eventProcessor.withContextSummarizer { $0.hasLoggedRequests }) == false guard case .isOffline = testContext.syncResult else { @@ -1662,7 +1659,7 @@ class DirectEventProcessorSpec: QuickSpec { let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) valueIsArray(published) { valueArray in expect(valueArray.count) == testContext.events.count - expect(valueArray) == testContext.events.map { encodeToLDValue($0) } + expect(unordered(valueArray)) == testContext.recordedEventsUnordered() } } } @@ -1738,18 +1735,60 @@ extension DirectEventProcessorSpec { expect(eventsLeftOnDisk().count).toEventually(equal(EventCommitter.pendingCommitThreshold), timeout: Constants.deliveryTimeout) } - itPersisting("has written the evaluations that came before a tracked event") { + itPersisting("writes only a tracked event itself, and leaves the evaluations before it to the commit queue") { + // Suspended, so that what is on disk when the track returns is what the track wrote. + let commits = DispatchQueue(label: "com.launchdarkly.tests.suspended") + commits.suspend() // A response date of its own, because the distant past that a fresh context defaults to also satisfies // the debug window comparison and would add a debug event to what is asserted below. - testContext = TestContext(lastEventResponseDate: Date()) - let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: false) + testContext = TestContext(lastEventResponseDate: Date(), commitQueue: commits) + let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: true) - // An untracked flag is only ever reported as a summary, so this is the case where the exposure exists - // nowhere but in memory until something forces it out. testContext.eventProcessor.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: testContext.context, includeReason: false) testContext.eventProcessor.record(CustomEvent(key: "fatal-error", context: testContext.context, data: nil)) - expect(eventsLeftOnDisk().compactMap { $0.kindField }) == ["custom", "summary"] + expect(eventsLeftOnDisk().compactMap { $0.kindField }) == ["custom"] + commits.resume() + expect(eventsLeftOnDisk().compactMap { $0.kindField }) + .toEventually(equal(["custom", "feature", "summary"]), timeout: Constants.deliveryTimeout) + } + + itPersisting("writes a tracked event without waiting for a commit in progress") { + var config = LDConfig.stub + config.eventCapacity = 100 + config.eventPersistence = .immediate + let serviceMock = DarklyServiceMock() + serviceMock.config = config + let inner = EventStore.temporary(capacity: config.eventCapacity) + defer { inner.deleteEverything() } + let commits = DispatchQueue(label: "com.launchdarkly.tests.commits") + let onCommits = DispatchSpecificKey() + commits.setSpecific(key: onCommits, value: true) + let store = ClosingHookStore(inner) + let stalled = DispatchSemaphore(value: 0) + let release = DispatchSemaphore(value: 0) + defer { release.signal() } + // A commit on the commit queue stalled on the disk, holding the commit lock while it is. + store.beforeCommitting = { + if DispatchQueue.getSpecific(key: onCommits) == true { + stalled.signal() + _ = release.wait(timeout: .now() + 10) + } + } + let reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, commitQueue: commits, pacing: .noWaits) + let context = LDContext.stub() + let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: true) + for _ in 0.. [String] { @@ -2189,10 +2221,6 @@ extension DirectEventProcessorSpec { expect(finished.wait(timeout: .now() + 10)) == .success expect(missingOnReturn).to(beEmpty()) - commitsLock.lock() - let totalCommits = commits - commitsLock.unlock() - expect(totalCommits) < trackers } itPersisting("returns from closing within its budget while the final commit is stalled on the disk, and still writes it") { @@ -3846,6 +3874,14 @@ private func expectEveryPartFitsAndEveryFlagIsCountedOnce(_ parts: [Data?], flag expect(Set(flagKeys).count) == flagCount } +/// Events in an order of their own, for comparing what was staged with what was recorded: at `.immediate` a custom or +/// identify event is staged ahead of the evaluations held before it. +private func unordered(_ events: [LDValue]) -> [String] { + let encoder = JSONEncoder() + encoder.outputFormatting = .sortedKeys + return events.map { String(decoding: (try? encoder.encode($0)) ?? Data(), as: UTF8.self) }.sorted() +} + /// A store that runs a hook on either side of closing a batch, and remembers the custom event keys of each batch closed. private final class ClosingHookStore: EventStoring { let inner: EventStoring From 9173f039a62167f3c0c29a298213d7949b4d5f2b Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Sun, 1 Nov 2026 20:59:53 -0800 Subject: [PATCH 141/158] Write only its own frame for a track at immediate persistence, from a reserved place A track or identify at immediate persistence committed the store, which wrote whatever a commit in flight had staged so far, up to megabytes of a run or a split summary, on the application thread. It now writes just its own frame and leaves the rest to the commit that staged it. It is also reserved where it is admitted, so a recording on another thread cannot take its place while it is encoded and have it refused as a loss. A commit now releases only its own unused reservations rather than all of them. Co-authored-by: Cursor --- .../ServiceObjects/EventCommitter.swift | 34 +++++++--- .../ServiceObjects/EventStore.swift | 64 +++++++++++++++++-- .../DirectEventProcessorSpec.swift | 10 +-- .../ServiceObjects/EventStoreSpec.swift | 41 ++++++++++++ 4 files changed, 129 insertions(+), 20 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift index 7378ca5e7..771ae104c 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift @@ -201,22 +201,31 @@ final class EventCommitter { /// /// Only the event itself, so the caller pays for one small encode and one write however much is held: a commit /// encodes the held run and every summary, and a summary with a large context or many flags takes megabytes, which - /// is a dropped frame on the main thread. They follow on `commitQueue`, queued here. The event is staged ahead of + /// is a dropped frame on the main thread. They follow on `commitQueue`, queued here. The event is written ahead of /// the run held before it, which LaunchDarkly does not mind: every event carries its own creation date. /// /// Not under `commitLock`, which a commit holds for the whole of its encoding, so this never waits for one. The - /// store takes its own locks, and the event goes through encoders of its own; the one wait left is for the store's - /// I/O lock, held by a write or by a delivery reading a batch back. + /// store writes the one frame rather than committing, which would also write whatever a commit in flight has staged + /// so far, and the event goes through encoders of its own; the one wait left is for the store's I/O lock, held by + /// a write or by a delivery reading a batch back. private func writeAlone(_ event: Event) { let admitted = recordLock.withLock { () -> Bool in takenSummaries.append(contentsOf: eventBuffer.takeSummaries()) - return store.admitsEvent(heldElsewhere: pending.count, capacity: eventCapacity) + guard store.admitsEvent(heldElsewhere: pending.count, capacity: eventCapacity) + else { return false } + // Reserved where it is admitted, as a run is where it leaves `pending`, so capacity counts it while it is + // encoded and a recording on another thread cannot take its place. + store.reserve(1) + return true } if admitted { - // Checked against capacity again, since it is counted nowhere between being admitted and being staged. - if let encoded = eventBuffer.encodeAlone(event), store.stage(encoded, bypassingCapacity: false) { - store.commit() + if let encoded = eventBuffer.encodeAlone(event) { + // Only the bytes are checked: a frame or byte capacity it turns out not to fit. + if !store.writeReservedAlone(encoded) { + reportLost() + } } else { + store.releaseReservations(1) reportLost() } } else { @@ -456,8 +465,8 @@ final class EventCommitter { return (run, summaries) } - stage(run) - store.releaseReservations() + // Only the run's own: a track or identify writing itself alone may hold one beside them. + store.releaseReservations(stage(run)) stageSummaries(summaries) store.commit() committedSequence = through @@ -471,12 +480,16 @@ final class EventCommitter { /// /// Requires `commitLock`: two threads draining separate runs would stage them in whichever order they finished /// encoding, which is not the order they were recorded in. - private func stage(_ run: [Event]) { + /// + /// - Returns: how many of the run's reservations are left unused, one for each event that did not encode. + private func stage(_ run: [Event]) -> Int { commitLock.assertOwned() + var unencoded = 0 for event in run { guard let encoded = eventBuffer.encode(event) else { reportLost() + unencoded += 1 continue } // The event capacity is bypassed here, so what is left is an event too large for a frame or for the bytes @@ -485,6 +498,7 @@ final class EventCommitter { reportLost() } } + return unencoded } /// Whether recording this kind of event at `.immediate` should encode and write the held run before it returns. diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index a9bff26c1..907171a19 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -77,12 +77,21 @@ protocol EventStoring { /// they were not known when it was accepted. func stageReserved(_ encodedEvent: Data) -> Bool - /// Gives back every reservation not used up by `stageReserved(_:)`, such as one for an event that failed to encode. - func releaseReservations() + /// Gives back `events` reservations not used up by `stageReserved(_:)` or `writeReservedAlone(_:)`, such as one for + /// an event that failed to encode. Only the caller's own, since another caller may hold reservations alongside. + func releaseReservations(_ events: Int) /// Hands every staged byte to the kernel, so that the events recorded so far survive the process dying. func commit() + /// Writes one encoded event to the log on its own, using up one reservation as `stageReserved(_:)` does, but leaving + /// what anything else has staged to the commit that staged it, so the caller pays for writing its own bytes and no + /// one else's. It may land ahead of events staged before it. Returns false if it is too large for a frame or for + /// the bytes left in the store. + /// + /// Without persistence, or once it has failed, the event is staged in memory instead, as a commit leaves it. + func writeReservedAlone(_ encodedEvent: Data) -> Bool + /// Commits, then closes the open log into a batch to deliver, and whatever is still staged in memory into another. /// /// Returns the batch closed from the log, or the first closed from memory where the log held nothing, and nil when @@ -374,8 +383,8 @@ final class EventStore: EventStoring { bufferLock.withLock { reservedEvents += events } } - func releaseReservations() { - bufferLock.withLock { reservedEvents = 0 } + func releaseReservations(_ events: Int) { + bufferLock.withLock { reservedEvents = max(0, reservedEvents - events) } } func stage(_ encodedEvent: Data, bypassingCapacity: Bool = false) -> Bool { @@ -434,6 +443,44 @@ final class EventStore: EventStoring { ioLock.withLock { writeStagedBytes() } } + func writeReservedAlone(_ encodedEvent: Data) -> Bool { + guard disk != nil + else { return stageReserved(encodedEvent) } + guard encodedEvent.count <= EventLogFormat.maxFrameSize + else { + os_log("%s dropping an event larger than a log frame allows", log: logger, type: .debug, typeName(and: #function)) + releaseReservations(1) + return false + } + let frame = EventLogFormat.frame(for: encodedEvent) + return ioLock.withLock { () -> Bool in + let (admitted, taken) = bufferLock.withLock { () -> (Bool, StagedFrames?) in + // In the same critical section as the count that replaces it, so the event is never counted twice or + // not at all. + reservedEvents = max(0, reservedEvents - 1) + guard countedBytes + frame.count <= eventByteCapacity + else { return (false, nil) } + // As a commit leaves staged bytes once persistence has failed, for a delivery to close from memory. + guard persistEvents + else { + bufferData.append(frame) + bufferedEventCount += 1 + return (true, nil) + } + writingEvents += 1 + writingBytes += frame.count + return (true, StagedFrames(frames: frame, summariesFrom: frame.count, events: 1, summaries: 0)) + } + if let taken { + write(taken) + } + if !admitted { + os_log("%s dropping an event the store has no room for", log: logger, type: .debug, typeName(and: #function)) + } + return admitted + } + } + func closeBatch() -> EventBatch? { guard let disk else { return closeInMemoryBatches().first } @@ -735,9 +782,14 @@ extension EventStore { else { return nil } return takeStaged() } - guard let taken - else { return } + if let taken { + write(taken) + } + } + /// Requires `ioLock`, and `taken` counted as being written. + private func write(_ taken: StagedFrames) { + ioLock.assertOwned() let bytes = taken.frames let base = bytes.startIndex var written = 0 diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index c7194ce25..f7489c0db 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -1787,8 +1787,8 @@ extension DirectEventProcessorSpec { reporter.record(CustomEvent(key: "fatal-error", context: context)) expect(Date().timeIntervalSince(started)) < 1 - // Alongside whatever the stalled commit had staged before it stalled, which the track's write takes too. - expect(inner.payloadsOnDisk.compactMap { try? JSONDecoder().decode(LDValue.self, from: $0).kindField }).to(contain("custom")) + // Without what the stalled commit had staged before it stalled, which is that commit's to write. + expect(inner.payloadsOnDisk.compactMap { try? JSONDecoder().decode(LDValue.self, from: $0).kindField }) == ["custom"] } itPersisting("persists feature and summary events synchronously on flush") { @@ -3902,11 +3902,12 @@ private final class ClosingHookStore: EventStoring { func stage(_ encodedEvent: Data, bypassingCapacity: Bool) -> Bool { inner.stage(encodedEvent, bypassingCapacity: bypassingCapacity) } func reserve(_ events: Int) { inner.reserve(events) } func stageReserved(_ encodedEvent: Data) -> Bool { inner.stageReserved(encodedEvent) } - func releaseReservations() { inner.releaseReservations() } + func releaseReservations(_ events: Int) { inner.releaseReservations(events) } func commit() { beforeCommitting?() inner.commit() } + func writeReservedAlone(_ encodedEvent: Data) -> Bool { inner.writeReservedAlone(encodedEvent) } var holdsUnclosedEvents: Bool { inner.holdsUnclosedEvents } func takeVanishedLogEvents() -> Int { inner.takeVanishedLogEvents() } func listPendingBatches() -> BatchListing { inner.listPendingBatches() } @@ -4017,8 +4018,9 @@ private final class SlowListingStore: EventStoring { func stage(_ encodedEvent: Data, bypassingCapacity: Bool) -> Bool { inner.stage(encodedEvent, bypassingCapacity: bypassingCapacity) } func reserve(_ events: Int) { inner.reserve(events) } func stageReserved(_ encodedEvent: Data) -> Bool { inner.stageReserved(encodedEvent) } - func releaseReservations() { inner.releaseReservations() } + func releaseReservations(_ events: Int) { inner.releaseReservations(events) } func commit() { inner.commit() } + func writeReservedAlone(_ encodedEvent: Data) -> Bool { inner.writeReservedAlone(encodedEvent) } func closeBatch() -> EventBatch? { inner.closeBatch() } var holdsUnclosedEvents: Bool { inner.holdsUnclosedEvents } func takeVanishedLogEvents() -> Int { inner.takeVanishedLogEvents() } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index ecaf0ecfe..3f56b535e 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -438,6 +438,37 @@ final class EventStoreSpec: QuickSpec { expect(EventStoreSpec.keys(in: store.payloadsOnDisk)) == ["event-0", "event-1", "event-2", "event-3", "event-4"] } + + it("writes an event on its own, leaving what is staged to its commit") { + _ = store.stage(EventStoreSpec.payload("staged")) + store.reserve(1) + + expect(store.writeReservedAlone(EventStoreSpec.payload("alone"))) == true + + expect(EventStoreSpec.keys(in: store.payloadsOnDisk)) == ["alone"] + expect(store.pendingEventCount) == 2 + store.commit() + expect(EventStoreSpec.keys(in: store.payloadsOnDisk)) == ["alone", "staged"] + } + + it("keeps the place of an event reserved to be written on its own") { + let full = EventStore.temporary(capacity: 1) + defer { full.deleteEverything() } + full.reserve(1) + + // What another thread recording meanwhile finds. + expect(full.stage(EventStoreSpec.payload("staged"))) == false + expect(full.writeReservedAlone(EventStoreSpec.payload("alone"))) == true + expect(full.pendingEventCount) == 1 + } + + it("releases only the reservations it is told to") { + store.reserve(1) + store.reserve(2) + store.releaseReservations(2) + + expect(store.pendingEventCount) == 1 + } } } @@ -2194,6 +2225,16 @@ final class EventStoreSpec: QuickSpec { expect(try store.body(of: held)) == (try written.body(of: onDisk)) } + it("delivers an event written on its own from memory when the write fails") { + store.reserve(1) + expect(store.writeReservedAlone(EventStoreSpec.payload("alone"))) == true + + expect(store.pendingEventCount) == 1 + let batch = store.closeBatch() + expect(batch?.eventCount) == 1 + expect(try store.body(of: batch!)) == Data("[{\"kind\":\"custom\",\"key\":\"alone\"}]".utf8) + } + it("keeps accepting events after persistence has been given up on") { _ = store.stage(EventStoreSpec.payload("before")) store.commit() From d0ee14b6b329a3832e64b4f2f7ccba349c80096b Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Mon, 2 Nov 2026 10:26:03 -0800 Subject: [PATCH 142/158] Drop a summary larger than a frame rather than splitting it A summary too large for one log frame is refused whole by the store, logged, and counted as lost, so the flush that committed it answers false. Reaching that takes one context evaluating dozens of the largest JSON flags between two commits, which did not justify the packing and halving that split it. Co-authored-by: Cursor --- .../FeatureFlag/FlagRequestTracker.swift | 32 ---- .../ServiceObjects/ContextEncodingCache.swift | 4 - .../ServiceObjects/EventCommitter.swift | 6 +- .../ServiceObjects/EventJSONWriter.swift | 3 - .../ServiceObjects/OutboundEventBuffer.swift | 101 +------------ .../FlagRequestTrackerSpec.swift | 26 ---- .../DirectEventProcessorSpec.swift | 141 +----------------- 7 files changed, 16 insertions(+), 297 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/Models/FeatureFlag/FlagRequestTracker.swift b/LaunchDarkly/LaunchDarkly/Models/FeatureFlag/FlagRequestTracker.swift index 2e39a8b40..61af453c5 100644 --- a/LaunchDarkly/LaunchDarkly/Models/FeatureFlag/FlagRequestTracker.swift +++ b/LaunchDarkly/LaunchDarkly/Models/FeatureFlag/FlagRequestTracker.swift @@ -11,38 +11,6 @@ struct FlagRequestTracker { self.startDate = startDate } - /// The flags split between two trackers over the same period. LaunchDarkly sums the counters of every summary it - /// receives, so the two count exactly what the one did. Nothing may count into this tracker any more: the halves - /// share its counters. - /// - /// - Returns: the two halves, or nil for a tracker of one flag, which cannot be split. - func halved() -> (FlagRequestTracker, FlagRequestTracker)? { - guard flagCounters.count > 1 - else { return nil } - let flagKeys = flagCounters.keys.sorted() - let middle = flagKeys.count / 2 - var first = FlagRequestTracker(logger: logger, startDate: startDate) - var second = FlagRequestTracker(logger: logger, startDate: startDate) - for (index, flagKey) in flagKeys.enumerated() { - if index < middle { - first.flagCounters[flagKey] = flagCounters[flagKey] - } else { - second.flagCounters[flagKey] = flagCounters[flagKey] - } - } - return (first, second) - } - - /// Only these flags, over the same period. Nothing may count into this tracker any more: the part shares its - /// counters. - func withFlags(_ flagKeys: Keys) -> FlagRequestTracker where Keys.Element == LDFlagKey { - var part = FlagRequestTracker(logger: logger, startDate: startDate) - for flagKey in flagKeys { - part.flagCounters[flagKey] = flagCounters[flagKey] - } - return part - } - mutating func trackRequest(flagKey: LDFlagKey, reportedValue: LDValue, featureFlag: FeatureFlag?, defaultValue: LDValue, context: LDContext) { if flagCounters[flagKey] == nil { flagCounters[flagKey] = FlagCounter(defaultValue: defaultValue) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/ContextEncodingCache.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/ContextEncodingCache.swift index bc79012e1..3d6798383 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/ContextEncodingCache.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/ContextEncodingCache.swift @@ -26,9 +26,6 @@ final class ContextEncodingCache { private var entries: [Entry?] = [nil, nil] - /// How many encodings have been stored, which is how many times a context was encoded rather than served. - private(set) var storeCount = 0 - /// The encoded bytes for `context` under `redactAnonymous`, or nil if that entry holds a different context. func encodedContext(for context: LDContext, redactAnonymous: Bool) -> [UInt8]? { let slot = ContextEncodingCache.slot(redactAnonymous) @@ -41,7 +38,6 @@ final class ContextEncodingCache { func store(_ context: LDContext, redactAnonymous: Bool, encoded: [UInt8]) { entries[ContextEncodingCache.slot(redactAnonymous)] = Entry(context: context, encoded: encoded) - storeCount += 1 } private static func slot(_ redactAnonymous: Bool) -> Int { diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift index 771ae104c..de2374dd4 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift @@ -519,8 +519,8 @@ final class EventCommitter { /// LaunchDarkly sums the counters of every summary it receives, so a session that produced several summaries is /// counted the same as one that produced a single summary covering the same evaluations. private func stageSummaries(_ summaries: [(tracker: FlagRequestTracker, context: LDContext)]) { - for encoded in summaries.flatMap({ eventBuffer.encodeSummary($0) }) { - guard let encoded = encoded + for summary in summaries { + guard let encoded = eventBuffer.encodeSummary(summary) else { reportLost() continue @@ -528,7 +528,7 @@ final class EventCommitter { // Summaries are an aggregate of evaluations that were already counted, so refusing one for capacity would // lose evaluations the SDK promised to report rather than shed new load. Only the byte capacity refuses // one, once even the room held back for summaries is used up, since nothing else bounds how many are - // staged while delivery keeps failing. + // staged while delivery keeps failing; and the frame size, for one too large for a frame. if !store.stage(encoded, bypassingCapacity: true) { reportLost() } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventJSONWriter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventJSONWriter.swift index fb6e8740f..8718520d7 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventJSONWriter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventJSONWriter.swift @@ -22,9 +22,6 @@ struct EventJSONWriter { globalPrivateAttributes: config.privateContextAttributes) } - /// How many times a context was encoded rather than served from the cache. - var contextEncodingCount: Int { contextCache.storeCount } - /// Nil if the event cannot be encoded, including when it holds a NaN or infinite number, which `JSONEncoder` also /// rejects. func encode(_ event: Event) -> Data? { diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift index bb3db6a8c..7f2bcfd82 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift @@ -40,12 +40,8 @@ final class OutboundEventBuffer { /// Only to be used while holding `aloneLock`. private let aloneJSONWriter: EventJSONWriter - /// For building the writer an oversized summary is measured with. - private let config: LDConfig - init(config: LDConfig, encoding: Encoding) { self.encoding = encoding - self.config = config self.logger = config.logger self.contextSummarizer = ContextSummarizer(logger: config.logger, maxContexts: config.eventCapacity) self.encoder = OutboundEventBuffer.makeEncoder(config: config) @@ -76,9 +72,6 @@ final class OutboundEventBuffer { return summaries } - /// How many times the writer events go through encoded a context rather than reusing its last encoding. - var contextEncodingCount: Int { eventJSONWriter.contextEncodingCount } - /// Requires the processor's `commitLock`. func encode(_ event: Event) -> Data? { logged(encoding == .codable ? try? encoder.encode(event) : eventJSONWriter.encode(event), event) @@ -103,95 +96,11 @@ final class OutboundEventBuffer { /// Encodes the summary of one context's counters. Requires the processor's `commitLock`. /// - /// A summary larger than one log frame holds is split between summaries of the same context and period, each with - /// as many of its flags as fit beside the context. Whole, it would be refused by the store and every evaluation in - /// it lost. Only a single flag too large to fit beside the context, and a summary whose context alone takes a frame, - /// are left whole, for the store to refuse. - /// - /// - Returns: at least one summary, nil for each part that could not be encoded. - func encodeSummary(_ summary: (tracker: FlagRequestTracker, context: LDContext)) -> [Data?] { - let tracker = summary.tracker - let context = summary.context - let endDate = Date() - let encoded = encode(SummaryEvent(flagRequestTracker: tracker, context: context, endDate: endDate)) - guard let size = encoded?.count, size > EventLogFormat.maxFrameSize, tracker.flagCounters.count > 1 - else { return [encoded] } - let parts = packIntoFrames(tracker, flagKeys: tracker.flagCounters.keys.sorted(), endDate: endDate, size: size) - guard parts.count > 1 - else { return [encoded] } - return parts.flatMap { flagKeys -> [Data?] in - let part = tracker.withFlags(flagKeys) - return halveUntilFits(part, encode(SummaryEvent(flagRequestTracker: part, context: context, endDate: endDate)), - context: context, endDate: endDate) - } - } - - private static let standInContext = try? LDContextBuilder(key: "k").build().get() - - /// Splits an oversized summary's flags into runs, in key order, that each fit a frame beside the context. Every - /// part carries the whole context, so how much room a part has for flags is a frame less the context. What each - /// flag takes is measured with a small stand-in for the context, which costs one small encoding per flag rather than - /// one of the context per part, and what is left of the whole summary is what every part repeats. Where the context - /// alone takes a frame, nothing fits, so the summary is one part, refused whole as one loss rather than one per flag. - /// - /// The stand-in goes through a writer of its own, which leaves the context's encoding in the cache of the writer - /// the parts go through, so that no part encodes the context again. - /// - /// - Parameters: - /// - flagKeys: the summary's flags, sorted, at least two. - /// - size: the whole summary's encoded size. - /// - Returns: the flags of each part. - private func packIntoFrames(_ tracker: FlagRequestTracker, flagKeys: [LDFlagKey], endDate: Date, size: Int) -> [ArraySlice] { - let whole = [flagKeys[...]] - guard let standIn = OutboundEventBuffer.standInContext - else { return whole } - let standInWriter = encoding == .handWritten ? EventJSONWriter(config: config) : nil - func standInSize(_ keys: ArraySlice) -> Int? { - let event = SummaryEvent(flagRequestTracker: tracker.withFlags(keys), context: standIn, endDate: endDate) - return (standInWriter.map { $0.encode(event) } ?? encode(event))?.count - } - var flagSizes: [Int] = [] - flagSizes.reserveCapacity(flagKeys.count) - for index in flagKeys.indices { - guard let single = standInSize(flagKeys[index...index]) - else { return whole } - flagSizes.append(single) - } - // Two flags measured alone and together give what a part of the stand-in repeats, less the separator a flag - // adds beside another, so that what a part takes is that plus the sum of its flags' sizes. - guard let pair = standInSize(flagKeys[0...1]) - else { return whole } - let standInRepeated = flagSizes[0] + flagSizes[1] - pair - flagSizes = flagSizes.map { $0 - standInRepeated } - let room = EventLogFormat.maxFrameSize - (size - flagSizes.reduce(0, +)) - guard room > 0 - else { return whole } - - var parts: [ArraySlice] = [] - var start = 0 - var used = 0 - for (index, flagSize) in flagSizes.enumerated() { - if index > start && used + flagSize > room { - parts.append(flagKeys[start.. [Data?] { - guard let size = encoded?.count, size > EventLogFormat.maxFrameSize, let halves = tracker.halved() - else { return [encoded] } - return [halves.0, halves.1].flatMap { half -> [Data?] in - halveUntilFits(half, encode(SummaryEvent(flagRequestTracker: half, context: context, endDate: endDate)), - context: context, endDate: endDate) - } + /// A summary larger than one log frame is not split: the store refuses it whole, and its evaluations are counted + /// as lost. Reaching that takes one context evaluating dozens of the largest JSON flags LaunchDarkly allows + /// between two commits. + func encodeSummary(_ summary: (tracker: FlagRequestTracker, context: LDContext)) -> Data? { + encode(SummaryEvent(flagRequestTracker: summary.tracker, context: summary.context)) } private static func makeEncoder(config: LDConfig) -> JSONEncoder { diff --git a/LaunchDarkly/LaunchDarklyTests/Models/FeatureFlag/FlagRequestTracking/FlagRequestTrackerSpec.swift b/LaunchDarkly/LaunchDarklyTests/Models/FeatureFlag/FlagRequestTracking/FlagRequestTrackerSpec.swift index 7c2edf951..9a67bb87c 100644 --- a/LaunchDarkly/LaunchDarklyTests/Models/FeatureFlag/FlagRequestTracking/FlagRequestTrackerSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/Models/FeatureFlag/FlagRequestTracking/FlagRequestTrackerSpec.swift @@ -63,32 +63,6 @@ final class FlagRequestTrackerSpec: XCTestCase { XCTAssertEqual(flagRequestTracker.flagCounters["alt-flag"], counter2) } - func testHalvedLeavesATrackerOfOneFlagWhole() { - var tracker = FlagRequestTracker(logger: .disabled) - tracker.trackRequest(flagKey: "only", reportedValue: true, featureFlag: FeatureFlag(flagKey: "only"), defaultValue: false, context: LDContext.stub()) - tracker.trackRequest(flagKey: "only", reportedValue: true, featureFlag: FeatureFlag(flagKey: "only"), defaultValue: false, context: LDContext.stub()) - XCTAssertNil(tracker.halved()) - } - - func testHalvedSharesTheFlagsBetweenTwoTrackersOverTheSamePeriod() { - var tracker = FlagRequestTracker(logger: .disabled) - for index in 0..<5 { - tracker.trackRequest(flagKey: "flag-\(index)", reportedValue: true, featureFlag: FeatureFlag(flagKey: "flag-\(index)"), defaultValue: false, context: LDContext.stub()) - } - guard let halves = tracker.halved() - else { return XCTFail("a tracker of five flags was not split") } - let (first, second) = halves - XCTAssertEqual(first.flagCounters.count, 2) - XCTAssertEqual(second.flagCounters.count, 3) - XCTAssertEqual(first.startDate, tracker.startDate) - XCTAssertEqual(second.startDate, tracker.startDate) - let rejoined = first.flagCounters.merging(second.flagCounters) { kept, _ in kept } - XCTAssertEqual(Set(rejoined.keys), Set(tracker.flagCounters.keys)) - for (flagKey, counter) in rejoined { - XCTAssertTrue(counter === tracker.flagCounters[flagKey]) - } - } - func testHasLoggedRequests() { var flagRequestTracker = FlagRequestTracker(logger: OSLog(subsystem: "com.launchdarkly", category: "tests")) flagRequestTracker.trackRequest(flagKey: "test-key", reportedValue: nil, featureFlag: FeatureFlag(flagKey: "test-key"), defaultValue: nil, context: LDContext.stub()) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index f7489c0db..1f5ee2146 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -1819,25 +1819,6 @@ extension DirectEventProcessorSpec { expect(eventsLeftOnDisk().compactMap { $0.kindField }) == ["custom"] } - itPersisting("has written the split of an oversized summary by the time a flush returns") { - // A flush promises everything recorded is written before it returns, the evaluations a summary too - // large for one frame counts included. Suspended, so the flush's own write is the only one there can be. - let commits = DispatchQueue(label: "com.launchdarkly.tests.suspended") - commits.suspend() - defer { commits.resume() } - testContext = TestContext(lastEventResponseDate: Date(), eventPersistence: .deferred, commitQueue: commits) - let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: false) - let large = LDValue.string(String(repeating: "v", count: 64 * 1024)) - // About 3 MiB of values, so two parts. - for index in 0..<48 { - testContext.eventProcessor.recordFlagEvaluationEvents(flagKey: "flag-\(index)", value: large, defaultValue: "b", featureFlag: flag, context: testContext.context, includeReason: false) - } - - testContext.eventProcessor.flush(completion: nil) - - expect(eventsLeftOnDisk().compactMap { $0.kindField }) == ["summary", "summary"] - } - it("does not write on the main thread when a flag is evaluated there") { // Suspended for the length of the evaluations, so that a write this thread was going to cause is a // write that cannot happen: an evaluation is expected to be a memory operation, and the thread @@ -3004,13 +2985,15 @@ extension DirectEventProcessorSpec { expect(bodies.flatMap(customKeys).sorted()) == (1...eventCount).map { "event-\($0)" }.sorted() } - it("delivers a summary larger than a frame in pieces that count every flag once") { + it("drops a summary larger than a frame whole and delivers the events beside it") { // Long keys, so that a few thousand flags take one context's summary past what a frame holds. let keyPrefix = String(repeating: "k", count: 1000) let flagCount = EventLogFormat.maxFrameSize / 1000 * 3 / 2 testContext = TestContext() let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: false) let context = LDContext.stub() + // First, ahead of the debug events the evaluations fill the capacity with. + testContext.eventProcessor.record(CustomEvent(key: "fine", context: context)) for index in 0.. [LDValue] in + let events = testContext.serviceMock.publishedEventPayloads.flatMap { body -> [LDValue] in guard case .array(let events)? = try? JSONDecoder().decode(LDValue.self, from: body) else { return [] } - return events.filter { $0.kindField == "summary" } - } - let flagKeys = summaries.flatMap { summary -> [String] in - guard case .object(let fields) = summary, case .object(let features)? = fields["features"] - else { return [] } - return Array(features.keys) - } - expect(summaries.count) > 1 - expect(flagKeys.count) == flagCount - expect(Set(flagKeys).count) == flagCount - } - - it("does not split a summary large for its context down to one flag per part") { - var builder = LDContextBuilder(key: "huge") - _ = builder.trySetValue("data", .string(String(repeating: "x", count: EventLogFormat.maxFrameSize))) - let context = try! builder.build().get() - let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: false) - let flagCount = 16 - var tracker = FlagRequestTracker(logger: .disabled) - for index in 0.. [String] in - guard case .object(let fields)? = try? JSONDecoder().decode(LDValue.self, from: part), - case .object(let features)? = fields["features"] - else { return [] } - return Array(features.keys) - } - expect(Set(flagKeys).count) == flagCount - expect(flagKeys.count) == flagCount - } - - it("splits a summary mostly context into parts that fit where the context does") { - var builder = LDContextBuilder(key: "large") - _ = builder.trySetValue("data", .string(String(repeating: "x", count: 1_700_000))) - let context = try! builder.build().get() - let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: false) - let flagCount = 16 - var tracker = FlagRequestTracker(logger: .disabled) - for index in 0.. [String] { From 972173010e77fc0c29f7c3612ec7e35c99a9e89e Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Mon, 2 Nov 2026 13:37:02 -0800 Subject: [PATCH 143/158] Commit everything held at a track or identify at immediate persistence again Writing only the tracked event left the evaluations before it to a commit queued on the commit queue, so a kill straight after the track, which is what immediate persistence is for, could take those exposures with it. With the summary split gone, the commit costs the caller a normal encode of the held run and the summaries, so the track commits it again before returning. This reverts 9173f039 and 5c612588. Co-authored-by: Cursor --- .../LaunchDarkly/Models/LDConfig.swift | 11 +- .../ServiceObjects/EventCommitter.swift | 109 ++++----------- .../ServiceObjects/EventStore.swift | 64 +-------- .../ServiceObjects/OutboundEventBuffer.swift | 27 +--- .../DirectEventProcessorSpec.swift | 128 +++++++----------- .../ServiceObjects/EventStoreSpec.swift | 41 ------ 6 files changed, 87 insertions(+), 293 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift index 9da641023..e5665d4de 100644 --- a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift @@ -48,11 +48,12 @@ public enum LDStreamingMode { case deferred /// Events are written to disk before `track` and `identify` return. /// - /// The write happens on the calling thread and covers that one event, costing tens to hundreds of microseconds - /// depending on the device, however many evaluations are waiting. Those, and the summaries of them, are written in - /// the background by a write the call sets going, so they follow moments later. In exchange there is no window: a - /// process that dies the instant after `track` returns still reports that event on the next launch. Evaluating a - /// flag stays in memory regardless. + /// The write happens on the calling thread, costing tens of microseconds depending on how many evaluations are + /// waiting to be encoded alongside it. Evaluations are usually written in the background before many build up. A + /// burst of them faster than that can leave up to `eventCapacity` waiting, and the `track` or `identify` that + /// follows encodes them all, which can take milliseconds. In exchange there is no window: a process that dies the + /// instant after `track` returns still reports that event on the next launch. Evaluating a flag stays in memory + /// regardless. /// /// This level is experimental; see `EventPersistence`. case immediate diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift index de2374dd4..bf2b3d45e 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift @@ -31,9 +31,8 @@ final class LostEventCount { /// /// Where the line falls is between the events an application asked for by name and the ones it did not. At `.immediate` /// recording a custom or identify event is a *commit point*, because an application reporting something it chose to -/// report is saying this matters more than the microseconds it costs, and the crash it describes may be moments away. It -/// writes itself before returning, and only itself: what is held, and the summaries of the evaluations so far, are left -/// to a commit queued in the same call, so the exposures leading up to the error follow it moments later. Everything +/// report is saying this matters more than the microseconds it costs, and the crash it describes may be moments away. A +/// commit takes the whole held run with it, so the exposures leading up to the error go down alongside it. Everything /// else is committed once `pendingCommitThreshold` events have accumulated, when a delivery starts, on entering the /// background, or on `flush`. /// @@ -42,23 +41,22 @@ final class LostEventCount { /// `.disabled` nothing commits early: the held run waits for the delivery, which encodes it. /// /// Threads. Recording runs on whichever thread evaluated or tracked. Commits run on `commitQueue`, or on the caller's -/// thread for a flush or entering the background, one at a time under `commitLock`. A delivery commits too, before it -/// closes a batch. A commit point at `.immediate` writes its event without `commitLock`. +/// thread at a commit point, one at a time under `commitLock`. A delivery commits too, before it closes a batch. final class EventCommitter { /// How many full events may be held unencoded before one of them pays for a commit. /// /// It sets two things at once. The first is how many evaluations a termination can take -- not everything a crash /// could take, since an application that flushes on its way out commits the whole run, but the window for the /// terminations that run nothing on the way out, such as the system reclaiming a backgrounded process. Recording a - /// custom or identify event at `.immediate` narrows it too, because that queues a commit. + /// custom or identify event at `.immediate` closes it too, because that commits. /// - /// The second is what a flush usually costs, since it encodes whatever is held before returning. The commit queue - /// keeps the run drained, so the common case is far below this; raising it trades that cost against the number of - /// writes. + /// The second is what a commit point usually costs, since it encodes whatever is held before returning. The commit + /// queue keeps the run drained, so the common case is far below this; raising it trades that cost against the + /// number of writes. /// /// Neither is a ceiling. Reaching this only schedules a commit, and recording does not wait for it, so where events /// arrive faster than the commit queue encodes them, or the queue is starved, the run grows up to the event - /// capacity, and the next flush encodes all of it. + /// capacity, and the next commit point encodes all of it. /// /// It is the only thing that writes events no caller asked to have written. The store writes only when told to, so /// every staged byte reaches the disk through a commit that started here, at a commit point, or at a delivery. @@ -85,21 +83,21 @@ final class EventCommitter { /// Held for the whole of a commit, so that only one runs at a time. /// - /// This is what a flush's guarantee rests on. Without it a commit already in flight could take the caller's events - /// out of `pending` before the caller got there, leaving the caller nothing to write and returning while those - /// bytes were still being produced somewhere else. Waiting here instead means that when the call returns the events - /// are on disk, whichever commit put them there. It also keeps two encoders from staging their runs in whichever - /// order they happened to finish. + /// This is what a commit point's guarantee rests on. Without it a commit already in flight could take the caller's + /// event out of `pending` before the caller got there, leaving the caller nothing to write and returning while + /// those bytes were still being produced somewhere else. Waiting here instead means that when the call returns the + /// event is on disk, whichever commit put it there. It also keeps two encoders from staging their runs in + /// whichever order they happened to finish. /// /// Taken before `recordLock` and before anything the store locks, never after. private let commitLock = UnfairLock() /// The last recording a finished commit covered. Only to be used while holding `commitLock`. /// - /// This is what makes concurrent flushes a group commit. The first caller's commit takes every recording made before - /// it took the run, including those of callers now waiting behind it. They find their recordings covered and - /// return, and the next caller still uncovered commits everything that arrived in the meantime. Without it each - /// waiter committed again in turn. + /// This is what makes concurrent `.immediate` commit points a group commit. The first caller's commit takes every + /// recording made before it took the run, including those of callers now waiting behind it. They find their + /// recordings covered and return, and the next caller still uncovered commits everything that arrived in the + /// meantime. Without it each waiter committed again in turn. private var committedSequence: UInt64 = 0 // MARK: Guarded by recordLock @@ -121,15 +119,6 @@ final class EventCommitter { /// happen where it is cheapest. private var pending: [Event] = [] - /// Counters a commit point at `.immediate` took from the summarizer for the next commit to encode. Only to be used - /// while holding `recordLock`. - /// - /// Taken there rather than left, so the summarizer's contexts are cleared at each of them as they were when they - /// committed the counters themselves: an application identifying its way through more contexts than capacity faster - /// than the commit queue keeps up would otherwise find the summarizer refusing evaluations. Every commit takes them - /// along with `pending`, so an evaluation's counter and its full event are still staged together. - private var takenSummaries: [(tracker: FlagRequestTracker, context: LDContext)] = [] - /// Numbers every recording, so a caller at a commit point can tell whether a commit has already covered it. Only to /// be used while holding `recordLock`. private var recordedSequence: UInt64 = 0 @@ -168,18 +157,13 @@ final class EventCommitter { // MARK: Recording - /// Records an event, and at `.immediate` writes it before returning if it is a commit point. + /// Records an event, and at `.immediate` commits before returning if it is a commit point. /// /// At `.deferred` a custom or identify event writes nothing itself (durability §13.3), and is committed with what /// is held at the pending threshold, a flush, a delivery or entering the background. A commit per call would cost /// an application that tracks every few seconds one encode and write per event, each taking `recordLock` from the /// threads evaluating flags and the store's locks from delivery. func record(_ event: Event) { - let isCommitPoint = persistence == .immediate && EventCommitter.isCommitPoint(event.kind) - if isCommitPoint && store.isPersisting { - writeAlone(event) - return - } var sequence: UInt64 = 0 let (dropped, needsCommit) = recordLock.withLock { () -> (Bool, Bool) in recordedSequence += 1 @@ -190,50 +174,13 @@ final class EventCommitter { if dropped { reportDropped() } - if isCommitPoint { + if persistence == .immediate && EventCommitter.isCommitPoint(event.kind) { commitThrough(sequence) } else if needsCommit { scheduleCommitWherePersisting() } } - /// Encodes and writes one event on the caller's thread, and queues a commit of everything else. - /// - /// Only the event itself, so the caller pays for one small encode and one write however much is held: a commit - /// encodes the held run and every summary, and a summary with a large context or many flags takes megabytes, which - /// is a dropped frame on the main thread. They follow on `commitQueue`, queued here. The event is written ahead of - /// the run held before it, which LaunchDarkly does not mind: every event carries its own creation date. - /// - /// Not under `commitLock`, which a commit holds for the whole of its encoding, so this never waits for one. The - /// store writes the one frame rather than committing, which would also write whatever a commit in flight has staged - /// so far, and the event goes through encoders of its own; the one wait left is for the store's I/O lock, held by - /// a write or by a delivery reading a batch back. - private func writeAlone(_ event: Event) { - let admitted = recordLock.withLock { () -> Bool in - takenSummaries.append(contentsOf: eventBuffer.takeSummaries()) - guard store.admitsEvent(heldElsewhere: pending.count, capacity: eventCapacity) - else { return false } - // Reserved where it is admitted, as a run is where it leaves `pending`, so capacity counts it while it is - // encoded and a recording on another thread cannot take its place. - store.reserve(1) - return true - } - if admitted { - if let encoded = eventBuffer.encodeAlone(event) { - // Only the bytes are checked: a frame or byte capacity it turns out not to fit. - if !store.writeReservedAlone(encoded) { - reportLost() - } - } else { - store.releaseReservations(1) - reportLost() - } - } else { - reportDropped() - } - scheduleCommit() - } - // swiftlint:disable:next function_parameter_count func recordFlagEvaluationEvents(flagKey: LDFlagKey, value: LDValue, @@ -266,8 +213,8 @@ final class EventCommitter { } // Deliberately no commit point. An evaluation is expected to cost what bookkeeping costs, and it is usually the // main thread doing it; where events are persisted, the run is encoded and written on the commit queue once - // enough of them have piled up, and at `.immediate` the next event recorded at a commit point queues that - // write as well. + // enough of them have piled up, and at `.immediate` the next event recorded at a commit point makes them + // durable along with itself. if needsCommit { scheduleCommitWherePersisting() } @@ -456,8 +403,7 @@ final class EventCommitter { let (run, summaries) = recordLock.withLock { () -> ([Event], [(tracker: FlagRequestTracker, context: LDContext)]) in let run = pending pending = [] - let summaries = takenSummaries + eventBuffer.takeSummaries() - takenSummaries = [] + let summaries = eventBuffer.takeSummaries() // Reserved in the same critical section the run leaves `pending` in, so capacity counts it in one place or // the other throughout the encode. store.reserve(run.count) @@ -465,8 +411,8 @@ final class EventCommitter { return (run, summaries) } - // Only the run's own: a track or identify writing itself alone may hold one beside them. - store.releaseReservations(stage(run)) + stage(run) + store.releaseReservations() stageSummaries(summaries) store.commit() committedSequence = through @@ -480,16 +426,12 @@ final class EventCommitter { /// /// Requires `commitLock`: two threads draining separate runs would stage them in whichever order they finished /// encoding, which is not the order they were recorded in. - /// - /// - Returns: how many of the run's reservations are left unused, one for each event that did not encode. - private func stage(_ run: [Event]) -> Int { + private func stage(_ run: [Event]) { commitLock.assertOwned() - var unencoded = 0 for event in run { guard let encoded = eventBuffer.encode(event) else { reportLost() - unencoded += 1 continue } // The event capacity is bypassed here, so what is left is an event too large for a frame or for the bytes @@ -498,7 +440,6 @@ final class EventCommitter { reportLost() } } - return unencoded } /// Whether recording this kind of event at `.immediate` should encode and write the held run before it returns. diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift index 907171a19..a9bff26c1 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -77,21 +77,12 @@ protocol EventStoring { /// they were not known when it was accepted. func stageReserved(_ encodedEvent: Data) -> Bool - /// Gives back `events` reservations not used up by `stageReserved(_:)` or `writeReservedAlone(_:)`, such as one for - /// an event that failed to encode. Only the caller's own, since another caller may hold reservations alongside. - func releaseReservations(_ events: Int) + /// Gives back every reservation not used up by `stageReserved(_:)`, such as one for an event that failed to encode. + func releaseReservations() /// Hands every staged byte to the kernel, so that the events recorded so far survive the process dying. func commit() - /// Writes one encoded event to the log on its own, using up one reservation as `stageReserved(_:)` does, but leaving - /// what anything else has staged to the commit that staged it, so the caller pays for writing its own bytes and no - /// one else's. It may land ahead of events staged before it. Returns false if it is too large for a frame or for - /// the bytes left in the store. - /// - /// Without persistence, or once it has failed, the event is staged in memory instead, as a commit leaves it. - func writeReservedAlone(_ encodedEvent: Data) -> Bool - /// Commits, then closes the open log into a batch to deliver, and whatever is still staged in memory into another. /// /// Returns the batch closed from the log, or the first closed from memory where the log held nothing, and nil when @@ -383,8 +374,8 @@ final class EventStore: EventStoring { bufferLock.withLock { reservedEvents += events } } - func releaseReservations(_ events: Int) { - bufferLock.withLock { reservedEvents = max(0, reservedEvents - events) } + func releaseReservations() { + bufferLock.withLock { reservedEvents = 0 } } func stage(_ encodedEvent: Data, bypassingCapacity: Bool = false) -> Bool { @@ -443,44 +434,6 @@ final class EventStore: EventStoring { ioLock.withLock { writeStagedBytes() } } - func writeReservedAlone(_ encodedEvent: Data) -> Bool { - guard disk != nil - else { return stageReserved(encodedEvent) } - guard encodedEvent.count <= EventLogFormat.maxFrameSize - else { - os_log("%s dropping an event larger than a log frame allows", log: logger, type: .debug, typeName(and: #function)) - releaseReservations(1) - return false - } - let frame = EventLogFormat.frame(for: encodedEvent) - return ioLock.withLock { () -> Bool in - let (admitted, taken) = bufferLock.withLock { () -> (Bool, StagedFrames?) in - // In the same critical section as the count that replaces it, so the event is never counted twice or - // not at all. - reservedEvents = max(0, reservedEvents - 1) - guard countedBytes + frame.count <= eventByteCapacity - else { return (false, nil) } - // As a commit leaves staged bytes once persistence has failed, for a delivery to close from memory. - guard persistEvents - else { - bufferData.append(frame) - bufferedEventCount += 1 - return (true, nil) - } - writingEvents += 1 - writingBytes += frame.count - return (true, StagedFrames(frames: frame, summariesFrom: frame.count, events: 1, summaries: 0)) - } - if let taken { - write(taken) - } - if !admitted { - os_log("%s dropping an event the store has no room for", log: logger, type: .debug, typeName(and: #function)) - } - return admitted - } - } - func closeBatch() -> EventBatch? { guard let disk else { return closeInMemoryBatches().first } @@ -782,14 +735,9 @@ extension EventStore { else { return nil } return takeStaged() } - if let taken { - write(taken) - } - } + guard let taken + else { return } - /// Requires `ioLock`, and `taken` counted as being written. - private func write(_ taken: StagedFrames) { - ioLock.assertOwned() let bytes = taken.frames let base = bytes.startIndex var written = 0 diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift index 7f2bcfd82..301fd93c5 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift @@ -4,8 +4,8 @@ import OSLog /// Turns what `DirectEventProcessor` records into what it stages: evaluations into summary counters, and events into the /// JSON the events endpoint accepts. Android's `OutboundEventBuffer` plays the same part. /// -/// The summarizer is only to be used while holding the processor's `recordLock`, and the encoders are only reached under -/// its `commitLock`, except the pair `encodeAlone(_:)` uses, which have a lock of their own. +/// It has no lock of its own. The summarizer is only to be used while holding the processor's `recordLock`, and the +/// encoders are only reached under its `commitLock`. final class OutboundEventBuffer { /// Which encoder recorded events go through. /// @@ -32,22 +32,12 @@ final class OutboundEventBuffer { /// `commitLock`. private let eventJSONWriter: EventJSONWriter - /// Guards the encoders a `track` or `identify` writing itself goes through, which are separate from the commit's so - /// that it never waits for a commit to finish encoding. - private let aloneLock = UnfairLock() - /// Only to be used while holding `aloneLock`. - private let aloneEncoder: JSONEncoder - /// Only to be used while holding `aloneLock`. - private let aloneJSONWriter: EventJSONWriter - init(config: LDConfig, encoding: Encoding) { self.encoding = encoding self.logger = config.logger self.contextSummarizer = ContextSummarizer(logger: config.logger, maxContexts: config.eventCapacity) self.encoder = OutboundEventBuffer.makeEncoder(config: config) self.eventJSONWriter = EventJSONWriter(config: config) - self.aloneEncoder = OutboundEventBuffer.makeEncoder(config: config) - self.aloneJSONWriter = EventJSONWriter(config: config) } /// Counts an evaluation into its context's summary. Requires the processor's `recordLock`. @@ -74,18 +64,7 @@ final class OutboundEventBuffer { /// Requires the processor's `commitLock`. func encode(_ event: Event) -> Data? { - logged(encoding == .codable ? try? encoder.encode(event) : eventJSONWriter.encode(event), event) - } - - /// `encode(_:)` for an event encoded outside a commit, on any thread. - func encodeAlone(_ event: Event) -> Data? { - let encoded = aloneLock.withLock { - encoding == .codable ? try? aloneEncoder.encode(event) : aloneJSONWriter.encode(event) - } - return logged(encoded, event) - } - - private func logged(_ encoded: Data?, _ event: Event) -> Data? { + let encoded: Data? = encoding == .codable ? try? encoder.encode(event) : eventJSONWriter.encode(event) guard let encoded = encoded else { os_log("%s Failed to serialize event for publication: %s", log: logger, type: .error, typeName(and: #function), String(describing: event)) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 1f5ee2146..7c8d90c70 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -106,11 +106,6 @@ class DirectEventProcessorSpec: QuickSpec { events.compactMap { encodeToLDValue($0) } } - /// `recordedEventsAsJSON()` through `unordered(_:)`. - func recordedEventsUnordered() -> [String] { - unordered(recordedEventsAsJSON()) - } - /// Flushes, and answers whether a flush made from the completion got anywhere. /// /// A completion is free to flush again, which only works where it is not running on the queue deliveries go @@ -647,7 +642,7 @@ class DirectEventProcessorSpec: QuickSpec { expect(testContext.eventProcessor.isOnline) == false expect(testContext.eventProcessor.isReportingActive) == false expect(testContext.serviceMock.publishEventDataCallCount) == 0 - expect(unordered(testContext.committedEvents())) == testContext.recordedEventsUnordered() + expect(testContext.committedEvents()) == testContext.recordedEventsAsJSON() } } } @@ -665,7 +660,7 @@ class DirectEventProcessorSpec: QuickSpec { expect(testContext.eventProcessor.isOnline) == false expect(testContext.eventProcessor.isReportingActive) == false expect(testContext.serviceMock.publishEventDataCallCount) == 0 - expect(unordered(testContext.committedEvents())) == testContext.recordedEventsUnordered() + expect(testContext.committedEvents()) == testContext.recordedEventsAsJSON() } it("does not record a dropped event to diagnosticCache") { expect(testContext.diagnosticCache.incrementDroppedEventCountCallCount) == 0 @@ -683,7 +678,7 @@ class DirectEventProcessorSpec: QuickSpec { expect(testContext.eventProcessor.isOnline) == false expect(testContext.eventProcessor.isReportingActive) == false expect(testContext.serviceMock.publishEventDataCallCount) == 0 - expect(unordered(testContext.committedEvents())) == testContext.recordedEventsUnordered() + expect(testContext.committedEvents()) == testContext.recordedEventsAsJSON() } it("records a dropped event to diagnosticCache") { expect(testContext.diagnosticCache.incrementDroppedEventCountCallCount) == 1 @@ -1095,8 +1090,10 @@ class DirectEventProcessorSpec: QuickSpec { let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) valueIsArray(published) { valueArray in expect(valueArray.count) == testContext.events.count + 1 - expect(unordered(valueArray.filter { $0.kindField != "summary" })) == testContext.recordedEventsUnordered() - expect(valueArray.filter { $0.kindField == "summary" }.count) == 1 + expect(Array(valueArray.prefix(testContext.events.count))) == testContext.events.map { encodeToLDValue($0) } + valueIsObject(valueArray[testContext.events.count]) { summaryObject in + expect(summaryObject["kind"]) == "summary" + } } expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 @@ -1122,7 +1119,7 @@ class DirectEventProcessorSpec: QuickSpec { erOnline() expect(testContext.serviceMock.publishEventDataCallCount) == 1 let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) - valueIsArray(published) { expect(unordered($0)) == testContext.recordedEventsUnordered() } + expect(published) == encodeToLDValue(testContext.events) expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == testContext.events.count expect(testContext.store.pendingEventCount) == 0 @@ -1273,8 +1270,10 @@ class DirectEventProcessorSpec: QuickSpec { let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) valueIsArray(published) { valueArray in expect(valueArray.count) == testContext.events.count + 1 - expect(unordered(valueArray.filter { $0.kindField != "summary" })) == testContext.recordedEventsUnordered() - expect(valueArray.filter { $0.kindField == "summary" }.count) == 1 + expect(Array(valueArray.prefix(testContext.events.count))) == testContext.events.map { encodeToLDValue($0) } + valueIsObject(valueArray[testContext.events.count]) { summaryObject in + expect(summaryObject["kind"]) == "summary" + } } expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 @@ -1318,8 +1317,10 @@ class DirectEventProcessorSpec: QuickSpec { let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) valueIsArray(published) { valueArray in expect(valueArray.count) == testContext.events.count + 1 - expect(unordered(valueArray.filter { $0.kindField != "summary" })) == testContext.recordedEventsUnordered() - expect(valueArray.filter { $0.kindField == "summary" }.count) == 1 + expect(Array(valueArray.prefix(testContext.events.count))) == testContext.events.map { encodeToLDValue($0) } + valueIsObject(valueArray[testContext.events.count]) { summaryObject in + expect(summaryObject["kind"]) == "summary" + } } expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 @@ -1366,8 +1367,10 @@ class DirectEventProcessorSpec: QuickSpec { let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) valueIsArray(published) { valueArray in expect(valueArray.count) == testContext.events.count + 1 - expect(unordered(valueArray.filter { $0.kindField != "summary" })) == testContext.recordedEventsUnordered() - expect(valueArray.filter { $0.kindField == "summary" }.count) == 1 + expect(Array(valueArray.prefix(testContext.events.count))) == testContext.events.map { encodeToLDValue($0) } + valueIsObject(valueArray[testContext.events.count]) { summaryObject in + expect(summaryObject["kind"]) == "summary" + } } expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 @@ -1446,8 +1449,8 @@ class DirectEventProcessorSpec: QuickSpec { // Being offline delays delivery, not durability: a flush is a commit point, so the recorded events and // the evaluations counted so far are on disk even though nothing can be sent. let pending = testContext.committedEvents() - expect(unordered(pending.filter { $0.kindField != "summary" })) == testContext.recordedEventsUnordered() - expect(pending.filter { $0.kindField == "summary" }.count) == 1 + expect(pending.dropLast().map { $0 }) == testContext.recordedEventsAsJSON() + expect(pending.last?.kindField) == "summary" expect(testContext.eventProcessor.withContextSummarizer { $0.hasLoggedRequests }) == false guard case .isOffline = testContext.syncResult else { @@ -1659,7 +1662,7 @@ class DirectEventProcessorSpec: QuickSpec { let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) valueIsArray(published) { valueArray in expect(valueArray.count) == testContext.events.count - expect(unordered(valueArray)) == testContext.recordedEventsUnordered() + expect(valueArray) == testContext.events.map { encodeToLDValue($0) } } } } @@ -1735,60 +1738,22 @@ extension DirectEventProcessorSpec { expect(eventsLeftOnDisk().count).toEventually(equal(EventCommitter.pendingCommitThreshold), timeout: Constants.deliveryTimeout) } - itPersisting("writes only a tracked event itself, and leaves the evaluations before it to the commit queue") { - // Suspended, so that what is on disk when the track returns is what the track wrote. + itPersisting("has written the evaluations that came before a tracked event") { + // A response date of its own, because the distant past that a fresh context defaults to also satisfies + // the debug window comparison and would add a debug event to what is asserted below. The commit queue is + // suspended, so the only write there can be is the one the track makes on this thread. let commits = DispatchQueue(label: "com.launchdarkly.tests.suspended") commits.suspend() - // A response date of its own, because the distant past that a fresh context defaults to also satisfies - // the debug window comparison and would add a debug event to what is asserted below. + defer { commits.resume() } testContext = TestContext(lastEventResponseDate: Date(), commitQueue: commits) - let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: true) + let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: false) + // An untracked flag is only ever reported as a summary, so this is the case where the exposure exists + // nowhere but in memory until something forces it out. testContext.eventProcessor.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: testContext.context, includeReason: false) testContext.eventProcessor.record(CustomEvent(key: "fatal-error", context: testContext.context, data: nil)) - expect(eventsLeftOnDisk().compactMap { $0.kindField }) == ["custom"] - commits.resume() - expect(eventsLeftOnDisk().compactMap { $0.kindField }) - .toEventually(equal(["custom", "feature", "summary"]), timeout: Constants.deliveryTimeout) - } - - itPersisting("writes a tracked event without waiting for a commit in progress") { - var config = LDConfig.stub - config.eventCapacity = 100 - config.eventPersistence = .immediate - let serviceMock = DarklyServiceMock() - serviceMock.config = config - let inner = EventStore.temporary(capacity: config.eventCapacity) - defer { inner.deleteEverything() } - let commits = DispatchQueue(label: "com.launchdarkly.tests.commits") - let onCommits = DispatchSpecificKey() - commits.setSpecific(key: onCommits, value: true) - let store = ClosingHookStore(inner) - let stalled = DispatchSemaphore(value: 0) - let release = DispatchSemaphore(value: 0) - defer { release.signal() } - // A commit on the commit queue stalled on the disk, holding the commit lock while it is. - store.beforeCommitting = { - if DispatchQueue.getSpecific(key: onCommits) == true { - stalled.signal() - _ = release.wait(timeout: .now() + 10) - } - } - let reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, commitQueue: commits, pacing: .noWaits) - let context = LDContext.stub() - let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: true) - for _ in 0.. [String] { @@ -2202,6 +2174,10 @@ extension DirectEventProcessorSpec { expect(finished.wait(timeout: .now() + 10)) == .success expect(missingOnReturn).to(beEmpty()) + commitsLock.lock() + let totalCommits = commits + commitsLock.unlock() + expect(totalCommits) < trackers } itPersisting("returns from closing within its budget while the final commit is stalled on the disk, and still writes it") { @@ -3749,14 +3725,6 @@ private final class EncodingGate: CustomEvent { } } -/// Events in an order of their own, for comparing what was staged with what was recorded: at `.immediate` a custom or -/// identify event is staged ahead of the evaluations held before it. -private func unordered(_ events: [LDValue]) -> [String] { - let encoder = JSONEncoder() - encoder.outputFormatting = .sortedKeys - return events.map { String(decoding: (try? encoder.encode($0)) ?? Data(), as: UTF8.self) }.sorted() -} - /// A store that runs a hook on either side of closing a batch, and remembers the custom event keys of each batch closed. private final class ClosingHookStore: EventStoring { let inner: EventStoring @@ -3777,12 +3745,11 @@ private final class ClosingHookStore: EventStoring { func stage(_ encodedEvent: Data, bypassingCapacity: Bool) -> Bool { inner.stage(encodedEvent, bypassingCapacity: bypassingCapacity) } func reserve(_ events: Int) { inner.reserve(events) } func stageReserved(_ encodedEvent: Data) -> Bool { inner.stageReserved(encodedEvent) } - func releaseReservations(_ events: Int) { inner.releaseReservations(events) } + func releaseReservations() { inner.releaseReservations() } func commit() { beforeCommitting?() inner.commit() } - func writeReservedAlone(_ encodedEvent: Data) -> Bool { inner.writeReservedAlone(encodedEvent) } var holdsUnclosedEvents: Bool { inner.holdsUnclosedEvents } func takeVanishedLogEvents() -> Int { inner.takeVanishedLogEvents() } func listPendingBatches() -> BatchListing { inner.listPendingBatches() } @@ -3893,9 +3860,8 @@ private final class SlowListingStore: EventStoring { func stage(_ encodedEvent: Data, bypassingCapacity: Bool) -> Bool { inner.stage(encodedEvent, bypassingCapacity: bypassingCapacity) } func reserve(_ events: Int) { inner.reserve(events) } func stageReserved(_ encodedEvent: Data) -> Bool { inner.stageReserved(encodedEvent) } - func releaseReservations(_ events: Int) { inner.releaseReservations(events) } + func releaseReservations() { inner.releaseReservations() } func commit() { inner.commit() } - func writeReservedAlone(_ encodedEvent: Data) -> Bool { inner.writeReservedAlone(encodedEvent) } func closeBatch() -> EventBatch? { inner.closeBatch() } var holdsUnclosedEvents: Bool { inner.holdsUnclosedEvents } func takeVanishedLogEvents() -> Int { inner.takeVanishedLogEvents() } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift index 3f56b535e..ecaf0ecfe 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -438,37 +438,6 @@ final class EventStoreSpec: QuickSpec { expect(EventStoreSpec.keys(in: store.payloadsOnDisk)) == ["event-0", "event-1", "event-2", "event-3", "event-4"] } - - it("writes an event on its own, leaving what is staged to its commit") { - _ = store.stage(EventStoreSpec.payload("staged")) - store.reserve(1) - - expect(store.writeReservedAlone(EventStoreSpec.payload("alone"))) == true - - expect(EventStoreSpec.keys(in: store.payloadsOnDisk)) == ["alone"] - expect(store.pendingEventCount) == 2 - store.commit() - expect(EventStoreSpec.keys(in: store.payloadsOnDisk)) == ["alone", "staged"] - } - - it("keeps the place of an event reserved to be written on its own") { - let full = EventStore.temporary(capacity: 1) - defer { full.deleteEverything() } - full.reserve(1) - - // What another thread recording meanwhile finds. - expect(full.stage(EventStoreSpec.payload("staged"))) == false - expect(full.writeReservedAlone(EventStoreSpec.payload("alone"))) == true - expect(full.pendingEventCount) == 1 - } - - it("releases only the reservations it is told to") { - store.reserve(1) - store.reserve(2) - store.releaseReservations(2) - - expect(store.pendingEventCount) == 1 - } } } @@ -2225,16 +2194,6 @@ final class EventStoreSpec: QuickSpec { expect(try store.body(of: held)) == (try written.body(of: onDisk)) } - it("delivers an event written on its own from memory when the write fails") { - store.reserve(1) - expect(store.writeReservedAlone(EventStoreSpec.payload("alone"))) == true - - expect(store.pendingEventCount) == 1 - let batch = store.closeBatch() - expect(batch?.eventCount) == 1 - expect(try store.body(of: batch!)) == Data("[{\"kind\":\"custom\",\"key\":\"alone\"}]".utf8) - } - it("keeps accepting events after persistence has been given up on") { _ = store.stage(EventStoreSpec.payload("before")) store.commit() From e3684a8d7b08e08345099e77ba2ac8d9439fd35d Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Wed, 4 Nov 2026 09:08:17 -0800 Subject: [PATCH 144/158] Mark the experimental persistence levels with a warning callout Co-authored-by: Cursor --- LaunchDarkly/LaunchDarkly/Models/LDConfig.swift | 16 +++++++++------- .../LaunchDarkly/ObjectiveC/ObjcLDConfig.swift | 6 ++++-- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift index e5665d4de..c4ee27de9 100644 --- a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift @@ -25,9 +25,9 @@ public enum LDStreamingMode { /// recorded since the last one — including the crash an application was reporting when it died. Writing events to /// disk is what closes that gap: they are delivered on a later launch instead of being lost. /// -/// `.deferred` and `.immediate` are not stable in this major version, and not subject to any backwards compatibility -/// guarantees or semantic versioning. They are experimental: their behavior, their cost on the calling thread, and -/// what they leave on disk may change in a minor release. +/// - Warning: Experimental. `.deferred` and `.immediate` are not stable in this major version, and not subject to any +/// backwards compatibility guarantees or semantic versioning: their behavior, their cost on the calling thread, and +/// what they leave on disk may change in a minor release. @objc(LDEventPersistence) public enum EventPersistence: Int { /// Events are kept in memory only, and nothing survives the process ending. /// @@ -44,7 +44,7 @@ public enum LDStreamingMode { /// a flush so that an application ending the process on purpose can rely on it, and entering the background because /// that is when the system is most likely to end it. /// - /// This level is experimental; see `EventPersistence`. + /// - Warning: Experimental; see `EventPersistence`. case deferred /// Events are written to disk before `track` and `identify` return. /// @@ -55,7 +55,7 @@ public enum LDStreamingMode { /// instant after `track` returns still reports that event on the next launch. Evaluating a flag stays in memory /// regardless. /// - /// This level is experimental; see `EventPersistence`. + /// - Warning: Experimental; see `EventPersistence`. case immediate } @@ -378,8 +378,10 @@ public struct LDConfig { /// Anything other than `.disabled` appends events to a log under Application Support and delivers them on a later /// run, which is what lets the crash an application was reporting when it died reach LaunchDarkly at all. /// `.immediate` additionally puts the write on the thread that called `track` or `identify`, so there is no window - /// in which the event exists only in memory. Both levels other than `.disabled` are experimental, and not subject to - /// any backwards compatibility guarantees or semantic versioning. (Default: `.disabled`) + /// in which the event exists only in memory. (Default: `.disabled`) + /// + /// - Warning: Experimental. Both levels other than `.disabled` are not stable, and not subject to any backwards + /// compatibility guarantees or semantic versioning. public var eventPersistence: EventPersistence = Defaults.eventPersistence /// The maximum number of bytes of events the SDK keeps, on disk and in memory together, whatever `eventPersistence` is. It is for diff --git a/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift b/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift index da985578f..c629ccc15 100644 --- a/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift @@ -46,8 +46,10 @@ public final class ObjcLDConfig: NSObject { /// Without this, an event lives in memory until it is delivered, and a process that dies before the next flush /// takes everything recorded since the last one, including the crash an application was reporting when it died. /// `.immediate` additionally puts the write on the thread that called `track` or `identify`, so there is no window - /// in which the event exists only in memory. Both levels other than `.disabled` are experimental, and not subject to - /// any backwards compatibility guarantees or semantic versioning. (Default: `.disabled`) + /// in which the event exists only in memory. (Default: `.disabled`) + /// + /// - Warning: Experimental. Both levels other than `.disabled` are not stable, and not subject to any backwards + /// compatibility guarantees or semantic versioning. @objc public var eventPersistence: EventPersistence { get { config.eventPersistence } set { config.eventPersistence = newValue } From 635daf1164eb35ffda18c3d54d960d405ca17e1b Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Wed, 4 Nov 2026 09:15:36 -0800 Subject: [PATCH 145/158] Build the event persistence test helpers in every configuration, not only Debug Co-authored-by: Cursor --- .../ServiceObjects/DirectEventProcessor.swift | 50 +++++++++---------- .../ServiceObjects/EventCommitter.swift | 26 +++++----- .../EventDeliveryScheduler.swift | 29 +++++------ .../ServiceObjects/EventDirectory.swift | 2 - LaunchDarkly/LaunchDarkly/UnfairLock.swift | 4 +- 5 files changed, 49 insertions(+), 62 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 3eacd6ee7..5b9095335 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -943,35 +943,33 @@ extension DirectEventProcessor { extension DirectEventProcessor: TypeIdentifying { } -#if DEBUG - extension DirectEventProcessor { - /// Full events accepted but not yet handed to a commit. - var pendingEventsForTesting: [Event] { - committer.pendingEventsForTesting - } +extension DirectEventProcessor { + /// Full events accepted but not yet handed to a commit. + var pendingEventsForTesting: [Event] { + committer.pendingEventsForTesting + } - func setLastEventResponseDate(_ date: Date) { - committer.setLastEventResponseDate(date) - } + func setLastEventResponseDate(_ date: Date) { + committer.setLastEventResponseDate(date) + } - /// Occupies the queue a delivery runs on, so a test can see what one that has not started yet collects. - func occupyQueue(until released: DispatchSemaphore) { - deliveryQueue.async { _ = released.wait(timeout: .now() + 10) } - } + /// Occupies the queue a delivery runs on, so a test can see what one that has not started yet collects. + func occupyQueue(until released: DispatchSemaphore) { + deliveryQueue.async { _ = released.wait(timeout: .now() + 10) } + } - /// Callers waiting on the queued delivery. - var queuedDeliveryWaiterCount: Int { - deliveryScheduler.queuedWaiterCount - } + /// Callers waiting on the queued delivery. + var queuedDeliveryWaiterCount: Int { + deliveryScheduler.queuedWaiterCount + } - /// Deliveries started, counting one for a run of flushes that joined each other. - var startedDeliveryCount: Int { - deliveryScheduler.startedCount - } + /// Deliveries started, counting one for a run of flushes that joined each other. + var startedDeliveryCount: Int { + deliveryScheduler.startedCount + } - /// As `EventCommitter.withContextSummarizer(_:)`. - func withContextSummarizer(_ read: (ContextSummarizer) -> T) -> T { - committer.withContextSummarizer(read) - } + /// As `EventCommitter.withContextSummarizer(_:)`. + func withContextSummarizer(_ read: (ContextSummarizer) -> T) -> T { + committer.withContextSummarizer(read) } -#endif +} diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift index bf2b3d45e..fe716e5c5 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift @@ -479,20 +479,18 @@ final class EventCommitter { extension EventCommitter: TypeIdentifying { } -#if DEBUG - extension EventCommitter { - /// Full events accepted but not yet handed to a commit. - var pendingEventsForTesting: [Event] { - recordLock.withLock { pending } - } +extension EventCommitter { + /// Full events accepted but not yet handed to a commit. + var pendingEventsForTesting: [Event] { + recordLock.withLock { pending } + } - func setLastEventResponseDate(_ date: Date) { - responseDateLock.withLock { responseDate = date } - } + func setLastEventResponseDate(_ date: Date) { + responseDateLock.withLock { responseDate = date } + } - /// Reads the summarizer under `recordLock`, which a scheduled commit may be holding to take its counters. - func withContextSummarizer(_ read: (ContextSummarizer) -> T) -> T { - recordLock.withLock { read(eventBuffer.contextSummarizer) } - } + /// Reads the summarizer under `recordLock`, which a scheduled commit may be holding to take its counters. + func withContextSummarizer(_ read: (ContextSummarizer) -> T) -> T { + recordLock.withLock { read(eventBuffer.contextSummarizer) } } -#endif +} diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift index dabe01aad..f117acebb 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift @@ -150,10 +150,9 @@ final class EventDeliveryScheduler { /// one rather than one each. Only to be used while holding `lock`. private var isRetryScheduled = false - #if DEBUG - /// How many deliveries have started, which is fewer than the flushes asked for where they were joined. - private var deliveryRunCount = 0 - #endif + /// How many deliveries have started, which is fewer than the flushes asked for where they were joined. Only to be + /// used while holding `lock`. + private var deliveryRunCount = 0 init(pacesDeliveries: Bool, gap: TimeInterval) { self.pacesDeliveries = pacesDeliveries @@ -214,9 +213,7 @@ final class EventDeliveryScheduler { isPaced: next.isPaced && pacesDeliveries && !next.isClosing, gap: gap) running = run - #if DEBUG - deliveryRunCount += 1 - #endif + deliveryRunCount += 1 return (run, next.waiters) } } @@ -252,15 +249,13 @@ final class EventDeliveryScheduler { lock.withLock { isRetryScheduled = false } } - #if DEBUG - /// Callers waiting on the queued delivery. - var queuedWaiterCount: Int { - lock.withLock { queued?.waiters.count ?? 0 } - } + /// Callers waiting on the queued delivery. + var queuedWaiterCount: Int { + lock.withLock { queued?.waiters.count ?? 0 } + } - /// Deliveries started, counting one for a run of flushes that joined each other. - var startedCount: Int { - lock.withLock { deliveryRunCount } - } - #endif + /// Deliveries started, counting one for a run of flushes that joined each other. + var startedCount: Int { + lock.withLock { deliveryRunCount } + } } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index 8090602ac..624e2dbf9 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -1005,12 +1005,10 @@ enum BatchLocks { } } - #if DEBUG /// The lock every call on a batch takes: its directory's, whichever store names the batch. static func lockForTesting(_ batch: URL) -> UnfairLock { directory(of: batch).lock } - #endif } extension EventDirectory { diff --git a/LaunchDarkly/LaunchDarkly/UnfairLock.swift b/LaunchDarkly/LaunchDarkly/UnfairLock.swift index 39adcc92d..44aad3249 100644 --- a/LaunchDarkly/LaunchDarkly/UnfairLock.swift +++ b/LaunchDarkly/LaunchDarkly/UnfairLock.swift @@ -35,9 +35,7 @@ final class UnfairLock { /// Traps unless the calling thread holds the lock. Debug builds only, since it is called on recording paths. @inline(__always) func assertOwned() { - #if DEBUG - os_unfair_lock_assert_owner(unfairLock) - #endif + assert({ os_unfair_lock_assert_owner(unfairLock); return true }()) } } #else From f2793c1acf7131cb36e9b3f6a12f3d21bb15dfec Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Wed, 4 Nov 2026 10:41:39 -0800 Subject: [PATCH 146/158] Keep the lock ownership check to Debug builds Co-authored-by: Cursor --- LaunchDarkly/LaunchDarkly/UnfairLock.swift | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/LaunchDarkly/LaunchDarkly/UnfairLock.swift b/LaunchDarkly/LaunchDarkly/UnfairLock.swift index 44aad3249..39adcc92d 100644 --- a/LaunchDarkly/LaunchDarkly/UnfairLock.swift +++ b/LaunchDarkly/LaunchDarkly/UnfairLock.swift @@ -35,7 +35,9 @@ final class UnfairLock { /// Traps unless the calling thread holds the lock. Debug builds only, since it is called on recording paths. @inline(__always) func assertOwned() { - assert({ os_unfair_lock_assert_owner(unfairLock); return true }()) + #if DEBUG + os_unfair_lock_assert_owner(unfairLock) + #endif } } #else From 6c68152ff9068ba5f6147e34c942b78460685e48 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Wed, 4 Nov 2026 12:11:52 -0800 Subject: [PATCH 147/158] Guard the diagnostic cache mock with a lock, as the SDK calls it from several threads Co-authored-by: Cursor --- LaunchDarkly.xcodeproj/project.pbxproj | 4 + .../GeneratedCode/mocks.generated.swift | 47 ------ .../Cache/DiagnosticCache.swift | 1 - .../Mocks/DiagnosticCachingMock.swift | 134 ++++++++++++++++++ 4 files changed, 138 insertions(+), 48 deletions(-) create mode 100644 LaunchDarkly/LaunchDarklyTests/Mocks/DiagnosticCachingMock.swift diff --git a/LaunchDarkly.xcodeproj/project.pbxproj b/LaunchDarkly.xcodeproj/project.pbxproj index 7999ec30a..32e9d1202 100644 --- a/LaunchDarkly.xcodeproj/project.pbxproj +++ b/LaunchDarkly.xcodeproj/project.pbxproj @@ -201,6 +201,7 @@ 83B9A082204F6022000C3F17 /* FlagsUnchangedObserver.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83B9A081204F6022000C3F17 /* FlagsUnchangedObserver.swift */; }; 83CFE7CE1F7AD81D0010544E /* DirectEventProcessorSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83CFE7CD1F7AD81D0010544E /* DirectEventProcessorSpec.swift */; }; 83CFE7D11F7AD8DC0010544E /* DarklyServiceMock.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83CFE7D01F7AD8DC0010544E /* DarklyServiceMock.swift */; }; + A1D1A6C12F0B000000C0FFEE /* DiagnosticCachingMock.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D1A6C02F0B000000C0FFEE /* DiagnosticCachingMock.swift */; }; 83D559741FD87CC9002D10C8 /* KeyedValueCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83D559731FD87CC9002D10C8 /* KeyedValueCache.swift */; }; 83D9EC752062DEAB004D7FA6 /* LDCommon.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83B6C4B51F4DE7630055351C /* LDCommon.swift */; }; 83D9EC762062DEAB004D7FA6 /* LDConfig.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8354EFDD1F26380700C05156 /* LDConfig.swift */; }; @@ -625,6 +626,7 @@ 83B9A081204F6022000C3F17 /* FlagsUnchangedObserver.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FlagsUnchangedObserver.swift; sourceTree = ""; }; 83CFE7CD1F7AD81D0010544E /* DirectEventProcessorSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DirectEventProcessorSpec.swift; sourceTree = ""; }; 83CFE7D01F7AD8DC0010544E /* DarklyServiceMock.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DarklyServiceMock.swift; sourceTree = ""; }; + A1D1A6C02F0B000000C0FFEE /* DiagnosticCachingMock.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DiagnosticCachingMock.swift; sourceTree = ""; }; 83D559731FD87CC9002D10C8 /* KeyedValueCache.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeyedValueCache.swift; sourceTree = ""; }; 83D9EC6B2062DBB7004D7FA6 /* LaunchDarkly_watchOS.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = LaunchDarkly_watchOS.framework; sourceTree = BUILT_PRODUCTS_DIR; }; 83DDBEF51FA24A7E00E428B6 /* Data.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Data.swift; sourceTree = ""; }; @@ -982,6 +984,7 @@ isa = PBXGroup; children = ( 83CFE7D01F7AD8DC0010544E /* DarklyServiceMock.swift */, + A1D1A6C02F0B000000C0FFEE /* DiagnosticCachingMock.swift */, 832307A71F7DA61B0029815A /* LDEventSourceMock.swift */, 832307A91F7ECA630029815A /* LDConfigStub.swift */, A33A5F7928466D04000C29C7 /* LDContextStub.swift */, @@ -1897,6 +1900,7 @@ B40B419C249ADA6B00CD0726 /* DiagnosticCacheSpec.swift in Sources */, 83F0A5641FB5F33800550A95 /* LDConfigSpec.swift in Sources */, 83CFE7D11F7AD8DC0010544E /* DarklyServiceMock.swift in Sources */, + A1D1A6C12F0B000000C0FFEE /* DiagnosticCachingMock.swift in Sources */, 832307AA1F7ECA630029815A /* LDConfigStub.swift in Sources */, A3C6F7622B7FA803005B3B61 /* SheddingQueueSpec.swift in Sources */, A33A5F7A28466D04000C29C7 /* LDContextStub.swift in Sources */, diff --git a/LaunchDarkly/GeneratedCode/mocks.generated.swift b/LaunchDarkly/GeneratedCode/mocks.generated.swift index a982005e9..63f7ab293 100644 --- a/LaunchDarkly/GeneratedCode/mocks.generated.swift +++ b/LaunchDarkly/GeneratedCode/mocks.generated.swift @@ -38,53 +38,6 @@ final class DarklyStreamingProviderMock: DarklyStreamingProvider { } } -// MARK: - DiagnosticCachingMock -final class DiagnosticCachingMock: DiagnosticCaching { - - var getDiagnosticIdCallCount = 0 - var getDiagnosticIdCallback: (() throws -> Void)? - var getDiagnosticIdReturnValue: DiagnosticId! - func getDiagnosticId() -> DiagnosticId { - getDiagnosticIdCallCount += 1 - try! getDiagnosticIdCallback?() - return getDiagnosticIdReturnValue - } - - var getCurrentStatsAndResetCallCount = 0 - var getCurrentStatsAndResetCallback: (() throws -> Void)? - var getCurrentStatsAndResetReturnValue: DiagnosticStats! - func getCurrentStatsAndReset() -> DiagnosticStats { - getCurrentStatsAndResetCallCount += 1 - try! getCurrentStatsAndResetCallback?() - return getCurrentStatsAndResetReturnValue - } - - var incrementDroppedEventCountCallCount = 0 - var incrementDroppedEventCountCallback: (() throws -> Void)? - func incrementDroppedEventCount() { - incrementDroppedEventCountCallCount += 1 - try! incrementDroppedEventCountCallback?() - } - - var recordEventsInLastBatchCallCount = 0 - var recordEventsInLastBatchCallback: (() throws -> Void)? - var recordEventsInLastBatchReceivedEventsInLastBatch: Int? - func recordEventsInLastBatch(eventsInLastBatch: Int) { - recordEventsInLastBatchCallCount += 1 - recordEventsInLastBatchReceivedEventsInLastBatch = eventsInLastBatch - try! recordEventsInLastBatchCallback?() - } - - var addStreamInitCallCount = 0 - var addStreamInitCallback: (() throws -> Void)? - var addStreamInitReceivedStreamInit: DiagnosticStreamInit? - func addStreamInit(streamInit: DiagnosticStreamInit) { - addStreamInitCallCount += 1 - addStreamInitReceivedStreamInit = streamInit - try! addStreamInitCallback?() - } -} - // MARK: - DiagnosticReportingMock final class DiagnosticReportingMock: DiagnosticReporting { diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/Cache/DiagnosticCache.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/Cache/DiagnosticCache.swift index d97ff5bfd..f628e2ffa 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/Cache/DiagnosticCache.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/Cache/DiagnosticCache.swift @@ -1,6 +1,5 @@ import Foundation -// sourcery: autoMockable protocol DiagnosticCaching { func getDiagnosticId() -> DiagnosticId func getCurrentStatsAndReset() -> DiagnosticStats diff --git a/LaunchDarkly/LaunchDarklyTests/Mocks/DiagnosticCachingMock.swift b/LaunchDarkly/LaunchDarklyTests/Mocks/DiagnosticCachingMock.swift new file mode 100644 index 000000000..cc818d990 --- /dev/null +++ b/LaunchDarkly/LaunchDarklyTests/Mocks/DiagnosticCachingMock.swift @@ -0,0 +1,134 @@ +import Foundation + +@testable import LaunchDarkly + +/// Written by hand rather than generated, because the SDK calls it from whichever thread records, commits or delivers +/// events while a test reads the counts. Every property is guarded by `lock`; callbacks run outside it. +final class DiagnosticCachingMock: DiagnosticCaching { + private let lock = NSLock() + + private var _getDiagnosticIdCallCount = 0 + private var _getDiagnosticIdCallback: (() throws -> Void)? + private var _getDiagnosticIdReturnValue: DiagnosticId! + + private var _getCurrentStatsAndResetCallCount = 0 + private var _getCurrentStatsAndResetCallback: (() throws -> Void)? + private var _getCurrentStatsAndResetReturnValue: DiagnosticStats! + + private var _incrementDroppedEventCountCallCount = 0 + private var _incrementDroppedEventCountCallback: (() throws -> Void)? + + private var _recordEventsInLastBatchCallCount = 0 + private var _recordEventsInLastBatchCallback: (() throws -> Void)? + private var _recordEventsInLastBatchReceivedEventsInLastBatch: Int? + + private var _addStreamInitCallCount = 0 + private var _addStreamInitCallback: (() throws -> Void)? + private var _addStreamInitReceivedStreamInit: DiagnosticStreamInit? + + private func synchronized(_ body: () -> T) -> T { + lock.lock() + defer { lock.unlock() } + return body() + } + + var getDiagnosticIdCallCount: Int { + get { synchronized { _getDiagnosticIdCallCount } } + set { synchronized { _getDiagnosticIdCallCount = newValue } } + } + var getDiagnosticIdCallback: (() throws -> Void)? { + get { synchronized { _getDiagnosticIdCallback } } + set { synchronized { _getDiagnosticIdCallback = newValue } } + } + var getDiagnosticIdReturnValue: DiagnosticId! { + get { synchronized { _getDiagnosticIdReturnValue } } + set { synchronized { _getDiagnosticIdReturnValue = newValue } } + } + func getDiagnosticId() -> DiagnosticId { + let (callback, returnValue) = synchronized { () -> ((() throws -> Void)?, DiagnosticId?) in + _getDiagnosticIdCallCount += 1 + return (_getDiagnosticIdCallback, _getDiagnosticIdReturnValue) + } + try! callback?() + return returnValue! + } + + var getCurrentStatsAndResetCallCount: Int { + get { synchronized { _getCurrentStatsAndResetCallCount } } + set { synchronized { _getCurrentStatsAndResetCallCount = newValue } } + } + var getCurrentStatsAndResetCallback: (() throws -> Void)? { + get { synchronized { _getCurrentStatsAndResetCallback } } + set { synchronized { _getCurrentStatsAndResetCallback = newValue } } + } + var getCurrentStatsAndResetReturnValue: DiagnosticStats! { + get { synchronized { _getCurrentStatsAndResetReturnValue } } + set { synchronized { _getCurrentStatsAndResetReturnValue = newValue } } + } + func getCurrentStatsAndReset() -> DiagnosticStats { + let (callback, returnValue) = synchronized { () -> ((() throws -> Void)?, DiagnosticStats?) in + _getCurrentStatsAndResetCallCount += 1 + return (_getCurrentStatsAndResetCallback, _getCurrentStatsAndResetReturnValue) + } + try! callback?() + return returnValue! + } + + var incrementDroppedEventCountCallCount: Int { + get { synchronized { _incrementDroppedEventCountCallCount } } + set { synchronized { _incrementDroppedEventCountCallCount = newValue } } + } + var incrementDroppedEventCountCallback: (() throws -> Void)? { + get { synchronized { _incrementDroppedEventCountCallback } } + set { synchronized { _incrementDroppedEventCountCallback = newValue } } + } + func incrementDroppedEventCount() { + let callback = synchronized { () -> (() throws -> Void)? in + _incrementDroppedEventCountCallCount += 1 + return _incrementDroppedEventCountCallback + } + try! callback?() + } + + var recordEventsInLastBatchCallCount: Int { + get { synchronized { _recordEventsInLastBatchCallCount } } + set { synchronized { _recordEventsInLastBatchCallCount = newValue } } + } + var recordEventsInLastBatchCallback: (() throws -> Void)? { + get { synchronized { _recordEventsInLastBatchCallback } } + set { synchronized { _recordEventsInLastBatchCallback = newValue } } + } + var recordEventsInLastBatchReceivedEventsInLastBatch: Int? { + get { synchronized { _recordEventsInLastBatchReceivedEventsInLastBatch } } + set { synchronized { _recordEventsInLastBatchReceivedEventsInLastBatch = newValue } } + } + func recordEventsInLastBatch(eventsInLastBatch: Int) { + let callback = synchronized { () -> (() throws -> Void)? in + _recordEventsInLastBatchCallCount += 1 + _recordEventsInLastBatchReceivedEventsInLastBatch = eventsInLastBatch + return _recordEventsInLastBatchCallback + } + try! callback?() + } + + var addStreamInitCallCount: Int { + get { synchronized { _addStreamInitCallCount } } + set { synchronized { _addStreamInitCallCount = newValue } } + } + var addStreamInitCallback: (() throws -> Void)? { + get { synchronized { _addStreamInitCallback } } + set { synchronized { _addStreamInitCallback = newValue } } + } + var addStreamInitReceivedStreamInit: DiagnosticStreamInit? { + get { synchronized { _addStreamInitReceivedStreamInit } } + set { synchronized { _addStreamInitReceivedStreamInit = newValue } } + } + func addStreamInit(streamInit: DiagnosticStreamInit) { + let callback = synchronized { () -> (() throws -> Void)? in + _addStreamInitCallCount += 1 + _addStreamInitReceivedStreamInit = streamInit + return _addStreamInitCallback + } + try! callback?() + } +} From 6d6875c2be23c9e24cbe0b96a210d4e4de9370e3 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Wed, 4 Nov 2026 14:43:13 -0800 Subject: [PATCH 148/158] Make the event delivery delay configurable, and hold the diagnostic init event whether or not events persist Co-authored-by: Cursor --- LaunchDarkly/LaunchDarkly/LDClient.swift | 6 +-- .../LaunchDarkly/Models/LDConfig.swift | 20 ++++++++++ .../ObjectiveC/ObjcLDConfig.swift | 9 +++++ .../ServiceObjects/DiagnosticReporter.swift | 2 +- .../ServiceObjects/DirectEventProcessor.swift | 34 ++++++++-------- .../Models/LDConfigSpec.swift | 18 +++++++++ .../DiagnosticReporterSpec.swift | 16 ++++++++ .../DirectEventProcessorSpec.swift | 40 ++++++++++++++----- 8 files changed, 114 insertions(+), 31 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/LDClient.swift b/LaunchDarkly/LaunchDarkly/LDClient.swift index 504d57371..baa28c6ad 100644 --- a/LaunchDarkly/LaunchDarkly/LDClient.swift +++ b/LaunchDarkly/LaunchDarkly/LDClient.swift @@ -1105,10 +1105,10 @@ public class LDClient { extension LDClient: TypeIdentifying { } extension LDClient { - /// Has the diagnostic init event wait for the event processor's launch gate, as a previous run's events do. Where - /// events do not persist there is no gate, and the init event goes out as it did before persistence. + /// Has the diagnostic init event wait for the event processor's launch gate, as a previous run's events do, whether + /// or not events persist. fileprivate func giveFlagValuesTheNetworkFirst() { - if let eventProcessor = eventProcessor as? DirectEventProcessor, eventProcessor.givesWayToFlags { + if let eventProcessor = eventProcessor as? DirectEventProcessor { (diagnosticReporter as? DiagnosticReporter)?.sendsInitAfter = eventProcessor.whenLaunchGateOpens } } diff --git a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift index c4ee27de9..6b77481c0 100644 --- a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift @@ -243,6 +243,11 @@ public struct LDConfig { /// The default behavior for the SDK is to keep events in memory only. static let eventPersistence = EventPersistence.disabled + /// The default time the SDK's own event and diagnostic traffic waits after first going online. (2 seconds) + static let eventDeliveryDelay: TimeInterval = 2 + /// The longest `eventDeliveryDelay` can be set to. (60 seconds) + static let maxEventDeliveryDelay: TimeInterval = 60 + /// The default timeout interval for flag requests and event reports. (10 seconds) static let connectionTimeout: TimeInterval = 10.0 /// The default time interval between event reports. (30 seconds) @@ -402,6 +407,20 @@ public struct LDConfig { /// It has no effect where `eventPersistence` is `.disabled`. public var eventMaxAge: TimeInterval = Defaults.eventMaxAge + private var _eventDeliveryDelay: TimeInterval = Defaults.eventDeliveryDelay + /// How long after the client first goes online the SDK holds back the traffic it starts on its own, so that the + /// first flag request has the network to itself. That is the diagnostic init payload, the events a previous run left + /// where `eventPersistence` keeps them, and the periodic delivery of this run's events. A flush the application asks + /// for is never held. Zero sends them as soon as the client is online. Values are kept between 0 and 60 seconds, so + /// an application that ends soon after every launch still sends what a previous run left. (Default: 2 seconds) + /// + /// - Warning: Experimental. This property is not stable, and not subject to any backwards compatibility guarantees + /// or semantic versioning. + public var eventDeliveryDelay: TimeInterval { + get { _eventDeliveryDelay } + set { _eventDeliveryDelay = newValue.isNaN ? Defaults.eventDeliveryDelay : min(max(newValue, 0), Defaults.maxEventDeliveryDelay) } + } + /// The timeout interval for flag requests and event reports. (Default: 10 seconds) public var connectionTimeout: TimeInterval = Defaults.connectionTimeout /// The time interval between event reports (Default: 30 seconds) @@ -634,6 +653,7 @@ extension LDConfig: Equatable { && lhs.eventPersistence == rhs.eventPersistence && lhs.eventCapacityBytes == rhs.eventCapacityBytes && lhs.eventMaxAge == rhs.eventMaxAge + && lhs.eventDeliveryDelay == rhs.eventDeliveryDelay && lhs.sendEvents == rhs.sendEvents && lhs.connectionTimeout == rhs.connectionTimeout && lhs.eventFlushInterval == rhs.eventFlushInterval diff --git a/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift b/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift index c629ccc15..89f12ca7b 100644 --- a/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift @@ -67,6 +67,15 @@ public final class ObjcLDConfig: NSObject { set { config.eventMaxAge = newValue } } + /// How long after the client first goes online the SDK holds back the traffic it starts on its own: the diagnostic init payload, the events a previous run left, and the periodic delivery of this run's events. A flush the application asks for is never held. Kept between 0 and 60 seconds. (Default: 2 seconds) + /// + /// - Warning: Experimental. This property is not stable, and not subject to any backwards compatibility guarantees + /// or semantic versioning. + @objc public var eventDeliveryDelay: TimeInterval { + get { config.eventDeliveryDelay } + set { config.eventDeliveryDelay = newValue } + } + /// The timeout interval for flag requests and event reports. (Default: 10 seconds) @objc public var connectionTimeout: TimeInterval { get { config.connectionTimeout } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DiagnosticReporter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DiagnosticReporter.swift index 55f792018..58d0e06fd 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DiagnosticReporter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DiagnosticReporter.swift @@ -19,7 +19,7 @@ class DiagnosticReporter: DiagnosticReporting { private let stateQueue = DispatchQueue(label: "com.launchdarkly.diagnosticReporter.state", qos: .utility) private let workQueue = DispatchQueue(label: "com.launchdarkly.diagnosticReporter.work", qos: .utility) /// Where set, what the init event waits for: the event processor's launch gate, so that it does not compete with the - /// first flag request. Set once the client has both. + /// first flag request. Set once the client has both; unset only where events are not sent, and so neither is this. var sendsInitAfter: ((@escaping () -> Void) -> Void)? init(service: DarklyServiceProvider, environmentReporting: EnvironmentReporting) { diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 5b9095335..fdae9e48c 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -11,20 +11,24 @@ import OSLog /// calls are commit points and which thread each commit runs on. This class delivers what is committed, and runs the /// reporter's life: flushes, going online and offline, and the flush at close. /// -/// Flag values come first. Where events persist, what a previous run left waits at launch until a fixed time after the -/// reporter first goes online, and the deliveries this reporter starts on its own send one batch at a time, a gap apart, -/// leaving room for flag requests. Both go by time alone: nothing here hears what the flag synchronizer is doing. A -/// delivery the application asks for is never held. +/// Flag values come first. What this reporter sends on its own waits at launch until `LDConfig.eventDeliveryDelay` after +/// it first goes online, and where events persist, the deliveries it starts on its own send one batch at a time, a gap +/// apart, leaving room for flag requests. Both go by time alone: nothing here hears what the flag synchronizer is doing. +/// A delivery the application asks for is never held. class DirectEventProcessor: EventProcessor { /// How long the deliveries this reporter starts on its own wait, so that flag values come first. struct Pacing { - static let specified = Pacing(launchDelay: 2, gap: 1) + /// The gap the specification gives, after the launch delay the application configured. + static func specified(launchDelay: TimeInterval) -> Pacing { + Pacing(launchDelay: launchDelay, gap: 1) + } /// No waits at all, for tests about something else. static let noWaits = Pacing(launchDelay: 0, gap: 0) - /// How long after first going online that a previous run's events, and the diagnostic init event, wait. It - /// gives the application's startup traffic, the first flag request among it, the network first, and is short - /// enough for an application that dies a few seconds after every launch to still send them. + /// How long after first going online that a previous run's events, this run's periodic delivery, and the + /// diagnostic init event wait. It gives the application's startup traffic, the first flag request among it, the + /// network first, and is short enough for an application that dies a few seconds after every launch to still + /// send them. let launchDelay: TimeInterval /// The least time between one paced batch's answer and the next batch's request. let gap: TimeInterval @@ -51,10 +55,6 @@ class DirectEventProcessor: EventProcessor { private let retryState: DeliveryRetryState private let pacing: Pacing - /// Whether events give way to flag values: only where they persist, like the rest of what persistence brings. - /// Without it there is no previous run to hold back, and delivery is what it was before persistence: no launch - /// gate and no pacing. - let givesWayToFlags: Bool var lastEventResponseDate: Date { committer.lastEventResponseDate @@ -144,8 +144,8 @@ class DirectEventProcessor: EventProcessor { private var hasEventsFromPreviousRun = false /// Whether the launch gate has opened. It opens once the launch delay has passed since first going online. Until then - /// neither a previous run's events nor the diagnostic init event are sent. Open from the start where events do not - /// give way. + /// neither a previous run's events, this run's periodic delivery, nor the diagnostic init event are sent. Open from + /// the start where there is no launch delay. private var launchGateOpen: Bool /// When the reporter first went online, which the gate counts from. private var wentOnlineAt: DispatchTime? @@ -158,7 +158,7 @@ class DirectEventProcessor: EventProcessor { encoding: OutboundEventBuffer.Encoding = .handWritten, commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.DirectEventProcessor.commitQueue", qos: .userInitiated), retryState: DeliveryRetryState? = nil, - pacing: Pacing = .specified) { + pacing: Pacing? = nil) { let eventStore = store ?? DirectEventProcessor.makeStore(config: service.config) let lostEvents = LostEventCount() self.service = service @@ -171,10 +171,10 @@ class DirectEventProcessor: EventProcessor { commitQueue: commitQueue, diagnosticCache: service.diagnosticCache, lostEvents: lostEvents) self.store = eventStore + let pacing = pacing ?? .specified(launchDelay: service.config.eventDeliveryDelay) self.pacing = pacing - self.givesWayToFlags = eventStore.isPersisting self.deliveryScheduler = EventDeliveryScheduler(pacesDeliveries: eventStore.isPersisting, gap: pacing.gap) - self.launchGateOpen = !eventStore.isPersisting + self.launchGateOpen = pacing.launchDelay <= 0 // A log left open by a previous run has to be closed before it can be delivered, but nothing waits on that: // doing it on the caller's thread would put file I/O in the way of the client starting up. Below the queue's diff --git a/LaunchDarkly/LaunchDarklyTests/Models/LDConfigSpec.swift b/LaunchDarkly/LaunchDarklyTests/Models/LDConfigSpec.swift index 0e18e078c..aac3c83d8 100644 --- a/LaunchDarkly/LaunchDarklyTests/Models/LDConfigSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/Models/LDConfigSpec.swift @@ -29,6 +29,7 @@ final class LDConfigSpec: XCTestCase { fileprivate static let maxCachedContexts = -1 fileprivate static let diagnosticOptOut = true fileprivate static let diagnosticRecordingInterval: TimeInterval = 600.0 + fileprivate static let eventDeliveryDelay: TimeInterval = 5 fileprivate static let wrapperName = "ReactNative" fileprivate static let wrapperVersion = "0.1.0" fileprivate static let additionalHeaders = ["Proxy-Authorization": "creds"] @@ -43,6 +44,7 @@ final class LDConfigSpec: XCTestCase { ("event persistence", Constants.eventPersistence, { c, v in c.eventPersistence = v as! EventPersistence }), ("event capacity bytes", Constants.eventCapacityBytes, { c, v in c.eventCapacityBytes = v as! Int }), ("event max age", Constants.eventMaxAge, { c, v in c.eventMaxAge = v as! TimeInterval }), + ("event delivery delay", Constants.eventDeliveryDelay, { c, v in c.eventDeliveryDelay = v as! TimeInterval }), ("connection timeout", Constants.connectionTimeout, { c, v in c.connectionTimeout = v as! TimeInterval }), ("event flush interval", Constants.eventFlushInterval, { c, v in c.eventFlushInterval = v as! TimeInterval }), ("poll interval", Constants.flagPollingInterval, { c, v in c.flagPollingInterval = v as! TimeInterval }), @@ -75,6 +77,7 @@ final class LDConfigSpec: XCTestCase { XCTAssertEqual(config.eventPersistence, LDConfig.Defaults.eventPersistence) XCTAssertEqual(config.eventCapacityBytes, 20 * 1024 * 1024) XCTAssertEqual(config.eventMaxAge, 7 * 24 * 60 * 60) + XCTAssertEqual(config.eventDeliveryDelay, 2) XCTAssertEqual(config.connectionTimeout, LDConfig.Defaults.connectionTimeout) XCTAssertEqual(config.eventFlushInterval, LDConfig.Defaults.eventFlushInterval) XCTAssertEqual(config.flagPollingInterval, LDConfig.Defaults.flagPollingInterval) @@ -108,6 +111,7 @@ final class LDConfigSpec: XCTestCase { XCTAssertEqual(config.eventCapacity, Constants.eventCapacity, "\(os)") XCTAssertEqual(config.eventCapacityBytes, Constants.eventCapacityBytes, "\(os)") XCTAssertEqual(config.eventMaxAge, Constants.eventMaxAge, "\(os)") + XCTAssertEqual(config.eventDeliveryDelay, Constants.eventDeliveryDelay, "\(os)") XCTAssertEqual(config.connectionTimeout, Constants.connectionTimeout, "\(os)") XCTAssertEqual(config.eventFlushInterval, Constants.eventFlushInterval, "\(os)") XCTAssertEqual(config.flagPollingInterval, Constants.flagPollingInterval, "\(os)") @@ -186,6 +190,20 @@ final class LDConfigSpec: XCTestCase { XCTAssertEqual(config.diagnosticRecordingInterval, config.minima.diagnosticRecordingInterval + 0.001) } + func testEventDeliveryDelayIsKeptBetweenZeroAndAMinute() { + var config = LDConfig(mobileKey: LDConfig.Constants.mockMobileKey, autoEnvAttributes: .disabled) + config.eventDeliveryDelay = 0 + XCTAssertEqual(config.eventDeliveryDelay, 0) + config.eventDeliveryDelay = -1 + XCTAssertEqual(config.eventDeliveryDelay, 0) + config.eventDeliveryDelay = 61 + XCTAssertEqual(config.eventDeliveryDelay, 60) + config.eventDeliveryDelay = .infinity + XCTAssertEqual(config.eventDeliveryDelay, 60) + config.eventDeliveryDelay = .nan + XCTAssertEqual(config.eventDeliveryDelay, 2) + } + func testEquals() { let defaultConfig = LDConfig(mobileKey: LDConfig.Constants.mockMobileKey, autoEnvAttributes: .disabled) // same config diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DiagnosticReporterSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DiagnosticReporterSpec.swift index 78dcf71ea..9d4b6f1d7 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DiagnosticReporterSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DiagnosticReporterSpec.swift @@ -112,6 +112,22 @@ final class DiagnosticReporterSpec: XCTestCase { tst.stop() } + func testInitEventWaitsForTheLaunchGateWhereEventsDoNotPersist() { + let tst = TestContext() + tst.queueResponse(status: 202) + let events = DirectEventProcessor(service: tst.service, onSyncComplete: nil, + store: EventStore.temporary(capacity: 10, persistEvents: false), + pacing: DirectEventProcessor.Pacing(launchDelay: 0.5, gap: 0)) + tst.subject.sendsInitAfter = events.whenLaunchGateOpens + events.isOnline = true + tst.subject.setMode(.foreground, online: true) + + tst.expectNoEvent() + XCTAssertEqual(tst.takeEvent().kind, .diagnosticInit) + events.isOnline = false + tst.stop() + } + func testRetries() { let tst = TestContext() tst.queueResponse(status: 500) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 7c8d90c70..508ff852b 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -452,22 +452,42 @@ class DirectEventProcessorSpec: QuickSpec { expect(testContext.serviceMock.publishEventDataCallCount) == 3 } - it("holds what waits for the launch gate only where events persist") { - let reporter = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: testContext.store, - pacing: DirectEventProcessor.Pacing(launchDelay: 1, gap: 0)) - nextRun = reporter - expect(reporter.givesWayToFlags) == self.persists + /// Whether what waits on `reporter`'s launch gate has run, `after` it went online. + func gateOpened(_ reporter: DirectEventProcessor, after: TimeInterval) -> Bool { let opened = UnfairLock() var hasOpened = false reporter.whenLaunchGateOpens { opened.withLock { hasOpened = true } } reporter.isOnline = true + Thread.sleep(forTimeInterval: after) + return opened.withLock { hasOpened } + } - if self.persists { - Thread.sleep(forTimeInterval: 0.6) - expect(opened.withLock { hasOpened }) == false - } + it("holds what waits for the launch gate until the launch delay has passed, whether or not events persist") { + let reporter = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: testContext.store, + pacing: DirectEventProcessor.Pacing(launchDelay: 1, gap: 0)) + nextRun = reporter + + expect(gateOpened(reporter, after: 0.6)) == false + expect(gateOpened(reporter, after: 0.8)) == true + } + + it("holds nothing at the launch gate where the launch delay is zero") { + let reporter = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: testContext.store, + pacing: DirectEventProcessor.Pacing(launchDelay: 0, gap: 0)) + nextRun = reporter + + expect(gateOpened(reporter, after: 0.2)) == true + } + + it("takes the launch delay from the configuration") { + var config = testContext.serviceMock.config + config.eventDeliveryDelay = 1 + testContext.serviceMock.config = config + let reporter = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: testContext.store) + nextRun = reporter - expect(opened.withLock { hasOpened }).toEventually(beTrue(), timeout: .seconds(2)) + expect(gateOpened(reporter, after: 0.6)) == false + expect(gateOpened(reporter, after: 0.8)) == true } itPersisting("does not let a report interval shorter than the launch delay send what a previous run left") { From b55af5df909f57d8ea6d9eb5328fa5170b619b8e Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 5 Nov 2026 13:29:07 -0800 Subject: [PATCH 149/158] Log undelivered events at suspension at info where nothing persists, and say where they are The message said the events remain on disk even where persistence is off, and was at debug level only. Where the store persists it stays at debug and says they remain on disk for the next launch; otherwise it is at info and says they are only in memory and lost if the application is terminated. Co-authored-by: Cursor --- LaunchDarkly/LaunchDarkly/LDClient.swift | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/LDClient.swift b/LaunchDarkly/LaunchDarkly/LDClient.swift index baa28c6ad..594bdf94c 100644 --- a/LaunchDarkly/LaunchDarkly/LDClient.swift +++ b/LaunchDarkly/LaunchDarkly/LDClient.swift @@ -269,7 +269,7 @@ public class LDClient { eventProcessor.commitWherePersisting() // A backgrounded process is suspended once idle, so deliver inside an activity assertion. Only where suspension // is real: macOS posts this notification whenever the application loses focus, and never suspends it for that. - // What does not get out is on disk for the next launch. + // What does not get out is on disk for the next launch where events persist, and otherwise only in memory. #if os(iOS) || os(tvOS) backgroundDelivery.run { [weak self] finished in guard let self = self @@ -279,7 +279,7 @@ public class LDClient { } self.eventProcessor.flushReportingOutcome { delivered in if !delivered { - os_log("%s events did not reach LaunchDarkly before suspension; they remain on disk", log: self.config.logger, type: .debug, self.typeName(and: #function)) + self.logUndeliveredAtSuspension() } finished() } @@ -1112,6 +1112,21 @@ extension LDClient { (diagnosticReporter as? DiagnosticReporter)?.sendsInitAfter = eventProcessor.whenLaunchGateOpens } } + + #if os(iOS) || os(tvOS) + /// At info where nothing persists, since the events are lost if the application is terminated while suspended. + /// Whether events persist is the store's answer rather than the configuration's: a store that failed to write has + /// stopped persisting. + fileprivate func logUndeliveredAtSuspension() { + if (eventProcessor as? DirectEventProcessor)?.store.isPersisting == true { + os_log("%s events did not reach LaunchDarkly before suspension; they remain on disk for the next launch", + log: config.logger, type: .debug, typeName(and: #function)) + } else { + os_log("%s events did not reach LaunchDarkly before suspension; they are only in memory and are lost if the application is terminated", + log: config.logger, type: .info, typeName(and: #function)) + } + } + #endif } #if DEBUG From 5711315e9a9950b417a47aeee3d6189f35775d23 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 5 Nov 2026 13:45:23 -0800 Subject: [PATCH 150/158] Gather the platform differences in backgrounding into SystemCapabilities, and event backgrounding into its own class Three places each had their own platform list: LDClient delivered under an assertion on iOS and tvOS, BackgroundActivity took assertions on iOS, tvOS and watchOS, and SystemCapabilities posted lifecycle notifications on iOS, tvOS and macOS. SystemCapabilities now says, per platform, whether a backgrounded process is suspended and how to get an assertion, beside the notifications. BackgroundEventDelivery owns what events do on backgrounding: commit, then deliver under an assertion where the platform suspends, and log what did not get out. LDClient and BackgroundActivity carry no platform conditions. watchOS is now stated rather than implied: it suspends and has assertions, but posts no notification the SDK observes, so nothing happens there. Co-authored-by: Cursor --- LaunchDarkly.xcodeproj/project.pbxproj | 14 ++++ LaunchDarkly/LaunchDarkly/LDClient.swift | 43 +----------- .../ServiceObjects/BackgroundActivity.swift | 19 ++--- .../BackgroundEventDelivery.swift | 65 +++++++++++++++++ .../SystemCapabilities.swift | 19 +++++ .../LaunchDarklyTests/LDClientSpec.swift | 14 ++-- .../BackgroundActivitySpec.swift | 18 ++--- .../BackgroundEventDeliverySpec.swift | 70 +++++++++++++++++++ 8 files changed, 192 insertions(+), 70 deletions(-) create mode 100644 LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundEventDelivery.swift create mode 100644 LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundEventDeliverySpec.swift diff --git a/LaunchDarkly.xcodeproj/project.pbxproj b/LaunchDarkly.xcodeproj/project.pbxproj index 32e9d1202..b21fdfcc0 100644 --- a/LaunchDarkly.xcodeproj/project.pbxproj +++ b/LaunchDarkly.xcodeproj/project.pbxproj @@ -40,6 +40,7 @@ 3D2406232E0D90E000F91253 /* SdkMetadata.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3D2406122E0D90E000F91253 /* SdkMetadata.swift */; }; 3D2406252E0D90EA00F91253 /* LDClientPluginsSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3D2406242E0D90EA00F91253 /* LDClientPluginsSpec.swift */; }; 3D34EEEF37158657D49335BB /* BackgroundActivity.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9A771B316E19ED598CD4FB0B /* BackgroundActivity.swift */; }; + 051BF088E9BDA17A3871A530 /* BackgroundEventDelivery.swift in Sources */ = {isa = PBXBuildFile; fileRef = EF2483453F8EF6F6D01A83B5 /* BackgroundEventDelivery.swift */; }; 3D3AB9432A4F16FE003AECF1 /* ReportingConsts.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3D3AB9422A4F16FE003AECF1 /* ReportingConsts.swift */; }; 3D3AB9442A4F16FE003AECF1 /* ReportingConsts.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3D3AB9422A4F16FE003AECF1 /* ReportingConsts.swift */; }; 3D3AB9452A4F16FE003AECF1 /* ReportingConsts.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3D3AB9422A4F16FE003AECF1 /* ReportingConsts.swift */; }; @@ -60,6 +61,7 @@ D4080613CEA27D51BF349075 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; ECBE45806B470ACC5187B7D6 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; 7185A3C30E7FFC4F725769F1 /* BackgroundActivity.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9A771B316E19ED598CD4FB0B /* BackgroundActivity.swift */; }; + 1830F6435EE141699F106451 /* BackgroundEventDelivery.swift in Sources */ = {isa = PBXBuildFile; fileRef = EF2483453F8EF6F6D01A83B5 /* BackgroundEventDelivery.swift */; }; 830BF933202D188E006DF9B1 /* HTTPURLRequest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 830BF932202D188E006DF9B1 /* HTTPURLRequest.swift */; }; 830DB3AC22380A3E00D65D25 /* HTTPHeadersSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 830DB3AB22380A3E00D65D25 /* HTTPHeadersSpec.swift */; }; 830DB3AE2239B54900D65D25 /* URLResponse.swift in Sources */ = {isa = PBXBuildFile; fileRef = 830DB3AD2239B54900D65D25 /* URLResponse.swift */; }; @@ -110,6 +112,7 @@ 831AAE2E20A9E4F600B46DBA /* Throttler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831AAE2B20A9E4F600B46DBA /* Throttler.swift */; }; 831AAE3020A9E75D00B46DBA /* ThrottlerSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831AAE2F20A9E75D00B46DBA /* ThrottlerSpec.swift */; }; 3F9AC0D80849C86866E03807 /* BackgroundActivitySpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = C3BDE2A66C982198D55602DA /* BackgroundActivitySpec.swift */; }; + 379F34CAB1EE19A0F85AA2C8 /* BackgroundEventDeliverySpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88A4CE0E204CB660F54E8A4B /* BackgroundEventDeliverySpec.swift */; }; 831D2AAF2061AAA000B4AC3C /* Thread.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831D2AAE2061AAA000B4AC3C /* Thread.swift */; }; 831D8B6F1F71532300ED65E8 /* HTTPHeaders.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831D8B6E1F71532300ED65E8 /* HTTPHeaders.swift */; }; 831D8B721F71D3E700ED65E8 /* DarklyService.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831D8B711F71D3E700ED65E8 /* DarklyService.swift */; }; @@ -287,6 +290,7 @@ A032A73E446055D14A546CA0 /* EventJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = B430884DBF2E541B919C08FA /* EventJSONWriter.swift */; }; A0BAD9EE25943C43A5154E6F /* EventJSONWriterTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 367A9182439001C3EBD68477 /* EventJSONWriterTests.swift */; }; A20146E7AF098560627C614E /* BackgroundActivity.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9A771B316E19ED598CD4FB0B /* BackgroundActivity.swift */; }; + BAB394DCCA3C189DD1B11C36 /* BackgroundEventDelivery.swift in Sources */ = {isa = PBXBuildFile; fileRef = EF2483453F8EF6F6D01A83B5 /* BackgroundEventDelivery.swift */; }; B6168D5F889EFC4C19F59135 /* JSONWriterTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1BECAB38F41B30401D3CD7EA /* JSONWriterTests.swift */; }; A3047D642A606B6000F568E0 /* SDKEnvironmentReporterSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = A3047D5D2A606B6000F568E0 /* SDKEnvironmentReporterSpec.swift */; }; A3047D652A606B6000F568E0 /* IOSEnvironmentReporterSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = A3047D5E2A606B6000F568E0 /* IOSEnvironmentReporterSpec.swift */; }; @@ -312,6 +316,7 @@ A31088282837DCA900184942 /* ReferenceSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = A31088252837DCA900184942 /* ReferenceSpec.swift */; }; A31088292837DCA900184942 /* KindSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = A31088262837DCA900184942 /* KindSpec.swift */; }; A32FCDB22D53263851DDB727 /* BackgroundActivity.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9A771B316E19ED598CD4FB0B /* BackgroundActivity.swift */; }; + 8F654B978A758FDFC94E4AF0 /* BackgroundEventDelivery.swift in Sources */ = {isa = PBXBuildFile; fileRef = EF2483453F8EF6F6D01A83B5 /* BackgroundEventDelivery.swift */; }; A33A5F7A28466D04000C29C7 /* LDContextStub.swift in Sources */ = {isa = PBXBuildFile; fileRef = A33A5F7928466D04000C29C7 /* LDContextStub.swift */; }; A3470C372B7C1ACE00951CEE /* LDValueDecoder.swift in Sources */ = {isa = PBXBuildFile; fileRef = A3470C362B7C1ACE00951CEE /* LDValueDecoder.swift */; }; A3470C382B7C1ACE00951CEE /* LDValueDecoder.swift in Sources */ = {isa = PBXBuildFile; fileRef = A3470C362B7C1ACE00951CEE /* LDValueDecoder.swift */; }; @@ -576,6 +581,7 @@ 831AAE2B20A9E4F600B46DBA /* Throttler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Throttler.swift; sourceTree = ""; }; 831AAE2F20A9E75D00B46DBA /* ThrottlerSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ThrottlerSpec.swift; sourceTree = ""; }; C3BDE2A66C982198D55602DA /* BackgroundActivitySpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BackgroundActivitySpec.swift; sourceTree = ""; }; + 88A4CE0E204CB660F54E8A4B /* BackgroundEventDeliverySpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BackgroundEventDeliverySpec.swift; sourceTree = ""; }; 831D2AAE2061AAA000B4AC3C /* Thread.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Thread.swift; sourceTree = ""; }; 831D8B6E1F71532300ED65E8 /* HTTPHeaders.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HTTPHeaders.swift; sourceTree = ""; }; 831D8B711F71D3E700ED65E8 /* DarklyService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DarklyService.swift; sourceTree = ""; }; @@ -651,6 +657,7 @@ 9A47F7492F3D4CBF0001FD9C /* ContextSummarizerSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContextSummarizerSpec.swift; sourceTree = ""; }; 9A47F74B2F3D4CCF0001FD9C /* ContextSummarizer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContextSummarizer.swift; sourceTree = ""; }; 9A771B316E19ED598CD4FB0B /* BackgroundActivity.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = BackgroundActivity.swift; sourceTree = ""; }; + EF2483453F8EF6F6D01A83B5 /* BackgroundEventDelivery.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = BackgroundEventDelivery.swift; sourceTree = ""; }; 9AE1D0A02F4A1B000001FD9C /* EvaluationExposureDeduper.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EvaluationExposureDeduper.swift; sourceTree = ""; }; 9AE1D0A52F4A1B000001FD9C /* EvaluationExposureDeduperSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EvaluationExposureDeduperSpec.swift; sourceTree = ""; }; 9AE1D0B02F4A1B000001FD9C /* HookDecorator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HookDecorator.swift; sourceTree = ""; }; @@ -819,6 +826,7 @@ 837406D321F760640087B22B /* LDTimerSpec.swift */, 831AAE2F20A9E75D00B46DBA /* ThrottlerSpec.swift */, C3BDE2A66C982198D55602DA /* BackgroundActivitySpec.swift */, + 88A4CE0E204CB660F54E8A4B /* BackgroundEventDeliverySpec.swift */, 8354AC75224316C700CDE602 /* Cache */, A3C6F7612B7FA803005B3B61 /* SheddingQueueSpec.swift */, EE6BAECBB739A0F24DB5A944 /* EventStoreSpec.swift */, @@ -1108,6 +1116,7 @@ 6F2A8C1D4E7B9053A1C4D8E6 /* JSONWriter.swift */, D93B7F5F6CF5C3DE79FB4286 /* ContextEncodingCache.swift */, 9A771B316E19ED598CD4FB0B /* BackgroundActivity.swift */, + EF2483453F8EF6F6D01A83B5 /* BackgroundEventDelivery.swift */, C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */, E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */, A1B2C3D4E5F60718293A4B5C /* EventDirectory.swift */, @@ -1647,6 +1656,7 @@ 8B3D9E2F5A6C1047B2D5E9F1 /* JSONWriter.swift in Sources */, 40F4D07AD934AF976687011D /* ContextEncodingCache.swift in Sources */, A32FCDB22D53263851DDB727 /* BackgroundActivity.swift in Sources */, + 8F654B978A758FDFC94E4AF0 /* BackgroundEventDelivery.swift in Sources */, CF9A1EB116D8AA3B7B5535B2 /* EventStore.swift in Sources */, E2F8080AB2B3CD6E9A9C1FCC /* DiskEventLog.swift in Sources */, A1B2C3D4E5F60718293A4B04 /* EventDirectory.swift in Sources */, @@ -1754,6 +1764,7 @@ 9C4E0F306B7D2158C3E6F0A2 /* JSONWriter.swift in Sources */, 4882B223C23B63F30BC60941 /* ContextEncodingCache.swift in Sources */, 7185A3C30E7FFC4F725769F1 /* BackgroundActivity.swift in Sources */, + 1830F6435EE141699F106451 /* BackgroundEventDelivery.swift in Sources */, 2CA9D36B3566D78850D6F0AF /* EventStore.swift in Sources */, B8F228923BE11E444D1ECB8E /* DiskEventLog.swift in Sources */, A1B2C3D4E5F60718293A4B01 /* EventDirectory.swift in Sources */, @@ -1860,6 +1871,7 @@ AD5F10417C8E3269D4F701B3 /* JSONWriter.swift in Sources */, D7A500FCA04E3412F931003F /* ContextEncodingCache.swift in Sources */, A20146E7AF098560627C614E /* BackgroundActivity.swift in Sources */, + BAB394DCCA3C189DD1B11C36 /* BackgroundEventDelivery.swift in Sources */, 5AEA1F832DCA974314E07D21 /* EventStore.swift in Sources */, D4225F1F05F2C55E9518941B /* DiskEventLog.swift in Sources */, A1B2C3D4E5F60718293A4B02 /* EventDirectory.swift in Sources */, @@ -1889,6 +1901,7 @@ 83B6E3F1222EFA3800FF2A6A /* ThreadSpec.swift in Sources */, 831AAE3020A9E75D00B46DBA /* ThrottlerSpec.swift in Sources */, 3F9AC0D80849C86866E03807 /* BackgroundActivitySpec.swift in Sources */, + 379F34CAB1EE19A0F85AA2C8 /* BackgroundEventDeliverySpec.swift in Sources */, 832D68AC224B3321005F052A /* CacheConverterSpec.swift in Sources */, 838F96741FB9F024009CFC45 /* LDClientSpec.swift in Sources */, 83A0E6B1203B557F00224298 /* FeatureFlagSpec.swift in Sources */, @@ -2034,6 +2047,7 @@ BE6011528D9F4370E50812C4 /* JSONWriter.swift in Sources */, BAE0B4F45562EBF4AC786B5D /* ContextEncodingCache.swift in Sources */, 3D34EEEF37158657D49335BB /* BackgroundActivity.swift in Sources */, + 051BF088E9BDA17A3871A530 /* BackgroundEventDelivery.swift in Sources */, B0B1AD7CADE8DBB596C2DE9B /* EventStore.swift in Sources */, DAA2C9B9C144F129217A7C55 /* DiskEventLog.swift in Sources */, A1B2C3D4E5F60718293A4B03 /* EventDirectory.swift in Sources */, diff --git a/LaunchDarkly/LaunchDarkly/LDClient.swift b/LaunchDarkly/LaunchDarkly/LDClient.swift index 594bdf94c..51a082267 100644 --- a/LaunchDarkly/LaunchDarkly/LDClient.swift +++ b/LaunchDarkly/LaunchDarkly/LDClient.swift @@ -263,28 +263,7 @@ public class LDClient { @objc private func didEnterBackground() { os_log("%s", log: config.logger, type: .debug, typeName(and: #function)) - // The OS may kill a backgrounded application without warning, so make its events durable now, before returning, - // at `.deferred` too. On every platform: macOS does not suspend an application that loses focus, but it can - // still be quit or killed. - eventProcessor.commitWherePersisting() - // A backgrounded process is suspended once idle, so deliver inside an activity assertion. Only where suspension - // is real: macOS posts this notification whenever the application loses focus, and never suspends it for that. - // What does not get out is on disk for the next launch where events persist, and otherwise only in memory. - #if os(iOS) || os(tvOS) - backgroundDelivery.run { [weak self] finished in - guard let self = self - else { - finished() - return - } - self.eventProcessor.flushReportingOutcome { delivered in - if !delivered { - self.logUndeliveredAtSuspension() - } - finished() - } - } - #endif + backgroundEvents.applicationDidEnterBackground() Thread.performOnMain { runMode = .background } @@ -297,8 +276,7 @@ public class LDClient { } } - /// One assertion per client, however many times the application is backgrounded while a delivery is running. - private let backgroundDelivery: BackgroundActivity + private let backgroundEvents: BackgroundEventDelivery let config: LDConfig /// Identifies this client's environment to a hook, without handing it the mobile key that identifies the @@ -1040,7 +1018,6 @@ public class LDClient { throttler = self.serviceFactory.makeThrottler(environmentReporter: environmentReporter) config = configuration - backgroundDelivery = BackgroundActivity(reason: "LaunchDarkly event delivery", logger: configuration.logger) context = startContext ?? LDContext() if config.autoEnvAttributes { @@ -1062,6 +1039,7 @@ public class LDClient { eventProcessor = self.serviceFactory.makeEventProcessor(config: config, service: service) { result in LDClient.logEventSyncComplete(result, logger) } + backgroundEvents = BackgroundEventDelivery(eventProcessor: eventProcessor, logger: logger) connectionInformation = self.serviceFactory.makeConnectionInformation() let cachedData = flagCache.getCachedData(cacheKey: context.fullyQualifiedHashedKey(), contextHash: context.contextHash()) @@ -1112,21 +1090,6 @@ extension LDClient { (diagnosticReporter as? DiagnosticReporter)?.sendsInitAfter = eventProcessor.whenLaunchGateOpens } } - - #if os(iOS) || os(tvOS) - /// At info where nothing persists, since the events are lost if the application is terminated while suspended. - /// Whether events persist is the store's answer rather than the configuration's: a store that failed to write has - /// stopped persisting. - fileprivate func logUndeliveredAtSuspension() { - if (eventProcessor as? DirectEventProcessor)?.store.isPersisting == true { - os_log("%s events did not reach LaunchDarkly before suspension; they remain on disk for the next launch", - log: config.logger, type: .debug, typeName(and: #function)) - } else { - os_log("%s events did not reach LaunchDarkly before suspension; they are only in memory and are lost if the application is terminated", - log: config.logger, type: .info, typeName(and: #function)) - } - } - #endif } #if DEBUG diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundActivity.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundActivity.swift index eda2bb543..fa26f3c76 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundActivity.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundActivity.swift @@ -72,12 +72,16 @@ final class BackgroundActivity { /// An assertion from the process, over an injectable system call and cap so that a test can drive both. /// - /// Without assertions on this platform, `whileHeld` runs on the calling thread. + /// Where the platform has no assertions, `perform` is nil and `whileHeld` runs on the calling thread. static func processAssertion(logger: OSLog, - perform: @escaping ExpiringActivity = BackgroundActivity.performExpiringActivity, + perform: ExpiringActivity? = SystemCapabilities.expiringActivity, maximumDuration: TimeInterval = BackgroundActivity.maximumDuration) -> Assertion { { reason, whileHeld in - #if os(iOS) || os(tvOS) || os(watchOS) + guard let perform + else { + whileHeld {} + return + } let finished = DispatchSemaphore(value: 0) // The work does not wait on the assertion, and does not run inside it. The system refuses one outright // under the very resource pressure this is here for, and a delivery it may suspend is better than no @@ -96,15 +100,6 @@ final class BackgroundActivity { _ = finished.wait(timeout: .now() + maximumDuration) } whileHeld { finished.signal() } - #else - whileHeld {} - #endif } } - - private static func performExpiringActivity(_ reason: String, _ whileGranted: @escaping (Bool) -> Void) { - #if os(iOS) || os(tvOS) || os(watchOS) - ProcessInfo.processInfo.performExpiringActivity(withReason: reason, using: whileGranted) - #endif - } } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundEventDelivery.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundEventDelivery.swift new file mode 100644 index 000000000..28b2cc495 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundEventDelivery.swift @@ -0,0 +1,65 @@ +import Foundation +import OSLog + +/// What an environment's events do as the application goes to the background. +/// +/// On every platform that notifies the SDK, they are made durable before the notification returns: the OS may kill a backgrounded +/// application without warning, and macOS, which suspends nothing, can still quit or kill it. Where a backgrounded +/// process is suspended once idle, they are also delivered, under a background activity assertion. What does not get +/// out is on disk for the next launch where events persist, and otherwise only in memory. +final class BackgroundEventDelivery { + private let eventProcessor: EventProcessor + private let suspendsWhenBackgrounded: Bool + /// One assertion per client, however many times the application is backgrounded while a delivery is running. + private let activity: BackgroundActivity + private let logger: OSLog + + init(eventProcessor: EventProcessor, suspendsWhenBackgrounded: Bool, activity: BackgroundActivity, logger: OSLog) { + self.eventProcessor = eventProcessor + self.suspendsWhenBackgrounded = suspendsWhenBackgrounded + self.activity = activity + self.logger = logger + } + + convenience init(eventProcessor: EventProcessor, logger: OSLog) { + self.init(eventProcessor: eventProcessor, + suspendsWhenBackgrounded: SystemCapabilities.suspendsWhenBackgrounded, + activity: BackgroundActivity(reason: "LaunchDarkly event delivery", logger: logger), + logger: logger) + } + + /// Commits before returning, at `.deferred` too, and where the process is about to be suspended starts a delivery. + func applicationDidEnterBackground() { + eventProcessor.commitWherePersisting() + guard suspendsWhenBackgrounded + else { return } + activity.run { [weak self] finished in + guard let self + else { + finished() + return + } + self.eventProcessor.flushReportingOutcome { delivered in + if !delivered { + self.logUndelivered() + } + finished() + } + } + } + + /// At info where nothing persists, since the events are lost if the application is terminated while suspended. + /// Whether events persist is the store's answer rather than the configuration's: a store that failed to write has + /// stopped persisting. + private func logUndelivered() { + if (eventProcessor as? DirectEventProcessor)?.store.isPersisting == true { + os_log("%s events did not reach LaunchDarkly before suspension; they remain on disk for the next launch", + log: logger, type: .debug, typeName(and: #function)) + } else { + os_log("%s events did not reach LaunchDarkly before suspension; they are only in memory and are lost if the application is terminated", + log: logger, type: .info, typeName(and: #function)) + } + } +} + +extension BackgroundEventDelivery: TypeIdentifying { } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EnvironmentReporting/SystemCapabilities.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EnvironmentReporting/SystemCapabilities.swift index 61a4e97c5..70317b5d3 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EnvironmentReporting/SystemCapabilities.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EnvironmentReporting/SystemCapabilities.swift @@ -8,26 +8,45 @@ import AppKit import WatchKit #endif +/// What differs between the platforms the SDK runs on, in one place, so that the code using it carries no platform +/// conditions of its own. class SystemCapabilities { #if os(iOS) static var backgroundNotification: Notification.Name? { UIApplication.didEnterBackgroundNotification } static var foregroundNotification: Notification.Name? { UIApplication.willEnterForegroundNotification } + static var suspendsWhenBackgrounded: Bool { true } + static var expiringActivity: BackgroundActivity.ExpiringActivity? { performExpiringActivity } static var systemName: String { UIDevice.current.systemName } static var operatingSystem: OperatingSystem { .iOS } #elseif os(watchOS) + /// None the SDK observes, so a watchOS client neither changes run mode nor does anything with its events on its way + /// to the background, although the platform suspends it and has the assertions to deliver under. static var backgroundNotification: Notification.Name? { nil } static var foregroundNotification: Notification.Name? { nil } + static var suspendsWhenBackgrounded: Bool { true } + static var expiringActivity: BackgroundActivity.ExpiringActivity? { performExpiringActivity } static var systemName: String { WKInterfaceDevice.current().systemName } static var operatingSystem: OperatingSystem { .watchOS } #elseif os(OSX) + /// Posted on every loss of focus, which suspends nothing. static var backgroundNotification: Notification.Name? { NSApplication.willResignActiveNotification } static var foregroundNotification: Notification.Name? { NSApplication.didBecomeActiveNotification } + static var suspendsWhenBackgrounded: Bool { false } + static var expiringActivity: BackgroundActivity.ExpiringActivity? { nil } static var systemName: String { "macOS" } static var operatingSystem: OperatingSystem { .macOS } #elseif os(tvOS) static var backgroundNotification: Notification.Name? { UIApplication.didEnterBackgroundNotification } static var foregroundNotification: Notification.Name? { UIApplication.willEnterForegroundNotification } + static var suspendsWhenBackgrounded: Bool { true } + static var expiringActivity: BackgroundActivity.ExpiringActivity? { performExpiringActivity } static var systemName: String { UIDevice.current.systemName } static var operatingSystem: OperatingSystem { .tvOS } #endif + + #if os(iOS) || os(tvOS) || os(watchOS) + private static func performExpiringActivity(_ reason: String, _ whileGranted: @escaping (Bool) -> Void) { + ProcessInfo.processInfo.performExpiringActivity(withReason: reason, using: whileGranted) + } + #endif } diff --git a/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift b/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift index 5512a9633..d084925e2 100644 --- a/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift @@ -1404,13 +1404,13 @@ final class LDClientSpec: QuickSpec { testContext.start() NotificationCenter.default.post(name: SystemCapabilities.backgroundNotification!, object: self) - #if os(iOS) || os(tvOS) - expect(testContext.eventProcessorMock.flushReportingOutcomeCallCount).toEventually(equal(1)) - #else - // macOS posts this on every loss of focus, which suspends nothing. - expect(testContext.subject.runMode).toEventually(equal(LDClientRunMode.background)) - expect(testContext.eventProcessorMock.flushReportingOutcomeCallCount) == 0 - #endif + if SystemCapabilities.suspendsWhenBackgrounded { + expect(testContext.eventProcessorMock.flushReportingOutcomeCallCount).toEventually(equal(1)) + } else { + // macOS posts this on every loss of focus, which suspends nothing. + expect(testContext.subject.runMode).toEventually(equal(LDClientRunMode.background)) + expect(testContext.eventProcessorMock.flushReportingOutcomeCallCount) == 0 + } } it("writes what it has before the notification returns, on every platform", onlyWhere: platformPostsLifecycleNotifications) { let testContext = TestContext(startOnline: true, enableBackgroundUpdates: false) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundActivitySpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundActivitySpec.swift index f3de67f69..c44ff50d5 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundActivitySpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundActivitySpec.swift @@ -77,7 +77,6 @@ final class BackgroundActivitySpec: QuickSpec { } } - #if os(iOS) || os(tvOS) || os(watchOS) describe("an assertion from the process") { /// Stands in for `performExpiringActivity`, which answers on a system queue rather than the caller's. /// @@ -140,14 +139,9 @@ final class BackgroundActivitySpec: QuickSpec { // Ended by the cap, rather than by the work or by never having been held. expect(Date().timeIntervalSince(started)) > 0.2 } - } - #else - describe("an assertion from the process") { - it("runs the work on the calling thread without asking the system, which has no assertions here") { - var asked = false - let assertion = BackgroundActivity.processAssertion(logger: .disabled, - perform: { _, _ in asked = true }, - maximumDuration: 30) + + it("runs the work on the calling thread where the platform has no assertions") { + let assertion = BackgroundActivity.processAssertion(logger: .disabled, perform: nil, maximumDuration: 30) var ran = false assertion("test") { finished in @@ -156,9 +150,11 @@ final class BackgroundActivitySpec: QuickSpec { } expect(ran) == true - expect(asked) == false + } + + it("asks the system for one only where the platform has them") { + expect(SystemCapabilities.expiringActivity != nil) == SystemCapabilities.suspendsWhenBackgrounded } } - #endif } } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundEventDeliverySpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundEventDeliverySpec.swift new file mode 100644 index 000000000..4cc599345 --- /dev/null +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundEventDeliverySpec.swift @@ -0,0 +1,70 @@ +import Foundation +import Quick +import Nimble +@testable import LaunchDarkly + +final class BackgroundEventDeliverySpec: QuickSpec { + override func spec() { + describe("going to the background") { + var eventProcessor: EventProcessorMock! + var assertions = 0 + var released = 0 + var activity: BackgroundActivity! + beforeEach { + eventProcessor = EventProcessorMock() + assertions = 0 + released = 0 + activity = BackgroundActivity(reason: "test") { _, work in + assertions += 1 + work { released += 1 } + } + } + + func delivery(suspendsWhenBackgrounded: Bool) -> BackgroundEventDelivery { + BackgroundEventDelivery(eventProcessor: eventProcessor, + suspendsWhenBackgrounded: suspendsWhenBackgrounded, + activity: activity, + logger: .disabled) + } + + it("commits, and delivers nothing, where the platform does not suspend") { + delivery(suspendsWhenBackgrounded: false).applicationDidEnterBackground() + + expect(eventProcessor.commitWherePersistingCallCount) == 1 + expect(eventProcessor.flushReportingOutcomeCallCount) == 0 + expect(assertions) == 0 + } + + it("commits before it delivers, where the platform suspends") { + var committedBeforeDelivering = false + eventProcessor.flushReportingOutcomeCallback = { + committedBeforeDelivering = eventProcessor.commitWherePersistingCallCount == 1 + } + + delivery(suspendsWhenBackgrounded: true).applicationDidEnterBackground() + + expect(eventProcessor.flushReportingOutcomeCallCount) == 1 + expect(committedBeforeDelivering) == true + } + + it("holds the assertion until the delivery answers") { + let subject = delivery(suspendsWhenBackgrounded: true) + subject.applicationDidEnterBackground() + + expect(assertions) == 1 + expect(released) == 0 + + eventProcessor.flushReportingOutcomeReceivedCompletion?(true) + expect(released) == 1 + } + + it("releases the assertion where the delivery fails too") { + let subject = delivery(suspendsWhenBackgrounded: true) + subject.applicationDidEnterBackground() + + eventProcessor.flushReportingOutcomeReceivedCompletion?(false) + expect(released) == 1 + } + } + } +} From b5e81ef5a4f1a78ac54c7eb76fb1aae7fa16493b Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 5 Nov 2026 20:48:12 -0800 Subject: [PATCH 151/158] Build the event pipeline's test accessors in Debug only The accessors only tests read -- the delivery counts, pending events, the summarizer and response-date hooks, the queue occupier and the batch lock -- move into DEBUG extensions. The delivery counter itself stays in every build: it is one increment under a lock already held. Co-authored-by: Cursor --- .../LaunchDarkly/ServiceObjects/DirectEventProcessor.swift | 2 ++ LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift | 2 ++ .../LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift | 4 ++++ LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift | 4 ++++ 4 files changed, 12 insertions(+) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index fdae9e48c..539f68359 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -943,6 +943,7 @@ extension DirectEventProcessor { extension DirectEventProcessor: TypeIdentifying { } +#if DEBUG extension DirectEventProcessor { /// Full events accepted but not yet handed to a commit. var pendingEventsForTesting: [Event] { @@ -973,3 +974,4 @@ extension DirectEventProcessor { committer.withContextSummarizer(read) } } +#endif diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift index fe716e5c5..2c9b1e7e5 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift @@ -479,6 +479,7 @@ final class EventCommitter { extension EventCommitter: TypeIdentifying { } +#if DEBUG extension EventCommitter { /// Full events accepted but not yet handed to a commit. var pendingEventsForTesting: [Event] { @@ -494,3 +495,4 @@ extension EventCommitter { recordLock.withLock { read(eventBuffer.contextSummarizer) } } } +#endif diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift index f117acebb..b4dc22682 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift @@ -248,7 +248,10 @@ final class EventDeliveryScheduler { func retryDue() { lock.withLock { isRetryScheduled = false } } +} +#if DEBUG +extension EventDeliveryScheduler { /// Callers waiting on the queued delivery. var queuedWaiterCount: Int { lock.withLock { queued?.waiters.count ?? 0 } @@ -259,3 +262,4 @@ final class EventDeliveryScheduler { lock.withLock { deliveryRunCount } } } +#endif diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift index 624e2dbf9..a87e94f96 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -1004,12 +1004,16 @@ enum BatchLocks { return (try? FileManager.default.removeItem(at: batch)) != nil } } +} +#if DEBUG +extension BatchLocks { /// The lock every call on a batch takes: its directory's, whichever store names the batch. static func lockForTesting(_ batch: URL) -> UnfairLock { directory(of: batch).lock } } +#endif extension EventDirectory { /// The directory the store for a mobile key belongs in. From 7fa137c5a662c4feb2ad02a230b83b8c093d5930 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 5 Nov 2026 21:19:41 -0800 Subject: [PATCH 152/158] One background observer for the events of every environment Backgrounding commits every started environment before the notification returns and, where the process is suspended, delivers them all under one background activity assertion, released once every environment has answered. Closed clients the application still holds are no longer reached. Run mode stays per client. Co-authored-by: Cursor --- LaunchDarkly/LaunchDarkly/LDClient.swift | 28 ++++- .../ServiceObjects/BackgroundActivity.swift | 9 +- .../BackgroundEventDelivery.swift | 103 ++++++++++++------ .../BackgroundEventDeliverySpec.swift | 73 ++++++++++--- 4 files changed, 158 insertions(+), 55 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/LDClient.swift b/LaunchDarkly/LaunchDarkly/LDClient.swift index 51a082267..e144b03f3 100644 --- a/LaunchDarkly/LaunchDarkly/LDClient.swift +++ b/LaunchDarkly/LaunchDarkly/LDClient.swift @@ -263,7 +263,6 @@ public class LDClient { @objc private func didEnterBackground() { os_log("%s", log: config.logger, type: .debug, typeName(and: #function)) - backgroundEvents.applicationDidEnterBackground() Thread.performOnMain { runMode = .background } @@ -276,8 +275,6 @@ public class LDClient { } } - private let backgroundEvents: BackgroundEventDelivery - let config: LDConfig /// Identifies this client's environment to a hook, without handing it the mobile key that identifies the /// environment to LaunchDarkly. Hashed once here rather than per evaluation, because a deduping hook asks for it on @@ -1039,7 +1036,6 @@ public class LDClient { eventProcessor = self.serviceFactory.makeEventProcessor(config: config, service: service) { result in LDClient.logEventSyncComplete(result, logger) } - backgroundEvents = BackgroundEventDelivery(eventProcessor: eventProcessor, logger: logger) connectionInformation = self.serviceFactory.makeConnectionInformation() let cachedData = flagCache.getCachedData(cacheKey: context.fullyQualifiedHashedKey(), contextHash: context.contextHash()) @@ -1052,6 +1048,7 @@ public class LDClient { if let backgroundNotification = SystemCapabilities.backgroundNotification { NotificationCenter.default.addObserver(self, selector: #selector(didEnterBackground), name: backgroundNotification, object: nil) } + LDClient.observeBackgroundingForEvents() if let foregroundNotification = SystemCapabilities.foregroundNotification { NotificationCenter.default.addObserver(self, selector: #selector(willEnterForeground), name: foregroundNotification, object: nil) } @@ -1090,6 +1087,29 @@ extension LDClient { (diagnosticReporter as? DiagnosticReporter)?.sendsInitAfter = eventProcessor.whenLaunchGateOpens } } + + /// Observes the notification once for the events of every environment, however many clients are made. + private static let backgroundEvents: BackgroundEventDelivery? = { + guard let backgroundNotification = SystemCapabilities.backgroundNotification + else { return nil } + let delivery = BackgroundEventDelivery(environments: LDClient.currentEnvironments) + // Run on the posting thread, so that every environment has committed by the time the notification returns. + NotificationCenter.default.addObserver(forName: backgroundNotification, object: nil, queue: nil) { _ in + delivery.applicationDidEnterBackground() + } + return delivery + }() + + fileprivate static func observeBackgroundingForEvents() { + _ = backgroundEvents + } + + private static func currentEnvironments() -> [BackgroundEventDelivery.Environment] { + let clients = instancesQueue.sync { instances } ?? [:] + return clients.map { name, client in + BackgroundEventDelivery.Environment(name: name, eventProcessor: client.eventProcessor, logger: client.config.logger) + } + } } #if DEBUG diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundActivity.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundActivity.swift index fa26f3c76..dc669aa76 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundActivity.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundActivity.swift @@ -32,8 +32,9 @@ final class BackgroundActivity { self.assertion = assertion } - convenience init(reason: String, logger: OSLog) { - self.init(reason: reason, assertion: BackgroundActivity.processAssertion(logger: logger)) + /// - parameter logger: Read each time it logs, so that it can be the logger of whichever client is current then. + convenience init(reason: String, logger: @autoclosure @escaping () -> OSLog) { + self.init(reason: reason, assertion: BackgroundActivity.processAssertion(logger: logger())) } /// Runs `work` under an assertion released when `work` calls `finished`, the system expires it, or @@ -73,7 +74,7 @@ final class BackgroundActivity { /// An assertion from the process, over an injectable system call and cap so that a test can drive both. /// /// Where the platform has no assertions, `perform` is nil and `whileHeld` runs on the calling thread. - static func processAssertion(logger: OSLog, + static func processAssertion(logger: @autoclosure @escaping () -> OSLog, perform: ExpiringActivity? = SystemCapabilities.expiringActivity, maximumDuration: TimeInterval = BackgroundActivity.maximumDuration) -> Assertion { { reason, whileHeld in @@ -92,7 +93,7 @@ final class BackgroundActivity { // The system taking the assertion back, or declining to give one. Nothing should stay blocked on // it, and the work goes ahead either way. os_log("%s unprotected by a background assertion: %s", - log: logger, type: .debug, String(describing: BackgroundActivity.self), reason) + log: logger(), type: .debug, String(describing: BackgroundActivity.self), reason) finished.signal() return } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundEventDelivery.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundEventDelivery.swift index 28b2cc495..c922df1f2 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundEventDelivery.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundEventDelivery.swift @@ -1,37 +1,48 @@ import Foundation import OSLog -/// What an environment's events do as the application goes to the background. +/// What the events of every environment do as the application goes to the background. /// -/// On every platform that notifies the SDK, they are made durable before the notification returns: the OS may kill a backgrounded -/// application without warning, and macOS, which suspends nothing, can still quit or kill it. Where a backgrounded -/// process is suspended once idle, they are also delivered, under a background activity assertion. What does not get -/// out is on disk for the next launch where events persist, and otherwise only in memory. +/// One for all environments rather than one per client, so that backgrounding takes one background activity assertion +/// however many environments there are, and reaches only the clients started now: a closed client the application +/// still holds is not among them. +/// +/// On every platform that notifies the SDK, events are made durable before the notification returns: the OS may kill a +/// backgrounded application without warning, and macOS, which suspends nothing, can still quit or kill it. Where a +/// backgrounded process is suspended once idle, they are also delivered, under the assertion, which is released once +/// every environment has answered. What does not get out is on disk for the next launch where events persist, and +/// otherwise only in memory. final class BackgroundEventDelivery { - private let eventProcessor: EventProcessor + struct Environment { + let name: String + let eventProcessor: EventProcessor + let logger: OSLog + } + + /// The environments started now, read at each backgrounding. + private let environments: () -> [Environment] private let suspendsWhenBackgrounded: Bool - /// One assertion per client, however many times the application is backgrounded while a delivery is running. private let activity: BackgroundActivity - private let logger: OSLog - init(eventProcessor: EventProcessor, suspendsWhenBackgrounded: Bool, activity: BackgroundActivity, logger: OSLog) { - self.eventProcessor = eventProcessor + init(environments: @escaping () -> [Environment], suspendsWhenBackgrounded: Bool, activity: BackgroundActivity) { + self.environments = environments self.suspendsWhenBackgrounded = suspendsWhenBackgrounded self.activity = activity - self.logger = logger } - convenience init(eventProcessor: EventProcessor, logger: OSLog) { - self.init(eventProcessor: eventProcessor, + convenience init(environments: @escaping () -> [Environment]) { + self.init(environments: environments, suspendsWhenBackgrounded: SystemCapabilities.suspendsWhenBackgrounded, - activity: BackgroundActivity(reason: "LaunchDarkly event delivery", logger: logger), - logger: logger) + activity: BackgroundActivity(reason: "LaunchDarkly event delivery", + logger: environments().first?.logger ?? .disabled)) } - /// Commits before returning, at `.deferred` too, and where the process is about to be suspended starts a delivery. + /// Commits every environment before returning, at `.deferred` too, and where the process is about to be suspended + /// starts their deliveries. func applicationDidEnterBackground() { - eventProcessor.commitWherePersisting() - guard suspendsWhenBackgrounded + let environments = self.environments() + environments.forEach { $0.eventProcessor.commitWherePersisting() } + guard suspendsWhenBackgrounded, !environments.isEmpty else { return } activity.run { [weak self] finished in guard let self @@ -39,27 +50,57 @@ final class BackgroundEventDelivery { finished() return } - self.eventProcessor.flushReportingOutcome { delivered in - if !delivered { - self.logUndelivered() + self.deliver(environments, then: finished) + } + } + + /// Delivers every environment at once, and calls `finished` once all of them have answered. + private func deliver(_ environments: [Environment], then finished: @escaping () -> Void) { + let lock = UnfairLock() + /// Only to be used while holding `lock`: each environment answers from whichever thread its delivery ended on. + var outstanding = Set(environments.indices) + var undelivered: [Environment] = [] + for (index, environment) in environments.enumerated() { + environment.eventProcessor.flushReportingOutcome { delivered in + // Nil until the last environment answers, so this ends once, and a reporter answering twice neither + // ends it early nor ends it again. + let ended: [Environment]? = lock.withLock { + guard outstanding.remove(index) != nil + else { return nil } + if !delivered { + undelivered.append(environment) + } + return outstanding.isEmpty ? undelivered : nil } + guard let ended + else { return } + self.logUndelivered(ended) finished() } } } - /// At info where nothing persists, since the events are lost if the application is terminated while suspended. - /// Whether events persist is the store's answer rather than the configuration's: a store that failed to write has - /// stopped persisting. - private func logUndelivered() { - if (eventProcessor as? DirectEventProcessor)?.store.isPersisting == true { - os_log("%s events did not reach LaunchDarkly before suspension; they remain on disk for the next launch", - log: logger, type: .debug, typeName(and: #function)) - } else { - os_log("%s events did not reach LaunchDarkly before suspension; they are only in memory and are lost if the application is terminated", - log: logger, type: .info, typeName(and: #function)) + /// At info for the environments where nothing persists, since their events are lost if the application is + /// terminated while suspended. Whether events persist is each store's answer rather than the configuration's: a + /// store that failed to write has stopped persisting. + private func logUndelivered(_ environments: [Environment]) { + guard let logger = environments.first?.logger + else { return } + let onDisk = environments.filter { ($0.eventProcessor as? DirectEventProcessor)?.store.isPersisting == true } + let inMemory = environments.filter { ($0.eventProcessor as? DirectEventProcessor)?.store.isPersisting != true } + if !onDisk.isEmpty { + os_log("%s events of %s did not reach LaunchDarkly before suspension; they remain on disk for the next launch", + log: logger, type: .debug, typeName(and: #function), BackgroundEventDelivery.names(onDisk)) + } + if !inMemory.isEmpty { + os_log("%s events of %s did not reach LaunchDarkly before suspension; they are only in memory and are lost if the application is terminated", + log: logger, type: .info, typeName(and: #function), BackgroundEventDelivery.names(inMemory)) } } + + private static func names(_ environments: [Environment]) -> String { + environments.map(\.name).sorted().joined(separator: ", ") + } } extension BackgroundEventDelivery: TypeIdentifying { } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundEventDeliverySpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundEventDeliverySpec.swift index 4cc599345..e5d4437bb 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundEventDeliverySpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundEventDeliverySpec.swift @@ -6,12 +6,17 @@ import Nimble final class BackgroundEventDeliverySpec: QuickSpec { override func spec() { describe("going to the background") { - var eventProcessor: EventProcessorMock! + var primary: EventProcessorMock! + var secondary: EventProcessorMock! + var environments: [BackgroundEventDelivery.Environment] = [] var assertions = 0 var released = 0 var activity: BackgroundActivity! beforeEach { - eventProcessor = EventProcessorMock() + primary = EventProcessorMock() + secondary = EventProcessorMock() + environments = [BackgroundEventDelivery.Environment(name: "primary", eventProcessor: primary, logger: .disabled), + BackgroundEventDelivery.Environment(name: "secondary", eventProcessor: secondary, logger: .disabled)] assertions = 0 released = 0 activity = BackgroundActivity(reason: "test") { _, work in @@ -21,50 +26,86 @@ final class BackgroundEventDeliverySpec: QuickSpec { } func delivery(suspendsWhenBackgrounded: Bool) -> BackgroundEventDelivery { - BackgroundEventDelivery(eventProcessor: eventProcessor, + BackgroundEventDelivery(environments: { environments }, suspendsWhenBackgrounded: suspendsWhenBackgrounded, - activity: activity, - logger: .disabled) + activity: activity) } - it("commits, and delivers nothing, where the platform does not suspend") { + it("commits every environment, and delivers nothing, where the platform does not suspend") { delivery(suspendsWhenBackgrounded: false).applicationDidEnterBackground() - expect(eventProcessor.commitWherePersistingCallCount) == 1 - expect(eventProcessor.flushReportingOutcomeCallCount) == 0 + expect(primary.commitWherePersistingCallCount) == 1 + expect(secondary.commitWherePersistingCallCount) == 1 + expect(primary.flushReportingOutcomeCallCount) == 0 + expect(secondary.flushReportingOutcomeCallCount) == 0 expect(assertions) == 0 } - it("commits before it delivers, where the platform suspends") { + it("commits every environment before it delivers any, where the platform suspends") { var committedBeforeDelivering = false - eventProcessor.flushReportingOutcomeCallback = { - committedBeforeDelivering = eventProcessor.commitWherePersistingCallCount == 1 + primary.flushReportingOutcomeCallback = { + committedBeforeDelivering = primary.commitWherePersistingCallCount == 1 + && secondary.commitWherePersistingCallCount == 1 } delivery(suspendsWhenBackgrounded: true).applicationDidEnterBackground() - expect(eventProcessor.flushReportingOutcomeCallCount) == 1 + expect(primary.flushReportingOutcomeCallCount) == 1 + expect(secondary.flushReportingOutcomeCallCount) == 1 expect(committedBeforeDelivering) == true } - it("holds the assertion until the delivery answers") { + it("takes one assertion for every environment, and holds it until all of them answer") { let subject = delivery(suspendsWhenBackgrounded: true) subject.applicationDidEnterBackground() expect(assertions) == 1 expect(released) == 0 - eventProcessor.flushReportingOutcomeReceivedCompletion?(true) + primary.flushReportingOutcomeReceivedCompletion?(true) + expect(released) == 0 + + secondary.flushReportingOutcomeReceivedCompletion?(true) + expect(released) == 1 + } + + it("releases the assertion where deliveries fail too") { + let subject = delivery(suspendsWhenBackgrounded: true) + subject.applicationDidEnterBackground() + + primary.flushReportingOutcomeReceivedCompletion?(false) + secondary.flushReportingOutcomeReceivedCompletion?(true) expect(released) == 1 } - it("releases the assertion where the delivery fails too") { + it("waits for every environment where one answers twice") { let subject = delivery(suspendsWhenBackgrounded: true) subject.applicationDidEnterBackground() - eventProcessor.flushReportingOutcomeReceivedCompletion?(false) + primary.flushReportingOutcomeReceivedCompletion?(true) + primary.flushReportingOutcomeReceivedCompletion?(true) + expect(released) == 0 + + secondary.flushReportingOutcomeReceivedCompletion?(true) + secondary.flushReportingOutcomeReceivedCompletion?(true) expect(released) == 1 } + + it("takes no assertion where no client is started") { + environments = [] + delivery(suspendsWhenBackgrounded: true).applicationDidEnterBackground() + + expect(assertions) == 0 + } + + it("reads the environments at each backgrounding") { + let subject = delivery(suspendsWhenBackgrounded: false) + environments.removeLast() + subject.applicationDidEnterBackground() + + expect(primary.commitWherePersistingCallCount) == 1 + expect(secondary.commitWherePersistingCallCount) == 0 + } } } } From d052e1cfc8385812b9442c8460e99b2326dc72f6 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 5 Nov 2026 21:35:42 -0800 Subject: [PATCH 153/158] One way to flush several environments and collect their outcomes flush(completion:), flushAndWait and the background delivery each started every environment's delivery and counted answers. They now share flushReportingOutcomes, which tracks each environment, so a reporter answering twice can no longer end flushAndWait before every environment has answered. Co-authored-by: Cursor --- LaunchDarkly/LaunchDarkly/LDClientFlush.swift | 63 +++++-------------- .../BackgroundEventDelivery.swift | 27 +------- .../ServiceObjects/EventProcessor.swift | 38 +++++++++++ .../LaunchDarklyTests/LDClientSpec.swift | 17 +++++ 4 files changed, 74 insertions(+), 71 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/LDClientFlush.swift b/LaunchDarkly/LaunchDarkly/LDClientFlush.swift index e7dcc944d..a5b23fdbc 100644 --- a/LaunchDarkly/LaunchDarkly/LDClientFlush.swift +++ b/LaunchDarkly/LaunchDarkly/LDClientFlush.swift @@ -56,25 +56,8 @@ extension LDClient { return } - let lock = UnfairLock() - /// Only to be used while holding `lock`: each environment answers from whichever thread its delivery ended on. - var delivered = true - var outstanding = clients.count - - for client in clients { - client.eventProcessor.flushReportingOutcome { result in - lock.lock() - delivered = delivered && result - outstanding -= 1 - // Nil until the last environment answers, so the completion runs once. A reporter answering twice - // takes the count past zero rather than back to it, which cannot match again. - let outcome = outstanding == 0 ? delivered : nil - lock.unlock() - - if let outcome { - completion(outcome) - } - } + clients.map(\.eventProcessor).flushReportingOutcomes { delivered in + completion(!delivered.contains(false)) } } @@ -143,46 +126,34 @@ extension LDClient { os_log("%s called on a closed client", log: config.logger, type: .debug, self.typeName(and: #function)) return false } - // Counts answers rather than tracking each delivery, so that a reporter answering twice costs a spare permit - // no one reads. A DispatchGroup would fit as well, but its `leave` traps without a matching `enter`, which - // turns a broken reporter into a crash in the application. + // Signaled once, by the last environment to answer, so a reporter answering twice cannot end the wait while + // another environment's delivery is still running. A DispatchGroup would fit as well, but its `leave` traps + // without a matching `enter`, which turns a broken reporter into a crash in the application. let answered = DispatchSemaphore(value: 0) let lock = UnfairLock() - /// Only to be used while holding `lock`: each environment answers from whichever thread its delivery ended on. - var delivered = true + /// Only to be used while holding `lock`: the last environment answers from whichever thread its delivery ended on. + var delivered = false // Every environment is started before any of them is waited on. Each has its own event reporter, so waiting on // one before starting the next would spend the caller's budget on deliveries that could have been running all // along, and leave the last environment with none of it. - for client in clients { - client.eventProcessor.flushBypassingBackoff { result in - lock.lock() - delivered = delivered && result - lock.unlock() - answered.signal() - } + clients.map(\.eventProcessor).flushReportingOutcomes(bypassingBackoff: true) { outcomes in + lock.withLock { delivered = !outcomes.contains(false) } + answered.signal() } // Started, then given up on. The deliveries are what makes a zero timeout worth calling at all: wherever events // persist, each `flushReportingOutcome` has queued their write, which no longer waits on this thread. Waiting - // is what is skipped, and a reporter that answered during the loop above must not turn that into a `true`. + // is what is skipped, and deliveries that already answered must not turn that into a `true`. guard timeout > 0 else { return false } - // Every wait is against the one deadline rather than a fresh copy of it, so the timeout the caller asked for - // is the time this call can take however many environments there are. A delivery still running when the wait - // gives up is left to finish rather than canceled: its events are already in flight, so abandoning it would - // only make losing them certain. - for _ in clients { - guard answered.wait(timeout: deadline) == .success - else { return false } - } - - // The waits count answers rather than environments, so a reporter answering twice can end them while another - // environment's answer is still being written. - lock.lock() - defer { lock.unlock() } - return delivered + // The deadline was taken on entry, so the timeout the caller asked for is the time this call can take however + // many environments there are. A delivery still running when the wait gives up is left to finish rather than + // canceled: its events are already in flight, so abandoning it would only make losing them certain. + guard answered.wait(timeout: deadline) == .success + else { return false } + return lock.withLock { delivered } } func internalFlush() { diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundEventDelivery.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundEventDelivery.swift index c922df1f2..e6a0557d3 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundEventDelivery.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundEventDelivery.swift @@ -50,31 +50,8 @@ final class BackgroundEventDelivery { finished() return } - self.deliver(environments, then: finished) - } - } - - /// Delivers every environment at once, and calls `finished` once all of them have answered. - private func deliver(_ environments: [Environment], then finished: @escaping () -> Void) { - let lock = UnfairLock() - /// Only to be used while holding `lock`: each environment answers from whichever thread its delivery ended on. - var outstanding = Set(environments.indices) - var undelivered: [Environment] = [] - for (index, environment) in environments.enumerated() { - environment.eventProcessor.flushReportingOutcome { delivered in - // Nil until the last environment answers, so this ends once, and a reporter answering twice neither - // ends it early nor ends it again. - let ended: [Environment]? = lock.withLock { - guard outstanding.remove(index) != nil - else { return nil } - if !delivered { - undelivered.append(environment) - } - return outstanding.isEmpty ? undelivered : nil - } - guard let ended - else { return } - self.logUndelivered(ended) + environments.map(\.eventProcessor).flushReportingOutcomes { delivered in + self.logUndelivered(zip(environments, delivered).filter { !$0.1 }.map(\.0)) finished() } } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift index e8e951d76..e99f65270 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift @@ -63,3 +63,41 @@ protocol EventProcessor { /// a write that was only queued may never happen. func commitWherePersisting() } + +extension Array where Element == EventProcessor { + /// Starts every processor's delivery at once, and gives `completion` each one's outcome, in order, once all of them + /// have answered. With no processors it answers at once, on the calling thread. + /// + /// The completion runs once, on the thread of the last answer. A reporter answering twice neither ends the wait + /// before every processor has answered nor ends it again: its second answer is ignored. + func flushReportingOutcomes(bypassingBackoff: Bool = false, completion: @escaping (_ delivered: [Bool]) -> Void) { + guard !isEmpty + else { + completion([]) + return + } + let lock = UnfairLock() + /// Only to be used while holding `lock`: each processor answers from whichever thread its delivery ended on. + var outcomes = [Bool?](repeating: nil, count: count) + var outstanding = count + for (index, processor) in enumerated() { + let answer: FlushOutcomeClosure = { delivered in + let ended: [Bool]? = lock.withLock { + guard outcomes[index] == nil + else { return nil } + outcomes[index] = delivered + outstanding -= 1 + return outstanding == 0 ? outcomes.compactMap { $0 } : nil + } + if let ended { + completion(ended) + } + } + if bypassingBackoff { + processor.flushBypassingBackoff(completion: answer) + } else { + processor.flushReportingOutcome(completion: answer) + } + } + } +} diff --git a/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift b/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift index d084925e2..031e42105 100644 --- a/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift @@ -1881,6 +1881,23 @@ final class LDClientSpec: QuickSpec { expect(testContext.subject.flushAndWait(timeout: 0.6)) == false expect(Date().timeIntervalSince(started)) < 0.85 } + + it("waits for every environment where one answers twice") { + let testContext = TestContext() + try testContext.config.setSecondaryMobileKeys(["alternate": Constants.alternateMockMobileKey]) + testContext.start() + testContext.eventProcessorMock.flushBypassingBackoffCallback = { [weak testContext] in + let completion = testContext?.eventProcessorMock.flushBypassingBackoffReceivedCompletion + completion?(true) + completion?(true) + } + let silent = LDClient.get(environment: "alternate")?.eventProcessor as? EventProcessorMock + expect(silent).toNot(beNil()) + silent?.flushBypassingBackoffCallback = nil + + // Two answers from one environment are not one from each, and the other's events are still unsent. + expect(testContext.subject.flushAndWait(timeout: 0.2)) == false + } } } From 793c3b9df10c6ad0e47b0c82a2d73da872e3f757 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 5 Nov 2026 22:12:36 -0800 Subject: [PATCH 154/158] Give up on an environment's answer after the background assertion's cap flushReportingOutcomes takes an optional deadline, after which the environments that have not answered count as undelivered and a late answer is ignored. The background delivery uses the assertion's 30s cap, so a delivery that never answers no longer keeps the activity held and every later backgrounding from delivering. Pins the once-only contract with never-answering and double-answering reporters. Co-authored-by: Cursor --- LaunchDarkly.xcodeproj/project.pbxproj | 4 + .../ServiceObjects/BackgroundActivity.swift | 2 +- .../BackgroundEventDelivery.swift | 12 ++- .../ServiceObjects/EventProcessor.swift | 30 ++++++- .../LaunchDarklyTests/LDClientSpec.swift | 23 +++++ .../BackgroundEventDeliverySpec.swift | 18 ++++ .../EventProcessorOutcomesSpec.swift | 89 +++++++++++++++++++ 7 files changed, 171 insertions(+), 7 deletions(-) create mode 100644 LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventProcessorOutcomesSpec.swift diff --git a/LaunchDarkly.xcodeproj/project.pbxproj b/LaunchDarkly.xcodeproj/project.pbxproj index 035c7d02e..d78acde9c 100644 --- a/LaunchDarkly.xcodeproj/project.pbxproj +++ b/LaunchDarkly.xcodeproj/project.pbxproj @@ -113,6 +113,7 @@ 831AAE3020A9E75D00B46DBA /* ThrottlerSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831AAE2F20A9E75D00B46DBA /* ThrottlerSpec.swift */; }; 3F9AC0D80849C86866E03807 /* BackgroundActivitySpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = C3BDE2A66C982198D55602DA /* BackgroundActivitySpec.swift */; }; 379F34CAB1EE19A0F85AA2C8 /* BackgroundEventDeliverySpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88A4CE0E204CB660F54E8A4B /* BackgroundEventDeliverySpec.swift */; }; + C863C417AB6004C6EE2F600C /* EventProcessorOutcomesSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 406B825B2AAD6C24F5467449 /* EventProcessorOutcomesSpec.swift */; }; 831D2AAF2061AAA000B4AC3C /* Thread.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831D2AAE2061AAA000B4AC3C /* Thread.swift */; }; 831D8B6F1F71532300ED65E8 /* HTTPHeaders.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831D8B6E1F71532300ED65E8 /* HTTPHeaders.swift */; }; 831D8B721F71D3E700ED65E8 /* DarklyService.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831D8B711F71D3E700ED65E8 /* DarklyService.swift */; }; @@ -582,6 +583,7 @@ 831AAE2F20A9E75D00B46DBA /* ThrottlerSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ThrottlerSpec.swift; sourceTree = ""; }; C3BDE2A66C982198D55602DA /* BackgroundActivitySpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BackgroundActivitySpec.swift; sourceTree = ""; }; 88A4CE0E204CB660F54E8A4B /* BackgroundEventDeliverySpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BackgroundEventDeliverySpec.swift; sourceTree = ""; }; + 406B825B2AAD6C24F5467449 /* EventProcessorOutcomesSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EventProcessorOutcomesSpec.swift; sourceTree = ""; }; 831D2AAE2061AAA000B4AC3C /* Thread.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Thread.swift; sourceTree = ""; }; 831D8B6E1F71532300ED65E8 /* HTTPHeaders.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HTTPHeaders.swift; sourceTree = ""; }; 831D8B711F71D3E700ED65E8 /* DarklyService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DarklyService.swift; sourceTree = ""; }; @@ -827,6 +829,7 @@ 831AAE2F20A9E75D00B46DBA /* ThrottlerSpec.swift */, C3BDE2A66C982198D55602DA /* BackgroundActivitySpec.swift */, 88A4CE0E204CB660F54E8A4B /* BackgroundEventDeliverySpec.swift */, + 406B825B2AAD6C24F5467449 /* EventProcessorOutcomesSpec.swift */, 8354AC75224316C700CDE602 /* Cache */, A3C6F7612B7FA803005B3B61 /* SheddingQueueSpec.swift */, EE6BAECBB739A0F24DB5A944 /* EventStoreSpec.swift */, @@ -1902,6 +1905,7 @@ 831AAE3020A9E75D00B46DBA /* ThrottlerSpec.swift in Sources */, 3F9AC0D80849C86866E03807 /* BackgroundActivitySpec.swift in Sources */, 379F34CAB1EE19A0F85AA2C8 /* BackgroundEventDeliverySpec.swift in Sources */, + C863C417AB6004C6EE2F600C /* EventProcessorOutcomesSpec.swift in Sources */, 832D68AC224B3321005F052A /* CacheConverterSpec.swift in Sources */, 838F96741FB9F024009CFC45 /* LDClientSpec.swift in Sources */, 83A0E6B1203B557F00224298 /* FeatureFlagSpec.swift in Sources */, diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundActivity.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundActivity.swift index dc669aa76..9e3bda96d 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundActivity.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundActivity.swift @@ -17,7 +17,7 @@ final class BackgroundActivity { typealias ExpiringActivity = (_ reason: String, _ whileGranted: @escaping (_ expired: Bool) -> Void) -> Void /// Upper bound for work that never calls `finished`. - private static let maximumDuration: TimeInterval = 30 + static let maximumDuration: TimeInterval = 30 private let reason: String private let assertion: Assertion diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundEventDelivery.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundEventDelivery.swift index e6a0557d3..1cab80ae6 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundEventDelivery.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundEventDelivery.swift @@ -23,11 +23,19 @@ final class BackgroundEventDelivery { private let environments: () -> [Environment] private let suspendsWhenBackgrounded: Bool private let activity: BackgroundActivity + /// How long the deliveries have to answer before the ones that have not count as undelivered. The activity holds + /// one assertion at a time, so a delivery that never answered would otherwise keep every later backgrounding from + /// delivering at all. + private let answerTimeout: TimeInterval - init(environments: @escaping () -> [Environment], suspendsWhenBackgrounded: Bool, activity: BackgroundActivity) { + init(environments: @escaping () -> [Environment], + suspendsWhenBackgrounded: Bool, + activity: BackgroundActivity, + answerTimeout: TimeInterval = BackgroundActivity.maximumDuration) { self.environments = environments self.suspendsWhenBackgrounded = suspendsWhenBackgrounded self.activity = activity + self.answerTimeout = answerTimeout } convenience init(environments: @escaping () -> [Environment]) { @@ -50,7 +58,7 @@ final class BackgroundEventDelivery { finished() return } - environments.map(\.eventProcessor).flushReportingOutcomes { delivered in + environments.map(\.eventProcessor).flushReportingOutcomes(deadline: .now() + self.answerTimeout) { delivered in self.logUndelivered(zip(environments, delivered).filter { !$0.1 }.map(\.0)) finished() } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift index e99f65270..1dce1412a 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift @@ -66,11 +66,19 @@ protocol EventProcessor { extension Array where Element == EventProcessor { /// Starts every processor's delivery at once, and gives `completion` each one's outcome, in order, once all of them - /// have answered. With no processors it answers at once, on the calling thread. + /// have answered or `deadline` has passed, whichever is first. A processor that has not answered by the deadline + /// counts as not delivered, and its answer, should it come later, is ignored. With no processors it answers at + /// once, on the calling thread. /// - /// The completion runs once, on the thread of the last answer. A reporter answering twice neither ends the wait - /// before every processor has answered nor ends it again: its second answer is ignored. - func flushReportingOutcomes(bypassingBackoff: Bool = false, completion: @escaping (_ delivered: [Bool]) -> Void) { + /// The completion runs once, on the thread of the last answer, or a global queue at the deadline. A reporter + /// answering twice neither ends the wait before every processor has answered nor ends it again: its second answer + /// is ignored. + /// + /// - parameter deadline: where a caller cannot wait on a reporter that never answers. Without one, the completion + /// waits for every answer. + func flushReportingOutcomes(bypassingBackoff: Bool = false, + deadline: DispatchTime? = nil, + completion: @escaping (_ delivered: [Bool]) -> Void) { guard !isEmpty else { completion([]) @@ -80,6 +88,20 @@ extension Array where Element == EventProcessor { /// Only to be used while holding `lock`: each processor answers from whichever thread its delivery ended on. var outcomes = [Bool?](repeating: nil, count: count) var outstanding = count + if let deadline { + DispatchQueue.global().asyncAfter(deadline: deadline) { + let ended: [Bool]? = lock.withLock { + guard outstanding > 0 + else { return nil } + outstanding = 0 + outcomes = outcomes.map { $0 ?? false } + return outcomes.compactMap { $0 } + } + if let ended { + completion(ended) + } + } + } for (index, processor) in enumerated() { let answer: FlushOutcomeClosure = { delivered in let ended: [Bool]? = lock.withLock { diff --git a/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift b/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift index 031e42105..f6d7f8b46 100644 --- a/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift @@ -1773,6 +1773,29 @@ final class LDClientSpec: QuickSpec { Thread.sleep(forTimeInterval: 0.2) expect(answers) == [false] } + + it("waits for every environment where one answers twice") { + let testContext = TestContext() + try testContext.config.setSecondaryMobileKeys(["alternate": Constants.alternateMockMobileKey]) + testContext.start() + testContext.eventProcessorMock.flushReportingOutcomeCallback = { [weak testContext] in + let completion = testContext?.eventProcessorMock.flushReportingOutcomeReceivedCompletion + DispatchQueue.global().async { + completion?(true) + completion?(true) + } + } + let answering = LDClient.get(environment: "alternate")?.eventProcessor as? EventProcessorMock + expect(answering).toNot(beNil()) + answer(answering!, with: false, after: 0.2) + + var answers: [Bool] = [] + testContext.subject.flush { answers.append($0) } + // Two answers from the primary are not one from each, so a `true` here would be about events still unsent. + expect(answers).toEventually(equal([false])) + Thread.sleep(forTimeInterval: 0.2) + expect(answers) == [false] + } } describe("flushAndWait") { diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundEventDeliverySpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundEventDeliverySpec.swift index e5d4437bb..6f245345e 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundEventDeliverySpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundEventDeliverySpec.swift @@ -91,6 +91,24 @@ final class BackgroundEventDeliverySpec: QuickSpec { expect(released) == 1 } + it("releases the assertion where an environment never answers, so a later backgrounding still delivers") { + var releases = 0 + let lock = UnfairLock() + let counted = BackgroundActivity(reason: "test") { _, work in + work { lock.withLock { releases += 1 } } + } + let subject = BackgroundEventDelivery(environments: { environments }, + suspendsWhenBackgrounded: true, + activity: counted, + answerTimeout: 0.1) + subject.applicationDidEnterBackground() + primary.flushReportingOutcomeReceivedCompletion?(true) + expect(lock.withLock { releases }).toEventually(equal(1)) + + subject.applicationDidEnterBackground() + expect(primary.flushReportingOutcomeCallCount) == 2 + } + it("takes no assertion where no client is started") { environments = [] delivery(suspendsWhenBackgrounded: true).applicationDidEnterBackground() diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventProcessorOutcomesSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventProcessorOutcomesSpec.swift new file mode 100644 index 000000000..abd8be8e4 --- /dev/null +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventProcessorOutcomesSpec.swift @@ -0,0 +1,89 @@ +import Foundation +import Quick +import Nimble +@testable import LaunchDarkly + +final class EventProcessorOutcomesSpec: QuickSpec { + override func spec() { + describe("flushing several processors") { + var first: EventProcessorMock! + var second: EventProcessorMock! + var answers: [[Bool]] = [] + var processors: [EventProcessor] { [first!, second!] } + beforeEach { + first = EventProcessorMock() + second = EventProcessorMock() + answers = [] + } + + it("answers with each processor's outcome, in order, once all of them have answered") { + processors.flushReportingOutcomes { answers.append($0) } + second.flushReportingOutcomeReceivedCompletion?(false) + expect(answers) == [] + + first.flushReportingOutcomeReceivedCompletion?(true) + expect(answers) == [[true, false]] + } + + it("answers at once where there are no processors") { + [EventProcessor]().flushReportingOutcomes { answers.append($0) } + expect(answers) == [[]] + } + + it("waits for every processor where one answers twice, and answers once") { + processors.flushReportingOutcomes { answers.append($0) } + first.flushReportingOutcomeReceivedCompletion?(true) + first.flushReportingOutcomeReceivedCompletion?(false) + expect(answers) == [] + + second.flushReportingOutcomeReceivedCompletion?(true) + second.flushReportingOutcomeReceivedCompletion?(true) + expect(answers) == [[true, true]] + } + + it("waits for a processor that never answers, where there is no deadline") { + processors.flushReportingOutcomes { answers.append($0) } + first.flushReportingOutcomeReceivedCompletion?(true) + Thread.sleep(forTimeInterval: 0.1) + expect(answers) == [] + } + + it("counts a processor that has not answered by the deadline as undelivered, and ignores its late answer") { + var answered: [[Bool]] = [] + let lock = UnfairLock() + processors.flushReportingOutcomes(deadline: .now() + 0.1) { outcome in + lock.withLock { answered.append(outcome) } + } + first.flushReportingOutcomeReceivedCompletion?(true) + + expect(lock.withLock { answered }).toEventually(equal([[true, false]])) + second.flushReportingOutcomeReceivedCompletion?(true) + expect(lock.withLock { answered }) == [[true, false]] + } + + it("answers once where every processor answers before the deadline") { + var answered: [[Bool]] = [] + let lock = UnfairLock() + processors.flushReportingOutcomes(deadline: .now() + 0.1) { outcome in + lock.withLock { answered.append(outcome) } + } + first.flushReportingOutcomeReceivedCompletion?(true) + second.flushReportingOutcomeReceivedCompletion?(true) + + Thread.sleep(forTimeInterval: 0.2) + expect(lock.withLock { answered }) == [[true, true]] + } + + it("posts through the backoff only where asked to") { + processors.flushReportingOutcomes(bypassingBackoff: true) { answers.append($0) } + expect(first.flushBypassingBackoffCallCount) == 1 + expect(second.flushBypassingBackoffCallCount) == 1 + expect(first.flushReportingOutcomeCallCount) == 0 + + first.flushBypassingBackoffReceivedCompletion?(true) + second.flushBypassingBackoffReceivedCompletion?(true) + expect(answers) == [[true, true]] + } + } + } +} From bcbde8a4fc55c4b5858489b34cfc28220dfba714 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Thu, 5 Nov 2026 22:13:32 -0800 Subject: [PATCH 155/158] Drop the long-timeout log from flushAndWait Co-authored-by: Cursor --- LaunchDarkly/LaunchDarkly/LDClientFlush.swift | 4 ---- 1 file changed, 4 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/LDClientFlush.swift b/LaunchDarkly/LaunchDarkly/LDClientFlush.swift index a5b23fdbc..19447856c 100644 --- a/LaunchDarkly/LaunchDarkly/LDClientFlush.swift +++ b/LaunchDarkly/LaunchDarkly/LDClientFlush.swift @@ -114,10 +114,6 @@ extension LDClient { public func flushAndWait(timeout: TimeInterval) -> Bool { // Taken first, so that the timeout covers the whole call, finding the clients included. let deadline = DispatchTime.now() + max(0, timeout) - if timeout > LDClient.longTimeoutInterval { - os_log("%s LDClient.flushAndWait was called with a timeout greater than %f seconds. We recommend a timeout of less than %f seconds.", log: config.logger, type: .info, self.typeName(and: #function), LDClient.longTimeoutInterval, LDClient.longTimeoutInterval) - } - let clients = instancesIncludingThisClient() guard !clients.isEmpty else { From 5a04c203aad532f0fccc4d9521a12961f63acdf0 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Fri, 6 Nov 2026 07:57:35 -0800 Subject: [PATCH 156/158] fix: pace the delivery a backoff's wait sets when it ends (R3-3) It was queued unpaced, so it sent a backlog back to back and ended the pacing of a delivery already running, competing with flag requests. Co-authored-by: Cursor --- .../ServiceObjects/DirectEventProcessor.swift | 5 +++-- .../EventDeliveryScheduler.swift | 3 ++- .../DirectEventProcessorSpec.swift | 18 ++++++++++++++++-- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 539f68359..0a9814290 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -399,7 +399,8 @@ extension DirectEventProcessor { queueScheduledDelivery() } - /// Queues the delivery the report timer and the launch gate start, which nobody asked for now and which is paced. + /// Queues the delivery the report timer, the launch gate and the end of a backoff's wait start, which nobody asked + /// for now and which is paced. Unpaced, it would also end the pacing of the delivery running. private func queueScheduledDelivery() { queueDelivery(nil, paced: true) } @@ -897,7 +898,7 @@ extension DirectEventProcessor { guard let self else { return } self.deliveryScheduler.retryDue() - self.queueDelivery(nil) + self.queueScheduledDelivery() } } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift index b4dc22682..365c55ac4 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift @@ -126,7 +126,8 @@ final class EventDeliveryScheduler { var isClosing: Bool /// Whether a caller bypassing the backoff asked for it, or joined it. var bypassesBackoff: Bool - /// Whether it was asked for only by the report timer or by the launch gate, and so is paced. + /// Whether it was asked for only by the report timer, the launch gate or the end of a backoff's wait, and so is + /// paced. var isPaced: Bool } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 508ff852b..442e80c88 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -351,14 +351,14 @@ class DirectEventProcessorSpec: QuickSpec { } /// A reporter on what a previous run left: `batches` batches of one event each. - func nextRunLeft(_ batches: Int, pacing: DirectEventProcessor.Pacing) -> DirectEventProcessor { + func nextRunLeft(_ batches: Int, pacing: DirectEventProcessor.Pacing, retryState: DeliveryRetryState? = nil) -> DirectEventProcessor { for _ in 0.. Date: Fri, 6 Nov 2026 08:04:40 -0800 Subject: [PATCH 157/158] Recover a previous run's events at utility A block on deliveryQueue cannot run below the queue's own userInitiated QoS, so asking for utility with .enforceQoS left the recovery at userInitiated. Run it on a utility global queue instead, with deliveryQueue suspended until it is done, so deliveries, the report timer and the launch gate still run after it. Co-authored-by: Cursor --- .../ServiceObjects/DirectEventProcessor.swift | 31 ++++++-- .../DirectEventProcessorSpec.swift | 71 +++++++++++++++++++ 2 files changed, 96 insertions(+), 6 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift index 0a9814290..1445b8c56 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -177,21 +177,40 @@ class DirectEventProcessor: EventProcessor { self.launchGateOpen = pacing.launchDelay <= 0 // A log left open by a previous run has to be closed before it can be delivered, but nothing waits on that: - // doing it on the caller's thread would put file I/O in the way of the client starting up. Below the queue's - // own priority, and so below the flag requests, which matter more at launch than what a previous run left. - // Only where events persist, since memory holds nothing from a previous run. + // doing it on the caller's thread would put file I/O in the way of the client starting up. At utility, below + // the flag requests, which matter more at launch than what a previous run left. Not as a block on + // `deliveryQueue`, whose own userInitiated QoS a block cannot go below; the queue is suspended instead, so + // everything enqueued on it meanwhile still runs after the recovery. Only where events persist, since memory + // holds nothing from a previous run. guard eventStore.isPersisting else { return } let store = eventStore - deliveryQueue.async(qos: .utility, flags: .enforceQoS) { [weak self] in + let queue = deliveryQueue + let recovered = RecoveredListing() + queue.suspend() + queue.async { [weak self] in + self?.reportVanished(recovered.vanishedEvents) + self?.hasEventsFromPreviousRun = recovered.hasBatches + } + // Enforced, since the caller is usually the main thread, whose QoS would otherwise be carried over. + DispatchQueue.global(qos: .utility).async(qos: .utility, flags: .enforceQoS) { [weak self] in + // Resumed whatever happens: a dispatch queue released while suspended traps. + defer { queue.resume() } store.preparePayloadsFromInterruptedLogs() self?.reportDiscarded(store.discardExpired()) let listing = store.listPendingBatches() - self?.reportVanished(listing.vanishedEvents) - self?.hasEventsFromPreviousRun = !listing.batches.isEmpty + recovered.vanishedEvents = listing.vanishedEvents + recovered.hasBatches = !listing.batches.isEmpty } } + /// What the recovery found, handed to `deliveryQueue`. Written before the queue resumes and read after, so the + /// resume orders the two. + private final class RecoveredListing { + var vanishedEvents = 0 + var hasBatches = false + } + private static func makeStore(config: LDConfig) -> EventStoring { let mobileKey = config.mobileKey return EventStore(locatingDirectory: { EventDirectory.defaultUrl(mobileKey: mobileKey) }, diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift index 442e80c88..294b14643 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -436,6 +436,33 @@ class DirectEventProcessorSpec: QuickSpec { expect(testContext.serviceMock.publishEventDataCallCount) == 3 } + itPersisting("recovers what a previous run left at utility, though it is started from the main thread") { + let store = RecoveryObservingStore(testContext.store.nextRun()) + nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: store, pacing: .noWaits) + + expect(store.recoveryQoS).toEventually(equal(QOS_CLASS_UTILITY), timeout: Constants.deliveryTimeout) + } + + itPersisting("holds a delivery asked for during the recovery until the recovery is done") { + testContext.recordEvents(1) + testContext.eventProcessor.commitRecordedEvents() + _ = testContext.store.closeBatch() + let store = RecoveryObservingStore(testContext.store.nextRun(), holdingRecovery: true) + defer { store.release.signal() } + let reporter = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: store, pacing: .noWaits) + nextRun = reporter + reporter.isOnline = true + + var delivered: Bool? + reporter.flushReportingOutcome { delivered = $0 } + Thread.sleep(forTimeInterval: 0.3) + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + + store.release.signal() + expect(delivered).toEventually(beTrue(), timeout: Constants.deliveryTimeout) + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + } + itPersisting("keeps the gap for the delivery a backoff's wait sets when it ends") { // A refusal waits the extended ceiling, longer than a pass sits out, so the pass ends and the delivery // is set for when the wait ends. @@ -3922,6 +3949,50 @@ private final class SlowListingStore: EventStoring { } } +/// A store that notes the QoS the previous run's events are recovered at, and can hold the recovery open. +private final class RecoveryObservingStore: EventStoring { + let inner: EventStoring + /// Signal to let a held recovery finish. + let release = DispatchSemaphore(value: 0) + private let holds: Bool + private let lock = UnfairLock() + private var observedQoS: qos_class_t? + + init(_ inner: EventStoring, holdingRecovery holds: Bool = false) { + self.inner = inner + self.holds = holds + } + + var recoveryQoS: qos_class_t? { lock.withLock { observedQoS } } + + var pendingEventCount: Int { inner.pendingEventCount } + var hasRoomForEventBytes: Bool { inner.hasRoomForEventBytes } + func admitsEvent(heldElsewhere: Int, capacity: Int) -> Bool { inner.admitsEvent(heldElsewhere: heldElsewhere, capacity: capacity) } + var isPersisting: Bool { inner.isPersisting } + func stage(_ encodedEvent: Data, bypassingCapacity: Bool) -> Bool { inner.stage(encodedEvent, bypassingCapacity: bypassingCapacity) } + func reserve(_ events: Int) { inner.reserve(events) } + func stageReserved(_ encodedEvent: Data) -> Bool { inner.stageReserved(encodedEvent) } + func releaseReservations() { inner.releaseReservations() } + func commit() { inner.commit() } + func closeBatch() -> EventBatch? { inner.closeBatch() } + var holdsUnclosedEvents: Bool { inner.holdsUnclosedEvents } + func takeVanishedLogEvents() -> Int { inner.takeVanishedLogEvents() } + func payload(of batch: EventBatch) throws -> BatchPayload? { try inner.payload(of: batch) } + func remove(_ batch: EventBatch) { inner.remove(batch) } + func startedSending(_ batch: EventBatch) { inner.startedSending(batch) } + func stoppedSending(_ batch: EventBatch) { inner.stoppedSending(batch) } + func discardExpired() -> Int { inner.discardExpired() } + func listPendingBatches() -> BatchListing { inner.listPendingBatches() } + + func preparePayloadsFromInterruptedLogs() { + lock.withLock { observedQoS = qos_class_self() } + if holds { + _ = release.wait(timeout: .now() + 5) + } + inner.preparePayloadsFromInterruptedLogs() + } +} + /// Waits while letting the main run loop turn, since that is where the report timer fires. private func waitLettingTimersFire(_ seconds: TimeInterval) { RunLoop.current.run(until: Date().addingTimeInterval(seconds)) From 3f8bab5bfafc1db4384a37e723928b9d3bd5f076 Mon Sep 17 00:00:00 2001 From: Andrey Belonogov Date: Fri, 6 Nov 2026 11:17:45 -0800 Subject: [PATCH 158/158] Close the clients outside instancesQueue close() held instancesQueue while each environment's flushAtClose() waited up to closeCommitBudget for its final commit, so LDClient.get(), flush() and every other call reading instances waited behind those writes. Take the clients and clear instances under the queue, then close them outside it. Co-authored-by: Cursor --- LaunchDarkly/LaunchDarkly/LDClient.swift | 9 ++++++--- .../LaunchDarklyTests/LDClientSpec.swift | 18 ++++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/LaunchDarkly/LaunchDarkly/LDClient.swift b/LaunchDarkly/LaunchDarkly/LDClient.swift index e144b03f3..f36ba9b32 100644 --- a/LaunchDarkly/LaunchDarkly/LDClient.swift +++ b/LaunchDarkly/LaunchDarkly/LDClient.swift @@ -247,10 +247,13 @@ public class LDClient { There is almost no reason to stop the LDClient. Normally, set the LDClient offline to stop communication with the LaunchDarkly servers. Stop the LDClient to stop recording events. There is no need to stop the LDClient prior to suspending, moving to the background, or terminating the app. The SDK will respond to these events as the system requires and as configured in LDConfig. */ public func close() { - LDClient.instancesQueue.sync(flags: .barrier) { - LDClient.instances?.forEach { $1.internalClose() } - LDClient.instances = nil + // Closed outside `instancesQueue`: each close can wait on its final event commit, and every other call that + // reads `instances` would wait behind it. + let clients = LDClient.instancesQueue.sync(flags: .barrier) { () -> [LDClient] in + defer { LDClient.instances = nil } + return LDClient.instances.map { Array($0.values) } ?? [] } + clients.forEach { $0.internalClose() } } private func internalClose() { diff --git a/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift b/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift index f6d7f8b46..6ff3c8146 100644 --- a/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift @@ -922,6 +922,24 @@ final class LDClientSpec: QuickSpec { expect(testContext.eventProcessorMock.flushAtCloseCallCount) == 1 expect(testContext.eventProcessorMock.flushCallCount) == 0 } + it("answers other calls while an environment's final commit is under way") { + let testContext = TestContext() + testContext.start() + var answeredDuringClose = false + testContext.eventProcessorMock.flushAtCloseCallback = { + // Standing in for a commit stalled on the disk, which can hold the closing thread for the budget. + let answered = DispatchSemaphore(value: 0) + DispatchQueue.global().async { + _ = LDClient.get() + answered.signal() + } + answeredDuringClose = answered.wait(timeout: .now() + 1) == .success + } + testContext.subject.close() + + expect(answeredDuringClose) == true + expect(LDClient.get()).to(beNil()) + } } }