diff --git a/LaunchDarkly.xcodeproj/project.pbxproj b/LaunchDarkly.xcodeproj/project.pbxproj index 8df9c3f7f..d78acde9c 100644 --- a/LaunchDarkly.xcodeproj/project.pbxproj +++ b/LaunchDarkly.xcodeproj/project.pbxproj @@ -7,6 +7,7 @@ objects = { /* Begin PBXBuildFile section */ + 1E65A889B0528E5AB03C517A /* EventStoreSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = EE6BAECBB739A0F24DB5A944 /* EventStoreSpec.swift */; }; 29F9D19E2812E005008D12C0 /* ObjcLDValue.swift in Sources */ = {isa = PBXBuildFile; fileRef = 29F9D19D2812E005008D12C0 /* ObjcLDValue.swift */; }; 29F9D19F2812E005008D12C0 /* ObjcLDValue.swift in Sources */ = {isa = PBXBuildFile; fileRef = 29F9D19D2812E005008D12C0 /* ObjcLDValue.swift */; }; 29F9D1A02812E005008D12C0 /* ObjcLDValue.swift in Sources */ = {isa = PBXBuildFile; fileRef = 29F9D19D2812E005008D12C0 /* ObjcLDValue.swift */; }; @@ -15,6 +16,12 @@ 29FE1299280413D4008CC918 /* Util.swift in Sources */ = {isa = PBXBuildFile; fileRef = 29FE1297280413D4008CC918 /* Util.swift */; }; 29FE129A280413D4008CC918 /* Util.swift in Sources */ = {isa = PBXBuildFile; fileRef = 29FE1297280413D4008CC918 /* Util.swift */; }; 29FE129B280413D4008CC918 /* Util.swift in Sources */ = {isa = PBXBuildFile; fileRef = 29FE1297280413D4008CC918 /* Util.swift */; }; + 2CA9D36B3566D78850D6F0AF /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; + B8F228923BE11E444D1ECB8E /* DiskEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */; }; + A1B2C3D4E5F60718293A4B01 /* EventDirectory.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D4E5F60718293A4B5C /* EventDirectory.swift */; }; + 3C55384C9E3485B493E1F86A /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; + C3F7E502BD916C40AE238F64 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; + AB4827D11084D58661E24B05 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; 3D2406142E0D90E000F91253 /* Plugin.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3D2406102E0D90E000F91253 /* Plugin.swift */; }; 3D2406152E0D90E000F91253 /* PluginMetadata.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3D2406112E0D90E000F91253 /* PluginMetadata.swift */; }; 3D2406162E0D90E000F91253 /* EnvironmentMetadata.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3D24060F2E0D90E000F91253 /* EnvironmentMetadata.swift */; }; @@ -33,6 +40,7 @@ 3D2406232E0D90E000F91253 /* SdkMetadata.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3D2406122E0D90E000F91253 /* SdkMetadata.swift */; }; 3D2406252E0D90EA00F91253 /* LDClientPluginsSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3D2406242E0D90EA00F91253 /* LDClientPluginsSpec.swift */; }; 3D34EEEF37158657D49335BB /* BackgroundActivity.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9A771B316E19ED598CD4FB0B /* BackgroundActivity.swift */; }; + 051BF088E9BDA17A3871A530 /* BackgroundEventDelivery.swift in Sources */ = {isa = PBXBuildFile; fileRef = EF2483453F8EF6F6D01A83B5 /* BackgroundEventDelivery.swift */; }; 3D3AB9432A4F16FE003AECF1 /* ReportingConsts.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3D3AB9422A4F16FE003AECF1 /* ReportingConsts.swift */; }; 3D3AB9442A4F16FE003AECF1 /* ReportingConsts.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3D3AB9422A4F16FE003AECF1 /* ReportingConsts.swift */; }; 3D3AB9452A4F16FE003AECF1 /* ReportingConsts.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3D3AB9422A4F16FE003AECF1 /* ReportingConsts.swift */; }; @@ -46,7 +54,14 @@ 50EE85C42EA0487F007CC662 /* TimeoutExecutor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50EE85C12EA0487F007CC662 /* TimeoutExecutor.swift */; }; 50EE85C52EA0487F007CC662 /* TimeoutExecutor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50EE85C12EA0487F007CC662 /* TimeoutExecutor.swift */; }; 50EE85C72EA0749C007CC662 /* TimeoutExecutorSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 50EE85C62EA0749C007CC662 /* TimeoutExecutorSpec.swift */; }; + 5AEA1F832DCA974314E07D21 /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; + D4225F1F05F2C55E9518941B /* DiskEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */; }; + A1B2C3D4E5F60718293A4B02 /* EventDirectory.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D4E5F60718293A4B5C /* EventDirectory.swift */; }; + DDA788E3056F8AC3CA203933 /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; + D4080613CEA27D51BF349075 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; + ECBE45806B470ACC5187B7D6 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; 7185A3C30E7FFC4F725769F1 /* BackgroundActivity.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9A771B316E19ED598CD4FB0B /* BackgroundActivity.swift */; }; + 1830F6435EE141699F106451 /* BackgroundEventDelivery.swift in Sources */ = {isa = PBXBuildFile; fileRef = EF2483453F8EF6F6D01A83B5 /* BackgroundEventDelivery.swift */; }; 830BF933202D188E006DF9B1 /* HTTPURLRequest.swift in Sources */ = {isa = PBXBuildFile; fileRef = 830BF932202D188E006DF9B1 /* HTTPURLRequest.swift */; }; 830DB3AC22380A3E00D65D25 /* HTTPHeadersSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 830DB3AB22380A3E00D65D25 /* HTTPHeadersSpec.swift */; }; 830DB3AE2239B54900D65D25 /* URLResponse.swift in Sources */ = {isa = PBXBuildFile; fileRef = 830DB3AD2239B54900D65D25 /* URLResponse.swift */; }; @@ -66,7 +81,13 @@ 831188522113ADF700D77CB5 /* KeyedValueCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83D559731FD87CC9002D10C8 /* KeyedValueCache.swift */; }; 831188562113AE0800D77CB5 /* FlagSynchronizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */; }; 831188572113AE0B00D77CB5 /* FlagChangeNotifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8358F25F1F476AD800ECE1AF /* FlagChangeNotifier.swift */; }; - 831188582113AE0F00D77CB5 /* EventReporter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* EventReporter.swift */; }; + 831188582113AE0F00D77CB5 /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; + E7C0A11E2F00000000000002 /* EventCommitter.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000001 /* EventCommitter.swift */; }; + E7C0A11E2F00000000000007 /* EventDeliveryScheduler.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000006 /* EventDeliveryScheduler.swift */; }; + B66ECB2014E0FC95C23DA0A2 /* AnalyticsEventSender.swift in Sources */ = {isa = PBXBuildFile; fileRef = 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */; }; + 629D5A9FE152DBC3E20D6DFC /* OutboundEventBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */; }; + 014901E06BE1F2B0FAAD6AA6 /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; + 86B71C91DB78D0F17DBE0E69 /* EventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 293A30078FEAA25BB4C3D26C /* EventProcessor.swift */; }; 831188592113AE1200D77CB5 /* FlagStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831D8B731F72994600ED65E8 /* FlagStore.swift */; }; 8311885A2113AE1500D77CB5 /* Log.swift in Sources */ = {isa = PBXBuildFile; fileRef = 837EF3732059C237009D628A /* Log.swift */; }; 8311885B2113AE1D00D77CB5 /* Throttler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831AAE2B20A9E4F600B46DBA /* Throttler.swift */; }; @@ -91,6 +112,8 @@ 831AAE2E20A9E4F600B46DBA /* Throttler.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831AAE2B20A9E4F600B46DBA /* Throttler.swift */; }; 831AAE3020A9E75D00B46DBA /* ThrottlerSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831AAE2F20A9E75D00B46DBA /* ThrottlerSpec.swift */; }; 3F9AC0D80849C86866E03807 /* BackgroundActivitySpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = C3BDE2A66C982198D55602DA /* BackgroundActivitySpec.swift */; }; + 379F34CAB1EE19A0F85AA2C8 /* BackgroundEventDeliverySpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88A4CE0E204CB660F54E8A4B /* BackgroundEventDeliverySpec.swift */; }; + C863C417AB6004C6EE2F600C /* EventProcessorOutcomesSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 406B825B2AAD6C24F5467449 /* EventProcessorOutcomesSpec.swift */; }; 831D2AAF2061AAA000B4AC3C /* Thread.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831D2AAE2061AAA000B4AC3C /* Thread.swift */; }; 831D8B6F1F71532300ED65E8 /* HTTPHeaders.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831D8B6E1F71532300ED65E8 /* HTTPHeaders.swift */; }; 831D8B721F71D3E700ED65E8 /* DarklyService.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831D8B711F71D3E700ED65E8 /* DarklyService.swift */; }; @@ -107,7 +130,13 @@ 831EF35120655E730001C643 /* KeyedValueCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83D559731FD87CC9002D10C8 /* KeyedValueCache.swift */; }; 831EF35520655E730001C643 /* FlagSynchronizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */; }; 831EF35620655E730001C643 /* FlagChangeNotifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8358F25F1F476AD800ECE1AF /* FlagChangeNotifier.swift */; }; - 831EF35720655E730001C643 /* EventReporter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* EventReporter.swift */; }; + 831EF35720655E730001C643 /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; + E7C0A11E2F00000000000003 /* EventCommitter.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000001 /* EventCommitter.swift */; }; + E7C0A11E2F00000000000008 /* EventDeliveryScheduler.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000006 /* EventDeliveryScheduler.swift */; }; + 367D374B5CD1B2A8596C0738 /* AnalyticsEventSender.swift in Sources */ = {isa = PBXBuildFile; fileRef = 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */; }; + 5A89648200D2566D3F8409BA /* OutboundEventBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */; }; + CFF357D68BBF2DC8395930F2 /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; + 850C49C9054E642CC006E730 /* EventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 293A30078FEAA25BB4C3D26C /* EventProcessor.swift */; }; 831EF35820655E730001C643 /* FlagStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831D8B731F72994600ED65E8 /* FlagStore.swift */; }; 831EF35920655E730001C643 /* Log.swift in Sources */ = {isa = PBXBuildFile; fileRef = 837EF3732059C237009D628A /* Log.swift */; }; 831EF35A20655E730001C643 /* HTTPHeaders.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831D8B6E1F71532300ED65E8 /* HTTPHeaders.swift */; }; @@ -174,8 +203,9 @@ 83B8C2471FE4071F0082B8A9 /* HTTPURLResponse.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83B8C2461FE4071F0082B8A9 /* HTTPURLResponse.swift */; }; 83B9A080204F56F4000C3F17 /* FlagChangeObserverSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83B9A07F204F56F4000C3F17 /* FlagChangeObserverSpec.swift */; }; 83B9A082204F6022000C3F17 /* FlagsUnchangedObserver.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83B9A081204F6022000C3F17 /* FlagsUnchangedObserver.swift */; }; - 83CFE7CE1F7AD81D0010544E /* EventReporterSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83CFE7CD1F7AD81D0010544E /* EventReporterSpec.swift */; }; + 83CFE7CE1F7AD81D0010544E /* DirectEventProcessorSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83CFE7CD1F7AD81D0010544E /* DirectEventProcessorSpec.swift */; }; 83CFE7D11F7AD8DC0010544E /* DarklyServiceMock.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83CFE7D01F7AD8DC0010544E /* DarklyServiceMock.swift */; }; + A1D1A6C12F0B000000C0FFEE /* DiagnosticCachingMock.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D1A6C02F0B000000C0FFEE /* DiagnosticCachingMock.swift */; }; 83D559741FD87CC9002D10C8 /* KeyedValueCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83D559731FD87CC9002D10C8 /* KeyedValueCache.swift */; }; 83D9EC752062DEAB004D7FA6 /* LDCommon.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83B6C4B51F4DE7630055351C /* LDCommon.swift */; }; 83D9EC762062DEAB004D7FA6 /* LDConfig.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8354EFDD1F26380700C05156 /* LDConfig.swift */; }; @@ -189,7 +219,13 @@ 83D9EC832062DEAB004D7FA6 /* KeyedValueCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83D559731FD87CC9002D10C8 /* KeyedValueCache.swift */; }; 83D9EC872062DEAB004D7FA6 /* FlagSynchronizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */; }; 83D9EC882062DEAB004D7FA6 /* FlagChangeNotifier.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8358F25F1F476AD800ECE1AF /* FlagChangeNotifier.swift */; }; - 83D9EC892062DEAB004D7FA6 /* EventReporter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* EventReporter.swift */; }; + 83D9EC892062DEAB004D7FA6 /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; + E7C0A11E2F00000000000004 /* EventCommitter.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000001 /* EventCommitter.swift */; }; + E7C0A11E2F00000000000009 /* EventDeliveryScheduler.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000006 /* EventDeliveryScheduler.swift */; }; + 321659EE1824B83BD0D0F22D /* AnalyticsEventSender.swift in Sources */ = {isa = PBXBuildFile; fileRef = 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */; }; + 73E23C0FA009141BB4CBC24C /* OutboundEventBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */; }; + 54745AEF1A43D165F5A6E08F /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; + 7F4004579B9094737F3C21D1 /* EventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 293A30078FEAA25BB4C3D26C /* EventProcessor.swift */; }; 83D9EC8A2062DEAB004D7FA6 /* FlagStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831D8B731F72994600ED65E8 /* FlagStore.swift */; }; 83D9EC8B2062DEAB004D7FA6 /* Log.swift in Sources */ = {isa = PBXBuildFile; fileRef = 837EF3732059C237009D628A /* Log.swift */; }; 83D9EC8C2062DEAB004D7FA6 /* HTTPHeaders.swift in Sources */ = {isa = PBXBuildFile; fileRef = 831D8B6E1F71532300ED65E8 /* HTTPHeaders.swift */; }; @@ -213,7 +249,13 @@ 83EF67931F9945E800403126 /* EventSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83EF67921F9945E800403126 /* EventSpec.swift */; }; 83F0A5641FB5F33800550A95 /* LDConfigSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83F0A5631FB5F33800550A95 /* LDConfigSpec.swift */; }; 83FEF8DD1F266742001CF12C /* FlagSynchronizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */; }; - 83FEF8DF1F2667E4001CF12C /* EventReporter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* EventReporter.swift */; }; + 83FEF8DF1F2667E4001CF12C /* DirectEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */; }; + E7C0A11E2F00000000000005 /* EventCommitter.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000001 /* EventCommitter.swift */; }; + E7C0A11E2F0000000000000A /* EventDeliveryScheduler.swift in Sources */ = {isa = PBXBuildFile; fileRef = E7C0A11E2F00000000000006 /* EventDeliveryScheduler.swift */; }; + E91F5DCAEEE5FC4AC408BA67 /* AnalyticsEventSender.swift in Sources */ = {isa = PBXBuildFile; fileRef = 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */; }; + 7D37AD3ECAA9D8E2927228C5 /* OutboundEventBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */; }; + 9FA0EE3A87678F58F4406B24 /* NullEventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */; }; + 7B38D33B59821BD9BCFFA731 /* EventProcessor.swift in Sources */ = {isa = PBXBuildFile; fileRef = 293A30078FEAA25BB4C3D26C /* EventProcessor.swift */; }; 8B3D9E2F5A6C1047B2D5E9F1 /* JSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6F2A8C1D4E7B9053A1C4D8E6 /* JSONWriter.swift */; }; 8E38AA8C306308E500C4C163 /* RetryState.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8E38AA8B306308E500C4C163 /* RetryState.swift */; }; 8E38AA8D306308E500C4C163 /* RetryState.swift in Sources */ = {isa = PBXBuildFile; fileRef = 8E38AA8B306308E500C4C163 /* RetryState.swift */; }; @@ -249,6 +291,7 @@ A032A73E446055D14A546CA0 /* EventJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = B430884DBF2E541B919C08FA /* EventJSONWriter.swift */; }; A0BAD9EE25943C43A5154E6F /* EventJSONWriterTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 367A9182439001C3EBD68477 /* EventJSONWriterTests.swift */; }; A20146E7AF098560627C614E /* BackgroundActivity.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9A771B316E19ED598CD4FB0B /* BackgroundActivity.swift */; }; + BAB394DCCA3C189DD1B11C36 /* BackgroundEventDelivery.swift in Sources */ = {isa = PBXBuildFile; fileRef = EF2483453F8EF6F6D01A83B5 /* BackgroundEventDelivery.swift */; }; B6168D5F889EFC4C19F59135 /* JSONWriterTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 1BECAB38F41B30401D3CD7EA /* JSONWriterTests.swift */; }; A3047D642A606B6000F568E0 /* SDKEnvironmentReporterSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = A3047D5D2A606B6000F568E0 /* SDKEnvironmentReporterSpec.swift */; }; A3047D652A606B6000F568E0 /* IOSEnvironmentReporterSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = A3047D5E2A606B6000F568E0 /* IOSEnvironmentReporterSpec.swift */; }; @@ -274,6 +317,7 @@ A31088282837DCA900184942 /* ReferenceSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = A31088252837DCA900184942 /* ReferenceSpec.swift */; }; A31088292837DCA900184942 /* KindSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = A31088262837DCA900184942 /* KindSpec.swift */; }; A32FCDB22D53263851DDB727 /* BackgroundActivity.swift in Sources */ = {isa = PBXBuildFile; fileRef = 9A771B316E19ED598CD4FB0B /* BackgroundActivity.swift */; }; + 8F654B978A758FDFC94E4AF0 /* BackgroundEventDelivery.swift in Sources */ = {isa = PBXBuildFile; fileRef = EF2483453F8EF6F6D01A83B5 /* BackgroundEventDelivery.swift */; }; A33A5F7A28466D04000C29C7 /* LDContextStub.swift in Sources */ = {isa = PBXBuildFile; fileRef = A33A5F7928466D04000C29C7 /* LDContextStub.swift */; }; A3470C372B7C1ACE00951CEE /* LDValueDecoder.swift in Sources */ = {isa = PBXBuildFile; fileRef = A3470C362B7C1ACE00951CEE /* LDValueDecoder.swift */; }; A3470C382B7C1ACE00951CEE /* LDValueDecoder.swift in Sources */ = {isa = PBXBuildFile; fileRef = A3470C362B7C1ACE00951CEE /* LDValueDecoder.swift */; }; @@ -295,6 +339,10 @@ A350387D2F4F5A380032BA9F /* Data+Compression.swift in Sources */ = {isa = PBXBuildFile; fileRef = A350387B2F4F5A380032BA9F /* Data+Compression.swift */; }; A350387E2F4F5A380032BA9F /* Data+Compression.swift in Sources */ = {isa = PBXBuildFile; fileRef = A350387B2F4F5A380032BA9F /* Data+Compression.swift */; }; A350387F2F4F5A380032BA9F /* Data+Compression.swift in Sources */ = {isa = PBXBuildFile; fileRef = A350387B2F4F5A380032BA9F /* Data+Compression.swift */; }; + A35038812F4F5A380032BA9F /* Data+CRC.swift in Sources */ = {isa = PBXBuildFile; fileRef = A35038802F4F5A380032BA9F /* Data+CRC.swift */; }; + A35038822F4F5A380032BA9F /* Data+CRC.swift in Sources */ = {isa = PBXBuildFile; fileRef = A35038802F4F5A380032BA9F /* Data+CRC.swift */; }; + A35038832F4F5A380032BA9F /* Data+CRC.swift in Sources */ = {isa = PBXBuildFile; fileRef = A35038802F4F5A380032BA9F /* Data+CRC.swift */; }; + A35038842F4F5A380032BA9F /* Data+CRC.swift in Sources */ = {isa = PBXBuildFile; fileRef = A35038802F4F5A380032BA9F /* Data+CRC.swift */; }; A35038812F4F965E0032BA9F /* LDClientIdentifyHookSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = A35038802F4F965E0032BA9F /* LDClientIdentifyHookSpec.swift */; }; A35038832F4F96680032BA9F /* LDClientEvaluationHookSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = A35038822F4F96680032BA9F /* LDClientEvaluationHookSpec.swift */; }; A35038852F4F96CA0032BA9F /* WatchOSEnvironmentReporterSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = A35038842F4F96CA0032BA9F /* WatchOSEnvironmentReporterSpec.swift */; }; @@ -374,6 +422,12 @@ A8449C0800DF492B8F0619E4 /* LDContextJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88DAB8D1864C14D9C7E06F11 /* LDContextJSONWriter.swift */; }; AD5F10417C8E3269D4F701B3 /* JSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 6F2A8C1D4E7B9053A1C4D8E6 /* JSONWriter.swift */; }; AFCE752CB1A75DEBABC23034 /* LDContextJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88DAB8D1864C14D9C7E06F11 /* LDContextJSONWriter.swift */; }; + B0B1AD7CADE8DBB596C2DE9B /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; + DAA2C9B9C144F129217A7C55 /* DiskEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */; }; + A1B2C3D4E5F60718293A4B03 /* EventDirectory.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D4E5F60718293A4B5C /* EventDirectory.swift */; }; + 44E0BE449714952A68628CEB /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; + E5191724DFB38E62C0451086 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; + CA2ADB3D20A229D70CA34F73 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; B40B419C249ADA6B00CD0726 /* DiagnosticCacheSpec.swift in Sources */ = {isa = PBXBuildFile; fileRef = B40B419B249ADA6B00CD0726 /* DiagnosticCacheSpec.swift */; }; B4265EB124E7390C001CFD2C /* TestUtil.swift in Sources */ = {isa = PBXBuildFile; fileRef = B4265EB024E7390C001CFD2C /* TestUtil.swift */; }; B468E71024B3C3AC00E0C883 /* ObjcLDEvaluationDetail.swift in Sources */ = {isa = PBXBuildFile; fileRef = B468E70F24B3C3AC00E0C883 /* ObjcLDEvaluationDetail.swift */; }; @@ -424,6 +478,12 @@ C443A41123186A4F00145710 /* ConnectionModeChangeObserver.swift in Sources */ = {isa = PBXBuildFile; fileRef = C443A40E23186A4F00145710 /* ConnectionModeChangeObserver.swift */; }; C443A41223186A4F00145710 /* ConnectionModeChangeObserver.swift in Sources */ = {isa = PBXBuildFile; fileRef = C443A40E23186A4F00145710 /* ConnectionModeChangeObserver.swift */; }; C4AF311E4DA8C3AB7040637D /* EventJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = B430884DBF2E541B919C08FA /* EventJSONWriter.swift */; }; + CF9A1EB116D8AA3B7B5535B2 /* EventStore.swift in Sources */ = {isa = PBXBuildFile; fileRef = C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */; }; + E2F8080AB2B3CD6E9A9C1FCC /* DiskEventLog.swift in Sources */ = {isa = PBXBuildFile; fileRef = E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */; }; + A1B2C3D4E5F60718293A4B04 /* EventDirectory.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1B2C3D4E5F60718293A4B5C /* EventDirectory.swift */; }; + F0F504E5EC32223905356098 /* EventLogFile.swift in Sources */ = {isa = PBXBuildFile; fileRef = 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */; }; + B2E6D4F1AC805B3F9D127E53 /* DeliveryClaims.swift in Sources */ = {isa = PBXBuildFile; fileRef = A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */; }; + 625ADAC0CCFE3BE324C45752 /* EventLogFormat.swift in Sources */ = {isa = PBXBuildFile; fileRef = B58551BB165F28784C785720 /* EventLogFormat.swift */; }; D7A500FCA04E3412F931003F /* ContextEncodingCache.swift in Sources */ = {isa = PBXBuildFile; fileRef = D93B7F5F6CF5C3DE79FB4286 /* ContextEncodingCache.swift */; }; DAF5695D58E28B8CF9221213 /* LDContextJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88DAB8D1864C14D9C7E06F11 /* LDContextJSONWriter.swift */; }; EBFA09F4DFC59FCB6C0AE209 /* LDContextJSONWriter.swift in Sources */ = {isa = PBXBuildFile; fileRef = 88DAB8D1864C14D9C7E06F11 /* LDContextJSONWriter.swift */; }; @@ -522,6 +582,8 @@ 831AAE2B20A9E4F600B46DBA /* Throttler.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Throttler.swift; sourceTree = ""; }; 831AAE2F20A9E75D00B46DBA /* ThrottlerSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ThrottlerSpec.swift; sourceTree = ""; }; C3BDE2A66C982198D55602DA /* BackgroundActivitySpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BackgroundActivitySpec.swift; sourceTree = ""; }; + 88A4CE0E204CB660F54E8A4B /* BackgroundEventDeliverySpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = BackgroundEventDeliverySpec.swift; sourceTree = ""; }; + 406B825B2AAD6C24F5467449 /* EventProcessorOutcomesSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EventProcessorOutcomesSpec.swift; sourceTree = ""; }; 831D2AAE2061AAA000B4AC3C /* Thread.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Thread.swift; sourceTree = ""; }; 831D8B6E1F71532300ED65E8 /* HTTPHeaders.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HTTPHeaders.swift; sourceTree = ""; }; 831D8B711F71D3E700ED65E8 /* DarklyService.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DarklyService.swift; sourceTree = ""; }; @@ -570,8 +632,9 @@ 83B8C2461FE4071F0082B8A9 /* HTTPURLResponse.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HTTPURLResponse.swift; sourceTree = ""; }; 83B9A07F204F56F4000C3F17 /* FlagChangeObserverSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FlagChangeObserverSpec.swift; sourceTree = ""; }; 83B9A081204F6022000C3F17 /* FlagsUnchangedObserver.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = FlagsUnchangedObserver.swift; sourceTree = ""; }; - 83CFE7CD1F7AD81D0010544E /* EventReporterSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EventReporterSpec.swift; sourceTree = ""; }; + 83CFE7CD1F7AD81D0010544E /* DirectEventProcessorSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DirectEventProcessorSpec.swift; sourceTree = ""; }; 83CFE7D01F7AD8DC0010544E /* DarklyServiceMock.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DarklyServiceMock.swift; sourceTree = ""; }; + A1D1A6C02F0B000000C0FFEE /* DiagnosticCachingMock.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DiagnosticCachingMock.swift; sourceTree = ""; }; 83D559731FD87CC9002D10C8 /* KeyedValueCache.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = KeyedValueCache.swift; sourceTree = ""; }; 83D9EC6B2062DBB7004D7FA6 /* LaunchDarkly_watchOS.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = LaunchDarkly_watchOS.framework; sourceTree = BUILT_PRODUCTS_DIR; }; 83DDBEF51FA24A7E00E428B6 /* Data.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Data.swift; sourceTree = ""; }; @@ -583,13 +646,20 @@ 83EF67921F9945E800403126 /* EventSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EventSpec.swift; sourceTree = ""; }; 83F0A5631FB5F33800550A95 /* LDConfigSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LDConfigSpec.swift; sourceTree = ""; }; 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = FlagSynchronizer.swift; sourceTree = ""; }; - 83FEF8DE1F2667E4001CF12C /* EventReporter.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = EventReporter.swift; sourceTree = ""; }; + 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = DirectEventProcessor.swift; sourceTree = ""; }; + E7C0A11E2F00000000000001 /* EventCommitter.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = EventCommitter.swift; sourceTree = ""; }; + E7C0A11E2F00000000000006 /* EventDeliveryScheduler.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = EventDeliveryScheduler.swift; sourceTree = ""; }; + 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = AnalyticsEventSender.swift; sourceTree = ""; }; + 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = OutboundEventBuffer.swift; sourceTree = ""; }; + BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = NullEventProcessor.swift; sourceTree = ""; }; + 293A30078FEAA25BB4C3D26C /* EventProcessor.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = EventProcessor.swift; sourceTree = ""; }; 88DAB8D1864C14D9C7E06F11 /* LDContextJSONWriter.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = LDContextJSONWriter.swift; sourceTree = ""; }; 8E38AA8B306308E500C4C163 /* RetryState.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RetryState.swift; sourceTree = ""; }; 8E38AA903063090600C4C163 /* RetryStateSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = RetryStateSpec.swift; sourceTree = ""; }; 9A47F7492F3D4CBF0001FD9C /* ContextSummarizerSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContextSummarizerSpec.swift; sourceTree = ""; }; 9A47F74B2F3D4CCF0001FD9C /* ContextSummarizer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContextSummarizer.swift; sourceTree = ""; }; 9A771B316E19ED598CD4FB0B /* BackgroundActivity.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = BackgroundActivity.swift; sourceTree = ""; }; + EF2483453F8EF6F6D01A83B5 /* BackgroundEventDelivery.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = BackgroundEventDelivery.swift; sourceTree = ""; }; 9AE1D0A02F4A1B000001FD9C /* EvaluationExposureDeduper.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EvaluationExposureDeduper.swift; sourceTree = ""; }; 9AE1D0A52F4A1B000001FD9C /* EvaluationExposureDeduperSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = EvaluationExposureDeduperSpec.swift; sourceTree = ""; }; 9AE1D0B02F4A1B000001FD9C /* HookDecorator.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = HookDecorator.swift; sourceTree = ""; }; @@ -612,6 +682,7 @@ A35038762F4F58660032BA9F /* LDClientIdentifyHook.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LDClientIdentifyHook.swift; sourceTree = ""; }; 86CB57159BC81898DE96A352 /* LDClientFlush.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LDClientFlush.swift; sourceTree = ""; }; A350387B2F4F5A380032BA9F /* Data+Compression.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Data+Compression.swift"; sourceTree = ""; }; + A35038802F4F5A380032BA9F /* Data+CRC.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = "Data+CRC.swift"; sourceTree = ""; }; A35038802F4F965E0032BA9F /* LDClientIdentifyHookSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LDClientIdentifyHookSpec.swift; sourceTree = ""; }; A35038822F4F96680032BA9F /* LDClientEvaluationHookSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LDClientEvaluationHookSpec.swift; sourceTree = ""; }; A35038842F4F96CA0032BA9F /* WatchOSEnvironmentReporterSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = WatchOSEnvironmentReporterSpec.swift; sourceTree = ""; }; @@ -648,12 +719,19 @@ B4C9D4322489C8FD004A9B03 /* DiagnosticCache.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DiagnosticCache.swift; sourceTree = ""; }; B4C9D4372489E20A004A9B03 /* DiagnosticReporter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DiagnosticReporter.swift; sourceTree = ""; }; B4F689132497B2FC004D3CE0 /* DiagnosticEventSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DiagnosticEventSpec.swift; sourceTree = ""; }; + C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventStore.swift; sourceTree = ""; }; + E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = DiskEventLog.swift; sourceTree = ""; }; + A1B2C3D4E5F60718293A4B5C /* EventDirectory.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventDirectory.swift; sourceTree = ""; }; + 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventLogFile.swift; sourceTree = ""; }; + A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = DeliveryClaims.swift; sourceTree = ""; }; + B58551BB165F28784C785720 /* EventLogFormat.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventLogFormat.swift; sourceTree = ""; }; C408884623033B3600420721 /* ConnectionInformationStore.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConnectionInformationStore.swift; sourceTree = ""; }; C408884823033B7500420721 /* ConnectionInformation.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConnectionInformation.swift; sourceTree = ""; }; C43C37E0236BA050003C1624 /* LDEvaluationDetail.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LDEvaluationDetail.swift; sourceTree = ""; }; C443A4092315AA4D00145710 /* NetworkReporter.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = NetworkReporter.swift; sourceTree = ""; }; C443A40E23186A4F00145710 /* ConnectionModeChangeObserver.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ConnectionModeChangeObserver.swift; sourceTree = ""; }; D93B7F5F6CF5C3DE79FB4286 /* ContextEncodingCache.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = ContextEncodingCache.swift; sourceTree = ""; }; + EE6BAECBB739A0F24DB5A944 /* EventStoreSpec.swift */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.swift; path = EventStoreSpec.swift; sourceTree = ""; }; FED0000000000000000000C0 /* TrackSeriesContext.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = TrackSeriesContext.swift; sourceTree = ""; }; FED0000000000000000000D0 /* LDClientTrackHookSpec.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = LDClientTrackHookSpec.swift; sourceTree = ""; }; /* End PBXFileReference section */ @@ -742,7 +820,7 @@ 50EE85C62EA0749C007CC662 /* TimeoutExecutorSpec.swift */, A3047D5B2A606A0000F568E0 /* EnvironmentReporting */, B46F344025E6DB7D0078D45F /* DiagnosticReporterSpec.swift */, - 83CFE7CD1F7AD81D0010544E /* EventReporterSpec.swift */, + 83CFE7CD1F7AD81D0010544E /* DirectEventProcessorSpec.swift */, 83B8C2441FE360CF0082B8A9 /* FlagChangeNotifierSpec.swift */, 83DDBEFF1FA2589900E428B6 /* FlagStoreSpec.swift */, 831D8B761F72A4B400ED65E8 /* FlagSynchronizerSpec.swift */, @@ -750,8 +828,11 @@ 837406D321F760640087B22B /* LDTimerSpec.swift */, 831AAE2F20A9E75D00B46DBA /* ThrottlerSpec.swift */, C3BDE2A66C982198D55602DA /* BackgroundActivitySpec.swift */, + 88A4CE0E204CB660F54E8A4B /* BackgroundEventDeliverySpec.swift */, + 406B825B2AAD6C24F5467449 /* EventProcessorOutcomesSpec.swift */, 8354AC75224316C700CDE602 /* Cache */, A3C6F7612B7FA803005B3B61 /* SheddingQueueSpec.swift */, + EE6BAECBB739A0F24DB5A944 /* EventStoreSpec.swift */, ); path = ServiceObjects; sourceTree = ""; @@ -914,6 +995,7 @@ isa = PBXGroup; children = ( 83CFE7D01F7AD8DC0010544E /* DarklyServiceMock.swift */, + A1D1A6C02F0B000000C0FFEE /* DiagnosticCachingMock.swift */, 832307A71F7DA61B0029815A /* LDEventSourceMock.swift */, 832307A91F7ECA630029815A /* LDConfigStub.swift */, A33A5F7928466D04000C29C7 /* LDContextStub.swift */, @@ -936,6 +1018,7 @@ isa = PBXGroup; children = ( A350387B2F4F5A380032BA9F /* Data+Compression.swift */, + A35038802F4F5A380032BA9F /* Data+CRC.swift */, 83DDBEF51FA24A7E00E428B6 /* Data.swift */, 83DDBEFD1FA24F9600E428B6 /* Date.swift */, 831D2AAE2061AAA000B4AC3C /* Thread.swift */, @@ -1018,7 +1101,13 @@ 83B1D7C82073F354006D1B1C /* CwlSysctl.swift */, B4C9D4372489E20A004A9B03 /* DiagnosticReporter.swift */, 831425B0206B030100F2EF36 /* EnvironmentReporter.swift */, - 83FEF8DE1F2667E4001CF12C /* EventReporter.swift */, + 83FEF8DE1F2667E4001CF12C /* DirectEventProcessor.swift */, + E7C0A11E2F00000000000001 /* EventCommitter.swift */, + E7C0A11E2F00000000000006 /* EventDeliveryScheduler.swift */, + 508A41CF2AAB55147AF2045D /* AnalyticsEventSender.swift */, + 77995B29BA49D033C433E799 /* OutboundEventBuffer.swift */, + BBC1B395A14BFACDE8798321 /* NullEventProcessor.swift */, + 293A30078FEAA25BB4C3D26C /* EventProcessor.swift */, 8358F25F1F476AD800ECE1AF /* FlagChangeNotifier.swift */, 831D8B731F72994600ED65E8 /* FlagStore.swift */, 83FEF8DC1F266742001CF12C /* FlagSynchronizer.swift */, @@ -1030,6 +1119,13 @@ 6F2A8C1D4E7B9053A1C4D8E6 /* JSONWriter.swift */, D93B7F5F6CF5C3DE79FB4286 /* ContextEncodingCache.swift */, 9A771B316E19ED598CD4FB0B /* BackgroundActivity.swift */, + EF2483453F8EF6F6D01A83B5 /* BackgroundEventDelivery.swift */, + C0E8C6E9182F7333C4A4E5A7 /* EventStore.swift */, + E21BC7A0E0F8D272785D7985 /* DiskEventLog.swift */, + A1B2C3D4E5F60718293A4B5C /* EventDirectory.swift */, + 03F6C30CC57DF77870B421F9 /* EventLogFile.swift */, + A1D5C3E09B7F4A2E8C016D42 /* DeliveryClaims.swift */, + B58551BB165F28784C785720 /* EventLogFormat.swift */, ); path = ServiceObjects; sourceTree = ""; @@ -1491,7 +1587,13 @@ A310881A2837DC0400184942 /* Reference.swift in Sources */, 3D3AB9462A4F16FE003AECF1 /* ReportingConsts.swift in Sources */, 831188522113ADF700D77CB5 /* KeyedValueCache.swift in Sources */, - 831188582113AE0F00D77CB5 /* EventReporter.swift in Sources */, + 831188582113AE0F00D77CB5 /* DirectEventProcessor.swift in Sources */, + E7C0A11E2F00000000000002 /* EventCommitter.swift in Sources */, + E7C0A11E2F00000000000007 /* EventDeliveryScheduler.swift in Sources */, + B66ECB2014E0FC95C23DA0A2 /* AnalyticsEventSender.swift in Sources */, + 629D5A9FE152DBC3E20D6DFC /* OutboundEventBuffer.swift in Sources */, + 014901E06BE1F2B0FAAD6AA6 /* NullEventProcessor.swift in Sources */, + 86B71C91DB78D0F17DBE0E69 /* EventProcessor.swift in Sources */, 50EE85C42EA0487F007CC662 /* TimeoutExecutor.swift in Sources */, A350386E2F4F4F610032BA9F /* IdentifySeriesContext.swift in Sources */, FED0000000000000000000C1 /* TrackSeriesContext.swift in Sources */, @@ -1505,6 +1607,7 @@ A35038772F4F58660032BA9F /* LDClientIdentifyHook.swift in Sources */, 99686398CCB7515A18247662 /* LDClientFlush.swift in Sources */, A350387C2F4F5A380032BA9F /* Data+Compression.swift in Sources */, + A35038812F4F5A380032BA9F /* Data+CRC.swift in Sources */, A3BA7CF62BD05A280000DB28 /* EvaluationSeriesContext.swift in Sources */, A3470C3A2B7C1ACE00951CEE /* LDValueDecoder.swift in Sources */, C443A41223186A4F00145710 /* ConnectionModeChangeObserver.swift in Sources */, @@ -1556,6 +1659,13 @@ 8B3D9E2F5A6C1047B2D5E9F1 /* JSONWriter.swift in Sources */, 40F4D07AD934AF976687011D /* ContextEncodingCache.swift in Sources */, A32FCDB22D53263851DDB727 /* BackgroundActivity.swift in Sources */, + 8F654B978A758FDFC94E4AF0 /* BackgroundEventDelivery.swift in Sources */, + CF9A1EB116D8AA3B7B5535B2 /* EventStore.swift in Sources */, + E2F8080AB2B3CD6E9A9C1FCC /* DiskEventLog.swift in Sources */, + A1B2C3D4E5F60718293A4B04 /* EventDirectory.swift in Sources */, + F0F504E5EC32223905356098 /* EventLogFile.swift in Sources */, + B2E6D4F1AC805B3F9D127E53 /* DeliveryClaims.swift in Sources */, + 625ADAC0CCFE3BE324C45752 /* EventLogFormat.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -1606,7 +1716,13 @@ A3BA7CF52BD05A280000DB28 /* EvaluationSeriesContext.swift in Sources */, 831EF35620655E730001C643 /* FlagChangeNotifier.swift in Sources */, A358D6D92A4DE6A500270C60 /* ApplicationInfoEnvironmentReporter.swift in Sources */, - 831EF35720655E730001C643 /* EventReporter.swift in Sources */, + 831EF35720655E730001C643 /* DirectEventProcessor.swift in Sources */, + E7C0A11E2F00000000000003 /* EventCommitter.swift in Sources */, + E7C0A11E2F00000000000008 /* EventDeliveryScheduler.swift in Sources */, + 367D374B5CD1B2A8596C0738 /* AnalyticsEventSender.swift in Sources */, + 5A89648200D2566D3F8409BA /* OutboundEventBuffer.swift in Sources */, + CFF357D68BBF2DC8395930F2 /* NullEventProcessor.swift in Sources */, + 850C49C9054E642CC006E730 /* EventProcessor.swift in Sources */, A350386F2F4F4F610032BA9F /* IdentifySeriesContext.swift in Sources */, FED0000000000000000000C2 /* TrackSeriesContext.swift in Sources */, 831EF35820655E730001C643 /* FlagStore.swift in Sources */, @@ -1615,6 +1731,7 @@ 831EF35A20655E730001C643 /* HTTPHeaders.swift in Sources */, 831EF35B20655E730001C643 /* DarklyService.swift in Sources */, A350387F2F4F5A380032BA9F /* Data+Compression.swift in Sources */, + A35038842F4F5A380032BA9F /* Data+CRC.swift in Sources */, 831EF35C20655E730001C643 /* HTTPURLResponse.swift in Sources */, C443A40723145FEE00145710 /* ConnectionInformationStore.swift in Sources */, 29FE129A280413D4008CC918 /* Util.swift in Sources */, @@ -1650,6 +1767,13 @@ 9C4E0F306B7D2158C3E6F0A2 /* JSONWriter.swift in Sources */, 4882B223C23B63F30BC60941 /* ContextEncodingCache.swift in Sources */, 7185A3C30E7FFC4F725769F1 /* BackgroundActivity.swift in Sources */, + 1830F6435EE141699F106451 /* BackgroundEventDelivery.swift in Sources */, + 2CA9D36B3566D78850D6F0AF /* EventStore.swift in Sources */, + B8F228923BE11E444D1ECB8E /* DiskEventLog.swift in Sources */, + A1B2C3D4E5F60718293A4B01 /* EventDirectory.swift in Sources */, + 3C55384C9E3485B493E1F86A /* EventLogFile.swift in Sources */, + C3F7E502BD916C40AE238F64 /* DeliveryClaims.swift in Sources */, + AB4827D11084D58661E24B05 /* EventLogFormat.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -1692,6 +1816,7 @@ A35038782F4F58660032BA9F /* LDClientIdentifyHook.swift in Sources */, B04B36074760CCA9F48E1851 /* LDClientFlush.swift in Sources */, A350387D2F4F5A380032BA9F /* Data+Compression.swift in Sources */, + A35038822F4F5A380032BA9F /* Data+CRC.swift in Sources */, A3BA7CF32BD05A280000DB28 /* EvaluationSeriesContext.swift in Sources */, A3470C372B7C1ACE00951CEE /* LDValueDecoder.swift in Sources */, C43C37E1236BA050003C1624 /* LDEvaluationDetail.swift in Sources */, @@ -1706,7 +1831,13 @@ 29F9D19E2812E005008D12C0 /* ObjcLDValue.swift in Sources */, A3599E882A4B4AD400DB5C67 /* Modifier.swift in Sources */, 8358F2601F476AD800ECE1AF /* FlagChangeNotifier.swift in Sources */, - 83FEF8DF1F2667E4001CF12C /* EventReporter.swift in Sources */, + 83FEF8DF1F2667E4001CF12C /* DirectEventProcessor.swift in Sources */, + E7C0A11E2F00000000000005 /* EventCommitter.swift in Sources */, + E7C0A11E2F0000000000000A /* EventDeliveryScheduler.swift in Sources */, + E91F5DCAEEE5FC4AC408BA67 /* AnalyticsEventSender.swift in Sources */, + 7D37AD3ECAA9D8E2927228C5 /* OutboundEventBuffer.swift in Sources */, + 9FA0EE3A87678F58F4406B24 /* NullEventProcessor.swift in Sources */, + 7B38D33B59821BD9BCFFA731 /* EventProcessor.swift in Sources */, 831D2AAF2061AAA000B4AC3C /* Thread.swift in Sources */, A358D6DD2A4DE7D600270C60 /* IOSEnvironmentReporter.swift in Sources */, 83B9A082204F6022000C3F17 /* FlagsUnchangedObserver.swift in Sources */, @@ -1743,6 +1874,13 @@ AD5F10417C8E3269D4F701B3 /* JSONWriter.swift in Sources */, D7A500FCA04E3412F931003F /* ContextEncodingCache.swift in Sources */, A20146E7AF098560627C614E /* BackgroundActivity.swift in Sources */, + BAB394DCCA3C189DD1B11C36 /* BackgroundEventDelivery.swift in Sources */, + 5AEA1F832DCA974314E07D21 /* EventStore.swift in Sources */, + D4225F1F05F2C55E9518941B /* DiskEventLog.swift in Sources */, + A1B2C3D4E5F60718293A4B02 /* EventDirectory.swift in Sources */, + DDA788E3056F8AC3CA203933 /* EventLogFile.swift in Sources */, + D4080613CEA27D51BF349075 /* DeliveryClaims.swift in Sources */, + ECBE45806B470ACC5187B7D6 /* EventLogFormat.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -1753,7 +1891,7 @@ 3D2406252E0D90EA00F91253 /* LDClientPluginsSpec.swift in Sources */, A3047D662A606B6000F568E0 /* EnvironmentReporterChainBaseSpec.swift in Sources */, A31088272837DCA900184942 /* LDContextSpec.swift in Sources */, - 83CFE7CE1F7AD81D0010544E /* EventReporterSpec.swift in Sources */, + 83CFE7CE1F7AD81D0010544E /* DirectEventProcessorSpec.swift in Sources */, 8392FFA32033565700320914 /* HTTPURLResponse.swift in Sources */, 83411A5F1FABDA8700E5CF39 /* mocks.generated.swift in Sources */, 83DDBF001FA2589900E428B6 /* FlagStoreSpec.swift in Sources */, @@ -1766,6 +1904,8 @@ 83B6E3F1222EFA3800FF2A6A /* ThreadSpec.swift in Sources */, 831AAE3020A9E75D00B46DBA /* ThrottlerSpec.swift in Sources */, 3F9AC0D80849C86866E03807 /* BackgroundActivitySpec.swift in Sources */, + 379F34CAB1EE19A0F85AA2C8 /* BackgroundEventDeliverySpec.swift in Sources */, + C863C417AB6004C6EE2F600C /* EventProcessorOutcomesSpec.swift in Sources */, 832D68AC224B3321005F052A /* CacheConverterSpec.swift in Sources */, 838F96741FB9F024009CFC45 /* LDClientSpec.swift in Sources */, 83A0E6B1203B557F00224298 /* FeatureFlagSpec.swift in Sources */, @@ -1777,6 +1917,7 @@ B40B419C249ADA6B00CD0726 /* DiagnosticCacheSpec.swift in Sources */, 83F0A5641FB5F33800550A95 /* LDConfigSpec.swift in Sources */, 83CFE7D11F7AD8DC0010544E /* DarklyServiceMock.swift in Sources */, + A1D1A6C12F0B000000C0FFEE /* DiagnosticCachingMock.swift in Sources */, 832307AA1F7ECA630029815A /* LDConfigStub.swift in Sources */, A3C6F7622B7FA803005B3B61 /* SheddingQueueSpec.swift in Sources */, A33A5F7A28466D04000C29C7 /* LDContextStub.swift in Sources */, @@ -1808,6 +1949,7 @@ 838F967A1FBA551A009CFC45 /* ClientServiceMockFactory.swift in Sources */, A31088292837DCA900184942 /* KindSpec.swift in Sources */, A0BAD9EE25943C43A5154E6F /* EventJSONWriterTests.swift in Sources */, + 1E65A889B0528E5AB03C517A /* EventStoreSpec.swift in Sources */, B6168D5F889EFC4C19F59135 /* JSONWriterTests.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; @@ -1851,6 +1993,7 @@ A350387A2F4F58660032BA9F /* LDClientIdentifyHook.swift in Sources */, 43DC6AA5DB0BE9A7B697C10F /* LDClientFlush.swift in Sources */, A350387E2F4F5A380032BA9F /* Data+Compression.swift in Sources */, + A35038832F4F5A380032BA9F /* Data+CRC.swift in Sources */, A3BA7CF42BD05A280000DB28 /* EvaluationSeriesContext.swift in Sources */, A3470C382B7C1ACE00951CEE /* LDValueDecoder.swift in Sources */, C43C37E6238DF22B003C1624 /* LDEvaluationDetail.swift in Sources */, @@ -1861,7 +2004,13 @@ 9AE1D0D42F4A1B000001FD9C /* UnfairLock.swift in Sources */, 83D9EC882062DEAB004D7FA6 /* FlagChangeNotifier.swift in Sources */, C443A40B2315AA4D00145710 /* NetworkReporter.swift in Sources */, - 83D9EC892062DEAB004D7FA6 /* EventReporter.swift in Sources */, + 83D9EC892062DEAB004D7FA6 /* DirectEventProcessor.swift in Sources */, + E7C0A11E2F00000000000004 /* EventCommitter.swift in Sources */, + E7C0A11E2F00000000000009 /* EventDeliveryScheduler.swift in Sources */, + 321659EE1824B83BD0D0F22D /* AnalyticsEventSender.swift in Sources */, + 73E23C0FA009141BB4CBC24C /* OutboundEventBuffer.swift in Sources */, + 54745AEF1A43D165F5A6E08F /* NullEventProcessor.swift in Sources */, + 7F4004579B9094737F3C21D1 /* EventProcessor.swift in Sources */, 83D9EC8A2062DEAB004D7FA6 /* FlagStore.swift in Sources */, 29F9D19F2812E005008D12C0 /* ObjcLDValue.swift in Sources */, A3599E892A4B4AD400DB5C67 /* Modifier.swift in Sources */, @@ -1902,6 +2051,13 @@ BE6011528D9F4370E50812C4 /* JSONWriter.swift in Sources */, BAE0B4F45562EBF4AC786B5D /* ContextEncodingCache.swift in Sources */, 3D34EEEF37158657D49335BB /* BackgroundActivity.swift in Sources */, + 051BF088E9BDA17A3871A530 /* BackgroundEventDelivery.swift in Sources */, + B0B1AD7CADE8DBB596C2DE9B /* EventStore.swift in Sources */, + DAA2C9B9C144F129217A7C55 /* DiskEventLog.swift in Sources */, + A1B2C3D4E5F60718293A4B03 /* EventDirectory.swift in Sources */, + 44E0BE449714952A68628CEB /* EventLogFile.swift in Sources */, + E5191724DFB38E62C0451086 /* DeliveryClaims.swift in Sources */, + CA2ADB3D20A229D70CA34F73 /* EventLogFormat.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; diff --git a/LaunchDarkly/GeneratedCode/mocks.generated.swift b/LaunchDarkly/GeneratedCode/mocks.generated.swift index d9a0c9f6c..1e79eab77 100644 --- a/LaunchDarkly/GeneratedCode/mocks.generated.swift +++ b/LaunchDarkly/GeneratedCode/mocks.generated.swift @@ -38,53 +38,6 @@ final class DarklyStreamingProviderMock: DarklyStreamingProvider { } } -// MARK: - DiagnosticCachingMock -final class DiagnosticCachingMock: DiagnosticCaching { - - var getDiagnosticIdCallCount = 0 - var getDiagnosticIdCallback: (() throws -> Void)? - var getDiagnosticIdReturnValue: DiagnosticId! - func getDiagnosticId() -> DiagnosticId { - getDiagnosticIdCallCount += 1 - try! getDiagnosticIdCallback?() - return getDiagnosticIdReturnValue - } - - var getCurrentStatsAndResetCallCount = 0 - var getCurrentStatsAndResetCallback: (() throws -> Void)? - var getCurrentStatsAndResetReturnValue: DiagnosticStats! - func getCurrentStatsAndReset() -> DiagnosticStats { - getCurrentStatsAndResetCallCount += 1 - try! getCurrentStatsAndResetCallback?() - return getCurrentStatsAndResetReturnValue - } - - var incrementDroppedEventCountCallCount = 0 - var incrementDroppedEventCountCallback: (() throws -> Void)? - func incrementDroppedEventCount() { - incrementDroppedEventCountCallCount += 1 - try! incrementDroppedEventCountCallback?() - } - - var recordEventsInLastBatchCallCount = 0 - var recordEventsInLastBatchCallback: (() throws -> Void)? - var recordEventsInLastBatchReceivedEventsInLastBatch: Int? - func recordEventsInLastBatch(eventsInLastBatch: Int) { - recordEventsInLastBatchCallCount += 1 - recordEventsInLastBatchReceivedEventsInLastBatch = eventsInLastBatch - try! recordEventsInLastBatchCallback?() - } - - var addStreamInitCallCount = 0 - var addStreamInitCallback: (() throws -> Void)? - var addStreamInitReceivedStreamInit: DiagnosticStreamInit? - func addStreamInit(streamInit: DiagnosticStreamInit) { - addStreamInitCallCount += 1 - addStreamInitReceivedStreamInit = streamInit - try! addStreamInitCallback?() - } -} - // MARK: - DiagnosticReportingMock final class DiagnosticReportingMock: DiagnosticReporting { @@ -174,8 +127,8 @@ final class EnvironmentReportingMock: EnvironmentReporting { } } -// MARK: - EventReportingMock -final class EventReportingMock: EventReporting { +// MARK: - EventProcessorMock +final class EventProcessorMock: EventProcessor { var isOnlineSetCount = 0 var setIsOnlineCallback: (() throws -> Void)? @@ -230,6 +183,36 @@ final class EventReportingMock: EventReporting { flushReportingOutcomeReceivedCompletion = completion try! flushReportingOutcomeCallback?() } + + var flushBypassingBackoffCallCount = 0 + var flushBypassingBackoffCallback: (() throws -> Void)? + var flushBypassingBackoffReceivedCompletion: (FlushOutcomeClosure)? + func flushBypassingBackoff(completion: @escaping FlushOutcomeClosure) { + flushBypassingBackoffCallCount += 1 + flushBypassingBackoffReceivedCompletion = completion + try! flushBypassingBackoffCallback?() + } + + var flushAtCloseCallCount = 0 + var flushAtCloseCallback: (() throws -> Void)? + func flushAtClose() { + flushAtCloseCallCount += 1 + try! flushAtCloseCallback?() + } + + var commitRecordedEventsCallCount = 0 + var commitRecordedEventsCallback: (() throws -> Void)? + func commitRecordedEvents() { + commitRecordedEventsCallCount += 1 + try! commitRecordedEventsCallback?() + } + + var commitWherePersistingCallCount = 0 + var commitWherePersistingCallback: (() throws -> Void)? + func commitWherePersisting() { + commitWherePersistingCallCount += 1 + try! commitWherePersistingCallback?() + } } // MARK: - FeatureFlagCachingMock diff --git a/LaunchDarkly/LaunchDarkly/Extensions/Data+CRC.swift b/LaunchDarkly/LaunchDarkly/Extensions/Data+CRC.swift new file mode 100644 index 000000000..159d9a069 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/Extensions/Data+CRC.swift @@ -0,0 +1,15 @@ +import Foundation + +extension Crc32 { + /// Advance the current checksum with bytes already in memory, without wrapping them in `Data` first. + /// - parameter bytes: bytes to advance the checksum + mutating func advance(withBytes bytes: UnsafeRawBufferPointer) { + guard let base = bytes.baseAddress?.assumingMemoryBound(to: UInt8.self) + else { return } + if let fastCrc32 = Crc32.zLibCrc32 { + checksum = fastCrc32(checksum, base, UInt32(bytes.count)) + } else { + checksum = slowCrc32(start: checksum, data: Data(bytes)) + } + } +} diff --git a/LaunchDarkly/LaunchDarkly/Extensions/Data+Compression.swift b/LaunchDarkly/LaunchDarkly/Extensions/Data+Compression.swift index 7ab1080ad..b746d0f15 100644 --- a/LaunchDarkly/LaunchDarkly/Extensions/Data+Compression.swift +++ b/LaunchDarkly/LaunchDarkly/Extensions/Data+Compression.swift @@ -258,12 +258,12 @@ internal extension Data { /// Struct based type representing a Crc32 checksum. struct Crc32: CustomStringConvertible { - private static let zLibCrc32: ZLibCrc32FuncPtr? = loadCrc32fromZLib() + static let zLibCrc32: ZLibCrc32FuncPtr? = loadCrc32fromZLib() init() {} // C convention function pointer type matching the signature of `libz::crc32` - private typealias ZLibCrc32FuncPtr = @convention(c) ( + typealias ZLibCrc32FuncPtr = @convention(c) ( _ cks: UInt32, _ buf: UnsafePointer, _ len: UInt32 @@ -300,7 +300,7 @@ struct Crc32: CustomStringConvertible { /// Rudimentary fallback implementation of the crc32 checksum. This is only a backup used /// when zlib can't be found under '/usr/lib/libz.dylib'. /// - returns: crc32 checksum (4 byte) - private func slowCrc32(start: UInt32, data: Data) -> UInt32 { + func slowCrc32(start: UInt32, data: Data) -> UInt32 { return ~data.reduce(~start) { (crc: UInt32, next: UInt8) -> UInt32 in let tableOffset = (crc ^ UInt32(next)) & 0xff return lookUpTable[Int(tableOffset)] ^ crc >> 8 diff --git a/LaunchDarkly/LaunchDarkly/LDClient.swift b/LaunchDarkly/LaunchDarkly/LDClient.swift index 4ca7fd11c..f36ba9b32 100644 --- a/LaunchDarkly/LaunchDarkly/LDClient.swift +++ b/LaunchDarkly/LaunchDarkly/LDClient.swift @@ -66,7 +66,7 @@ public class LDClient { let oldValue = _isOnline _isOnline = newValue flagSynchronizer.isOnline = _isOnline - eventReporter.isOnline = _isOnline + eventProcessor.isOnline = _isOnline diagnosticReporter.setMode(runMode, online: _isOnline) if _isOnline != oldValue { connectionInformation = ConnectionInformation.onlineSetCheck(connectionInformation: connectionInformation, ldClient: self, config: config, online: _isOnline) @@ -247,15 +247,18 @@ public class LDClient { There is almost no reason to stop the LDClient. Normally, set the LDClient offline to stop communication with the LaunchDarkly servers. Stop the LDClient to stop recording events. There is no need to stop the LDClient prior to suspending, moving to the background, or terminating the app. The SDK will respond to these events as the system requires and as configured in LDConfig. */ public func close() { - LDClient.instancesQueue.sync(flags: .barrier) { - LDClient.instances?.forEach { $1.internalClose() } - LDClient.instances = nil + // Closed outside `instancesQueue`: each close can wait on its final event commit, and every other call that + // reads `instances` would wait behind it. + let clients = LDClient.instancesQueue.sync(flags: .barrier) { () -> [LDClient] in + defer { LDClient.instances = nil } + return LDClient.instances.map { Array($0.values) } ?? [] } + clients.forEach { $0.internalClose() } } private func internalClose() { os_log("%s stopping", log: config.logger, type: .debug, typeName(and: #function)) - internalFlush() + eventProcessor.flushAtClose() internalSetOnline(false) hasStarted = false os_log("%s stopped", log: config.logger, type: .debug, typeName(and: #function)) @@ -263,23 +266,6 @@ public class LDClient { @objc private func didEnterBackground() { os_log("%s", log: config.logger, type: .debug, typeName(and: #function)) - // A backgrounded process is suspended once idle, so deliver inside an activity assertion. Only where suspension - // is real: macOS posts this notification whenever the application loses focus, and never suspends it for that. - #if os(iOS) || os(tvOS) - backgroundDelivery.run { [weak self] finished in - guard let self = self - else { - finished() - return - } - self.eventReporter.flushReportingOutcome { delivered in - if !delivered { - os_log("%s events did not reach LaunchDarkly before suspension", log: self.config.logger, type: .debug, self.typeName(and: #function)) - } - finished() - } - } - #endif Thread.performOnMain { runMode = .background } @@ -292,9 +278,6 @@ public class LDClient { } } - /// One assertion per client, however many times the application is backgrounded while a delivery is running. - private let backgroundDelivery: BackgroundActivity - let config: LDConfig /// Identifies this client's environment to a hook, without handing it the mobile key that identifies the /// environment to LaunchDarkly. Hashed once here rather than per evaluation, because a deduping hook asks for it on @@ -475,7 +458,7 @@ public class LDClient { internalIdentifyQueue.sync { if self.context == updatedContext { - self.eventReporter.record(IdentifyEvent(context: self.context)) + self.eventProcessor.record(IdentifyEvent(context: self.context)) completion?() return } @@ -525,7 +508,7 @@ public class LDClient { onSyncComplete: self.onFlagSyncComplete) if self.hasStarted { - self.eventReporter.record(IdentifyEvent(context: self.context)) + self.eventProcessor.record(IdentifyEvent(context: self.context)) } if self.isOnline && self.isInSupportedRunMode { @@ -769,7 +752,7 @@ public class LDClient { key, String(describing: data), String(describing: metricValue)) - eventReporter.record(event) + eventProcessor.record(event) executeAfterTrackHooks(key: key, data: data, metricValue: metricValue) } @@ -979,7 +962,7 @@ public class LDClient { private(set) var flagCache: FeatureFlagCaching private(set) var flagSynchronizer: LDFlagSynchronizing var flagChangeNotifier: FlagChangeNotifying - private(set) var eventReporter: EventReporting + private(set) var eventProcessor: EventProcessor private(set) var environmentReporter: EnvironmentReporting private(set) var throttler: Throttling private(set) var diagnosticReporter: DiagnosticReporting @@ -1035,14 +1018,13 @@ public class LDClient { throttler = self.serviceFactory.makeThrottler(environmentReporter: environmentReporter) config = configuration - backgroundDelivery = BackgroundActivity(reason: "LaunchDarkly event delivery", logger: configuration.logger) context = startContext ?? LDContext() if config.autoEnvAttributes { context = AutoEnvContextModifier(environmentReporter: environmentReporter, logger: config.logger).modifyContext(context) } - var hookState: IdentifyHookState? = nil + var hookState: IdentifyHookState? if !hooks.isEmpty { let seriesContext = IdentifySeriesContext(context: context, methodName: "init") let seriesData = hooks.map { hook in hook.beforeIdentify(seriesContext: seriesContext, seriesData: EvaluationSeriesData()) } @@ -1054,7 +1036,7 @@ public class LDClient { // Reports over the logger rather than over this client, which is not usable until every stored property // below has a value, so that the reporter can be built once rather than built and replaced. let logger = configuration.logger - eventReporter = self.serviceFactory.makeEventReporter(config: config, service: service) { result in + eventProcessor = self.serviceFactory.makeEventProcessor(config: config, service: service) { result in LDClient.logEventSyncComplete(result, logger) } connectionInformation = self.serviceFactory.makeConnectionInformation() @@ -1069,6 +1051,7 @@ public class LDClient { if let backgroundNotification = SystemCapabilities.backgroundNotification { NotificationCenter.default.addObserver(self, selector: #selector(didEnterBackground), name: backgroundNotification, object: nil) } + LDClient.observeBackgroundingForEvents() if let foregroundNotification = SystemCapabilities.foregroundNotification { NotificationCenter.default.addObserver(self, selector: #selector(willEnterForeground), name: foregroundNotification, object: nil) } @@ -1078,12 +1061,13 @@ public class LDClient { service.resetFlagResponseCache(etag: cachedData.etag) // Reports to this client, so it could not be given at construction: `self` is only usable from here. flagSynchronizer.onSyncComplete = onFlagSyncComplete + giveFlagValuesTheNetworkFirst() if let cachedFlags = cachedData.items, !cachedFlags.isEmpty { flagStore.replaceStore(newStoredItems: cachedFlags) } - eventReporter.record(IdentifyEvent(context: context)) + eventProcessor.record(IdentifyEvent(context: context)) self.connectionInformation = ConnectionInformation.uncacheConnectionInformation(config: config, ldClient: self, clientServiceFactory: self.serviceFactory) internalSetOnline(configuration.startOnline) { @@ -1098,12 +1082,35 @@ public class LDClient { extension LDClient: TypeIdentifying { } -private extension LDClient { - func onEventSyncComplete(result: SynchronizingError?) { - if let synchronizingError = result { - os_log("%s result: %s", log: config.logger, type: .debug, typeName(and: #function), String(describing: synchronizingError)) - } else { - os_log("%s result: success", log: config.logger, type: .debug, typeName(and: #function)) +extension LDClient { + /// Has the diagnostic init event wait for the event processor's launch gate, as a previous run's events do, whether + /// or not events persist. + fileprivate func giveFlagValuesTheNetworkFirst() { + if let eventProcessor = eventProcessor as? DirectEventProcessor { + (diagnosticReporter as? DiagnosticReporter)?.sendsInitAfter = eventProcessor.whenLaunchGateOpens + } + } + + /// Observes the notification once for the events of every environment, however many clients are made. + private static let backgroundEvents: BackgroundEventDelivery? = { + guard let backgroundNotification = SystemCapabilities.backgroundNotification + else { return nil } + let delivery = BackgroundEventDelivery(environments: LDClient.currentEnvironments) + // Run on the posting thread, so that every environment has committed by the time the notification returns. + NotificationCenter.default.addObserver(forName: backgroundNotification, object: nil, queue: nil) { _ in + delivery.applicationDidEnterBackground() + } + return delivery + }() + + fileprivate static func observeBackgroundingForEvents() { + _ = backgroundEvents + } + + private static func currentEnvironments() -> [BackgroundEventDelivery.Environment] { + let clients = instancesQueue.sync { instances } ?? [:] + return clients.map { name, client in + BackgroundEventDelivery.Environment(name: name, eventProcessor: client.eventProcessor, logger: client.config.logger) } } } diff --git a/LaunchDarkly/LaunchDarkly/LDClientFlush.swift b/LaunchDarkly/LaunchDarkly/LDClientFlush.swift index cbd7181fc..19447856c 100644 --- a/LaunchDarkly/LaunchDarkly/LDClientFlush.swift +++ b/LaunchDarkly/LaunchDarkly/LDClientFlush.swift @@ -4,19 +4,30 @@ import OSLog // MARK: - Flush extension LDClient { /** - Tells the SDK to immediately send any currently queued events to LaunchDarkly. + Writes down all pending events and sends them to LaunchDarkly. There should not normally be a need to call this function. While online, the LDClient automatically reports events on an interval defined by `LDConfig.eventFlushInterval`. Note that this function does not block until events are - sent, it only triggers a background task to send events immediately. + sent, it only triggers a background task to send them immediately. Because it does not wait, it cannot report how the delivery went. Use `flush(completion:)` where that matters but blocking does not suit, or `flushAndWait` where blocking is acceptable. + + Where `LDConfig.eventPersistence` is on, it also writes before it returns, on the calling thread, at `.deferred` + as well as at `.immediate`. Recording an event does not on its own make it outlive the process: events are written + in runs, so one recorded shortly before the process ends may never have been written at all. This call writes + everything recorded so far, and those events then survive whether or not the delivery does. Where it is off, + events live in memory only and nothing survives the process, whether or not this was called. + + That is what makes it worth calling where the process is about to end deliberately. It is not a crash-time + mechanism — Apple gives the SDK no crash hook, and a trap or a `SIGKILL` loses whatever was recorded since the + last write. What was written before then is delivered on the next launch. */ public func flush() { - LDClient.instancesQueue.sync(flags: .barrier) { - LDClient.instances?.forEach { $1.internalFlush() } - } + // Taken under the queue and flushed after it is released, since each flush writes, and every other client call + // in the process that reads the instances would otherwise wait behind every environment's write. + let clients = LDClient.instancesQueue.sync { LDClient.instances.map { Array($0.values) } } ?? [] + clients.forEach { $0.internalFlush() } } /** @@ -39,29 +50,14 @@ extension LDClient { guard !clients.isEmpty else { os_log("%s called on a closed client", log: config.logger, type: .debug, self.typeName(and: #function)) - completion(false) + // Answered off the caller's thread even with nothing to wait for, so that the contract holds whether or + // not the client turned out to be closed. + DispatchQueue.global().async { completion(false) } return } - let lock = UnfairLock() - /// Only to be used while holding `lock`: each environment answers from whichever thread its delivery ended on. - var delivered = true - var outstanding = clients.count - - for client in clients { - client.eventReporter.flushReportingOutcome { result in - lock.lock() - delivered = delivered && result - outstanding -= 1 - // Nil until the last environment answers, so the completion runs once. A reporter answering twice - // takes the count past zero rather than back to it, which cannot match again. - let outcome = outstanding == 0 ? delivered : nil - lock.unlock() - - if let outcome { - completion(outcome) - } - } + clients.map(\.eventProcessor).flushReportingOutcomes { delivered in + completion(!delivered.contains(false)) } } @@ -92,6 +88,15 @@ extension LDClient { This is not a crash-time mechanism. It is safe to call from the main thread, but keep the budget well below 15 seconds there. + It posts even while delivery is backing off after a failure, as closing the client does, since it is for a moment + after which the SDK may not get to send. After a refusal such as a 401 or 403 the SDK waits about eight and a half + minutes before posting again, doubling up to an hour, and a call that sat that out would make no attempt at all. + A post that fails then leaves the wait as it was, so `flush`, `flush(completion:)` and the periodic delivery go on + waiting out the same one rather than a longer one. That also means nothing slows these posts down: a call made + routinely, rather than just before the process is suspended or ends, posts once per call for as long as + LaunchDarkly keeps refusing. Keep it for those last moments; the SDK already flushes on every move to the + background. + - Warning: Experimental. This method is not stable, and not subject to any backwards compatibility guarantees or semantic versioning. @@ -99,18 +104,16 @@ extension LDClient { no upper limit: a delivery always ends, by its own request timeouts if nothing else, so a very long timeout waits for that. - returns: `true` if LaunchDarkly accepted the events, or there were none to send. `false` if the timeout expired - first, the client is offline or closed, or any of the events were lost: refused by LaunchDarkly, still failing - after one retry, or unable to be serialized. Lost events are not kept for a later flush, so calling this again - does not resend them. A refusal also stops event delivery until the client is next set online, and until then - this returns `false` even with nothing to send. A `false` because the timeout expired does not mean the events - were not sent: the delivery is left running, and may still arrive. + first, the client is offline or closed, or the events did not get out. Events the service refused or that + failed to post are kept, and posted again once the backoff allows, or by the next call to this method. Events + that could not be serialized, or whose payload LaunchDarkly refused twice in a row, are lost, and calling this + again does not resend them. A `false` because the timeout expired does not mean the events were not sent: the + delivery is left running, and may still arrive. */ @discardableResult public func flushAndWait(timeout: TimeInterval) -> Bool { - if timeout > LDClient.longTimeoutInterval { - os_log("%s LDClient.flushAndWait was called with a timeout greater than %f seconds. We recommend a timeout of less than %f seconds.", log: config.logger, type: .info, self.typeName(and: #function), LDClient.longTimeoutInterval, LDClient.longTimeoutInterval) - } - + // Taken first, so that the timeout covers the whole call, finding the clients included. + let deadline = DispatchTime.now() + max(0, timeout) let clients = instancesIncludingThisClient() guard !clients.isEmpty else { @@ -119,44 +122,37 @@ extension LDClient { os_log("%s called on a closed client", log: config.logger, type: .debug, self.typeName(and: #function)) return false } - let deadline = DispatchTime.now() + max(0, timeout) - // Counts answers rather than tracking each delivery, so that a reporter answering twice costs a spare permit - // no one reads. A DispatchGroup would fit as well, but its `leave` traps without a matching `enter`, which - // turns a broken reporter into a crash in the application. + // Signaled once, by the last environment to answer, so a reporter answering twice cannot end the wait while + // another environment's delivery is still running. A DispatchGroup would fit as well, but its `leave` traps + // without a matching `enter`, which turns a broken reporter into a crash in the application. let answered = DispatchSemaphore(value: 0) let lock = UnfairLock() - /// Only to be used while holding `lock`: each environment answers from whichever thread its delivery ended on. - var delivered = true + /// Only to be used while holding `lock`: the last environment answers from whichever thread its delivery ended on. + var delivered = false // Every environment is started before any of them is waited on. Each has its own event reporter, so waiting on // one before starting the next would spend the caller's budget on deliveries that could have been running all // along, and leave the last environment with none of it. - for client in clients { - client.eventReporter.flushReportingOutcome { result in - lock.lock() - delivered = delivered && result - lock.unlock() - answered.signal() - } - } - - // Every wait is against the one deadline rather than a fresh copy of it, so the timeout the caller asked for - // is the time this call can take however many environments there are. A delivery still running when the wait - // gives up is left to finish rather than canceled: its events are already in flight, so abandoning it would - // only make losing them certain. - for _ in clients { - guard answered.wait(timeout: deadline) == .success - else { return false } + clients.map(\.eventProcessor).flushReportingOutcomes(bypassingBackoff: true) { outcomes in + lock.withLock { delivered = !outcomes.contains(false) } + answered.signal() } - // The waits count answers rather than environments, so a reporter answering twice can end them while another - // environment's answer is still being written. - lock.lock() - defer { lock.unlock() } - return delivered + // Started, then given up on. The deliveries are what makes a zero timeout worth calling at all: wherever events + // persist, each `flushReportingOutcome` has queued their write, which no longer waits on this thread. Waiting + // is what is skipped, and deliveries that already answered must not turn that into a `true`. + guard timeout > 0 + else { return false } + + // The deadline was taken on entry, so the timeout the caller asked for is the time this call can take however + // many environments there are. A delivery still running when the wait gives up is left to finish rather than + // canceled: its events are already in flight, so abandoning it would only make losing them certain. + guard answered.wait(timeout: deadline) == .success + else { return false } + return lock.withLock { delivered } } func internalFlush() { - eventReporter.flush(completion: nil) + eventProcessor.flush(completion: nil) } } diff --git a/LaunchDarkly/LaunchDarkly/LDClientVariation.swift b/LaunchDarkly/LaunchDarkly/LDClientVariation.swift index 77bd60f85..76c1f4836 100644 --- a/LaunchDarkly/LaunchDarkly/LDClientVariation.swift +++ b/LaunchDarkly/LaunchDarkly/LDClientVariation.swift @@ -170,7 +170,7 @@ extension LDClient { } private func variationDetailInternal(_ flagKey: LDFlagKey, _ defaultValue: T, needsReason: Bool, methodName: String) -> LDEvaluationDetail where T: Decodable, T: LDValueConvertible { - var visited: Set? = nil + var visited: Set? return variationDetailInternal(flagKey, defaultValue, needsReason: needsReason, methodName: methodName, visited: &visited) } @@ -220,7 +220,7 @@ extension LDClient { result = LDEvaluationDetail(value: defaultValue, variationIndex: nil, reason: ["kind": "ERROR", "errorKind": "FLAG_NOT_FOUND"]) } - eventReporter.recordFlagEvaluationEvents(flagKey: flagKey, + eventProcessor.recordFlagEvaluationEvents(flagKey: flagKey, value: result.value.toLDValue(), defaultValue: defaultValue.toLDValue(), featureFlag: featureFlag, diff --git a/LaunchDarkly/LaunchDarkly/LDCommon.swift b/LaunchDarkly/LaunchDarkly/LDCommon.swift index af2628422..5d82e1895 100644 --- a/LaunchDarkly/LaunchDarkly/LDCommon.swift +++ b/LaunchDarkly/LaunchDarkly/LDCommon.swift @@ -184,3 +184,15 @@ public enum LDValue: Codable, } } } + +extension LDValue { + /// Whether this holds NaN or an infinity anywhere, which JSON cannot represent. + var containsNonFiniteNumber: Bool { + switch self { + case .number(let number): return !number.isFinite + case .array(let values): return values.contains { $0.containsNonFiniteNumber } + case .object(let values): return values.values.contains { $0.containsNonFiniteNumber } + case .null, .bool, .string: return false + } + } +} diff --git a/LaunchDarkly/LaunchDarkly/Models/Context/LDContext.swift b/LaunchDarkly/LaunchDarkly/Models/Context/LDContext.swift index 1088e64a2..dd763d4b5 100644 --- a/LaunchDarkly/LaunchDarkly/Models/Context/LDContext.swift +++ b/LaunchDarkly/LaunchDarkly/Models/Context/LDContext.swift @@ -652,18 +652,6 @@ extension LDContext { attributes.values.contains { $0.containsNonFiniteNumber } || contexts.contains { $0.containsNonFiniteNumber() } } } - -private extension LDValue { - var containsNonFiniteNumber: Bool { - switch self { - case .number(let number): return !number.isFinite - case .array(let values): return values.contains { $0.containsNonFiniteNumber } - case .object(let values): return values.values.contains { $0.containsNonFiniteNumber } - case .null, .bool, .string: return false - } - } -} - enum LDContextBuilderKey { case generateKey case key(String) diff --git a/LaunchDarkly/LaunchDarkly/Models/FeatureFlag/FlagRequestTracker.swift b/LaunchDarkly/LaunchDarkly/Models/FeatureFlag/FlagRequestTracker.swift index 9480aa32c..61af453c5 100644 --- a/LaunchDarkly/LaunchDarkly/Models/FeatureFlag/FlagRequestTracker.swift +++ b/LaunchDarkly/LaunchDarkly/Models/FeatureFlag/FlagRequestTracker.swift @@ -2,12 +2,13 @@ import Foundation import OSLog struct FlagRequestTracker { - let startDate = Date() + let startDate: Date var flagCounters: [LDFlagKey: FlagCounter] = [:] let logger: OSLog - init(logger: OSLog) { + init(logger: OSLog, startDate: Date = Date()) { self.logger = logger + self.startDate = startDate } mutating func trackRequest(flagKey: LDFlagKey, reportedValue: LDValue, featureFlag: FeatureFlag?, defaultValue: LDValue, context: LDContext) { diff --git a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift index f4edfa3bf..6b77481c0 100644 --- a/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/Models/LDConfig.swift @@ -19,6 +19,46 @@ public enum LDStreamingMode { case polling } +/// How far the SDK goes to make a recorded event outlive the process that recorded it. +/// +/// An event lives in memory until it is delivered, so a process that dies before the next flush takes everything +/// recorded since the last one — including the crash an application was reporting when it died. Writing events to +/// disk is what closes that gap: they are delivered on a later launch instead of being lost. +/// +/// - Warning: Experimental. `.deferred` and `.immediate` are not stable in this major version, and not subject to any +/// backwards compatibility guarantees or semantic versioning: their behavior, their cost on the calling thread, and +/// what they leave on disk may change in a minor release. +@objc(LDEventPersistence) public enum EventPersistence: Int { + /// Events are kept in memory only, and nothing survives the process ending. + /// + /// Events an earlier launch wrote to disk with persistence turned on are left there, and delivered by the next + /// launch that has it turned on. + case disabled + /// Events are written to disk, off the thread that recorded them. + /// + /// Recording stays as cheap as it is without persistence. Neither `track`, `identify` nor evaluating a flag + /// schedules a write of its own: events wait in memory until enough of them have built up, or until the next + /// flush, delivery or move to the background writes them. A process that dies before then loses them. + /// + /// A flush and entering the background are the exceptions. Both write on the calling thread, as `.immediate` does: + /// a flush so that an application ending the process on purpose can rely on it, and entering the background because + /// that is when the system is most likely to end it. + /// + /// - Warning: Experimental; see `EventPersistence`. + case deferred + /// Events are written to disk before `track` and `identify` return. + /// + /// The write happens on the calling thread, costing tens of microseconds depending on how many evaluations are + /// waiting to be encoded alongside it. Evaluations are usually written in the background before many build up. A + /// burst of them faster than that can leave up to `eventCapacity` waiting, and the `track` or `identify` that + /// follows encodes them all, which can take milliseconds. In exchange there is no window: a process that dies the + /// instant after `track` returns still reports that event on the next launch. Evaluating a flag stays in memory + /// regardless. + /// + /// - Warning: Experimental; see `EventPersistence`. + case immediate +} + /** Enable / disable options for Auto Environment Attributes functionality. When enabled, the SDK will automatically provide data about the mobile environment where the application is running. This data makes it simpler to target @@ -192,7 +232,21 @@ public struct LDConfig { static let sendEvents = true /// The default maximum number of events the LDClient can store - static let eventCapacity = 100 + static let eventCapacity = 2000 + + /// The default maximum number of bytes of events the SDK can persist, for all environments together. (20 MiB) + static let eventCapacityBytes = 20 * 1024 * 1024 + + /// The default maximum age of persisted events. (7 days) + static let eventMaxAge: TimeInterval = 7 * 24 * 60 * 60 + + /// The default behavior for the SDK is to keep events in memory only. + static let eventPersistence = EventPersistence.disabled + + /// The default time the SDK's own event and diagnostic traffic waits after first going online. (2 seconds) + static let eventDeliveryDelay: TimeInterval = 2 + /// The longest `eventDeliveryDelay` can be set to. (60 seconds) + static let maxEventDeliveryDelay: TimeInterval = 60 /// The default timeout interval for flag requests and event reports. (10 seconds) static let connectionTimeout: TimeInterval = 10.0 @@ -314,9 +368,59 @@ public struct LDConfig { /// only the sending of client-side events, not streaming or polling for events from the server. public var sendEvents: Bool = Defaults.sendEvents - /// The maximum number of analytics events the LDClient can store. When the LDClient event store reaches the eventCapacity, the SDK discards events until it successfully reports them to LaunchDarkly. (Default: 100) + /// The maximum number of analytics events the LDClient can store. When the LDClient event store reaches the eventCapacity, the SDK discards events until it successfully reports them to LaunchDarkly. The same number bounds how many distinct contexts flag evaluations are summarized for between two reports; evaluations for contexts beyond it are not counted until the next report. (Default: 2000) + /// + /// Where `eventPersistence` keeps events across launches, this includes the events an earlier launch left undelivered, from the + /// moment the SDK has read them, which it does in the background just after starting. Events recorded before then are + /// counted without them, so for that moment the SDK can hold up to this many on top of what the earlier launch left. + /// + /// Events being sent stop counting once the request is out, so a flush makes room straight away. If the request fails they are kept + /// to send again and count again, so after a failed send the SDK can hold up to this many plus the events that request carried. public var eventCapacity: Int = Defaults.eventCapacity + /// How far the SDK goes to make a recorded event outlive the process that recorded it. + /// + /// Anything other than `.disabled` appends events to a log under Application Support and delivers them on a later + /// run, which is what lets the crash an application was reporting when it died reach LaunchDarkly at all. + /// `.immediate` additionally puts the write on the thread that called `track` or `identify`, so there is no window + /// in which the event exists only in memory. (Default: `.disabled`) + /// + /// - Warning: Experimental. Both levels other than `.disabled` are not stable, and not subject to any backwards + /// compatibility guarantees or semantic versioning. + public var eventPersistence: EventPersistence = Defaults.eventPersistence + + /// The maximum number of bytes of events the SDK keeps, on disk and in memory together, whatever `eventPersistence` is. It is for + /// every environment the application configures together: with `secondaryMobileKeys`, each environment gets an equal share of it, + /// unlike `eventCapacity`, which each environment has in full. When an environment reaches its share, the SDK discards new events + /// until it successfully reports the stored ones, as it does when `eventCapacity` is reached; it never deletes stored events to make + /// room. A fifth of each share is kept for the summaries of flag evaluations, so that evaluations are still counted after full + /// events start being discarded. A value of zero or less means the default. (Default: 20 MiB) + /// + /// Ordinary events reach `eventCapacity` long before this. It binds where events are unusually large, or where delivery keeps + /// failing and summaries, which `eventCapacity` never refuses, accumulate. + public var eventCapacityBytes: Int = Defaults.eventCapacityBytes + + /// How long the SDK keeps events it has persisted but not yet reported, where `eventPersistence` keeps events across launches. Events + /// older than this are discarded rather than reported, and so is everything the SDK stored for an environment whose mobile key the + /// application stopped using. A value of zero or less means the default. (Default: 7 days) + /// + /// It has no effect where `eventPersistence` is `.disabled`. + public var eventMaxAge: TimeInterval = Defaults.eventMaxAge + + private var _eventDeliveryDelay: TimeInterval = Defaults.eventDeliveryDelay + /// How long after the client first goes online the SDK holds back the traffic it starts on its own, so that the + /// first flag request has the network to itself. That is the diagnostic init payload, the events a previous run left + /// where `eventPersistence` keeps them, and the periodic delivery of this run's events. A flush the application asks + /// for is never held. Zero sends them as soon as the client is online. Values are kept between 0 and 60 seconds, so + /// an application that ends soon after every launch still sends what a previous run left. (Default: 2 seconds) + /// + /// - Warning: Experimental. This property is not stable, and not subject to any backwards compatibility guarantees + /// or semantic versioning. + public var eventDeliveryDelay: TimeInterval { + get { _eventDeliveryDelay } + set { _eventDeliveryDelay = newValue.isNaN ? Defaults.eventDeliveryDelay : min(max(newValue, 0), Defaults.maxEventDeliveryDelay) } + } + /// The timeout interval for flag requests and event reports. (Default: 10 seconds) public var connectionTimeout: TimeInterval = Defaults.connectionTimeout /// The time interval between event reports (Default: 30 seconds) @@ -546,6 +650,10 @@ extension LDConfig: Equatable { && lhs.eventsUrl == rhs.eventsUrl && lhs.streamUrl == rhs.streamUrl && lhs.eventCapacity == rhs.eventCapacity + && lhs.eventPersistence == rhs.eventPersistence + && lhs.eventCapacityBytes == rhs.eventCapacityBytes + && lhs.eventMaxAge == rhs.eventMaxAge + && lhs.eventDeliveryDelay == rhs.eventDeliveryDelay && lhs.sendEvents == rhs.sendEvents && lhs.connectionTimeout == rhs.connectionTimeout && lhs.eventFlushInterval == rhs.eventFlushInterval diff --git a/LaunchDarkly/LaunchDarkly/Networking/DarklyService.swift b/LaunchDarkly/LaunchDarkly/Networking/DarklyService.swift index 73569116b..baef4e38f 100644 --- a/LaunchDarkly/LaunchDarkly/Networking/DarklyService.swift +++ b/LaunchDarkly/LaunchDarkly/Networking/DarklyService.swift @@ -196,7 +196,7 @@ final class DarklyService: DarklyServiceProvider { func publishEventData(_ eventData: Data, _ payloadId: String, completion: ServiceCompletionHandler?) { guard hasMobileKey(#function) else { - // Answered rather than dropped: the reporter holds every later delivery until this one answers. + // Answered rather than dropped: until this answers, its batch stays claimed and its flush never reports. completion?((nil, nil, URLError(.userAuthenticationRequired), nil)) return } diff --git a/LaunchDarkly/LaunchDarkly/Networking/HTTPURLResponse.swift b/LaunchDarkly/LaunchDarkly/Networking/HTTPURLResponse.swift index 2f7c07ad0..4e88b3989 100644 --- a/LaunchDarkly/LaunchDarkly/Networking/HTTPURLResponse.swift +++ b/LaunchDarkly/LaunchDarkly/Networking/HTTPURLResponse.swift @@ -5,6 +5,7 @@ extension HTTPURLResponse { struct HeaderKeys { static let date = "Date" static let etag = "Etag" + static let retryAfter = "Retry-After" } struct StatusCodes { @@ -16,6 +17,9 @@ extension HTTPURLResponse { static let unauthorized = 401 static let methodNotAllowed = 405 static let requestTimeout = 408 + static let contentTooLarge = 413 + static let unsupportedMediaType = 415 + static let unprocessableContent = 422 static let tooManyRequests = 429 static let internalServerError = 500 static let notImplemented = 501 @@ -27,6 +31,13 @@ extension HTTPURLResponse { static func isTerminalStatusCode(_ statusCode: Int) -> Bool { (400..<500).contains(statusCode) && !recoverableClientErrors.contains(statusCode) } + + /// Whether the events service refused an event payload for what it holds rather than for who sent it: one it + /// cannot read, too large for it, of a type it does not take, or read but not accepted. Posting the same body + /// again will not change the answer, though another body may be taken. + static func isEventPayloadRefused(_ statusCode: Int) -> Bool { + [badRequest, contentTooLarge, unsupportedMediaType, unprocessableContent].contains(statusCode) + } } var headerDate: Date? { @@ -38,4 +49,17 @@ extension HTTPURLResponse { var headerEtag: String? { self.allHeaderFields[HeaderKeys.etag] as? String } + + /// The wait a `Retry-After` header asks for, in delta-seconds or as an HTTP date; nil where there is none, or it + /// cannot be read. A date already past is no wait at all. + var headerRetryAfter: TimeInterval? { + guard let value = value(forHTTPHeaderField: HeaderKeys.retryAfter)?.trimmingCharacters(in: .whitespaces) + else { return nil } + if let seconds = Int(value) { + return seconds >= 0 ? TimeInterval(seconds) : nil + } + guard let date = DateFormatter.httpUrlHeaderFormatter.date(from: value) + else { return nil } + return max(0, date.timeIntervalSinceNow) + } } diff --git a/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDClient.swift b/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDClient.swift index 2b9e571cd..7eb0528a2 100644 --- a/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDClient.swift +++ b/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDClient.swift @@ -575,6 +575,14 @@ public final class ObjcLDClient: NSObject { This is not a crash-time mechanism. It is safe to call from the main thread, but keep the budget well below 15 seconds there. + It posts even while delivery is backing off after a failure, as closing the client does, since it is for a moment + after which the SDK may not get to send. After a refusal such as a 401 or 403 the SDK waits about eight and a half + minutes before posting again, doubling up to an hour, and a call that sat that out would make no attempt at all. + A post that fails then leaves the wait as it was, so `flush` and the periodic delivery go on waiting out the same + one rather than a longer one. That also means nothing slows these posts down: a call made routinely, rather than + just before the process is suspended or ends, posts once per call for as long as LaunchDarkly keeps refusing. + Keep it for those last moments; the SDK already flushes on every move to the background. + - Warning: Experimental. This method is not stable, and not subject to any backwards compatibility guarantees or semantic versioning. @@ -582,10 +590,10 @@ public final class ObjcLDClient: NSObject { no upper limit: a delivery always ends, by its own request timeouts if nothing else, so a very long timeout waits for that. - returns: `YES` if LaunchDarkly accepted the events, or there were none to send. `NO` if the timeout expired - first, the client is offline or closed, or any of the events were lost: refused by LaunchDarkly, still failing - after one retry, or unable to be serialized. Lost events are not kept for a later flush, so calling this again - does not resend them. A refusal also stops event delivery until the client is next set online, and until then - this returns `NO` even with nothing to send. A `NO` because the timeout expired does not mean the events were + first, the client is offline or closed, or the events did not get out. Events the service refused or that + failed to post are kept, and posted again once the backoff allows, or by the next call to this method. Events + that could not be serialized, or whose payload LaunchDarkly refused twice in a row, are lost, and calling this + again does not resend them. A `NO` because the timeout expired does not mean the events were not sent: the delivery is left running, and may still arrive. */ @objc public func flushAndWait(timeout: TimeInterval) -> Bool { diff --git a/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift b/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift index 78eb5706c..e3eae1aa2 100644 --- a/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift +++ b/LaunchDarkly/LaunchDarkly/ObjectiveC/ObjcLDConfig.swift @@ -35,12 +35,47 @@ public final class ObjcLDConfig: NSObject { set { config.streamUrl = newValue } } - /// The maximum number of analytics events the LDClient can store. When the LDClient event store reaches the eventCapacity, the SDK discards events until it successfully reports them to LaunchDarkly. (Default: 100) + /// The maximum number of analytics events the LDClient can store. When the LDClient event store reaches the eventCapacity, the SDK discards events until it successfully reports them to LaunchDarkly. The same number bounds how many distinct contexts flag evaluations are summarized for between two reports; evaluations for contexts beyond it are not counted until the next report. (Default: 2000) @objc public var eventCapacity: Int { get { config.eventCapacity } set { config.eventCapacity = newValue } } + /// How far the SDK goes to make a recorded event outlive the process that recorded it. + /// + /// Without this, an event lives in memory until it is delivered, and a process that dies before the next flush + /// takes everything recorded since the last one, including the crash an application was reporting when it died. + /// `.immediate` additionally puts the write on the thread that called `track` or `identify`, so there is no window + /// in which the event exists only in memory. (Default: `.disabled`) + /// + /// - Warning: Experimental. Both levels other than `.disabled` are not stable, and not subject to any backwards + /// compatibility guarantees or semantic versioning. + @objc public var eventPersistence: EventPersistence { + get { config.eventPersistence } + set { config.eventPersistence = newValue } + } + + /// The maximum number of bytes of events the SDK keeps, on disk and in memory together, whatever `eventPersistence` is, for every environment together: each environment in `secondaryMobileKeys` gets an equal share of it, unlike `eventCapacity`. When an environment reaches its share, the SDK discards new events until it reports the stored ones; a fifth of each share is kept for the summaries of flag evaluations. (Default: 20 MiB) + @objc public var eventCapacityBytes: Int { + get { config.eventCapacityBytes } + set { config.eventCapacityBytes = newValue } + } + + /// How long the SDK keeps persisted events it has not yet reported before discarding them, which also removes what it stored for a mobile key the application stopped using. No effect where `eventPersistence` is `.disabled`. (Default: 7 days) + @objc public var eventMaxAge: TimeInterval { + get { config.eventMaxAge } + set { config.eventMaxAge = newValue } + } + + /// How long after the client first goes online the SDK holds back the traffic it starts on its own: the diagnostic init payload, the events a previous run left, and the periodic delivery of this run's events. A flush the application asks for is never held. Kept between 0 and 60 seconds. (Default: 2 seconds) + /// + /// - Warning: Experimental. This property is not stable, and not subject to any backwards compatibility guarantees + /// or semantic versioning. + @objc public var eventDeliveryDelay: TimeInterval { + get { config.eventDeliveryDelay } + set { config.eventDeliveryDelay = newValue } + } + /// The timeout interval for flag requests and event reports. (Default: 10 seconds) @objc public var connectionTimeout: TimeInterval { get { config.connectionTimeout } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/AnalyticsEventSender.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/AnalyticsEventSender.swift new file mode 100644 index 000000000..b31aebb64 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/AnalyticsEventSender.swift @@ -0,0 +1,77 @@ +import Foundation +import OSLog + +/// Posts a batch of events to LaunchDarkly once and says what the response means. Android's `AnalyticsEventSender` +/// plays the same part. +/// +/// It keeps no state. What an outcome means for the SDK -- the server's time, when to post again, removing the batch -- +/// is left to `DirectEventProcessor`, which owns that state. +final class AnalyticsEventSender { + enum Outcome { + /// LaunchDarkly took the events. Carries the time the response gave, if it gave one. + case accepted(serverTime: Date?) + /// The post failed. The events are kept whatever the failure; it decides only how long to wait. + case failed(Failure) + } + + struct Failure { + let error: SynchronizingError + /// The response's status, or nil where there was no response. + let statusCode: Int? + /// Whether the failure is unlikely to pass soon: a 4xx other than a refusal of the payload + /// (`isEventPayloadRefused`), 408 and 429, or a TLS or certificate failure. + let unexpected: Bool + /// The wait the response asked for, or nil where it gave none. + let retryAfter: TimeInterval? + } + + private static let tlsErrorCodes: Set = [ + .secureConnectionFailed, + .serverCertificateHasBadDate, + .serverCertificateUntrusted, + .serverCertificateHasUnknownRoot, + .serverCertificateNotYetValid, + .clientCertificateRejected, + .clientCertificateRequired + ] + + private let service: DarklyServiceProvider + + init(service: DarklyServiceProvider) { + self.service = service + } + + /// Posts `body` under `payloadId`. The completion runs once, with the outcome. + func send(_ body: Data, payloadId: String, eventCount: Int, completion: @escaping (Outcome) -> Void) { + service.publishEventData(body, payloadId) { response in + completion(self.outcome(sentEvents: eventCount, response: response.urlResponse as? HTTPURLResponse, error: response.error)) + } + } + + private func outcome(sentEvents: Int, response: HTTPURLResponse?, error: Error?) -> Outcome { + if error == nil && (200..<300).contains(response?.statusCode ?? 0) { + os_log("%s Completed sending %d event(s)", log: service.config.logger, type: .debug, typeName(and: #function), sentEvents) + return .accepted(serverTime: response?.headerDate) + } + + os_log("%s Sending events failed with error: %s response: %s", log: service.config.logger, type: .debug, typeName(and: #function), String(describing: error), String(describing: response)) + let statusCode = response?.statusCode + let unexpected: Bool + if let statusCode, (400..<600).contains(statusCode) { + // A refusal of the payload waits like a 400: the processor drops the batch after a second in a row, and + // the batches behind it are not to be held back an extended wait meanwhile. + unexpected = HTTPURLResponse.StatusCodes.isTerminalStatusCode(statusCode) + && !HTTPURLResponse.StatusCodes.isEventPayloadRefused(statusCode) + } else if let urlError = error as? URLError { + unexpected = AnalyticsEventSender.tlsErrorCodes.contains(urlError.code) + } else { + unexpected = false + } + return .failed(Failure(error: error.map { .request($0) } ?? .response(response), + statusCode: statusCode, + unexpected: unexpected, + retryAfter: response?.headerRetryAfter)) + } +} + +extension AnalyticsEventSender: TypeIdentifying { } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundActivity.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundActivity.swift index eda2bb543..9e3bda96d 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundActivity.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundActivity.swift @@ -17,7 +17,7 @@ final class BackgroundActivity { typealias ExpiringActivity = (_ reason: String, _ whileGranted: @escaping (_ expired: Bool) -> Void) -> Void /// Upper bound for work that never calls `finished`. - private static let maximumDuration: TimeInterval = 30 + static let maximumDuration: TimeInterval = 30 private let reason: String private let assertion: Assertion @@ -32,8 +32,9 @@ final class BackgroundActivity { self.assertion = assertion } - convenience init(reason: String, logger: OSLog) { - self.init(reason: reason, assertion: BackgroundActivity.processAssertion(logger: logger)) + /// - parameter logger: Read each time it logs, so that it can be the logger of whichever client is current then. + convenience init(reason: String, logger: @autoclosure @escaping () -> OSLog) { + self.init(reason: reason, assertion: BackgroundActivity.processAssertion(logger: logger())) } /// Runs `work` under an assertion released when `work` calls `finished`, the system expires it, or @@ -72,12 +73,16 @@ final class BackgroundActivity { /// An assertion from the process, over an injectable system call and cap so that a test can drive both. /// - /// Without assertions on this platform, `whileHeld` runs on the calling thread. - static func processAssertion(logger: OSLog, - perform: @escaping ExpiringActivity = BackgroundActivity.performExpiringActivity, + /// Where the platform has no assertions, `perform` is nil and `whileHeld` runs on the calling thread. + static func processAssertion(logger: @autoclosure @escaping () -> OSLog, + perform: ExpiringActivity? = SystemCapabilities.expiringActivity, maximumDuration: TimeInterval = BackgroundActivity.maximumDuration) -> Assertion { { reason, whileHeld in - #if os(iOS) || os(tvOS) || os(watchOS) + guard let perform + else { + whileHeld {} + return + } let finished = DispatchSemaphore(value: 0) // The work does not wait on the assertion, and does not run inside it. The system refuses one outright // under the very resource pressure this is here for, and a delivery it may suspend is better than no @@ -88,7 +93,7 @@ final class BackgroundActivity { // The system taking the assertion back, or declining to give one. Nothing should stay blocked on // it, and the work goes ahead either way. os_log("%s unprotected by a background assertion: %s", - log: logger, type: .debug, String(describing: BackgroundActivity.self), reason) + log: logger(), type: .debug, String(describing: BackgroundActivity.self), reason) finished.signal() return } @@ -96,15 +101,6 @@ final class BackgroundActivity { _ = finished.wait(timeout: .now() + maximumDuration) } whileHeld { finished.signal() } - #else - whileHeld {} - #endif } } - - private static func performExpiringActivity(_ reason: String, _ whileGranted: @escaping (Bool) -> Void) { - #if os(iOS) || os(tvOS) || os(watchOS) - ProcessInfo.processInfo.performExpiringActivity(withReason: reason, using: whileGranted) - #endif - } } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundEventDelivery.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundEventDelivery.swift new file mode 100644 index 000000000..1cab80ae6 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/BackgroundEventDelivery.swift @@ -0,0 +1,91 @@ +import Foundation +import OSLog + +/// What the events of every environment do as the application goes to the background. +/// +/// One for all environments rather than one per client, so that backgrounding takes one background activity assertion +/// however many environments there are, and reaches only the clients started now: a closed client the application +/// still holds is not among them. +/// +/// On every platform that notifies the SDK, events are made durable before the notification returns: the OS may kill a +/// backgrounded application without warning, and macOS, which suspends nothing, can still quit or kill it. Where a +/// backgrounded process is suspended once idle, they are also delivered, under the assertion, which is released once +/// every environment has answered. What does not get out is on disk for the next launch where events persist, and +/// otherwise only in memory. +final class BackgroundEventDelivery { + struct Environment { + let name: String + let eventProcessor: EventProcessor + let logger: OSLog + } + + /// The environments started now, read at each backgrounding. + private let environments: () -> [Environment] + private let suspendsWhenBackgrounded: Bool + private let activity: BackgroundActivity + /// How long the deliveries have to answer before the ones that have not count as undelivered. The activity holds + /// one assertion at a time, so a delivery that never answered would otherwise keep every later backgrounding from + /// delivering at all. + private let answerTimeout: TimeInterval + + init(environments: @escaping () -> [Environment], + suspendsWhenBackgrounded: Bool, + activity: BackgroundActivity, + answerTimeout: TimeInterval = BackgroundActivity.maximumDuration) { + self.environments = environments + self.suspendsWhenBackgrounded = suspendsWhenBackgrounded + self.activity = activity + self.answerTimeout = answerTimeout + } + + convenience init(environments: @escaping () -> [Environment]) { + self.init(environments: environments, + suspendsWhenBackgrounded: SystemCapabilities.suspendsWhenBackgrounded, + activity: BackgroundActivity(reason: "LaunchDarkly event delivery", + logger: environments().first?.logger ?? .disabled)) + } + + /// Commits every environment before returning, at `.deferred` too, and where the process is about to be suspended + /// starts their deliveries. + func applicationDidEnterBackground() { + let environments = self.environments() + environments.forEach { $0.eventProcessor.commitWherePersisting() } + guard suspendsWhenBackgrounded, !environments.isEmpty + else { return } + activity.run { [weak self] finished in + guard let self + else { + finished() + return + } + environments.map(\.eventProcessor).flushReportingOutcomes(deadline: .now() + self.answerTimeout) { delivered in + self.logUndelivered(zip(environments, delivered).filter { !$0.1 }.map(\.0)) + finished() + } + } + } + + /// At info for the environments where nothing persists, since their events are lost if the application is + /// terminated while suspended. Whether events persist is each store's answer rather than the configuration's: a + /// store that failed to write has stopped persisting. + private func logUndelivered(_ environments: [Environment]) { + guard let logger = environments.first?.logger + else { return } + let onDisk = environments.filter { ($0.eventProcessor as? DirectEventProcessor)?.store.isPersisting == true } + let inMemory = environments.filter { ($0.eventProcessor as? DirectEventProcessor)?.store.isPersisting != true } + if !onDisk.isEmpty { + os_log("%s events of %s did not reach LaunchDarkly before suspension; they remain on disk for the next launch", + log: logger, type: .debug, typeName(and: #function), BackgroundEventDelivery.names(onDisk)) + } + if !inMemory.isEmpty { + os_log("%s events of %s did not reach LaunchDarkly before suspension; they are only in memory and are lost if the application is terminated", + log: logger, type: .info, typeName(and: #function), BackgroundEventDelivery.names(inMemory)) + } + } + + private static func names(_ environments: [Environment]) -> String { + environments.map(\.name).sorted().joined(separator: ", ") + } +} + +extension BackgroundEventDelivery: TypeIdentifying { } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/Cache/DiagnosticCache.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/Cache/DiagnosticCache.swift index d97ff5bfd..f628e2ffa 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/Cache/DiagnosticCache.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/Cache/DiagnosticCache.swift @@ -1,6 +1,5 @@ import Foundation -// sourcery: autoMockable protocol DiagnosticCaching { func getDiagnosticId() -> DiagnosticId func getCurrentStatsAndReset() -> DiagnosticStats diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/ClientServiceFactory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/ClientServiceFactory.swift index b97e18ce3..0ead86d9a 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/ClientServiceFactory.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/ClientServiceFactory.swift @@ -15,8 +15,10 @@ protocol ClientServiceCreating { service: DarklyServiceProvider, onSyncComplete: FlagSyncCompleteClosure?) -> LDFlagSynchronizing func makeFlagChangeNotifier() -> FlagChangeNotifying - func makeEventReporter(config: LDConfig, service: DarklyServiceProvider) -> EventReporting - func makeEventReporter(config: LDConfig, service: DarklyServiceProvider, onSyncComplete: EventSyncCompleteClosure?) -> EventReporting + func makeEventProcessor(config: LDConfig, service: DarklyServiceProvider) -> EventProcessor + func makeEventProcessor(config: LDConfig, + service: DarklyServiceProvider, + onSyncComplete: EventSyncCompleteClosure?) -> EventProcessor func makeStreamingProvider(url: URL, httpHeaders: [String: String], connectMethod: String, connectBody: Data?, handler: EventHandler, delegate: RequestHeaderTransform?, errorHandler: ConnectionErrorHandler?) -> DarklyStreamingProvider func makeEnvironmentReporter(config: LDConfig) -> EnvironmentReporting func makeThrottler(environmentReporter: EnvironmentReporting) -> Throttling @@ -67,15 +69,17 @@ final class ClientServiceFactory: ClientServiceCreating { FlagChangeNotifier(logger: logger) } - func makeEventReporter(config: LDConfig, service: DarklyServiceProvider) -> EventReporting { - makeEventReporter(config: config, service: service, onSyncComplete: nil) + func makeEventProcessor(config: LDConfig, service: DarklyServiceProvider) -> EventProcessor { + makeEventProcessor(config: config, service: service, onSyncComplete: nil) } - func makeEventReporter(config: LDConfig, service: DarklyServiceProvider, onSyncComplete: EventSyncCompleteClosure? = nil) -> EventReporting { + func makeEventProcessor(config: LDConfig, + service: DarklyServiceProvider, + onSyncComplete: EventSyncCompleteClosure? = nil) -> EventProcessor { if config.sendEvents { - return EventReporter(service: service, onSyncComplete: onSyncComplete) + return DirectEventProcessor(service: service, onSyncComplete: onSyncComplete) } else { - return NullEventReporter() + return NullEventProcessor() } } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DeliveryClaims.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DeliveryClaims.swift new file mode 100644 index 000000000..c5b7aef56 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DeliveryClaims.swift @@ -0,0 +1,48 @@ +import Foundation + +/// What one delivery's answer is made of. +/// +/// A commit only stages events; whichever delivery runs next is what closes them into a batch. Deliveries run one at a +/// time, but should a batch holding a caller's events belong to a delivery still under way, answering that caller means +/// hearing from it. These are the two small pieces that arrange it: a `Claim` is a batch somebody is waiting on, and a +/// `PassAnswer` gathers the outcomes one pass of a delivery depends on. +extension DirectEventProcessor { + /// A batch one delivery has taken, which another delivery's answer may depend on. + final class Claim { + /// This batch's result, or nil while the delivery holding it is still waiting. Only to be used while holding + /// the reporter's `claimLock`. + var outcome: Bool? + /// Deliveries whose answer waits on this batch. Only to be used while holding the reporter's `claimLock`. + var waiters: [FlushOutcomeClosure] = [] + } + + /// Gathers the outcomes one pass of a delivery depends on, and reports the worst of them: blocked where any batch + /// was, lost where any was lost, and delivered only where all of them were. + /// + /// How many there are is settled before the first can arrive, so this needs no notion of being complete beyond + /// counting down. + final class PassAnswer { + private let lock = UnfairLock() + /// Outcomes still to come, and the answer to make once there are none. Only to be used while holding `lock`. + private var outstanding: Int + private var outcome = PassOutcome.delivered + private var answer: ((PassOutcome) -> Void)? + + init(outstanding: Int, _ answer: @escaping (PassOutcome) -> Void) { + self.outstanding = outstanding + self.answer = answer + } + + func report(_ outcome: PassOutcome) { + let (answer, worst) = lock.withLock { () -> (((PassOutcome) -> Void)?, PassOutcome) in + self.outcome = max(self.outcome, outcome) + outstanding -= 1 + guard outstanding <= 0 + else { return (nil, self.outcome) } + defer { self.answer = nil } + return (self.answer, self.outcome) + } + answer?(worst) + } + } +} diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DiagnosticReporter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DiagnosticReporter.swift index 92bdb1aba..58d0e06fd 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/DiagnosticReporter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DiagnosticReporter.swift @@ -18,6 +18,9 @@ class DiagnosticReporter: DiagnosticReporting { private var sentInit: Bool private let stateQueue = DispatchQueue(label: "com.launchdarkly.diagnosticReporter.state", qos: .utility) private let workQueue = DispatchQueue(label: "com.launchdarkly.diagnosticReporter.work", qos: .utility) + /// Where set, what the init event waits for: the event processor's launch gate, so that it does not compete with the + /// first flag request. Set once the client has both; unset only where events are not sent, and so neither is this. + var sendsInitAfter: ((@escaping () -> Void) -> Void)? init(service: DarklyServiceProvider, environmentReporting: EnvironmentReporting) { self.service = service @@ -43,7 +46,12 @@ class DiagnosticReporter: DiagnosticReporting { environmentReporting: environmentReporting, diagnosticId: cache.getDiagnosticId(), creationDate: Date().millisSince1970) - sendDiagnosticEventAsync(diagnosticEvent: initEvent) + let send = { self.sendDiagnosticEventAsync(diagnosticEvent: initEvent) } + if let sendsInitAfter { + sendsInitAfter(send) + } else { + send() + } } timer = LDTimer(withTimeInterval: service.config.diagnosticRecordingInterval, fireQueue: workQueue) { diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift new file mode 100644 index 000000000..1445b8c56 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DirectEventProcessor.swift @@ -0,0 +1,997 @@ +import Foundation +import OSLog + +/// Records analytics events and delivers them to LaunchDarkly. +/// +/// Recorded events end up in an `EventStore`, an on-disk log, rather than held in memory until a delivery succeeds. +/// That is what lets an application that dies moments after recording an event still report it: the events are on disk +/// before the process is gone, and the next run of the application delivers them. +/// +/// What the application records is accepted, held and committed to the store by `EventCommitter`, which says which +/// calls are commit points and which thread each commit runs on. This class delivers what is committed, and runs the +/// reporter's life: flushes, going online and offline, and the flush at close. +/// +/// Flag values come first. What this reporter sends on its own waits at launch until `LDConfig.eventDeliveryDelay` after +/// it first goes online, and where events persist, the deliveries it starts on its own send one batch at a time, a gap +/// apart, leaving room for flag requests. Both go by time alone: nothing here hears what the flag synchronizer is doing. +/// A delivery the application asks for is never held. +class DirectEventProcessor: EventProcessor { + /// How long the deliveries this reporter starts on its own wait, so that flag values come first. + struct Pacing { + /// The gap the specification gives, after the launch delay the application configured. + static func specified(launchDelay: TimeInterval) -> Pacing { + Pacing(launchDelay: launchDelay, gap: 1) + } + /// No waits at all, for tests about something else. + static let noWaits = Pacing(launchDelay: 0, gap: 0) + + /// How long after first going online that a previous run's events, this run's periodic delivery, and the + /// diagnostic init event wait. It gives the application's startup traffic, the first flag request among it, the + /// network first, and is short enough for an application that dies a few seconds after every launch to still + /// send them. + let launchDelay: TimeInterval + /// The least time between one paced batch's answer and the next batch's request. + let gap: TimeInterval + } + + /// How long `flushAtClose()` waits for its commit, on a thread that for an application closing the client is + /// usually the main one. The same two seconds Android allows its whole `close()`, well inside what the system + /// tolerates from a main thread that has stopped answering. + static let closeCommitBudget: TimeInterval = 2 + + // MARK: Set in init and never changed + + let service: DarklyServiceProvider + let store: EventStoring + let committer: EventCommitter + private let lostEvents: LostEventCount + private let deliveryScheduler: EventDeliveryScheduler + + private let onSyncComplete: EventSyncCompleteClosure? + + private let sender: AnalyticsEventSender + + /// When a post is allowed after a failure. Nothing is posted while it says to wait. + private let retryState: DeliveryRetryState + + private let pacing: Pacing + + var lastEventResponseDate: Date { + committer.lastEventResponseDate + } + + // MARK: Guarded by stateLock + + /// Guards which losses have been told to a caller, and the counts of refused payloads and failed reads. + private let stateLock = UnfairLock() + + /// The mark up to which losses have already been reported to a caller, so that none is told twice. Only to be + /// used while holding `stateLock`. + private var reportedLostEvents: UInt64 = 0 + + /// How many refusals of the payload in a row drop a batch: a 400, 413, 415 or 422, in any mix + /// (`isEventPayloadRefused`). Each says the service will not take this body, which posting it again will not + /// change, and a kept batch is the oldest and so the next posted: kept for good, it would hold every batch behind + /// it. Two rather than one, so one from a proxy or an incident costs nothing. + static let badRequestsBeforeDropping = 2 + + /// The batch that last had its payload refused and how many times in a row: delivery goes oldest first and stops at + /// a kept failure, so the next post is that batch's unless capacity dropped it. Any response other than a refusal + /// ends the run; a post that got no response leaves it. Held only in memory, so a new run of the application counts + /// again. Only to be used while holding `stateLock`. + private var badRequests: (payloadId: String, inARow: Int)? + + /// How many counted failed reads in a row drop a batch. A read that fails is mostly one that will go on failing, a + /// permission taken away or a directory named like a batch, and expiry cannot delete a file it cannot open, so the + /// batch would be read, and fail, at every delivery for good. More than one, and counted at most once per + /// `readFailureSpacing`, so a passing failure such as running out of file descriptors costs nothing. + static let readFailuresBeforeDropping = 3 + + /// The least time between two failed reads of a batch that both count toward dropping it: the flush interval, so a + /// drop takes failures across at least two of them. Every pass that reaches the batch reads it, and passes can come + /// milliseconds apart, from flushes called in a row or the blocking `flushAndWait` posting through the backoff; + /// counted by pass, a failure lasting a moment would drop an intact batch. A failed read closer than this to the + /// last counted one passes the batch by without counting. Settable only for tests. + var readFailureSpacing: TimeInterval { + get { stateLock.withLock { readFailureSpacingValue } } + set { stateLock.withLock { readFailureSpacingValue = newValue } } + } + // Only to be used while holding `stateLock`. + private var readFailureSpacingValue: TimeInterval + + /// Counted failed reads in a row, by payload ID, of the batches whose last read failed, and when the last of them + /// was. Unlike refusals, these are counted for several batches at once, since a batch whose read failed is passed + /// by. Held only in memory, so a new run of the application counts again. Only to be used while holding + /// `stateLock`. + private var readFailures: [String: (counted: Int, lastCounted: DispatchTime)] = [:] + + // MARK: Guarded by claimLock + + /// Guards `batchesInFlight`. + /// + /// Its own lock rather than `stateLock`, which an evaluating thread takes to read `lastEventResponseDate`: a claim + /// is made next to a listing of the store, and nothing a flag evaluation waits on may be held near the disk. + private let claimLock = UnfairLock() + + /// The batches a delivery has taken and not yet had an answer for, by payload ID. Only to be used while holding + /// `claimLock`. + /// + /// A batch stays on disk until its response arrives. `EventDeliveryScheduler` keeps a second delivery from listing it + /// inside that window; this is what keeps it from being sent twice, under the same payload ID, should two ever overlap. + /// A delivery that finds a batch claimed waits on that claim's outcome rather than sending it. + private var batchesInFlight: [String: Claim] = [:] + + // MARK: Confined to timerQueue + + var isOnline: Bool { + get { timerQueue.sync { eventReportTimer != nil } } + set { timerQueue.sync { newValue ? startReporting() : stopReporting() } } + } + + private var timerQueue = DispatchQueue(label: "com.launchdarkly.DirectEventProcessor.timerQueue") + private var eventReportTimer: TimeResponding? + var isReportingActive: Bool { eventReportTimer != nil } + + // MARK: Confined to deliveryQueue + + /// Deliveries run here, off whatever thread recorded an event. + private let deliveryQueue = DispatchQueue(label: "com.launchdarkly.eventSyncQueue", qos: .userInitiated) + + /// Whether the store held events from a previous run of the application when this reporter started. + /// + /// Only to be used on `deliveryQueue`, which is what makes it safe without a lock: the recovery that answers the + /// question and the delivery that acts on it are both enqueued there, in that order. + private var hasEventsFromPreviousRun = false + + /// Whether the launch gate has opened. It opens once the launch delay has passed since first going online. Until then + /// neither a previous run's events, this run's periodic delivery, nor the diagnostic init event are sent. Open from + /// the start where there is no launch delay. + private var launchGateOpen: Bool + /// When the reporter first went online, which the gate counts from. + private var wentOnlineAt: DispatchTime? + /// What waits for the gate to open. + private var launchGateWaiters: [() -> Void] = [] + + init(service: DarklyServiceProvider, + onSyncComplete: EventSyncCompleteClosure?, + store: EventStoring? = nil, + encoding: OutboundEventBuffer.Encoding = .handWritten, + commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.DirectEventProcessor.commitQueue", qos: .userInitiated), + retryState: DeliveryRetryState? = nil, + pacing: Pacing? = nil) { + let eventStore = store ?? DirectEventProcessor.makeStore(config: service.config) + let lostEvents = LostEventCount() + self.service = service + self.onSyncComplete = onSyncComplete + self.sender = AnalyticsEventSender(service: service) + self.retryState = retryState ?? DeliveryRetryState(flushInterval: service.config.eventFlushInterval) + self.readFailureSpacingValue = service.config.eventFlushInterval + self.lostEvents = lostEvents + self.committer = EventCommitter(config: service.config, store: eventStore, encoding: encoding, + commitQueue: commitQueue, diagnosticCache: service.diagnosticCache, + lostEvents: lostEvents) + self.store = eventStore + let pacing = pacing ?? .specified(launchDelay: service.config.eventDeliveryDelay) + self.pacing = pacing + self.deliveryScheduler = EventDeliveryScheduler(pacesDeliveries: eventStore.isPersisting, gap: pacing.gap) + self.launchGateOpen = pacing.launchDelay <= 0 + + // A log left open by a previous run has to be closed before it can be delivered, but nothing waits on that: + // doing it on the caller's thread would put file I/O in the way of the client starting up. At utility, below + // the flag requests, which matter more at launch than what a previous run left. Not as a block on + // `deliveryQueue`, whose own userInitiated QoS a block cannot go below; the queue is suspended instead, so + // everything enqueued on it meanwhile still runs after the recovery. Only where events persist, since memory + // holds nothing from a previous run. + guard eventStore.isPersisting + else { return } + let store = eventStore + let queue = deliveryQueue + let recovered = RecoveredListing() + queue.suspend() + queue.async { [weak self] in + self?.reportVanished(recovered.vanishedEvents) + self?.hasEventsFromPreviousRun = recovered.hasBatches + } + // Enforced, since the caller is usually the main thread, whose QoS would otherwise be carried over. + DispatchQueue.global(qos: .utility).async(qos: .utility, flags: .enforceQoS) { [weak self] in + // Resumed whatever happens: a dispatch queue released while suspended traps. + defer { queue.resume() } + store.preparePayloadsFromInterruptedLogs() + self?.reportDiscarded(store.discardExpired()) + let listing = store.listPendingBatches() + recovered.vanishedEvents = listing.vanishedEvents + recovered.hasBatches = !listing.batches.isEmpty + } + } + + /// What the recovery found, handed to `deliveryQueue`. Written before the queue resumes and read after, so the + /// resume orders the two. + private final class RecoveredListing { + var vanishedEvents = 0 + var hasBatches = false + } + + private static func makeStore(config: LDConfig) -> EventStoring { + let mobileKey = config.mobileKey + return EventStore(locatingDirectory: { EventDirectory.defaultUrl(mobileKey: mobileKey) }, + capacity: config.eventCapacity, + byteCapacity: shareOfCapacityBytes(config: config), + maxAge: config.eventMaxAge, + persistEvents: config.eventPersistence != .disabled, + logger: config.logger) + } + + /// This environment's share of the byte capacity, which is for every environment the application configured together, + /// where the event capacity is for each one. + static func shareOfCapacityBytes(config: LDConfig) -> Int { + let total = config.eventCapacityBytes > 0 ? config.eventCapacityBytes : LDConfig.Defaults.eventCapacityBytes + // At least one byte, since the store reads zero as the default. + return max(1, total / (1 + config.getSecondaryMobileKeys().count)) + } + + // MARK: Recording + + func record(_ event: Event) { + committer.record(event) + } + + // swiftlint:disable:next function_parameter_count + func recordFlagEvaluationEvents(flagKey: LDFlagKey, + value: LDValue, + defaultValue: LDValue, + featureFlag: FeatureFlag?, + context: LDContext, + includeReason: Bool) { + committer.recordFlagEvaluationEvents(flagKey: flagKey, value: value, defaultValue: defaultValue, + featureFlag: featureFlag, context: context, includeReason: includeReason) + } + + func commitRecordedEvents() { + committer.commitRecordedEvents() + } + + func commitWherePersisting() { + committer.commitWherePersisting() + } + + /// Records stored events discarded for being older than the maximum age. + /// + /// Counted as dropped rather than lost: a flush is about the caller's events, and the ones discarded are long + /// past anything a caller is waiting on. + private func reportDiscarded(_ events: Int) { + guard events > 0 + else { return } + os_log("%s discarded %d stored event(s) older than the maximum age", log: service.config.logger, type: .debug, typeName(and: #function), events) + for _ in 0.. Bool { + stateLock.withLock { + guard mark > reportedLostEvents + else { return false } + reportedLostEvents = mark + return true + } + } + +} + +// MARK: Reporting + +extension DirectEventProcessor { + private func startReporting() { + guard eventReportTimer == nil + else { return } + eventReportTimer = LDTimer(withTimeInterval: service.config.eventFlushInterval, fireQueue: deliveryQueue) { [weak self] in + self?.reportIntervalEnded() + } + + // Events a previous run left behind are already late by however long the application was gone, so they do not + // wait out a report interval on top of that, only the launch gate: an application that crashed reports it as + // soon as it is next able to. Only the first time online brings a delivery forward, so going online again + // later, as a network comes and goes, keeps to the ordinary cadence. + deliveryQueue.async { [weak self] in + guard let self + else { return } + if self.wentOnlineAt == nil { + self.wentOnlineAt = .now() + if !self.launchGateOpen { + self.checkLaunchGate(after: self.pacing.launchDelay) + } + } + guard self.launchGateOpen, self.hasEventsFromPreviousRun + else { return } + self.hasEventsFromPreviousRun = false + self.queueScheduledDelivery() + } + } + + private func stopReporting() { + eventReportTimer?.cancel() + eventReportTimer = nil + deliveryScheduler.endPacing() + } + + /// Runs `block`, on a global queue, once the launch gate is open. + func whenLaunchGateOpens(_ block: @escaping () -> Void) { + deliveryQueue.async { [weak self] in + guard let self, !self.launchGateOpen + else { + DispatchQueue.global().async(execute: block) + return + } + self.launchGateWaiters.append(block) + } + } + + private func checkLaunchGate(after delay: TimeInterval) { + deliveryQueue.asyncAfter(deadline: .now() + max(0, delay)) { [weak self] in self?.checkLaunchGate() } + } + + /// Opens the launch gate once the launch delay has passed since first going online, and otherwise looks again then. + /// On `deliveryQueue`. + private func checkLaunchGate() { + guard !launchGateOpen, let wentOnlineAt + else { return } + let sinceOnline = TimeInterval(DispatchTime.now().uptimeNanoseconds - wentOnlineAt.uptimeNanoseconds) / 1_000_000_000 + guard sinceOnline >= pacing.launchDelay + else { + checkLaunchGate(after: pacing.launchDelay - sinceOnline) + return + } + launchGateOpen = true + let waiters = launchGateWaiters + launchGateWaiters = [] + waiters.forEach { DispatchQueue.global().async(execute: $0) } + guard hasEventsFromPreviousRun + else { return } + os_log("%s Sending events from a previous run, %dms after going online", log: service.config.logger, type: .debug, typeName(and: #function), Int(pacing.launchDelay * 1000)) + // Offline, they wait for the reporter to go online, which finds the gate open. + guard isOnline + else { return } + hasEventsFromPreviousRun = false + queueScheduledDelivery() + } + + func flush(completion: CompletionClosure?) { + // Nil stays nil, so a flush nobody waits on joins a queued delivery without adding to its waiters. + commitThenQueueDelivery(completion.map { done in { _ in done() } }) + } + + func flushReportingOutcome(completion: @escaping FlushOutcomeClosure) { + committer.queueCommitWherePersisting() + queueDelivery(completion) + } + + func flushBypassingBackoff(completion: @escaping FlushOutcomeClosure) { + committer.queueCommitWherePersisting() + queueDelivery(completion, bypassingBackoff: true) + } + + func flushAtClose() { + committer.commitWithinBudget(DirectEventProcessor.closeCommitBudget) + // Offline, it is an ordinary delivery, which finds the reporter offline and sends nothing: offline is the + // application keeping the SDK off the network, and closing does not revoke that. + queueDelivery(nil, closing: isOnline) + } + + /// Queues a delivery the way a caller asking for one expects, which is as a commit point. + /// + /// Wherever events persist, everything accepted before the call is on disk by the time it returns, even where the + /// delivery itself cannot run because the client is offline. What it does not do is close those events into a + /// batch: only a delivery does that, so a delivery already under way may be the one holding them. That is why the + /// answer waits on the claims it skips rather than only on what it sent itself. + private func commitThenQueueDelivery(_ completion: FlushOutcomeClosure?) { + commitWherePersisting() + queueDelivery(completion) + } + + /// The report timer's delivery, on `deliveryQueue`. Until the launch gate opens it only commits: a delivery sends the + /// oldest batches first, which are what a previous run left, so one now would send them ahead of the gate. What this + /// run recorded goes with the first delivery after the gate opens. + private func reportIntervalEnded() { + guard launchGateOpen + else { + committer.queueCommitWherePersisting() + return + } + queueScheduledDelivery() + } + + /// Queues the delivery the report timer, the launch gate and the end of a backoff's wait start, which nobody asked + /// for now and which is paced. Unpaced, it would also end the pacing of the delivery running. + private func queueScheduledDelivery() { + queueDelivery(nil, paced: true) + } + + /// As `EventDeliveryScheduler.queue(_:closing:bypassingBackoff:paced:)`, starting the delivery where nothing is + /// running. + private func queueDelivery(_ completion: FlushOutcomeClosure?, + closing: Bool = false, + bypassingBackoff: Bool = false, + paced: Bool = false) { + if deliveryScheduler.queue(completion, closing: closing, bypassingBackoff: bypassingBackoff, paced: paced) { + deliveryQueue.async { self.runDelivery() } + } + } + + private func runDelivery() { + let (run, waiters) = deliveryScheduler.begin() + reportEvents(run) { delivered, lossMark in + // Taken once for the delivery rather than once per waiter, so that every caller joined to it hears about + // a loss instead of only whichever happened to be answered first. Taken before the next delivery starts, + // which could otherwise answer first and take this one's loss with its own. + let kept = waiters.isEmpty || !self.takeEventsLost(upTo: lossMark) + // Before the waiters are answered, so a completion that flushes again finds this delivery over. + self.startQueuedDeliveryOrStop() + guard !waiters.isEmpty + else { return } + // Answered away from `deliveryQueue`, and from whichever thread the delivery ended on, so that every + // completion reaches its caller the same way. A completion is free to flush again: it is never running + // on the queue the delivery it asks for would be queued on. + DispatchQueue.global().async { + for waiter in waiters { + waiter(delivered && kept) + } + } + } + } + + /// Ends the delivery that just had its answer, starting the one queued behind it if there is one. + private func startQueuedDeliveryOrStop() { + if deliveryScheduler.finish() { + deliveryQueue.async { self.runDelivery() } + } + } + + /// - Parameter run: the delivery; a closing one was online when it was asked for. + private func reportEvents(_ run: EventDeliveryRun, completion: ((_ delivered: Bool, _ lossMark: UInt64) -> Void)?) { + guard run.isClosing || isOnline + else { + os_log("%s aborted. DirectEventProcessor is offline", log: service.config.logger, type: .debug, typeName(and: #function)) + reportSyncComplete(.isOffline) + completion?(false, 0) + return + } + + // The batches already waiting go first, and what has been committed since is closed into a batch of its own + // only once none of them is left blocked. Closing first would add a batch for every pass a failure lasts: every + // flush, periodic delivery and trip to the background of an outage would leave one more file, or one more batch + // held in memory, each with the summary its commit staged. Held back, what is committed meanwhile goes on into + // the open log, which closes once it is full or once the batches ahead of it are gone, so an outage costs a + // batch per `EventLogFormat.maxBatchSize` rather than one per pass. + let waiting = claimDeliverableBatches(discardingExpired: true) + // Settled before this delivery's own outcome is reported, which is what the answer waits for last. + var settled = (lossMark: UInt64(0), accountsForEverything: false) + // This delivery's own outcome, plus that of every batch another delivery is already sending. The caller's + // events may be in either: a delivery that started after they were recorded closed them into its batch. + let answer = PassAnswer(outstanding: waiting.waitingOn.count + 1) { outcome in + completion?(outcome == .delivered && settled.accountsForEverything, settled.lossMark) + } + // Waited on for the answer, but not before closing. + waitOn(waiting.waitingOn, answer) + + deliverOwn(waiting.deliverable, run, lostAnything: waiting.lostAnything) { waitingOutcome in + guard waitingOutcome != .blocked + else { + // Committed all the same, since a delivery is a commit point. Nothing was closed, so the caller's events + // did not go, and the mark is as of this commit, the last point anything the caller recorded was staged. + settled = (self.committer.commitLeavingTheLogOpen(), false) + os_log("%s will report failure: an earlier batch could not be delivered", log: self.service.config.logger, type: .debug, self.typeName(and: #function)) + answer.report(.blocked) + return + } + + let (lossMark, closedEverything) = self.committer.commitAndCloseBatch() + let new = self.claimDeliverableBatches(discardingExpired: false) + // Events left in a log that would not close, or in a batch a listing that failed could not see, are still on + // the disk for a later delivery. This one delivers what it has, and cannot say the caller's events got out. + settled = (lossMark, closedEverything && waiting.isComplete && new.isComplete) + if !settled.accountsForEverything { + os_log("%s will report failure: %s", log: self.service.config.logger, type: .debug, self.typeName(and: #function), + closedEverything ? "the stored events could not be listed" : "the event log could not be closed") + } + if waiting.deliverable.isEmpty && waiting.waitingOn.isEmpty && new.deliverable.isEmpty && new.waitingOn.isEmpty { + os_log("%s aborted. Event store is empty", log: self.service.config.logger, type: .debug, self.typeName(and: #function)) + self.reportSyncComplete(nil) + } + + let newAnswer = PassAnswer(outstanding: new.waitingOn.count + 1) { newOutcome in + answer.report(max(waitingOutcome, newOutcome)) + } + self.waitOn(new.waitingOn, newAnswer) + self.deliverOwn(new.deliverable, run, lostAnything: new.lostAnything, newAnswer.report) + } + } + + /// Delivers the batches this delivery claimed, releasing the claims once it has an outcome. + /// + /// - Parameter lostAnything: whether the listing that found them found other batches deleted unsent. + private func deliverOwn(_ batches: [EventBatch], _ run: EventDeliveryRun, lostAnything: Bool, _ completion: @escaping (PassOutcome) -> Void) { + guard !batches.isEmpty + else { + completion(lostAnything ? .lost : .delivered) + return + } + os_log("%s starting", log: service.config.logger, type: .debug, typeName(and: #function)) + deliver(batches, run, lostAnything: lostAnything, retried: false) { outcome in + self.releaseClaim(on: batches, delivered: outcome == .delivered) + completion(outcome) + } + } + + /// Reports the outcome of each batch another delivery is sending to `answer`, once it has one. + private func waitOn(_ claims: [Claim], _ answer: PassAnswer) { + for claim in claims { + whenSettled(claim) { answer.report($0 ? .delivered : .blocked) } + } + } + + /// What `claimDeliverableBatches(discardingExpired:)` took. + private struct ClaimedBatches { + let deliverable: [EventBatch] + /// The claims on the batches left out. One may hold this caller's events -- `commitRecordedEvents` only stages + /// them, and whichever delivery runs next is what closes them into a batch -- so the answer has to wait for + /// those claims rather than assume they are someone else's business. + let waitingOn: [Claim] + let isComplete: Bool + /// Whether the listing found batches deleted unsent. + let lostAnything: Bool + } + + /// Takes the batches this delivery will send, leaving out any another delivery is still waiting on a response for. + /// + /// - Parameter discardingExpired: whether to discard what is too old to report first, which a second pass moments + /// after the first has no need to do again. + private func claimDeliverableBatches(discardingExpired: Bool) -> ClaimedBatches { + // Listed before the lock is taken, because listing reads the disk. Taking what appeared after it is the next + // delivery's business; all this has to be atomic is deciding which of these batches are this delivery's. + // Expired first, so that a batch too old to report is not sent. + if discardingExpired { + reportDiscarded(store.discardExpired()) + } + let listing = store.listPendingBatches() + reportVanished(listing.vanishedEvents) + return claimLock.withLock { + var deliverable: [EventBatch] = [] + var waitingOn: [Claim] = [] + for batch in listing.batches { + if let claimed = batchesInFlight[batch.payloadId] { + waitingOn.append(claimed) + } else { + batchesInFlight[batch.payloadId] = Claim() + deliverable.append(batch) + } + } + return ClaimedBatches(deliverable: deliverable, waitingOn: waitingOn, isComplete: listing.isComplete, + lostAnything: listing.vanishedEvents > 0) + } + } + + /// Counts as lost the events of batches a listing found deleted unsent, or of an open log a close found gone. + private func reportVanished(_ events: Int) { + guard events > 0 + else { return } + os_log("%s %d stored event(s) were deleted before they could be sent", log: service.config.logger, type: .error, typeName(and: #function), events) + reportStoredEventsLost(events) + } + + /// Releases the claims once the delivery has an answer, whether or not every batch was sent: `deliver` stops at the + /// first retryable failure, and what it did not reach has to be deliverable again. + /// + /// Every batch in the pass is settled with the pass's own result, rather than its own. A pass that stopped partway + /// did not hear back about what it never reached, so the batches it is giving up are not known to have arrived. + private func releaseClaim(on batches: [EventBatch], delivered: Bool) { + let waiters = claimLock.withLock { () -> [FlushOutcomeClosure] in + batches.flatMap { batch -> [FlushOutcomeClosure] in + guard let claimed = batchesInFlight.removeValue(forKey: batch.payloadId) + else { return [] } + claimed.outcome = delivered + return claimed.waiters + } + } + + for waiter in waiters { + waiter(delivered) + } + } + + /// Calls back with a claimed batch's outcome, once there is one. + /// + /// Settling and registering both happen under `claimLock`, so a claim released between the listing and this call + /// answers from what it recorded rather than leaving a waiter nobody will ever come back to. + private func whenSettled(_ claim: Claim, _ completion: @escaping FlushOutcomeClosure) { + let settled = claimLock.withLock { () -> Bool? in + guard let outcome = claim.outcome + else { + claim.waiters.append(completion) + return nil + } + return outcome + } + + if let settled { + completion(settled) + } + } + + /// How a pass over some batches went, from best to worst. + enum PassOutcome: Int, Comparable { + /// Every batch was accepted. + case delivered + /// Every batch was attempted, and at least one was lost for good: unreadable, damaged, deleted unsent, or had its + /// payload refused `badRequestsBeforeDropping` times in a row. + case lost + /// Every batch was attempted or passed by, and at least one was passed by: its read failed, and it is kept for a + /// later delivery to read again, while the batches after it were still attempted. + case passedBy + /// A batch is still there to be sent again: its request failed, or the backoff did not allow it to be posted + /// yet. The batches after it were not attempted. + case blocked + + static func < (lhs: PassOutcome, rhs: PassOutcome) -> Bool { + lhs.rawValue < rhs.rawValue + } + } + + /// Delivers batches oldest first, stopping at the first one whose post failed. One whose read failed is passed by, + /// and is not read again in this delivery. + /// + /// Stopping matters: the batches that are left keep their place in the log, and a later delivery attempts them + /// again rather than the SDK spending the rest of the session's requests on a service that is refusing them. A read + /// says nothing about the service, so it stops nothing. + /// + /// A failure whose wait is no longer than the initial delay is retried here, once a pass, so a caller waiting on + /// the pass hears from the retry; a longer wait ends the pass at once, with a delivery set for when the wait ends. + /// + /// `lostAnything` carries whether a batch already taken in this pass was lost for good, so that the caller hears + /// about it even where the batches after it were accepted. `retried` whether this pass has had its retry. + /// + /// A paced delivery holds each batch after its first until a gap has passed since the last one was answered. A + /// backlog this sends unasked otherwise takes the network for as long as it lasts, an `identify`'s flag request + /// included. `hadTurn` is whether the first batch has been held already. + private func deliver(_ batches: [EventBatch], _ run: EventDeliveryRun, lostAnything: Bool, retried: Bool, hadTurn: Bool = false, _ completion: @escaping (PassOutcome) -> Void) { + var remaining = batches + guard let next = remaining.first + else { + completion(run.passedAnyBy ? .passedBy : lostAnything ? .lost : .delivered) + return + } + guard !run.passedBy(next.payloadId) + else { + deliver(Array(remaining.dropFirst()), run, lostAnything: lostAnything, retried: retried, hadTurn: hadTurn, completion) + return + } + + if !hadTurn, run.awaitTurn(then: { + self.deliver(batches, run, lostAnything: lostAnything, retried: retried, hadTurn: true, completion) + }) { + return + } + + guard mayPost(run) + else { + completion(.blocked) + return + } + + let wait = run.bypassesBackoff ? 0 : retryState.remainingWait + guard wait <= 0 + else { + // Not an attempt, so it moves nothing in the backoff. A wait no longer than a retry's is sat out, as this + // pass's retry; a longer one ends the pass. + if !retried && wait <= retryState.initialDelay { + DispatchQueue.global().asyncAfter(deadline: .now() + wait) { + self.deliver(batches, run, lostAnything: lostAnything, retried: true, completion) + } + } else { + scheduleDeliveryWhenWaitEnds() + completion(.blocked) + } + return + } + + let batch = remaining.removeFirst() + let read: BatchPayload? + do { + read = try store.payload(of: batch) + } catch is UnreadableBatch { + // Lost: nothing will ever send it, so it is removed, the rest are still attempted, and the caller hears + // that this pass lost events. + os_log("%s dropping %d stored event(s) this version cannot read", log: service.config.logger, type: .error, typeName(and: #function), batch.eventCount) + store.remove(batch) + reportBatchLost(batch) + deliver(remaining, run, lostAnything: true, retried: retried, completion) + return + } catch is VanishedBatch { + os_log("%s %d stored event(s) were deleted before they could be sent", log: service.config.logger, type: .error, typeName(and: #function), batch.eventCount) + store.remove(batch) + reportBatchLost(batch) + deliver(remaining, run, lostAnything: true, retried: retried, completion) + return + } catch { + let dropped = passByOrDrop(batch, readFailedWith: error, run) + deliver(remaining, run, lostAnything: lostAnything || dropped, retried: retried, completion) + return + } + endReadFailures(batch) + + guard let payload = read + else { + // Delivered by another process or store, or holding no events. Either way nothing is lost and there is + // nothing to keep. + store.remove(batch) + deliver(remaining, run, lostAnything: lostAnything, retried: retried, completion) + return + } + + // Damage after the first frame: what came before it is sent, and the rest is gone with the file. Counted once + // the batch is removed rather than now, since one kept for a retry is read, and found short, again. + let leftOut = max(0, batch.eventCount - payload.eventCount) + service.diagnosticCache?.recordEventsInLastBatch(eventsInLastBatch: payload.eventCount) + post(batch, payload, run, leftOut: leftOut, remaining: remaining, lostAnything: lostAnything, retried: retried, completion) + } + + /// Passes by a batch whose read failed, or drops it as lost once that is `readFailuresBeforeDropping` in a row. + /// + /// Kept, since a read that failed may not fail next time, but passed by rather than stopped at. Stopped at, it held + /// back every batch behind it, the open log's included, on every pass for as long as the read kept failing: for good + /// where expiry cannot open the file to delete it either. + /// + /// - Returns: whether it dropped the batch. + private func passByOrDrop(_ batch: EventBatch, readFailedWith error: Error, _ run: EventDeliveryRun) -> Bool { + guard countReadFailure(batch) < DirectEventProcessor.readFailuresBeforeDropping + else { + os_log("%s dropping %d stored event(s) that could not be read %d times in a row: %s", log: service.config.logger, type: .error, typeName(and: #function), batch.eventCount, DirectEventProcessor.readFailuresBeforeDropping, String(describing: error)) + store.remove(batch) + reportBatchLost(batch) + return true + } + os_log("%s could not read stored events, will try again at the next delivery: %s", log: service.config.logger, type: .info, typeName(and: #function), String(describing: error)) + run.passBy(batch.payloadId) + return false + } + + /// Whether the delivery running may post: online, or `flushAtClose()`'s. Going offline stops a delivery before its + /// next post, its retry's included, and leaves what it has not sent for a later one. + private func mayPost(_ run: EventDeliveryRun) -> Bool { + run.isClosing || isOnline + } + + /// Posts a batch already read, then goes on to the rest. Requires the backoff to allow a post. + // swiftlint:disable:next function_parameter_count + private func post(_ batch: EventBatch, + _ payload: BatchPayload, + _ run: EventDeliveryRun, + leftOut: Int, + remaining: [EventBatch], + lostAnything: Bool, + retried: Bool, + _ completion: @escaping (PassOutcome) -> Void) { + publish(batch, payload, run) { outcome in + switch outcome { + case .accepted: + self.endBadRequests() + guard leftOut > 0 + else { + self.deliver(remaining, run, lostAnything: lostAnything, retried: retried, completion) + return + } + os_log("%s dropped %d damaged stored event(s) from a batch it delivered", log: self.service.config.logger, type: .error, self.typeName(and: #function), leftOut) + self.reportStoredEventsLost(leftOut) + self.deliver(remaining, run, lostAnything: true, retried: retried, completion) + case .failed(let failure): + // Only a run bypassing the backoff posts during its wait. + let wait = run.bypassesBackoff + ? self.retryState.recordFailureDuringWait(unexpected: failure.unexpected, retryAfter: failure.retryAfter) + : self.retryState.recordFailure(unexpected: failure.unexpected, retryAfter: failure.retryAfter) + if self.countBadRequest(batch, statusCode: failure.statusCode) >= DirectEventProcessor.badRequestsBeforeDropping { + // The rest are still attempted, once the backoff allows. + os_log("%s dropping %d event(s) whose payload the service refused %d times in a row, last with HTTP %d", log: self.service.config.logger, type: .error, self.typeName(and: #function), batch.eventCount, DirectEventProcessor.badRequestsBeforeDropping, failure.statusCode ?? 0) + self.store.remove(batch) + self.reportBatchLost(batch) + self.reportSyncComplete(failure.error) + self.deliver(remaining, run, lostAnything: true, retried: retried, completion) + return + } + guard !retried, wait <= self.retryState.initialDelay + else { + // Told once the pass gives the batch up, not for a failure its retry may still answer. + self.reportSyncComplete(failure.error) + self.scheduleDeliveryWhenWaitEnds() + completion(.blocked) + return + } + os_log("%s Retrying event post after %.1fs", log: self.service.config.logger, type: .debug, self.typeName(and: #function), wait) + DispatchQueue.global().asyncAfter(deadline: .now() + wait) { + // Another delivery's failure may have set a longer wait meanwhile, or the reporter gone offline. + guard self.mayPost(run) + else { + self.reportSyncComplete(failure.error) + completion(.blocked) + return + } + guard self.retryState.remainingWait <= 0 + else { + self.reportSyncComplete(failure.error) + self.scheduleDeliveryWhenWaitEnds() + completion(.blocked) + return + } + self.post(batch, payload, run, leftOut: leftOut, remaining: remaining, lostAnything: lostAnything, retried: true, completion) + } + } + } + } + + /// - Parameter statusCode: the response's status, or nil where there was no response. + /// - Returns: how many refusals of the payload in a row this batch has had, counting this response; zero for any + /// other status, which ends the run, and once the batch is dropped. + private func countBadRequest(_ batch: EventBatch, statusCode: Int?) -> Int { + stateLock.withLock { + guard let statusCode + else { + // A network or TLS failure leaves the run as it was. The service did not answer, so it said nothing + // about the payload: had it ended the run, refusals with network failures between them would never + // reach two, and a batch the service will never take would stay the oldest, holding back every + // batch behind it. + if let previous = badRequests, previous.payloadId == batch.payloadId { + return previous.inARow + } + return 0 + } + guard HTTPURLResponse.StatusCodes.isEventPayloadRefused(statusCode) + else { + badRequests = nil + return 0 + } + var inARow = 1 + if let previous = badRequests, previous.payloadId == batch.payloadId { + inARow = previous.inARow + 1 + } + badRequests = inARow >= DirectEventProcessor.badRequestsBeforeDropping ? nil : (batch.payloadId, inARow) + return inARow + } + } + + /// Ends the run of refusals of the payload, as an accepted post does. + private func endBadRequests() { + stateLock.withLock { badRequests = nil } + } + + /// - Returns: how many counted reads in a row of this batch have failed, this one included if it is at least + /// `readFailureSpacing` after the last counted one; once that reaches `readFailuresBeforeDropping` the count is + /// ended, since the batch is dropped. + private func countReadFailure(_ batch: EventBatch) -> Int { + let now = DispatchTime.now() + return stateLock.withLock { + if let last = readFailures[batch.payloadId], now < last.lastCounted + readFailureSpacingValue { + return last.counted + } + let counted = (readFailures[batch.payloadId]?.counted ?? 0) + 1 + readFailures[batch.payloadId] = counted >= DirectEventProcessor.readFailuresBeforeDropping + ? nil : (counted, now) + return counted + } + } + + /// Ends a batch's run of failed reads, at a read that worked. + private func endReadFailures(_ batch: EventBatch) { + stateLock.withLock { _ = readFailures.removeValue(forKey: batch.payloadId) } + } + + /// Sets a delivery for when the backoff's wait ends, unless one is set already, so a batch kept through a failure + /// goes out then rather than at the next report interval after it. + /// + /// A delivery that fails again sets the next one, each a longer wait away, which is what keeps this from posting + /// in a loop. Going offline leaves the one set to find nothing to do. + private func scheduleDeliveryWhenWaitEnds() { + let wait = retryState.remainingWait + guard wait > 0 + else { return } + guard deliveryScheduler.claimRetry() + else { return } + + os_log("%s Will retry posting events in %.1fs after a failure", log: service.config.logger, type: .debug, typeName(and: #function), wait) + DispatchQueue.global().asyncAfter(deadline: .now() + wait) { [weak self] in + guard let self + else { return } + self.deliveryScheduler.retryDue() + self.queueScheduledDelivery() + } + } + + /// Sends one batch and acts on what LaunchDarkly made of it, before handing the outcome on. + private func publish(_ batch: EventBatch, + _ payload: BatchPayload, + _ run: EventDeliveryRun, + _ completion: @escaping (AnalyticsEventSender.Outcome) -> Void) { + store.startedSending(batch) + // The store is settled before observers are told, so that one looking at it on hearing the outcome sees it. + sender.send(payload.body, payloadId: batch.payloadId, eventCount: payload.eventCount) { outcome in + run.postAnswered() + switch outcome { + case .accepted(let serverTime): + self.retryState.recordSuccess() + if let serverTime { + self.committer.recordResponseDate(serverTime) + } + self.store.remove(batch) + self.store.stoppedSending(batch) + self.reportSyncComplete(nil) + case .failed: + self.store.stoppedSending(batch) + // Kept whatever the failure, a 401 included: a valid key is sometimes refused for minutes, and deleting + // on a refusal would lose every event recorded meanwhile. A later delivery, in this run of the + // application or the next one, sends it under the same payload ID, so LaunchDarkly can tell it is not + // new. + os_log("%s Keeping %d event(s) to retry later", log: self.service.config.logger, type: .debug, self.typeName(and: #function), batch.eventCount) + } + completion(outcome) + } + } + + private func reportSyncComplete(_ result: SynchronizingError?) { + // The eventProcessor is created when the LDClient singleton is created, and kept for the app's lifetime. So while the use of self in the async block does setup a retain cycle, it's not going to cause a memory leak + guard let onSyncComplete = onSyncComplete + else { return } + DispatchQueue.main.async { + onSyncComplete(result) + } + } +} + +extension DirectEventProcessor: TypeIdentifying { } + +#if DEBUG +extension DirectEventProcessor { + /// Full events accepted but not yet handed to a commit. + var pendingEventsForTesting: [Event] { + committer.pendingEventsForTesting + } + + func setLastEventResponseDate(_ date: Date) { + committer.setLastEventResponseDate(date) + } + + /// Occupies the queue a delivery runs on, so a test can see what one that has not started yet collects. + func occupyQueue(until released: DispatchSemaphore) { + deliveryQueue.async { _ = released.wait(timeout: .now() + 10) } + } + + /// Callers waiting on the queued delivery. + var queuedDeliveryWaiterCount: Int { + deliveryScheduler.queuedWaiterCount + } + + /// Deliveries started, counting one for a run of flushes that joined each other. + var startedDeliveryCount: Int { + deliveryScheduler.startedCount + } + + /// As `EventCommitter.withContextSummarizer(_:)`. + func withContextSummarizer(_ read: (ContextSummarizer) -> T) -> T { + committer.withContextSummarizer(read) + } +} +#endif diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift new file mode 100644 index 000000000..0844aefb9 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/DiskEventLog.swift @@ -0,0 +1,280 @@ +import Foundation +import OSLog + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#elseif canImport(Musl) +import Musl +#endif + +/// The file of events on disk a store appends to, not to be confused with diagnostic logging. +/// +/// One per store rather than one per directory, so that only one writer ever appends to a given log. Each is named, by +/// `EventDirectory` as every file there is, for a UUID picked when it is opened, which is what keeps two writers apart: +/// nothing about it comes from the application, and no two processes, or two stores in one, ever pick the same one. +/// +/// A log nobody is appending to belongs to no one alive, whoever wrote it, and recovery takes it. The logs open in this +/// process are remembered, which is all it takes where only this process uses the directory. On a Mac, which runs a +/// second instance of an application over the same directory on request, the owner also holds its log locked for as +/// long as it is open, and the kernel releases the lock when the descriptor closes, including when the process dies. +/// Elsewhere a lock is not compiled in at all: iOS kills a suspended application that holds a lock on a file in a shared +/// container. +/// +/// A log is given its header under a name recovery does not look for, and only then moved to one it does, so that a log +/// being opened is never taken for one whose owner is gone. +/// +/// Belongs to an `EventStore`, which makes every call holding its `ioLock`. The lock is handed in so that each call can +/// check it is held, rather than this taking a lock of its own that would have to be ordered against the store's. +final class DiskEventLog { + /// The paths of the logs open in this process, which recovery and expiry leave alone. Guarded by + /// `openInThisProcessLock`. + private static var openInThisProcess: Set = [] + private static let openInThisProcessLock = UnfairLock() + + private let directory: URL + private let ioLock: UnfairLock + private let logger: OSLog + /// How bytes reach the log, injectable so a test can make a write fail partway through a session, as a full disk + /// does. Answers with the errno it stopped on, or nil where every byte landed. + private let writeLog: (EventLogFile, Data) -> Int32? + + /// Where the log is, whether or not it exists yet. + private(set) var url: URL + /// Nil until the first append opens it, and again once it is closed. + private var file: EventLogFile? + + #if os(macOS) || targetEnvironment(macCatalyst) + /// Whether the log is locked while it is open, so that another live instance can tell it is taken. Only to be + /// changed before the log is first opened. + var isLocked = true + #endif + + init(directory: URL, + ioLock: UnfairLock, + logger: OSLog, + writeLog: @escaping (EventLogFile, Data) -> Int32?) { + self.directory = directory + self.ioLock = ioLock + self.logger = logger + self.writeLog = writeLog + self.url = EventDirectory.newLogUrl(in: directory) + } + + /// Whether some live process is appending to a log, and recovery and expiry must leave it alone. + /// + /// A log open in this process is answered from memory. On a Mac any other is answered by its lock, and one that + /// cannot be looked at counts as held: leaving a dead process's log for a later run costs nothing, while taking a + /// live one's would lose what it appends next. + static func isHeldByLiveProcess(_ log: URL) -> Bool { + if isOpenInThisProcess(log) { + return true + } + #if os(macOS) || targetEnvironment(macCatalyst) + return EventLogFile.isLockedByLiveProcess(log) + #else + return false + #endif + } + + /// Requires `ioLock`. + var isOpen: Bool { + ioLock.assertOwned() + return file != nil + } + + /// Requires `ioLock`. + var exists: Bool { + ioLock.assertOwned() + return FileManager.default.fileExists(atPath: url.path) + } + + /// Appends to the log, opening it first where it is not open. False where it could not be opened or written to, in + /// which case none of `bytes` is left in the log. + /// + /// Requires `ioLock`. + func append(_ bytes: Data) -> Bool { + ioLock.assertOwned() + guard let file = fileForAppending() + else { return false } + + let start = file.size + guard write(bytes, to: file) + else { + // A write that fails partway can leave whole frames behind. The caller keeps every one of `bytes` in memory + // to deliver from there, so frames left here as well would be sent twice. Where they cannot be cut back, the + // batch this log is closed into still delivers only the events it was closed holding. + if let start, !file.truncate(to: start) { + os_log("%s could not cut back a failed write: errno %d", log: logger, type: .error, typeName(and: #function), errno) + // A log with no events before this write is never closed into a batch, and a later run recovering it + // would send what landed under no count that leaves it out. Memory holds all of it. + if start <= EventLogFormat.fileHeader.count { + discardFailedFirstWrite() + } + } + return false + } + return true + } + + /// Moves the log to `destination` and closes it, so that the next append starts a new one. + /// + /// A move that fails leaves the log open, held and this store's, so that the next append continues it and the next + /// close moves all of it. Closed and forgotten instead, it would be a log recovery may take at any moment, and the + /// next append would start another beside it: one tally of committed events for two files. + /// + /// Requires `ioLock`. + func close(movingTo destination: URL) throws { + ioLock.assertOwned() + // Closed after the move rather than before it, so that the log is still held while it moves. Another store + // recovering logs could otherwise take it in between. + try FileManager.default.moveItem(at: url, to: destination) + closeDescriptor() + } + + /// Closes the descriptor, leaving the log where it is. Requires `ioLock`. + /// + /// No longer held, it is from here on a log like one a process that died left behind, which recovery may take at + /// any moment. So this log is never opened again: the next append starts a new one. + func closeDescriptor() { + ioLock.assertOwned() + guard let file + else { return } + file.close() + self.file = nil + DiskEventLog.forget(url) + } + + /// Claims a log whose writer is gone, for recovery to read and then move or remove; nil where a live process holds + /// it. Needs no lock, so that recovery can read the log without holding `ioLock`. + /// + /// On a Mac the claim is the log's lock, held until `release()`, so that another instance recovering at the same + /// time leaves it be. Elsewhere every log not open in this process is its writer's no longer. + static func claimForRecovery(_ log: URL) -> RecoveryClaim? { + #if os(macOS) || targetEnvironment(macCatalyst) + guard let held = EventLogFile.openLocked(log, flags: O_RDONLY) + else { return nil } + return RecoveryClaim(url: log, held: held) + #else + return RecoveryClaim(url: log) + #endif + } + + /// A log recovery has claimed; see `claimForRecovery(_:)`. + struct RecoveryClaim { + let url: URL + #if os(macOS) || targetEnvironment(macCatalyst) + fileprivate let held: EventLogFile + #endif + + /// To be called once the log has been moved or removed, or left for a later run. + func release() { + #if os(macOS) || targetEnvironment(macCatalyst) + held.close() + #endif + } + } + + /// Requires `ioLock`. + private func fileForAppending() -> EventLogFile? { + ioLock.assertOwned() + if let file { + return file + } + + do { + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + } catch { + os_log("%s could not create the event directory: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) + return nil + } + EventLogFile.excludeFromBackup(directory) + + if FileManager.default.fileExists(atPath: url.path) { + // Left by an earlier open this session and closed since, so no longer this store's to write. + url = EventDirectory.newLogUrl(in: directory) + } + guard let opened = open() + else { return nil } + file = opened + return opened + } + + private func write(_ bytes: Data, to file: EventLogFile) -> Bool { + guard let failure = writeLog(file, bytes) + else { return true } + os_log("%s could not write to the event log: errno %d", log: logger, type: .debug, typeName(and: #function), failure) + return false + } + + /// Opens a new log: created under `EventDirectory.openingLogPrefix`, locked where logs are, given its header, and + /// only then moved to the name recovery looks for. Requires `ioLock`. + private func open() -> EventLogFile? { + ioLock.assertOwned() + let opening = EventDirectory.openingUrl(for: url) + // O_APPEND is what makes each write land at the end of the file as one step, so that a commit racing a reader + // cannot produce a spliced frame. + let flags = O_WRONLY | O_APPEND | O_CREAT | O_EXCL + DiskEventLog.remember(url) + guard let opened = openFile(at: opening, flags: flags) + else { + os_log("%s could not open the event log: errno %d", log: logger, type: .debug, typeName(and: #function), errno) + DiskEventLog.forget(url) + return nil + } + + guard write(EventLogFormat.fileHeader, to: opened) + else { return abandon(opened, at: opening) } + do { + try FileManager.default.moveItem(at: opening, to: url) + } catch { + os_log("%s could not name the event log: %s", log: logger, type: .debug, typeName(and: #function), String(describing: error)) + return abandon(opened, at: opening) + } + return opened + } + + /// Closes and removes a log whose only events are those of a first write that failed. Requires `ioLock`. + private func discardFailedFirstWrite() { + do { + try FileManager.default.removeItem(at: url) + } catch { + os_log("%s could not remove a log holding only a failed write: %s", log: logger, type: .error, typeName(and: #function), String(describing: error)) + } + closeDescriptor() + } + + /// Closes and removes a log that never got the name recovery looks for. Requires `ioLock`. + private func abandon(_ opened: EventLogFile, at opening: URL) -> EventLogFile? { + opened.close() + try? FileManager.default.removeItem(at: opening) + DiskEventLog.forget(url) + return nil + } + + /// Opens a file for a new log, locked where logs are. Requires `ioLock`. + private func openFile(at url: URL, flags: Int32) -> EventLogFile? { + #if os(macOS) || targetEnvironment(macCatalyst) + if isLocked { + return EventLogFile.openLocked(url, flags: flags) + } + #endif + return EventLogFile.open(at: url, flags: flags) + } + + /// Whether a store in this process has `log` open, which recovery and expiry leave alone. + static func isOpenInThisProcess(_ log: URL) -> Bool { + openInThisProcessLock.withLock { openInThisProcess.contains(log.standardizedFileURL.path) } + } + + private static func remember(_ log: URL) { + openInThisProcessLock.withLock { _ = openInThisProcess.insert(log.standardizedFileURL.path) } + } + + private static func forget(_ log: URL) { + openInThisProcessLock.withLock { _ = openInThisProcess.remove(log.standardizedFileURL.path) } + } +} + +extension DiskEventLog: TypeIdentifying { } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EnvironmentReporting/SystemCapabilities.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EnvironmentReporting/SystemCapabilities.swift index 61a4e97c5..70317b5d3 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EnvironmentReporting/SystemCapabilities.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EnvironmentReporting/SystemCapabilities.swift @@ -8,26 +8,45 @@ import AppKit import WatchKit #endif +/// What differs between the platforms the SDK runs on, in one place, so that the code using it carries no platform +/// conditions of its own. class SystemCapabilities { #if os(iOS) static var backgroundNotification: Notification.Name? { UIApplication.didEnterBackgroundNotification } static var foregroundNotification: Notification.Name? { UIApplication.willEnterForegroundNotification } + static var suspendsWhenBackgrounded: Bool { true } + static var expiringActivity: BackgroundActivity.ExpiringActivity? { performExpiringActivity } static var systemName: String { UIDevice.current.systemName } static var operatingSystem: OperatingSystem { .iOS } #elseif os(watchOS) + /// None the SDK observes, so a watchOS client neither changes run mode nor does anything with its events on its way + /// to the background, although the platform suspends it and has the assertions to deliver under. static var backgroundNotification: Notification.Name? { nil } static var foregroundNotification: Notification.Name? { nil } + static var suspendsWhenBackgrounded: Bool { true } + static var expiringActivity: BackgroundActivity.ExpiringActivity? { performExpiringActivity } static var systemName: String { WKInterfaceDevice.current().systemName } static var operatingSystem: OperatingSystem { .watchOS } #elseif os(OSX) + /// Posted on every loss of focus, which suspends nothing. static var backgroundNotification: Notification.Name? { NSApplication.willResignActiveNotification } static var foregroundNotification: Notification.Name? { NSApplication.didBecomeActiveNotification } + static var suspendsWhenBackgrounded: Bool { false } + static var expiringActivity: BackgroundActivity.ExpiringActivity? { nil } static var systemName: String { "macOS" } static var operatingSystem: OperatingSystem { .macOS } #elseif os(tvOS) static var backgroundNotification: Notification.Name? { UIApplication.didEnterBackgroundNotification } static var foregroundNotification: Notification.Name? { UIApplication.willEnterForegroundNotification } + static var suspendsWhenBackgrounded: Bool { true } + static var expiringActivity: BackgroundActivity.ExpiringActivity? { performExpiringActivity } static var systemName: String { UIDevice.current.systemName } static var operatingSystem: OperatingSystem { .tvOS } #endif + + #if os(iOS) || os(tvOS) || os(watchOS) + private static func performExpiringActivity(_ reason: String, _ whileGranted: @escaping (Bool) -> Void) { + ProcessInfo.processInfo.performExpiringActivity(withReason: reason, using: whileGranted) + } + #endif } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift new file mode 100644 index 000000000..2c9b1e7e5 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventCommitter.swift @@ -0,0 +1,498 @@ +import Foundation +import OSLog + +/// How many events the SDK accepted and then lost for good, which `DirectEventProcessor` and its `EventCommitter` both +/// add to. +/// +/// A batch LaunchDarkly refuses is kept and retried, so it is not a loss, unless the refusal is a second refusal of the +/// payload in a row. Otherwise what is lost is what the store never received: an event or a summary that could not be +/// serialized when it was committed, or that serialized too large to store. A delivery that finds nothing to send cannot +/// tell those apart from events that arrived, so without this it would report success for them. +/// +/// A count rather than a flag. Each delivery takes the count as it closes its batch, and answers for the losses up to that +/// mark, so a loss is told to the caller whose delivery it happened before rather than to whichever delivery answers next. +final class LostEventCount { + private let lock = UnfairLock() + /// Only to be used while holding `lock`. + private var count: UInt64 = 0 + + var value: UInt64 { lock.withLock { count } } + + func add(_ events: Int) { + lock.withLock { count += UInt64(events) } + } +} + +/// Where `DirectEventProcessor` accepts events, holds them and commits them to the store. +/// +/// Recording an event summarizes it and, if it has to be delivered in full, holds it as an `Event`. Nothing is encoded +/// on the thread that recorded it; a commit turns the whole held run into bytes at once. So an evaluation of an +/// untracked flag costs a counter update, and an evaluation of a tracked one costs that plus an array append. +/// +/// Where the line falls is between the events an application asked for by name and the ones it did not. At `.immediate` +/// recording a custom or identify event is a *commit point*, because an application reporting something it chose to +/// report is saying this matters more than the microseconds it costs, and the crash it describes may be moments away. A +/// commit takes the whole held run with it, so the exposures leading up to the error go down alongside it. Everything +/// else is committed once `pendingCommitThreshold` events have accumulated, when a delivery starts, on entering the +/// background, or on `flush`. +/// +/// Which of the two `track` gets is the application's choice, through `LDConfig.eventPersistence`. Only at `.immediate` +/// can `track` promise the event is on disk by the time it returns; at `.deferred` it costs what an evaluation does. At +/// `.disabled` nothing commits early: the held run waits for the delivery, which encodes it. +/// +/// Threads. Recording runs on whichever thread evaluated or tracked. Commits run on `commitQueue`, or on the caller's +/// thread at a commit point, one at a time under `commitLock`. A delivery commits too, before it closes a batch. +final class EventCommitter { + /// How many full events may be held unencoded before one of them pays for a commit. + /// + /// It sets two things at once. The first is how many evaluations a termination can take -- not everything a crash + /// could take, since an application that flushes on its way out commits the whole run, but the window for the + /// terminations that run nothing on the way out, such as the system reclaiming a backgrounded process. Recording a + /// custom or identify event at `.immediate` closes it too, because that commits. + /// + /// The second is what a commit point usually costs, since it encodes whatever is held before returning. The commit + /// queue keeps the run drained, so the common case is far below this; raising it trades that cost against the + /// number of writes. + /// + /// Neither is a ceiling. Reaching this only schedules a commit, and recording does not wait for it, so where events + /// arrive faster than the commit queue encodes them, or the queue is starved, the run grows up to the event + /// capacity, and the next commit point encodes all of it. + /// + /// It is the only thing that writes events no caller asked to have written. The store writes only when told to, so + /// every staged byte reaches the disk through a commit that started here, at a commit point, or at a delivery. + static let pendingCommitThreshold = 32 + + // MARK: Set in init and never changed + + private let store: EventStoring + private let persistence: EventPersistence + private let logger: OSLog + private let diagnosticCache: DiagnosticCaching? + private let lostEvents: LostEventCount + + /// How many events the SDK will hold in total, across `pending` and the store. + private let eventCapacity: Int + + /// Where a commit runs when no caller is waiting on it. + private let commitQueue: DispatchQueue + + /// Its summarizer is guarded by `recordLock`, and its encoders are reached only under `commitLock`. + private let eventBuffer: OutboundEventBuffer + + // MARK: Guarded by commitLock + + /// Held for the whole of a commit, so that only one runs at a time. + /// + /// This is what a commit point's guarantee rests on. Without it a commit already in flight could take the caller's + /// event out of `pending` before the caller got there, leaving the caller nothing to write and returning while + /// those bytes were still being produced somewhere else. Waiting here instead means that when the call returns the + /// event is on disk, whichever commit put it there. It also keeps two encoders from staging their runs in + /// whichever order they happened to finish. + /// + /// Taken before `recordLock` and before anything the store locks, never after. + private let commitLock = UnfairLock() + + /// The last recording a finished commit covered. Only to be used while holding `commitLock`. + /// + /// This is what makes concurrent `.immediate` commit points a group commit. The first caller's commit takes every + /// recording made before it took the run, including those of callers now waiting behind it. They find their + /// recordings covered and return, and the next caller still uncovered commits everything that arrived in the + /// meantime. Without it each waiter committed again in turn. + private var committedSequence: UInt64 = 0 + + // MARK: Guarded by recordLock + + /// Guards everything one recording writes: the held events and the summarizer. + /// + /// One evaluation can produce a counter, a full event and a debug event, and the three have to land together. Taken + /// separately, a commit landing between them splits one evaluation across two payloads. + /// + /// Held for the appends and the handover of the run, never across the encoder or a syscall. It is a lock rather + /// than a queue because it is taken once per evaluation, and a `DispatchQueue.sync` costs enough at that rate to be + /// worth avoiding: the same measurement that made `EvaluationExposureDeduper` a lock applies here. + private let recordLock = UnfairLock() + + /// Full events recorded but not yet encoded. Only to be used while holding `recordLock`. + /// + /// Held rather than encoded because encoding early would not make them durable: the store stages bytes into memory + /// too, and only a commit reaches the file. Both forms are equally lost to a crash, so the encode may as well + /// happen where it is cheapest. + private var pending: [Event] = [] + + /// Numbers every recording, so a caller at a commit point can tell whether a commit has already covered it. Only to + /// be used while holding `recordLock`. + private var recordedSequence: UInt64 = 0 + + /// Whether a commit is already queued, so a run of recordings past the threshold asks for one write rather than + /// one each. Only to be used while holding `recordLock`. + private var isCommitScheduled = false + + // MARK: Guarded by responseDateLock + + /// When LaunchDarkly last answered a post, by its own clock, which is how long debug events go on for. + var lastEventResponseDate: Date { + responseDateLock.withLock { responseDate } + } + + /// Its own lock rather than `recordLock`, since an evaluation reads it before it takes that one. + private let responseDateLock = UnfairLock() + /// Only to be used while holding `responseDateLock`. + private var responseDate = Date() + + init(config: LDConfig, + store: EventStoring, + encoding: OutboundEventBuffer.Encoding, + commitQueue: DispatchQueue, + diagnosticCache: DiagnosticCaching?, + lostEvents: LostEventCount) { + self.store = store + self.persistence = config.eventPersistence + self.logger = config.logger + self.diagnosticCache = diagnosticCache + self.lostEvents = lostEvents + self.eventCapacity = config.eventCapacity + self.commitQueue = commitQueue + self.eventBuffer = OutboundEventBuffer(config: config, encoding: encoding) + } + + // MARK: Recording + + /// Records an event, and at `.immediate` commits before returning if it is a commit point. + /// + /// At `.deferred` a custom or identify event writes nothing itself (durability §13.3), and is committed with what + /// is held at the pending threshold, a flush, a delivery or entering the background. A commit per call would cost + /// an application that tracks every few seconds one encode and write per event, each taking `recordLock` from the + /// threads evaluating flags and the store's locks from delivery. + func record(_ event: Event) { + var sequence: UInt64 = 0 + let (dropped, needsCommit) = recordLock.withLock { () -> (Bool, Bool) in + recordedSequence += 1 + sequence = recordedSequence + return (!admitToPending(event), pending.count >= EventCommitter.pendingCommitThreshold) + } + + if dropped { + reportDropped() + } + if persistence == .immediate && EventCommitter.isCommitPoint(event.kind) { + commitThrough(sequence) + } else if needsCommit { + scheduleCommitWherePersisting() + } + } + + // swiftlint:disable:next function_parameter_count + func recordFlagEvaluationEvents(flagKey: LDFlagKey, + value: LDValue, + defaultValue: LDValue, + featureFlag: FeatureFlag?, + context: LDContext, + includeReason: Bool) { + let recordingFeatureEvent = featureFlag?.trackEvents == true + let recordingDebugEvent = featureFlag?.shouldCreateDebugEvents(lastEventReportResponseTime: lastEventResponseDate) ?? false + // Built before the lock is taken, so that the critical section is only the writes. + let featureEvent = recordingFeatureEvent ? FeatureEvent(key: flagKey, context: context, value: value, defaultValue: defaultValue, featureFlag: featureFlag, includeReason: includeReason, isDebug: false) : nil + let debugEvent = recordingDebugEvent ? FeatureEvent(key: flagKey, context: context, value: value, defaultValue: defaultValue, featureFlag: featureFlag, includeReason: includeReason, isDebug: true) : nil + + var dropped = 0 + let (counted, needsCommit) = recordLock.withLock { () -> (Bool, Bool) in + recordedSequence += 1 + let counted = eventBuffer.summarize(flagKey: flagKey, reportedValue: value, featureFlag: featureFlag, defaultValue: defaultValue, context: context) + if let featureEvent, !admitToPending(featureEvent) { dropped += 1 } + if let debugEvent, !admitToPending(debugEvent) { dropped += 1 } + return (counted, pending.count >= EventCommitter.pendingCommitThreshold) + } + + // A refused evaluation is a loss like a refused event, since nothing later reconstructs its counter. Its full + // event is still decided by the event capacity alone. + if !counted { + diagnosticCache?.incrementDroppedEventCount() + } + for _ in 0.. responseDate { + responseDate = serverTime + } + } + } + + /// Adds an event to `pending` for the next commit to encode, returning false if the SDK is already full. Requires + /// `recordLock`. + /// + /// Capacity is consulted before anything else, so an event that will not be kept is never encoded. That ordering is + /// what bounds an application re-evaluating a tracked flag in a render loop: once the limit is reached an + /// evaluation costs no more than its summary counter, however fast the loop runs. + /// + /// The byte capacity can only be consulted as far as whether any room is left, since the event has no size until + /// it is encoded; whether it fits is decided when it is staged. + private func admitToPending(_ event: Event) -> Bool { + recordLock.assertOwned() + guard store.admitsEvent(heldElsewhere: pending.count, capacity: eventCapacity) + else { return false } + + pending.append(event) + return true + } + + private func reportDropped() { + os_log("%s aborted. Event store is full", log: logger, type: .debug, typeName(and: #function)) + diagnosticCache?.incrementDroppedEventCount() + } + + /// Records the loss of an event the SDK accepted and then could not keep, which no later attempt recovers. + private func reportLost() { + os_log("%s dropping an event the store would not take", log: logger, type: .error, typeName(and: #function)) + diagnosticCache?.incrementDroppedEventCount() + lostEvents.add(1) + } + + /// Counts as lost the events of an open log a close found gone. + private func reportVanished(_ events: Int) { + guard events > 0 + else { return } + os_log("%s %d stored event(s) were deleted before they could be sent", log: logger, type: .error, typeName(and: #function), events) + lostEvents.add(events) + } + + // MARK: Committing + + /// Queues a commit where there is a disk for it to reach. + /// + /// Without persistence a commit makes nothing durable. All it would do is move the encode into the middle of the + /// application's evaluations, where it competes with them for the CPU and splits their counters across summaries; + /// left alone, the run waits for the delivery, which encodes it anyway, and capacity still bounds how much is held. + private func scheduleCommitWherePersisting() { + if store.isPersisting { + scheduleCommit() + } + } + + /// Queues a commit unless one is already queued, so a run of recordings asks for one write rather than one each. + private func scheduleCommit() { + let alreadyScheduled = recordLock.withLock { () -> Bool in + defer { isCommitScheduled = true } + return isCommitScheduled + } + guard !alreadyScheduled + else { return } + + commitQueue.async { [weak self] in + guard let self + else { return } + // Cleared before the commit rather than after it, so that an event held while this one is in flight can ask + // for another commit instead of finding one apparently already on its way. + self.recordLock.withLock { self.isCommitScheduled = false } + + self.commitRecordedEvents() + } + } + + /// Writes on the caller's thread at `.deferred` as well as at `.immediate`, unlike `track` and `identify`. + /// + /// A flush is what an application calls when it is about to end the process on purpose, and entering the background + /// is when the system is most likely to end it; a write only queued is one the process may end before. Deferring is + /// for the recordings an application makes all the time, which neither is. Where nothing persists, a commit makes + /// nothing durable, and is left to the delivery. + func commitWherePersisting() { + if persistence == .immediate || store.isPersisting { + commitThrough(recordLock.withLock { recordedSequence }) + } + } + + /// Commits everything recorded so far for `flushAtClose()`, waiting for it no longer than `budget`. + /// + /// On `commitQueue` rather than the caller's thread, which for an application closing the client is usually the + /// main one. Each step of a commit can block for as long as the device takes: on `commitLock` behind a commit + /// stalled on the disk, on the store's I/O lock, and on the write itself. A commit that outlasts the budget is left + /// to finish, and the delivery queued after it waits for it on the delivery queue, not on the caller's thread. + /// Where nothing persists there is nothing to write: the final delivery encodes what it sends. + func commitWithinBudget(_ budget: TimeInterval) { + guard store.isPersisting + else { return } + let through = recordLock.withLock { recordedSequence } + let committed = DispatchSemaphore(value: 0) + commitQueue.async { + self.commitThrough(through) + committed.signal() + } + if committed.wait(timeout: .now() + budget) == .timedOut { + os_log("%s gave up waiting for the final event commit after %dms; it continues in the background", log: logger, type: .info, typeName(and: #function), Int(budget * 1000)) + } + } + + /// `commitWherePersisting()` queued on `commitQueue` rather than written on the caller's thread, for a flush whose + /// caller has a deadline. + /// + /// `flushAndWait` can only hold to its timeout once every environment has handed back, and each step of a commit + /// can block for as long as the device takes: on `commitLock` behind a commit stalled on the disk, on the store's + /// I/O lock, and on the write itself. Queued rather than left to the delivery, because the delivery queue can be + /// held by a request and its retry long after the caller stopped waiting, while `commitQueue` never waits on the + /// network. The answer still covers the write: the delivery commits again before it sends, so `true` means the + /// events were written first. A commit still running when the caller stops waiting is left to finish. + func queueCommitWherePersisting() { + if persistence == .immediate || store.isPersisting { + scheduleCommit() + } + } + + /// Encodes everything recorded since the last commit, stages it, and commits it. + /// + /// The run and the counters are taken in one critical section, so that an evaluation's counter and its full event + /// are staged by the same commit, and encoded outside it, so recording does not wait on the encoder. + func commitRecordedEvents() { + commitLock.withLock { encodeAndCommitPending() } + } + + /// Commits unless a finished commit already covers recording `sequence`, checked once the lock is held, since the + /// commit a caller waited behind is the one most likely to have covered it. + private func commitThrough(_ sequence: UInt64) { + commitLock.withLock { + guard committedSequence < sequence + else { return } + encodeAndCommitPending() + } + } + + /// Commits what is held and closes it into a batch, as one step with respect to other commits. + /// + /// Closing takes everything staged so far, and a commit stages its run one event at a time, so a commit running + /// between the two would be closed partway: an evaluation's event in this batch and its summary in the next. + /// + /// - Returns: the loss count as of the close, which is the mark this delivery answers for, and whether the close + /// took every committed event into a batch. Both taken under the same lock, so they cover the closed batch and + /// nothing a later commit does. + func commitAndCloseBatch() -> (lossMark: UInt64, closedEverything: Bool) { + commitLock.withLock { + encodeAndCommitPending() + _ = store.closeBatch() + reportVanished(store.takeVanishedLogEvents()) + return (lostEvents.value, !store.holdsUnclosedEvents) + } + } + + /// `commitAndCloseBatch()` for a pass that found an earlier batch blocked, and so closes nothing. + /// + /// - Returns: the loss count as of the commit. + func commitLeavingTheLogOpen() -> UInt64 { + commitLock.withLock { + encodeAndCommitPending() + // A commit that fills the log closes it, and can find it gone. + reportVanished(store.takeVanishedLogEvents()) + return lostEvents.value + } + } + + /// Requires `commitLock`. + private func encodeAndCommitPending() { + commitLock.assertOwned() + var through: UInt64 = 0 + let (run, summaries) = recordLock.withLock { () -> ([Event], [(tracker: FlagRequestTracker, context: LDContext)]) in + let run = pending + pending = [] + let summaries = eventBuffer.takeSummaries() + // Reserved in the same critical section the run leaves `pending` in, so capacity counts it in one place or + // the other throughout the encode. + store.reserve(run.count) + through = recordedSequence + return (run, summaries) + } + + stage(run) + store.releaseReservations() + stageSummaries(summaries) + store.commit() + committedSequence = through + } + + /// Encodes the held events as one run and stages the bytes. + /// + /// Staging uses the reservation taken with the run rather than checking capacity, because the decision to keep + /// these events was made when they were held, and refusing them here would drop events the SDK has already counted + /// as accepted. + /// + /// Requires `commitLock`: two threads draining separate runs would stage them in whichever order they finished + /// encoding, which is not the order they were recorded in. + private func stage(_ run: [Event]) { + commitLock.assertOwned() + for event in run { + guard let encoded = eventBuffer.encode(event) + else { + reportLost() + continue + } + // The event capacity is bypassed here, so what is left is an event too large for a frame or for the bytes + // left in the store, which were not known until now. Either way it was accepted and is not kept. + if !store.stageReserved(encoded) { + reportLost() + } + } + } + + /// Whether recording this kind of event at `.immediate` should encode and write the held run before it returns. + /// + /// Custom and identify events are recorded because the application did something it chose to report, which is both + /// rare enough to afford a write and the moment it is least acceptable to lose. Evaluations are neither. + private static func isCommitPoint(_ kind: Event.Kind) -> Bool { + switch kind { + case .custom, .identify: return true + case .feature, .debug, .summary: return false + } + } + + /// Turns the counters taken with the run into summary events and stages them. + /// + /// Counters live only in memory, so a crash takes whatever has not been summarized with it. Summarizing at each + /// commit point rather than only at a delivery is what bounds that loss, and it costs nothing in accuracy: + /// LaunchDarkly sums the counters of every summary it receives, so a session that produced several summaries is + /// counted the same as one that produced a single summary covering the same evaluations. + private func stageSummaries(_ summaries: [(tracker: FlagRequestTracker, context: LDContext)]) { + for summary in summaries { + guard let encoded = eventBuffer.encodeSummary(summary) + else { + reportLost() + continue + } + // Summaries are an aggregate of evaluations that were already counted, so refusing one for capacity would + // lose evaluations the SDK promised to report rather than shed new load. Only the byte capacity refuses + // one, once even the room held back for summaries is used up, since nothing else bounds how many are + // staged while delivery keeps failing; and the frame size, for one too large for a frame. + if !store.stage(encoded, bypassingCapacity: true) { + reportLost() + } + } + } +} + +extension EventCommitter: TypeIdentifying { } + +#if DEBUG +extension EventCommitter { + /// Full events accepted but not yet handed to a commit. + var pendingEventsForTesting: [Event] { + recordLock.withLock { pending } + } + + func setLastEventResponseDate(_ date: Date) { + responseDateLock.withLock { responseDate = date } + } + + /// Reads the summarizer under `recordLock`, which a scheduled commit may be holding to take its counters. + func withContextSummarizer(_ read: (ContextSummarizer) -> T) -> T { + recordLock.withLock { read(eventBuffer.contextSummarizer) } + } +} +#endif diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift new file mode 100644 index 000000000..365c55ac4 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDeliveryScheduler.swift @@ -0,0 +1,266 @@ +import Foundation + +/// One delivery, from taking its batches until it has an answer for everything it sent, its retry included. +/// +/// Made by `EventDeliveryScheduler.begin()` and passed through the delivery, so that what is true of the delivery running is +/// held by it, and starts afresh with each one. +final class EventDeliveryRun { + /// A paced batch waiting out its gap. + private final class PacedTurn { + let proceed: () -> Void + + init(_ proceed: @escaping () -> Void) { + self.proceed = proceed + } + } + + /// Whether it is `flushAtClose()`'s, which runs although closing has turned `isOnline` off since. The delivery one of its failures sets for the end of the wait is another one, held + /// by the backoff, and finds the reporter offline as any other. + let isClosing: Bool + /// Whether it posts whatever wait the backoff is in: `flushAtClose()`'s, and `flushBypassingBackoff(completion:)`'s. + let bypassesBackoff: Bool + + /// The least time between one paced batch's answer and the next batch's request. + private let gap: TimeInterval + + /// Guards the pacing, which a delivery somebody asks for, or going offline, ends from another thread, and the + /// batches passed by, which the delivery notes from whichever thread it goes on from. + private let lock = UnfairLock() + /// Whether it is paced: until a delivery somebody asked for is queued, after which it sends the rest back to back, + /// or the reporter goes offline, after which it sends nothing more. Only to be used while holding `lock`. + private var isPaced: Bool + /// When it last had an answer for a post, or nil before its first. Only to be used while holding `lock`. + private var lastPostAnswered: DispatchTime? + /// The batch waiting out its gap, which ending the pacing lets go at once. Only to be used while holding `lock`. + private var waitingTurn: PacedTurn? + /// The batches whose read failed in this delivery, which it passes by rather than reading again. Only to be used + /// while holding `lock`. + private var passedBy: Set = [] + + init(isClosing: Bool, bypassesBackoff: Bool, isPaced: Bool, gap: TimeInterval) { + self.isClosing = isClosing + self.bypassesBackoff = bypassesBackoff + self.isPaced = isPaced + self.gap = gap + } + + /// Notes that a post had its answer, which the gap before the next one counts from. + func postAnswered() { + lock.withLock { + if isPaced { + lastPostAnswered = .now() + } + } + } + + /// Notes that a batch's read failed, so this delivery passes it by from here on: read again moments later, it would + /// most likely fail as it did, and count as two failures in a row. + func passBy(_ payloadId: String) { + lock.withLock { _ = passedBy.insert(payloadId) } + } + + /// Whether this delivery passed a batch by after its read failed. + func passedBy(_ payloadId: String) -> Bool { + lock.withLock { passedBy.contains(payloadId) } + } + + /// Whether this delivery passed any batch by after its read failed, and so left one unsent. + var passedAnyBy: Bool { + lock.withLock { !passedBy.isEmpty } + } + + /// Ends the pacing, so that the rest go back to back, or, offline, stop. + func endPacing() { + let turn = lock.withLock { () -> PacedTurn? in + defer { + isPaced = false + waitingTurn = nil + } + return waitingTurn + } + if let turn { + DispatchQueue.global().async(execute: turn.proceed) + } + } + + /// Holds the next batch until the gap has passed since the last one was answered, where the delivery is paced and + /// has posted before, and then calls `proceed` on a global queue; or sooner, once the pacing ends. + /// + /// - Returns: whether it holds the batch; false where the batch may go now, and `proceed` is not called. + func awaitTurn(then proceed: @escaping () -> Void) -> Bool { + let turn = PacedTurn(proceed) + // Under the same lock that ends the pacing, so that the pacing cannot end between the check and the wait. + guard let gapEnds = lock.withLock({ () -> DispatchTime? in + guard isPaced, let answered = lastPostAnswered + else { return nil } + waitingTurn = turn + return answered + gap + }) + else { return false } + DispatchQueue.global().asyncAfter(deadline: gapEnds) { + let taken = self.lock.withLock { () -> Bool in + guard self.waitingTurn === turn + else { return false } + self.waitingTurn = nil + return true + } + if taken { + turn.proceed() + } + } + return true + } +} + +/// Decides which delivery runs next, one at a time: queues one, joins a caller to one that is queued and has not +/// started, starts the queued one once the one running has its answer, and sets one for when the backoff's wait ends. +/// +/// Its lock is taken on a caller's thread, on the delivery queue and where a delivery's posts are answered, and never +/// held across any of them. +final class EventDeliveryScheduler { + /// A delivery queued that has not taken its batches yet. + private struct Queued { + /// The callers waiting on it. + var waiters: [FlushOutcomeClosure] + /// Whether it is `flushAtClose()`'s, or has been joined by it. + var isClosing: Bool + /// Whether a caller bypassing the backoff asked for it, or joined it. + var bypassesBackoff: Bool + /// Whether it was asked for only by the report timer, the launch gate or the end of a backoff's wait, and so is + /// paced. + var isPaced: Bool + } + + /// Whether deliveries nobody asked for are paced: only where events give way to flag values. + private let pacesDeliveries: Bool + private let gap: TimeInterval + + private let lock = UnfairLock() + /// Only to be used while holding `lock`. + private var queued: Queued? + /// Whether a delivery has started, or is about to, and not yet had an answer for everything it sent, its retry + /// included. Only to be used while holding `lock`. + /// + /// A delivery returns from the delivery queue as soon as its first request is out, so the queue alone does not keep + /// two apart. Two at once would each leave the batch it is sending out of both capacities, and a run of flushes, + /// each held up by a slow response, could take the store that many batches past them. + private var isDelivering = false + /// The delivery running, whose pacing a delivery somebody asks for ends. Only to be used while holding `lock`. + private var running: EventDeliveryRun? + /// Whether a delivery is already set to run when the backoff's wait ends, so a run of passes inside the wait sets + /// one rather than one each. Only to be used while holding `lock`. + private var isRetryScheduled = false + + /// How many deliveries have started, which is fewer than the flushes asked for where they were joined. Only to be + /// used while holding `lock`. + private var deliveryRunCount = 0 + + init(pacesDeliveries: Bool, gap: TimeInterval) { + self.pacesDeliveries = pacesDeliveries + self.gap = gap + } + + /// Queues a delivery, or joins one that is queued and has not started. + /// + /// A delivery that has not started yet will send everything recorded before it starts, which includes this + /// caller's events, so joining it is as good as queueing another. Without this, flushes arriving faster than a + /// post completes each queue their own, and the one that matters -- the flush at shutdown -- waits behind all of + /// them. + /// + /// A delivery that has already started is not joined. It has closed and taken its batches, so it cannot speak for + /// anything recorded since. Nor does another start beside it: the one queued waits until the running one has an + /// answer, then sends what that one left behind as well as what was recorded since, so its caller hears `true` + /// only if both got out. + /// + /// - Parameters: + /// - closing: whether it is `flushAtClose()`'s, which a queued delivery it joins becomes. + /// - bypassingBackoff: whether it posts whatever wait the backoff is in, which a queued delivery it joins does too. + /// - paced: whether nobody asked for it, so that it gives way to flag requests. One that is not ends the pacing of + /// the delivery running, so that it is not held behind it. + /// - Returns: whether the caller is to start it, which is where no delivery is running. + func queue(_ completion: FlushOutcomeClosure?, closing: Bool, bypassingBackoff: Bool, paced: Bool) -> Bool { + let (starts, unpaced) = lock.withLock { () -> (Bool, EventDeliveryRun?) in + let unpaced = paced ? nil : running + if var joined = queued { + if let completion { + joined.waiters.append(completion) + } + joined.isClosing = joined.isClosing || closing + joined.bypassesBackoff = joined.bypassesBackoff || bypassingBackoff + joined.isPaced = joined.isPaced && paced + queued = joined + return (false, unpaced) + } + queued = Queued(waiters: completion.map { [$0] } ?? [], isClosing: closing, bypassesBackoff: bypassingBackoff, + isPaced: paced) + let starts = !isDelivering + isDelivering = true + return (starts, unpaced) + } + unpaced?.endPacing() + return starts + } + + /// Starts the delivery queued. Requests arriving from here on need a delivery of their own: this one is about to + /// close and take its batches, and what it takes is all it can speak for. + /// + /// - Returns: the delivery, and the callers waiting on it. + func begin() -> (run: EventDeliveryRun, waiters: [FlushOutcomeClosure]) { + lock.withLock { + let next = queued ?? Queued(waiters: [], isClosing: false, bypassesBackoff: false, isPaced: false) + queued = nil + let run = EventDeliveryRun(isClosing: next.isClosing, + bypassesBackoff: next.isClosing || next.bypassesBackoff, + isPaced: next.isPaced && pacesDeliveries && !next.isClosing, + gap: gap) + running = run + deliveryRunCount += 1 + return (run, next.waiters) + } + } + + /// Ends the delivery running, which has had its answer. + /// + /// - Returns: whether one queued behind it is to start now. + func finish() -> Bool { + lock.withLock { + running = nil + isDelivering = queued != nil + return isDelivering + } + } + + /// Ends the pacing of the delivery running, which then sends the rest back to back, or, offline, stops. + func endPacing() { + lock.withLock { running }?.endPacing() + } + + /// Takes the one delivery set for when the backoff's wait ends. + /// + /// - Returns: false where one is set already. + func claimRetry() -> Bool { + lock.withLock { + defer { isRetryScheduled = true } + return !isRetryScheduled + } + } + + /// Notes that the wait has ended and the delivery set for it is being queued, so a later failure can set another. + func retryDue() { + lock.withLock { isRetryScheduled = false } + } +} + +#if DEBUG +extension EventDeliveryScheduler { + /// Callers waiting on the queued delivery. + var queuedWaiterCount: Int { + lock.withLock { queued?.waiters.count ?? 0 } + } + + /// Deliveries started, counting one for a run of flushes that joined each other. + var startedCount: Int { + lock.withLock { deliveryRunCount } + } +} +#endif diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift new file mode 100644 index 000000000..a87e94f96 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventDirectory.swift @@ -0,0 +1,1046 @@ +import Foundation +import OSLog + +/// How many events, and how many bytes, something stored holds: the two things capacity counts. With how many of the +/// events are summaries, which capacity leaves out, where this store wrote them. +struct EventTally: Equatable { + var events = 0 + var bytes = 0 + /// How many of `events` are known to be summaries; none for a batch this store did not close. + var summaries = 0 + + static func += (lhs: inout EventTally, rhs: EventTally) { + lhs.events += rhs.events + lhs.bytes += rhs.bytes + lhs.summaries += rhs.summaries + } +} + +/// The directory one store keeps its events in: the log this process appends to, and beside it the batches that have +/// been closed off for delivery. +/// +/// Everything about the filesystem lives here -- where the directory is, what the files in it are called, and how they +/// are listed, read, moved and deleted. A store the application told to keep events off the disk has no +/// `EventDirectory`, rather than one it is careful never to use. +/// +/// Where the directory is, is worked out on first use rather than at construction, so that a store which never reaches +/// for the disk makes no filesystem call at all, not even to find out where it would have written. +/// +/// Belongs to an `EventStore`, which makes every call holding its `ioLock`, exactly as `DiskEventLog` belongs to this. +/// The lock is handed down so that each call can check it is held, rather than each level taking a lock of its own +/// that would have to be ordered against the others. +final class EventDirectory { + /// Names a log that has been closed off for delivery, whichever version named it. What follows is the version of the + /// rest of the name, `batchNameVersion`, then the batch's payload ID and how many events are in it: + /// `ready-v1-.`. Anything more a batch needs to say belongs in a frame type readers skip, rather + /// than in the name. + /// + /// The count is in the name so that listing a batch this process did not close costs a directory entry rather than + /// a read of the whole file. That is what every batch is at startup: the ones a previous run left behind, and the + /// ones belonging to another process sharing the environment. Reading them all is done holding `ioLock`, which a + /// commit at a commit point then waits behind. + /// + /// A batch is never appended to once it is closed, so the count is as good as the file is. + private static let batchPrefix = "ready-" + /// The version of how the rest of a batch's name reads, so that a later version may change it without this one + /// misreading the count. A batch another version named is never listed, counted or sent: reading it is that + /// version's to do. Expiry still removes it, as it does any batch, so that a file this version does not read cannot + /// stay on the device for good. A payload ID is a UUID, which cannot start with a `v`. + private static let batchNameVersion = "v1-" + /// Names a batch this version named, and so reads. + private static let ownBatchPrefix = batchPrefix + batchNameVersion + private static let nameFieldSeparator: Character = "." + /// Names a log whose events are not yet in a batch: one its owner is appending to, or one left by an owner that is + /// gone, for recovery to close off. The rest of the name is the UUID it was opened under. + static let logPrefix = "pending-" + /// Names a log while it is being opened, before it has its header. Recovery never looks at these; one left by a + /// process that died opening it holds no events, and expiry removes it. + static let openingLogPrefix = "new-" + + /// A name for a new log in `directory`, under a UUID no other writer picks. + static func newLogUrl(in directory: URL) -> URL { + directory.appendingPathComponent(logPrefix + UUID().uuidString) + } + + /// Where `log` is created and given its header, before it is moved to its own name. + static func openingUrl(for log: URL) -> URL { + log.deletingLastPathComponent() + .appendingPathComponent(openingLogPrefix + log.lastPathComponent.dropFirst(logPrefix.count)) + } + + /// The largest file this version reads. It writes none larger than `EventLogFormat.maxBatchSize`, so one past this + /// is not its own, whatever it holds. Read anyway, it would be held in memory whole, and one large enough would + /// take the application down with it. Twice the batch size, so that a later version may raise its own a little + /// without an earlier one losing what it wrote. + static let maxReadSize = 2 * EventLogFormat.maxBatchSize + + private let locateDirectory: () -> URL + private let ioLock: UnfairLock + private let logger: OSLog + /// How a stored log is read, injectable so a test can make a read fail while the file is intact. + private let readFile: (URL) throws -> Data + /// How bytes reach the open log, injectable so a test can make a write fail partway through a session. + private let writeLog: (EventLogFile, Data) -> Int32? + + /// Only to be used while holding `ioLock`. + private var located: (directory: URL, log: DiskEventLog)? + + /// A batch in the directory: the file it is, how many events it holds, how big it is, when it was last written, + /// which is what orders batches oldest first, and how many of its events this store knows to be summaries. + private struct Stored { + let url: URL + let events: Int + let bytes: Int + let modified: Date + var summaries = 0 + + var tally: EventTally { EventTally(events: events, bytes: bytes, summaries: summaries) } + } + + /// What this process knows about each batch in the directory, by payload ID, which is how everything outside this + /// class names one. + /// + /// The file is remembered rather than composed from the payload ID, because the count is part of the name and a + /// batch closed by a version that did not write one has no count to compose. The count is remembered so that such + /// a batch is read at most once. + /// + /// Only to be used while holding `ioLock`. + private var stored: [String: Stored] = [:] + + /// Batches LaunchDarkly has accepted whose file the filesystem would not let go of. + /// + /// Left out of every listing, because a batch that is delivered and merely still on the disk would otherwise be + /// found again by the next delivery, and the one after that, and sent each time. The delete is tried once more on + /// each listing, so a directory that stops being read-only clears itself. + /// + /// Only for this session: a later run, which may well be able to delete the file, treats it as it would any other + /// batch a previous run left behind. LaunchDarkly discards a repeated payload ID, so the cost of that is a wasted + /// request rather than events counted twice. + /// + /// Only to be used while holding `ioLock`. + private var deliveredButUndeleted: Set = [] + + /// The batches this store has a request out for, by payload ID: the file each was locked under, so the lock is + /// released from that file even once the batch has been removed or dropped from a listing, and how many of this + /// store's deliveries are sending it. + /// + /// Only to be used while holding `ioLock`. + private var sending: [String: (url: URL, senders: Int)] = [:] + + /// What `openInThisProcess` counts this store under, from locating the directory until closing the log, kept so + /// that the count is taken back under the same path even once the directory is gone. Only to be used while holding + /// `ioLock`. + private var countedAsOpen: String? + + /// The directories of the stores open in this process, by resolved path, with how many stores have each open. Another + /// environment's sweep leaves these alone, however long since they were written: a store that is open may be about + /// to write, or be sending a batch older than the cutoff. Guarded by `openInThisProcessLock`. + private static var openInThisProcess: [String: Int] = [:] + private static let openInThisProcessLock = UnfairLock() + + #if os(macOS) || targetEnvironment(macCatalyst) + /// Whether the open log is locked while it is open. Only to be changed before the store first touches the disk. + /// + /// Kept here rather than asked of the log, so that setting it does not build the log and with it work out where + /// the directory is. Only to be used while holding `ioLock`. + var locksOpenLog = true { + didSet { located?.log.isLocked = locksOpenLog } + } + #endif + + /// Whether other processes of the application may use this directory, and so deliver its batches. Only a Mac runs + /// a second instance of an application over the same directory; elsewhere a batch that is gone, and that no store + /// in this process removed, was deleted unsent. Only to be used while holding `ioLock`. + #if os(macOS) || targetEnvironment(macCatalyst) + var sharesDirectoryWithOtherProcesses = true + #else + var sharesDirectoryWithOtherProcesses = false + #endif + + /// Whether another process could recover the log this store holds open, and so deliver it: only one sharing the + /// directory, and on a Mac only where the log is not locked. Requires `ioLock`. + private var otherProcessesCanTakeTheOpenLog: Bool { + #if os(macOS) || targetEnvironment(macCatalyst) + sharesDirectoryWithOtherProcesses && !locksOpenLog + #else + sharesDirectoryWithOtherProcesses + #endif + } + + /// The payload IDs of the batches stores in this process last removed, delivered or expired, oldest first, so that + /// another store on the same directory -- one opened after `LDClient` was closed, say -- does not take a batch it + /// listed and then found gone for one deleted unsent. Only the most recent are kept: what matters is a removal + /// between another store's listing and its read, which is never many batches ago. Guarded by `removedLock`. + private static var removedInThisProcess: [String] = [] + private static let removedInThisProcessLimit = 1024 + private static let removedLock = UnfairLock() + + private static func rememberRemoved(_ payloadId: String) { + removedLock.withLock { + removedInThisProcess.append(payloadId) + if removedInThisProcess.count > removedInThisProcessLimit { + removedInThisProcess.removeFirst(removedInThisProcess.count - removedInThisProcessLimit) + } + } + } + + private static func wasRemovedInThisProcess(_ payloadId: String) -> Bool { + removedLock.withLock { removedInThisProcess.contains(payloadId) } + } + + /// - Parameter locatingDirectory: answers where the events go. Asked once, and only on first use. + init(locatingDirectory: @escaping () -> URL, + ioLock: UnfairLock, + logger: OSLog, + readFile: @escaping (URL) throws -> Data, + writeLog: @escaping (EventLogFile, Data) -> Int32?) { + self.locateDirectory = locatingDirectory + self.ioLock = ioLock + self.logger = logger + self.readFile = readFile + self.writeLog = writeLog + } + + /// The directory and the log in it, worked out the first time either is reached for. Requires `ioLock`. + private var disk: (directory: URL, log: DiskEventLog) { + ioLock.assertOwned() + if let located { + return located + } + let directory = locateDirectory() + let log = DiskEventLog(directory: directory, + ioLock: ioLock, + logger: logger, + writeLog: writeLog) + #if os(macOS) || targetEnvironment(macCatalyst) + log.isLocked = locksOpenLog + #endif + let disk = (directory: directory, log: log) + located = disk + let path = directory.resolvingSymlinksInPath().path + countedAsOpen = path + EventDirectory.openInThisProcessLock.withLock { EventDirectory.openInThisProcess[path, default: 0] += 1 } + return disk + } + + private static func isOpenInThisProcess(_ directory: URL) -> Bool { + let path = directory.resolvingSymlinksInPath().path + return openInThisProcessLock.withLock { openInThisProcess[path] != nil } + } + + /// Where the batches are kept. Requires `ioLock`. + var url: URL { disk.directory } + + /// Where the log this process appends to is, for a test or a diagnosis to look at. Requires `ioLock`. + var openLogUrl: URL { disk.log.url } + + /// A batch is identified by its payload ID rather than by a path, so that a batch listed from the directory and + /// the same batch as it was closed are one thing. Requires `ioLock`. + /// + /// - Parameter events: what the caller holding the batch knows it to contain, which is what names it where this + /// process has neither closed nor listed it -- as when it delivers a batch another process closed. + private func url(of payloadId: String, holding events: Int) -> URL { + stored[payloadId]?.url ?? newUrl(of: payloadId, holding: events) + } + + /// Requires `ioLock`. + private func newUrl(of payloadId: String, holding events: Int) -> URL { + disk.directory.appendingPathComponent( + "\(EventDirectory.ownBatchPrefix)\(payloadId)\(EventDirectory.nameFieldSeparator)\(events)") + } + + /// A name for a batch of events not yet counted, which the first listing to read it counts. Requires `ioLock`. + private func newUncountedUrl() -> URL { + disk.directory.appendingPathComponent("\(EventDirectory.ownBatchPrefix)\(UUID().uuidString)") + } + + /// Whether a file is a batch this version named, and so reads. + private static func isOwnBatch(_ name: String) -> Bool { + name.hasPrefix(ownBatchPrefix) + } + + /// Whether a file is a batch some other version named, which this one leaves to expiry. + private static func isAnotherVersionsBatch(_ name: String) -> Bool { + name.hasPrefix(batchPrefix) && !name.hasPrefix(ownBatchPrefix) + } + + /// A batch this version named, `ready-v1-.`, as its name gives it: each field by its position. + /// + /// Fields are split at `nameFieldSeparator`, which a payload ID never contains, so nothing about the count's own + /// characters is needed to tell it apart. Fields after it are ignored. + /// + /// - Returns: the payload ID, and the count, which is nil where the name has none or it does not parse -- a batch a + /// recovery could not read, which is read once to count it. + private static func parse(batchName name: String) -> (payloadId: String, events: Int?) { + let fields = name.dropFirst(ownBatchPrefix.count) + .split(separator: nameFieldSeparator, omittingEmptySubsequences: false) + let payloadId = String(fields[0]) + guard fields.count > 1, !fields[1].isEmpty, fields[1].allSatisfy({ $0.isASCII && $0.isNumber }), + let events = Int32(fields[1]) + else { return (payloadId, nil) } + return (payloadId, Int(events)) + } + + // MARK: Writing + + /// Appends to the open log, opening a new one where none is open. Requires `ioLock`. + /// + /// - Returns: whether every byte landed. False means none of `bytes` is in the log, and that the caller should give + /// up on the disk. + func append(_ bytes: Data) -> Bool { + ioLock.assertOwned() + return disk.log.append(bytes) + } + + /// What closing the open log produced. + enum LogClose { + case closed(EventBatch) + /// The log is still there and still open, for the next append to continue and the next close to try again. + case failed + /// The log is gone with its events -- a purged cache directory on tvOS, say -- and they have to stop counting + /// against capacity or the store would refuse events for the rest of the session. `deletedUnsent` where + /// nothing could have delivered them: the log was still held, and no other process could have taken it. + case vanished(deletedUnsent: Bool) + } + + /// Closes the open log into a batch, so the next append starts a new one. Requires `ioLock`. + /// + /// - Parameter held: what the log holds, which the caller has counted as it staged and wrote it. + func closeLog(holding held: EventTally) -> LogClose { + ioLock.assertOwned() + let events = held.events + let payloadId = UUID().uuidString + let file = newUrl(of: payloadId, holding: events) + do { + try disk.log.close(movingTo: file) + } catch { + os_log("%s could not close the event log, will try again at the next delivery: %s", log: logger, type: .info, typeName(and: #function), String(describing: error)) + guard disk.log.exists + else { + // One no longer held -- persistence given up on closes it where it is -- may have been moved into a + // batch by recovery, which delivers it. + let deletedUnsent = disk.log.isOpen && !otherProcessesCanTakeTheOpenLog + disk.log.closeDescriptor() + return .vanished(deletedUnsent: deletedUnsent) + } + return .failed + } + + stored[payloadId] = Stored(url: file, events: events, bytes: held.bytes, modified: Date(), summaries: held.summaries) + return .closed(EventBatch(payloadId: payloadId, eventCount: events)) + } + + /// Closes the descriptor, leaving everything where it is, and stops counting the directory as open in this process. + /// Requires `ioLock`. + func closeOpenLog() { + ioLock.assertOwned() + // Not `disk`: a store that never wrote has never worked out where it lives, and shutting down or giving up on + // the disk is no reason to start. + located?.log.closeDescriptor() + guard let path = countedAsOpen + else { return } + countedAsOpen = nil + EventDirectory.openInThisProcessLock.withLock { + let stores = EventDirectory.openInThisProcess[path] ?? 0 + EventDirectory.openInThisProcess[path] = stores > 1 ? stores - 1 : nil + } + } + + /// Locks a batch against expiry, in this process and every other, for as long as a request for it is out. Each + /// call is ended by one `stoppedSending(_:)`. Requires `ioLock`. + /// + /// - Parameter events: what the caller holding the batch knows it to contain, as for `payload(of:)`. + func startedSending(_ payloadId: String, holding events: Int) { + ioLock.assertOwned() + var batch = sending[payloadId] ?? (url: url(of: payloadId, holding: events), senders: 0) + batch.senders += 1 + sending[payloadId] = batch + BatchLocks.startedSending(batch.url) + } + + /// Ends one `startedSending(_:holding:)`. Requires `ioLock`. + func stoppedSending(_ payloadId: String) { + ioLock.assertOwned() + guard var batch = sending[payloadId] + else { return } + BatchLocks.stoppedSending(batch.url) + batch.senders -= 1 + sending[payloadId] = batch.senders > 0 ? batch : nil + } + + // MARK: Listing and reading + + /// The batches in the directory, oldest first. Requires `ioLock`. + /// + /// Sizes come from the same listing, so that counting the directory's bytes reads no file either. + /// + /// A directory that is not there holds nothing. One that is there and cannot be listed -- out of file descriptors, + /// or no longer readable -- is not empty, so it answers with the batches the last listing or close counted, as they + /// were counted, and says the listing is incomplete. + func listBatches() -> BatchListing { + ioLock.assertOwned() + let contents: [URL] + do { + contents = try FileManager.default.contentsOfDirectory(at: disk.directory, + includingPropertiesForKeys: EventDirectory.listedKeys, + options: [.skipsHiddenFiles]) + } catch { + guard FileManager.default.fileExists(atPath: disk.directory.path) + else { + let vanished = vanishedEvents(listed: []) + stored = [:] + return BatchListing(batches: [], isComplete: true, vanishedEvents: vanished) + } + os_log("%s could not list stored events, will try again at the next delivery: %s", log: logger, type: .info, typeName(and: #function), String(describing: error)) + let known = stored + .sorted { $0.value.modified < $1.value.modified } + .map { EventBatch(payloadId: $0.key, eventCount: $0.value.events) } + return BatchListing(batches: known, isComplete: false) + } + + let batches = contents + .filter { EventDirectory.isOwnBatch($0.lastPathComponent) } + .compactMap { file -> (EventBatch, Date)? in + let (payloadId, counted) = EventDirectory.parse(batchName: file.lastPathComponent) + guard !isDeliveredButUndeleted(payloadId, at: file) + else { return nil } + // Reading is only for a batch whose name does not count it, which a version that named them without a + // count may have left behind, and then only the once. + let read = counted == nil && stored[payloadId] == nil ? countOfEvents(in: file) : nil + guard let events = counted ?? stored[payloadId]?.events ?? read?.events + else { return nil } + let listed = EventDirectory.listed(file) + let modified = listed.modified ?? Date.distantPast + // What the close knew of its summaries is kept; nothing in the file says. + let previous = stored[payloadId] + // A batch just read is as big as what was read. The size the listing came with can be missing, and no + // batch is empty: it has a header at least. + let bytes = read?.bytes ?? listed.bytes.flatMap { $0 > 0 ? $0 : nil } + ?? previous?.bytes ?? EventLogFormat.fileHeaderSize + stored[payloadId] = Stored(url: file, events: events, bytes: bytes, + modified: modified, + summaries: previous.map { $0.events == events ? $0.summaries : 0 } ?? 0) + return (EventBatch(payloadId: payloadId, eventCount: events), modified) + } + .sorted { $0.1 < $1.1 } + .map { $0.0 } + + // A batch that left the directory without going through `remove` -- a purged cache directory on tvOS, say -- + // would otherwise keep its count for the rest of the session. + let listed = Set(batches.map { $0.payloadId }) + let vanished = vanishedEvents(listed: listed) + stored = stored.filter { listed.contains($0.key) } + + return BatchListing(batches: batches, isComplete: true, vanishedEvents: vanished) + } + + /// The events in the batches counted here that are not in `listed` and that nothing could have delivered: no other + /// process uses the directory, and no store in this one removed them. Requires `ioLock`. + private func vanishedEvents(listed: Set) -> Int { + ioLock.assertOwned() + guard !sharesDirectoryWithOtherProcesses + else { return 0 } + return stored + .filter { !listed.contains($0.key) && !EventDirectory.wasRemovedInThisProcess($0.key) } + .reduce(0) { $0 + $1.value.events } + } + + private static let listedKeys: [URLResourceKey] = [.contentModificationDateKey, .fileSizeKey] + + /// What a directory listing says about a file, without opening it. + private static func listed(_ file: URL) -> (bytes: Int?, modified: Date?) { + let values = try? file.resourceValues(forKeys: Set(listedKeys)) + return (values?.fileSize, values?.contentModificationDate) + } + + /// The bytes in every batch the last listing or close counted. Requires `ioLock`. + var storedBytes: Int { + ioLock.assertOwned() + return stored.values.reduce(0) { $0 + $1.bytes } + } + + /// The summaries known to be in the batches the last listing or close counted. Requires `ioLock`. + var storedSummaries: Int { + ioLock.assertOwned() + return stored.values.reduce(0) { $0 + $1.summaries } + } + + /// What this process counts for a batch, or nil if it counts nothing for it: one removed, or dropped by a listing it + /// was no longer in. Requires `ioLock`. + func counted(_ payloadId: String) -> EventTally? { + ioLock.assertOwned() + return stored[payloadId]?.tally + } + + /// Whether a batch has already been delivered and only its file is left, trying the delete that failed once more + /// on the way. Requires `ioLock`. + private func isDeliveredButUndeleted(_ payloadId: String, at file: URL) -> Bool { + guard deliveredButUndeleted.contains(payloadId) + else { return false } + try? FileManager.default.removeItem(at: file) + if !FileManager.default.fileExists(atPath: file.path) { + deliveredButUndeleted.remove(payloadId) + } + return true + } + + /// The `[...]` payload for a batch, assembled in the bytes the file was read into; or nil if it holds no events, or + /// is gone where another process or store may have delivered it. + /// + /// Requires `ioLock`. + /// + /// - throws: `UnreadableBatch` if the batch is there but holds nothing this version can send; `VanishedBatch` if it + /// held events and is gone where nothing could have delivered it; anything else if it could not be read this + /// time, which a later attempt may manage. + func payload(of batch: EventBatch) throws -> BatchPayload? { + ioLock.assertOwned() + // Taken out of the read's result rather than bound inside the switch, so that nothing else refers to it and it + // is assembled where it is rather than copied first. + var file: Data + switch read(url(of: batch.payloadId, holding: batch.eventCount)) { + case .missing: + guard batch.eventCount > 0, !sharesDirectoryWithOtherProcesses, + !EventDirectory.wasRemovedInThisProcess(batch.payloadId) + else { return nil } + throw VanishedBatch(payloadId: batch.payloadId) + case .failed(let error): throw error + case .tooLarge: throw UnreadableBatch(payloadId: batch.payloadId) + case .bytes(let bytes): file = bytes + } + // No more than the batch was closed holding. A write that failed and could not be truncated leaves whole frames + // after them, whose events are delivered from memory instead, and would otherwise be sent twice. + let assembled = EventLogFormat.assembleBody(inPlace: &file, maxEvents: batch.eventCount > 0 ? batch.eventCount : .max) + if assembled > 0 { + return BatchPayload(body: file, eventCount: assembled) + } + // A log this version reads that holds no events has nothing to lose, unless its first frame is damaged. Where + // there is no payload the log is as it was read. + guard EventLogFormat.eventCount(in: file) == 0, !EventLogFormat.isDamaged(file) + else { throw UnreadableBatch(payloadId: batch.payloadId) } + return nil + } + + /// Forgets a batch. Requires `ioLock`. + /// + /// - Returns: what it stopped counting, which is nothing for a batch that was already gone -- one another process + /// delivered, say, which a listing has stopped counting already. + func remove(_ payloadId: String, holding events: Int) -> EventTally { + ioLock.assertOwned() + let file = url(of: payloadId, holding: events) + try? FileManager.default.removeItem(at: file) + EventDirectory.rememberRemoved(payloadId) + if FileManager.default.fileExists(atPath: file.path) { + // Its events stop counting all the same: LaunchDarkly has them, and holding capacity for a file this + // process may never be allowed to delete would cost the session events it could still have recorded. + os_log("%s could not delete a delivered batch; it will not be sent again this session", log: logger, type: .debug, typeName(and: #function)) + deliveredButUndeleted.insert(payloadId) + } + return stored.removeValue(forKey: payloadId)?.tally ?? EventTally() + } + + /// Every event in the directory, encoded exactly as it will be sent. Requires `ioLock`. + /// + /// For tests and for diagnosing a store rather than for the recording path: it reads every file. + func storedPayloads() -> [Data] { + ioLock.assertOwned() + let contents = (try? FileManager.default.contentsOfDirectory(at: disk.directory, + includingPropertiesForKeys: nil, + options: [.skipsHiddenFiles])) ?? [] + var logs = contents.filter { EventDirectory.isOwnBatch($0.lastPathComponent) }.sorted { $0.path < $1.path } + if disk.log.exists { + logs.append(disk.log.url) + } + + var payloads: [Data] = [] + for log in logs { + guard case .bytes(let file) = read(log) + else { continue } + _ = EventLogFormat.forEachFrame(in: file) { type, payload in + if type == EventLogFormat.eventFrame { + payloads.append(payload) + } + } + } + return payloads + } + + /// What reading a stored log produced. + /// + /// A read that failed is kept apart from a file that is not there, because the two arrive the same way: a process + /// out of file descriptors fails to open a file that is perfectly intact with the error a missing one gets. Only + /// what is missing, or what this version cannot make sense of, is gone for good. + private enum LogRead { + case bytes(Data) + case missing + case failed(Error) + /// Larger than `maxReadSize`, and so never read: it can never be delivered. + case tooLarge(size: Int) + } + + private func read(_ url: URL) -> LogRead { + if let size = EventDirectory.sizeIfTooLarge(url) { + return .tooLarge(size: size) + } + do { + return .bytes(try readFile(url)) + } catch { + return FileManager.default.fileExists(atPath: url.path) ? .failed(error) : .missing + } + } + + /// The size of a file larger than `maxReadSize`, or nil for any other, including one whose size could not be + /// found, which the read then fails on or reads. + private static func sizeIfTooLarge(_ url: URL) -> Int? { + guard let size = (try? FileManager.default.attributesOfItem(atPath: url.path))?[.size] as? Int, + size > maxReadSize + else { return nil } + return size + } + + /// How many events a batch file holds, and how many bytes were read to count them, or nil when it is not one to + /// list this time. + /// + /// Requires `ioLock`. + private func countOfEvents(in file: URL) -> EventTally? { + ioLock.assertOwned() + switch read(file) { + case .missing: + // Gone since the directory was listed. + return nil + case .failed(let error): + // Left for a later listing rather than deleted: the read may pass then, and the events in it may have been + // intact all along. Without a count it cannot be delivered now. + os_log("%s could not read stored events, will try again at the next delivery: %s", log: logger, type: .info, typeName(and: #function), String(describing: error)) + return nil + case .tooLarge(let size): + // Never read, so never counted. Listed as holding one event, the fewest it can, so that the delivery + // reaching it drops it and counts the loss. + return EventTally(events: 1, bytes: size) + case .bytes(let log): + guard let events = EventLogFormat.eventCount(in: log) + else { + // Written by a version of the SDK whose format this one does not read, or damaged beyond what the + // torn tail recovery tolerates. Either way it can never be delivered, but it held events. Listed as + // holding one, the fewest it can, so that the delivery reaching it drops it and counts the loss, as + // for one too large to read. + return EventTally(events: 1, bytes: log.count) + } + // Damaged from its first frame, it counts as no events, and would be dropped as a loss of none. + return EventTally(events: events == 0 && EventLogFormat.isDamaged(log) ? 1 : events, bytes: log.count) + } + } + + // MARK: Recovery + + /// A log whose writer is gone, and what reading it found. + struct InterruptedLog { + fileprivate enum Contents { + case missing + case failed(Error) + /// Not a log this version reads. + case unreadable + case events(EventTally) + } + + fileprivate let url: URL + fileprivate let contents: Contents + } + + /// The logs recovery is to close off, so their events join the batches to deliver: every one no store in this + /// process has open, of which `DiskEventLog.claimForRecovery(_:)` then takes those no live process holds. + /// + /// The events in them were recorded by a store that is gone, so there is no one left to add to them: a previous run + /// of this process, or another process that has since ended, whatever it was called. None of them becomes open here + /// later: a log is opened under a name of its own and never reopened once closed. + /// + /// Requires `ioLock`. + func interruptedLogs() -> [URL] { + ioLock.assertOwned() + let contents = (try? FileManager.default.contentsOfDirectory(at: disk.directory, + includingPropertiesForKeys: nil, + options: [.skipsHiddenFiles])) ?? [] + return contents.filter { + $0.lastPathComponent.hasPrefix(EventDirectory.logPrefix) && !DiskEventLog.isOpenInThisProcess($0) + } + } + + /// Reads and counts a log recovery has claimed. + /// + /// Needs no lock, and is meant to be called without `ioLock`: a log can be as large as a batch, and a commit at a + /// commit point would otherwise wait for every log a crash left. Nothing here is this directory's state, and the + /// log's writer is gone, so it is not changing either. + func read(interruptedLog leftOpen: URL) -> InterruptedLog { + switch read(leftOpen) { + case .missing: + return InterruptedLog(url: leftOpen, contents: .missing) + case .failed(let error): + return InterruptedLog(url: leftOpen, contents: .failed(error)) + case .tooLarge: + return InterruptedLog(url: leftOpen, contents: .unreadable) + case .bytes(let log): + guard let events = EventLogFormat.eventCount(in: log) + else { return InterruptedLog(url: leftOpen, contents: .unreadable) } + return InterruptedLog(url: leftOpen, contents: .events(EventTally(events: events, bytes: log.count))) + } + } + + /// Closes off a log `read(interruptedLog:)` read. Requires `ioLock`, which is what keeps a listing from counting + /// the batch it becomes on its own as well. + /// + /// - Returns: what it closed off, or nothing where there was nothing to close off. + func preparePayload(from interrupted: InterruptedLog) -> EventTally { + ioLock.assertOwned() + let leftOpen = interrupted.url + let events: Int + let bytes: Int + switch interrupted.contents { + case .missing: + return EventTally() + case .failed(let error): + // Deleting it would lose events that may be intact, and left under this name it would wait for the next + // run's recovery. Closed off uncounted instead, it is read and counted by whichever listing first manages + // to. + // Named without a count, which is what a batch whose events have never been counted is: the listing that + // first manages to read it is what counts it. + let uncounted = newUncountedUrl() + if (try? FileManager.default.moveItem(at: leftOpen, to: uncounted)) != nil { + os_log("%s could not read events from a previous run, will try again at the next delivery: %s", log: logger, type: .info, typeName(and: #function), String(describing: error)) + } else { + // Left where it is, unlocked and appended to by no one, for the next launch to recover. + os_log("%s could not read events from a previous run, will try again at the next launch: %s", log: logger, type: .info, typeName(and: #function), String(describing: error)) + } + return EventTally() + case .unreadable: + // A log that cannot be read cannot be appended to either. + try? FileManager.default.removeItem(at: leftOpen) + return EventTally() + case .events(let counted): + events = counted.events + bytes = counted.bytes + } + + guard events > 0 + else { + try? FileManager.default.removeItem(at: leftOpen) + return EventTally() + } + + let payloadId = UUID().uuidString + let file = newUrl(of: payloadId, holding: events) + guard (try? FileManager.default.moveItem(at: leftOpen, to: file)) != nil + else { + // Gone since it was read is another store recovering it first, which where logs are not locked can be one + // in this process. + if FileManager.default.fileExists(atPath: leftOpen.path) { + os_log("%s could not prepare %d event(s) from a previous run, will try again at the next launch", log: logger, type: .info, typeName(and: #function), events) + } + return EventTally() + } + + let recovered = Stored(url: file, events: events, bytes: bytes, modified: EventDirectory.listed(file).modified ?? Date()) + stored[payloadId] = recovered + return recovered.tally + } +} + +// MARK: Expiry + +extension EventDirectory { + /// Deletes the batches in this directory last written before `cutoff`, and the logs no live process holds that have + /// not been written since then either. Requires `ioLock`. + /// + /// A batch's modification time is when its newest event was written, since a batch is renamed rather than rewritten + /// when it is closed, so a batch is discarded only once everything in it is that old. A log is discarded only once + /// nothing is appending to it, however long ago it was written: a live process may have gone that long without an + /// event. + /// + /// A batch whose request is out is left alone however old it is, whichever store or process sent it; see + /// `BatchLocks`. A batch another version named is deleted without being read or counted: it was never counted + /// against capacity, and its events were never this version's to lose. + /// + /// - Parameter sending: batches whose request is out from this store. + /// - Returns: how many events were discarded, and how much this process had counted of the batches that left the + /// directory, which is what stops counting against capacity. + func discard(lastWrittenBefore cutoff: Date, sparing sending: Set) -> (events: Int, counted: EventTally) { + ioLock.assertOwned() + let contents = (try? FileManager.default.contentsOfDirectory(at: disk.directory, + includingPropertiesForKeys: EventDirectory.listedKeys, + options: [.skipsHiddenFiles])) ?? [] + var events = 0 + var counted = EventTally() + for file in contents where EventDirectory.listed(file).modified.map({ $0 < cutoff }) ?? false { + let name = file.lastPathComponent + if EventDirectory.isAnotherVersionsBatch(name) { + _ = BatchLocks.deleteUnlessBeingSent(file) + } else if EventDirectory.isOwnBatch(name) { + let (payloadId, named) = EventDirectory.parse(batchName: name) + guard !sending.contains(payloadId), !deliveredButUndeleted.contains(payloadId), + !BatchLocks.isSentInThisProcess(file) + else { continue } + let held = named ?? stored[payloadId]?.events ?? EventDirectory.eventsReading(file) + guard BatchLocks.deleteUnlessBeingSent(file) + else { + if !FileManager.default.fileExists(atPath: file.path) { + // Delivered by another process since the listing: no longer counted, and not dropped. + counted += stored.removeValue(forKey: payloadId)?.tally ?? EventTally() + } + continue + } + EventDirectory.rememberRemoved(payloadId) + events += held + counted += stored.removeValue(forKey: payloadId)?.tally ?? EventTally() + } else { + events += EventDirectory.discardIfAbandonedLog(file) + } + } + return (events, counted) + } + + /// Empties and removes the directories of other environments that nothing has written to since `cutoff`: those + /// of a mobile key the application no longer uses, which no client will look in again. + /// + /// Age decides rather than the keys the application configures, because another process of the same application + /// may be using a key this one does not configure. Nothing is touched here that this store reads or writes, so it + /// needs no lock. A directory is removed only if it is empty, so a log another process starts in it meanwhile + /// keeps the directory and itself. + /// + /// Only directories carrying this application's bundle identifier are considered. An unsandboxed Mac application + /// shares Application Support with every other one, whose events are not this application's to delete. + /// + /// The directory of a store open in this process is left alone, and so is a batch some live process is sending; + /// see `BatchLocks`. + /// + /// - Returns: how many events were discarded. + static func discardAbandonedEnvironments(besides current: URL, lastWrittenBefore cutoff: Date) -> Int { + let root = current.deletingLastPathComponent() + let ownName = current.lastPathComponent + let isSameApplication: (String) -> Bool + if let dot = ownName.firstIndex(of: ".") { + let prefix = ownName[...dot] + isSameApplication = { $0.hasPrefix(prefix) } + } else { + isSameApplication = { !$0.contains(".") } + } + + let siblings = (try? FileManager.default.contentsOfDirectory(at: root, + includingPropertiesForKeys: [.isDirectoryKey], + options: [.skipsHiddenFiles])) ?? [] + var events = 0 + for directory in siblings where directory.lastPathComponent != ownName && isSameApplication(directory.lastPathComponent) { + guard (try? directory.resourceValues(forKeys: [.isDirectoryKey]))?.isDirectory == true, + !isOpenInThisProcess(directory) + else { continue } + let contents = (try? FileManager.default.contentsOfDirectory(at: directory, + includingPropertiesForKeys: listedKeys, + options: [])) ?? [] + for file in contents where listed(file).modified.map({ $0 < cutoff }) ?? false { + let name = file.lastPathComponent + if isAnotherVersionsBatch(name) { + _ = BatchLocks.deleteUnlessBeingSent(file) + } else if isOwnBatch(name) { + guard !BatchLocks.isSentInThisProcess(file) + else { continue } + let held = parse(batchName: name).events ?? eventsReading(file) + if BatchLocks.deleteUnlessBeingSent(file) { + events += held + } + } else { + events += discardIfAbandonedLog(file) + } + } + EventLogFile.removeDirectoryIfEmpty(directory) + } + return events + } + + /// Deletes a log no live process is appending to, or one a process died opening, returning how many events it + /// held. Anything else is left alone. + private static func discardIfAbandonedLog(_ file: URL) -> Int { + let name = file.lastPathComponent + if name.hasPrefix(EventDirectory.openingLogPrefix) { + // Opening a log takes moments, so one this old was never finished, and holds no events. + delete(file) + return 0 + } + guard name.hasPrefix(EventDirectory.logPrefix), !DiskEventLog.isHeldByLiveProcess(file) + else { return 0 } + let held = eventsReading(file) + return delete(file) ? held : 0 + } + + private static func eventsReading(_ file: URL) -> Int { + // Never read, so held to be the one event it holds at the fewest. + guard sizeIfTooLarge(file) == nil + else { return 1 } + return (try? Data(contentsOf: file)).flatMap { EventLogFormat.eventCount(in: $0) } ?? 0 + } + + @discardableResult + private static func delete(_ file: URL) -> Bool { + try? FileManager.default.removeItem(at: file) + return !FileManager.default.fileExists(atPath: file.path) + } +} + +/// What tells expiry which batches have a request out, so that it never deletes a batch someone is still sending: were +/// that request to fail, there would be no file left to retry from. +/// +/// On a Mac, where processes of one application can share an environment, a sender holds a shared lock on the batch +/// for as long as its request is out, and expiry deletes a batch only while holding an exclusive lock on it, which it +/// cannot have while anyone holds the shared one. The kernel drops a lock when its process dies, so a batch whose +/// sender is gone is expired like any other. Elsewhere a process has its directory to itself, as it does its logs. +/// +/// The batches sent in this process are also remembered, and expiry here asks that rather than opening the file. The +/// lock is taken once the batch has been read for sending; a batch another process expires in the moment between is +/// still sent from what was read, but has no file for a retry. +enum BatchLocks { + /// A batch this process is sending, the descriptor its lock is held through, and how many send it. + private final class Held { + var file: EventLogFile? + var senders = 0 + } + + /// The batches of one directory this process has a request out for, by name. Guarded by `lock`, which covers the + /// probes and deletions below as well, so that a store in this process cannot start sending a batch another is + /// expiring. One per directory rather than one for all, since a lock taken or dropped on one file leaves every other + /// file's alone, so stores of different environments have nothing to wait on each other for. + private final class Directory { + var heldInThisProcess: [String: Held] = [:] + let lock = UnfairLock() + } + + /// Every directory this process has sent from or expired in, by `key(_:)`. Process-wide, because two stores in one + /// process can share a directory and must share its lock. Never shrinks: a directory dropped while a call still held + /// it would let the next call take a second lock for the same files. There is one entry per environment directory + /// the process has touched, which stays a handful. + private static var directories: [String: Directory] = [:] + private static let directoriesLock = UnfairLock() + + /// One name for a batch's directory however its URL was come by, from a listing or put together from the directory: + /// only the directory is resolved, since the batch itself may be gone by the time its send stops. + private static func key(_ batch: URL) -> String { + batch.deletingLastPathComponent().resolvingSymlinksInPath().path + } + + private static func directory(of batch: URL) -> Directory { + let path = key(batch) + return directoriesLock.withLock { + if let known = directories[path] { + return known + } + let created = Directory() + directories[path] = created + return created + } + } + + /// Locks a batch for as long as a request for it is out. Each call is ended by one `stoppedSending(_:)`; a batch + /// two deliveries send at once stays locked until both have stopped. + static func startedSending(_ batch: URL) { + let directory = directory(of: batch) + let name = batch.lastPathComponent + directory.lock.withLock { + let held = directory.heldInThisProcess[name] ?? Held() + directory.heldInThisProcess[name] = held + held.senders += 1 + #if os(macOS) || targetEnvironment(macCatalyst) + // Only fails to open where the batch is gone already, leaving nothing for expiry to take. Still remembered + // where it cannot be locked, which keeps expiry in this process away from it; only other processes cannot + // tell. + guard held.file == nil, let opened = EventLogFile.open(at: batch, flags: O_RDONLY) + else { return } + if opened.tryLock(shared: true) { + held.file = opened + } else { + opened.close() + } + #endif + } + } + + /// Ends one `startedSending(_:)`, releasing the lock once the last sender has stopped. + static func stoppedSending(_ batch: URL) { + let directory = directory(of: batch) + let name = batch.lastPathComponent + directory.lock.withLock { + guard let held = directory.heldInThisProcess[name] + else { return } + held.senders -= 1 + guard held.senders <= 0 + else { return } + directory.heldInThisProcess[name] = nil + // Releases the lock with it. + held.file?.close() + } + } + + /// Whether a store in this process is sending a batch, answered without opening the file. + static func isSentInThisProcess(_ batch: URL) -> Bool { + let directory = directory(of: batch) + let name = batch.lastPathComponent + return directory.lock.withLock { directory.heldInThisProcess[name] != nil } + } + + /// Deletes a batch for expiry, unless some live process is sending it. + /// + /// - Returns: whether this call deleted it; false for one being sent, one that could not be looked at, and one + /// already gone, whose events are someone else's to count. + static func deleteUnlessBeingSent(_ batch: URL) -> Bool { + let directory = directory(of: batch) + let name = batch.lastPathComponent + return directory.lock.withLock { + guard directory.heldInThisProcess[name] == nil + else { return false } + #if os(macOS) || targetEnvironment(macCatalyst) + // Opened without being created: a batch delivered since it was listed has to stay gone, since a delivery + // reading one brought back empty would report its events lost. + guard let probe = EventLogFile.open(at: batch, flags: O_RDONLY) + else { return false } + defer { probe.close() } + guard probe.tryLock(shared: false) + else { return false } + #endif + // Deleted while still locked, so no sender can take its lock between the look and the delete. + return (try? FileManager.default.removeItem(at: batch)) != nil + } + } +} + +#if DEBUG +extension BatchLocks { + /// The lock every call on a batch takes: its directory's, whichever store names the batch. + static func lockForTesting(_ batch: URL) -> UnfairLock { + directory(of: batch).lock + } +} +#endif + +extension EventDirectory { + /// The directory the store for a mobile key belongs in. + /// + /// tvOS is the reason this is not simply Application Support: a tvOS application is only allowed to persist to + /// purgeable caches, so there the events are kept somewhere the system may reclaim, which is the most durability + /// the platform offers. A directory that turns out not to be writable is no different from a full device: the store + /// gives up on the disk and holds events in memory. + static func defaultUrl(mobileKey: String) -> URL { + #if os(tvOS) + let searchPath = FileManager.SearchPathDirectory.cachesDirectory + #else + let searchPath = FileManager.SearchPathDirectory.applicationSupportDirectory + #endif + + let root = FileManager.default.urls(for: searchPath, in: .userDomainMask).first + ?? FileManager.default.temporaryDirectory + + var keyComponent = Util.sha256(mobileKey).base64UrlEncodedString + if let bundleId = Bundle.main.bundleIdentifier { + keyComponent = "\(Util.sha256(bundleId).base64UrlEncodedString).\(keyComponent)" + } + + return root + .appendingPathComponent("com.launchdarkly.events", isDirectory: true) + .appendingPathComponent(keyComponent, isDirectory: true) + } +} + +extension EventDirectory: TypeIdentifying { } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventJSONWriter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventJSONWriter.swift index 814a4346e..8718520d7 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventJSONWriter.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventJSONWriter.swift @@ -2,8 +2,8 @@ import Foundation /// Writes an `Event` as the JSON the events endpoint accepts, producing the same JSON as `Event.encode(to:)`. /// -/// Create one per batch. The buffer and the context cache are reused across the events of that batch, and neither is -/// safe to share between threads. +/// The buffer and the context cache are reused from one event to the next, so the longer an instance lives the more +/// they save. Neither is safe to share between threads. struct EventJSONWriter { private let allAttributesPrivate: Bool private let globalPrivateAttributes: [Reference] diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFile.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFile.swift new file mode 100644 index 000000000..0dff684b0 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFile.swift @@ -0,0 +1,196 @@ +import Foundation + +#if canImport(Darwin) +import Darwin +#elseif canImport(Glibc) +import Glibc +#elseif canImport(Musl) +import Musl +#endif + +/// An open file an event log is written through, on a descriptor rather than Foundation's file handles. +/// +/// Foundation's FileHandle is deliberately not used: it raises an Objective-C exception when a write fails, which is not +/// something a Swift caller can catch, and has taken host applications down with it when a device ran out of space. +/// +/// A value, so a copy names the same open file. Whoever opened it closes it once, and uses no copy after that. +struct EventLogFile { + let descriptor: Int32 + + /// Opens a file, creating it readable and writable by its owner alone. Nil where it could not, with `errno` saying + /// why. + static func open(at url: URL, flags: Int32) -> EventLogFile? { + let opened = url.withUnsafeFileSystemRepresentation { path -> Int32 in + guard let path = path + else { return -1 } + return openFile(path, flags, 0o600) + } + return opened >= 0 ? EventLogFile(descriptor: opened) : nil + } + + #if os(macOS) || targetEnvironment(macCatalyst) + /// Opens a file and takes its lock without waiting, or nil where another live process holds it. + /// + /// The lock has to be on the file that is still at `url`: one renamed or removed between the open and the lock, + /// by another process closing off or recovering it, is no longer a log anyone appends to. + static func openLocked(_ url: URL, flags: Int32) -> EventLogFile? { + for _ in 0..<3 { + guard let opened = open(at: url, flags: flags) + else { return nil } + guard lockFile(opened.descriptor) + else { + opened.close() + return nil + } + if isFile(opened.descriptor, at: url) { + return opened + } + opened.close() + } + return nil + } + + /// Whether a live process holds the lock on a log, which is to say is still appending to it. + static func isLockedByLiveProcess(_ url: URL) -> Bool { + guard let opened = openLocked(url, flags: O_RDONLY) + else { return FileManager.default.fileExists(atPath: url.path) } + opened.close() + return false + } + + /// Takes the file's lock without waiting: shared, which any number can hold at once, or exclusive, which nobody + /// else can hold alongside it. False where someone else's lock is in the way. + /// + /// The lock belongs to this open file rather than to the process, so closing another descriptor on the same file + /// leaves it in place, and another descriptor in this process contends for it as another process would. + func tryLock(shared: Bool) -> Bool { + flock(descriptor, (shared ? LOCK_SH : LOCK_EX) | LOCK_NB) == 0 + } + #endif + + /// Removes a directory only if it is empty, so that nothing put there since it was last looked at goes with it. + @discardableResult + static func removeDirectoryIfEmpty(_ url: URL) -> Bool { + url.withUnsafeFileSystemRepresentation { path -> Bool in + guard let path = path + else { return false } + return removeDirectory(path) == 0 + } + } + + static func excludeFromBackup(_ directory: URL) { + #if canImport(Darwin) + var url = directory + var values = URLResourceValues() + values.isExcludedFromBackup = true + try? url.setResourceValues(values) + #endif + } + + /// Writes every byte, returning the errno it stopped on or nil where they all landed. + func writeAll(_ bytes: Data) -> Int32? { + var offset = 0 + return bytes.withUnsafeBytes { raw -> Int32? in + guard let base = raw.baseAddress + else { return nil } + while offset < raw.count { + let result = writeBytes(descriptor, base.advanced(by: offset), raw.count - offset) + if result > 0 { + offset += result + continue + } + // A signal interrupting the write is not a failure; anything else is. + if errno == EINTR { + continue + } + return errno + } + return nil + } + } + + var isEmpty: Bool { + lseek(descriptor, 0, SEEK_END) == 0 + } + + /// How long the file is, or nil where that could not be found. + var size: off_t? { + let end = lseek(descriptor, 0, SEEK_END) + return end >= 0 ? end : nil + } + + /// Cuts the file back to `length`. False where it could not, with `errno` saying why. + func truncate(to length: off_t) -> Bool { + truncateFile(descriptor, length) == 0 + } + + func close() { + closeFile(descriptor) + } +} + +// The POSIX calls are wrapped so that the names cannot be shadowed by a member of the type using them, and so the +// platform differences stay in one place. +#if canImport(Darwin) +private func openFile(_ path: UnsafePointer, _ flags: Int32, _ mode: mode_t) -> Int32 { + Darwin.open(path, flags, mode) +} + +private func writeBytes(_ descriptor: Int32, _ bytes: UnsafeRawPointer, _ count: Int) -> Int { + Darwin.write(descriptor, bytes, count) +} + +private func truncateFile(_ descriptor: Int32, _ length: off_t) -> Int32 { + Darwin.ftruncate(descriptor, length) +} + +private func closeFile(_ descriptor: Int32) { + _ = Darwin.close(descriptor) +} + +private func removeDirectory(_ path: UnsafePointer) -> Int32 { + Darwin.rmdir(path) +} +#else +private func openFile(_ path: UnsafePointer, _ flags: Int32, _ mode: mode_t) -> Int32 { + open(path, flags, mode) +} + +private func writeBytes(_ descriptor: Int32, _ bytes: UnsafeRawPointer, _ count: Int) -> Int { + write(descriptor, bytes, count) +} + +private func truncateFile(_ descriptor: Int32, _ length: off_t) -> Int32 { + ftruncate(descriptor, length) +} + +private func closeFile(_ descriptor: Int32) { + _ = close(descriptor) +} + +private func removeDirectory(_ path: UnsafePointer) -> Int32 { + rmdir(path) +} +#endif + +#if os(macOS) || targetEnvironment(macCatalyst) +/// Takes an exclusive lock without waiting. Released by the kernel when the descriptor closes, including when the +/// process dies, so a crashed process never leaves one behind. +private func lockFile(_ descriptor: Int32) -> Bool { + flock(descriptor, LOCK_EX | LOCK_NB) == 0 +} + +/// Whether a descriptor is open on the file currently at `url`, rather than one since renamed or removed. +private func isFile(_ descriptor: Int32, at url: URL) -> Bool { + var opened = stat() + var named = stat() + guard fstat(descriptor, &opened) == 0 + else { return false } + let found = url.withUnsafeFileSystemRepresentation { path -> Bool in + guard let path = path + else { return false } + return stat(path, &named) == 0 + } + return found && opened.st_ino == named.st_ino && opened.st_dev == named.st_dev +} +#endif diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift new file mode 100644 index 000000000..e1345af17 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventLogFormat.swift @@ -0,0 +1,228 @@ +import Foundation + +/// The on-disk shape of an event log. +/// +/// A log opens with a magic and a format version, and holds a sequence of frames: +/// +/// +- 4 bytes -+- 2 bytes -+ +- 2 bytes -+- 4 bytes -+- n bytes -+- 4 bytes -+ +/// | LDEV | version | ... | type | size (n) | payload | CRC-32 | +/// +-----------+-----------+ +-----------+-----------+-----------+-----------+ +/// +/// The size prefix is what makes a log recoverable: a reader can tell a frame the writer finished from one it did not, +/// without having to parse the event to find where it ends. The checksum, over the type, the size and the payload -- +/// everything before it in the frame -- is what tells a frame the writer finished from one the storage has since +/// damaged; a reader treats either the same way. +/// The version is what makes an upgrade safe -- a log this version does not understand is discarded rather than +/// misread -- and the frame type leaves room for a later version to write something new into a log this one still +/// reads. +enum EventLogFormat { + /// Bump this whenever the framing or the meaning of a frame changes. + static let version: UInt16 = 1 + static let magic = Data("LDEV".utf8) + static let fileHeaderSize = 6 + /// The type and the size, before a frame's payload. + static let frameHeaderSize = 6 + /// The checksum, after a frame's payload. + static let frameTrailerSize = 4 + /// What a frame takes beyond its payload. + static let frameOverhead = frameHeaderSize + frameTrailerSize + /// The only frame type written today; a reader skips a frame whose type it does not know. + static let eventFrame: UInt16 = 1 + /// The most a batch may hold, file header included, so that no request it becomes is larger than the events + /// service accepts. + static let maxBatchSize = 2 * 1024 * 1024 + /// A ceiling on a single frame's payload: the most that still fits in a batch on its own. It is also what keeps a + /// corrupt size from making recovery allocate wildly. + static let maxFrameSize = maxBatchSize - fileHeaderSize - frameOverhead + + static var fileHeader: Data { + var header = magic + header.append(bigEndian(version)) + return header + } + + static func frame(for encodedEvent: Data) -> Data { + var frame = Data(capacity: frameOverhead + encodedEvent.count) + withUnsafeBytes(of: eventFrame.bigEndian) { frame.append(contentsOf: $0) } + withUnsafeBytes(of: UInt32(encodedEvent.count).bigEndian) { frame.append(contentsOf: $0) } + frame.append(encodedEvent) + var checksum = Crc32() + frame.withUnsafeBytes { checksum.advance(withBytes: $0) } + withUnsafeBytes(of: checksum.checksum.bigEndian) { frame.append(contentsOf: $0) } + return frame + } + + /// What walking a log's frames found. + struct FrameWalk { + /// False when the walk stopped at a frame that is torn or fails its checksum, rather than at the end of the + /// file. + let reachedEnd: Bool + } + + /// Walks the frames of a log, stopping at the first one that is not wholly there or fails its checksum. + /// + /// A process that died partway through a write leaves a torn frame at the end. Everything before it is intact, so + /// recovery keeps that and drops only the tail. A frame the storage damaged afterwards ends the walk the same way, + /// because its size can no longer be trusted to say where the next frame starts. Returns nil when the file is not a + /// log this version reads. + @discardableResult + static func forEachFrame(in file: Data, _ visit: (UInt16, Data) -> Void) -> FrameWalk? { + forEachFramePayload(in: file) { type, payload in + visit(type, file.subdata(in: (file.startIndex + payload.lowerBound)..<(file.startIndex + payload.upperBound))) + } + } + + /// `forEachFrame(in:_:)` handing over where each payload is, as offsets into the file, rather than a copy of it. + @discardableResult + static func forEachFramePayload(in file: Data, _ visit: (UInt16, Range) -> Void) -> FrameWalk? { + guard isReadable(file) + else { return nil } + return file.withUnsafeBytes { walkFrames($0, visit) } + } + + private static func isReadable(_ file: Data) -> Bool { + file.count >= fileHeaderSize && file.prefix(magic.count) == magic && readUInt16(file, at: magic.count) == version + } + + /// Hands over each frame once it has been checked, and before the next one is read, so that `visit` may write + /// anywhere before where the frame ends. + private static func walkFrames(_ bytes: UnsafeRawBufferPointer, _ visit: (UInt16, Range) -> Void) -> FrameWalk { + var cursor = fileHeaderSize + while cursor + frameOverhead <= bytes.count { + let type = UInt16(bytes[cursor]) << 8 | UInt16(bytes[cursor + 1]) + let length = Int(readUInt32(bytes, at: cursor + 2)) + let start = cursor + frameHeaderSize + let end = start + length + guard length > 0, length <= maxFrameSize, end + frameTrailerSize <= bytes.count + else { break } + + var checksum = Crc32() + checksum.advance(withBytes: UnsafeRawBufferPointer(rebasing: bytes[cursor.. UInt32 { + UInt32(bytes[offset]) << 24 | UInt32(bytes[offset + 1]) << 16 | UInt32(bytes[offset + 2]) << 8 | UInt32(bytes[offset + 3]) + } + + /// How many events a log holds, or nil where it is not one this version reads. + static func eventCount(in file: Data) -> Int? { + var events = 0 + guard forEachFramePayload(in: file, { type, _ in + if type == eventFrame { + events += 1 + } + }) != nil + else { return nil } + return events + } + + /// Whether a log holds frames that cannot be read back: torn, or failing their checksum. False for a log this + /// version does not read at all. + static func isDamaged(_ file: Data) -> Bool { + forEachFramePayload(in: file) { _, _ in }?.reachedEnd == false + } + + /// Turns a log into the JSON array LaunchDarkly expects, in the log's own bytes, without parsing the events + /// themselves: they were serialized on the way in and are shipped exactly as they were recorded. + /// + /// There is always room: the file header becomes the "[", and each frame's header and trailer the "," or "]" after + /// its payload. + /// Each payload moves towards the start only once its frame has been checked, and never past where that frame + /// began, so no frame still to be checked is written over. A log nothing else refers to is changed where it is, + /// rather than copied first. + /// + /// Only the first `maxEvents` events go into it; any after them are left out. So is everything from a frame that is + /// torn or fails its checksum on, which is why the count it returns can be fewer than the log was closed holding. + /// + /// - Returns: how many events went into it. Where none did, or the log is not one this version reads, it is left as + /// it was. + static func assembleBody(inPlace file: inout Data, maxEvents: Int = .max) -> Int { + guard isReadable(file) + else { return 0 } + + let (length, assembled) = file.withUnsafeMutableBytes { bytes -> (Int, Int) in + var cursor = 1 + var events = 0 + _ = walkFrames(UnsafeRawBufferPointer(bytes)) { type, payload in + guard type == eventFrame, events < maxEvents + else { return } + memmove(bytes.baseAddress! + cursor, bytes.baseAddress! + payload.lowerBound, payload.count) + cursor += payload.count + bytes[cursor] = UInt8(ascii: ",") + cursor += 1 + events += 1 + } + guard events > 0 + else { return (0, 0) } + bytes[0] = UInt8(ascii: "[") + bytes[cursor - 1] = UInt8(ascii: "]") + return (cursor, events) + } + guard assembled > 0 + else { return 0 } + file.count = length + return assembled + } + + /// `assembleBody(inPlace:maxEvents:)` into bytes of its own, leaving the log as it was. + static func assembleBody(from file: Data, maxEvents: Int = .max) -> Data? { + var body = file + return assembleBody(inPlace: &body, maxEvents: maxEvents) > 0 ? body : nil + } + + static func bigEndian(_ value: UInt16) -> Data { + Data([UInt8(truncatingIfNeeded: value >> 8), UInt8(truncatingIfNeeded: value)]) + } + + static func bigEndian(_ value: UInt32) -> Data { + Data([UInt8(truncatingIfNeeded: value >> 24), + UInt8(truncatingIfNeeded: value >> 16), + UInt8(truncatingIfNeeded: value >> 8), + UInt8(truncatingIfNeeded: value)]) + } + + /// The size, header and trailer included, of the frame at `start` in frames this process wrote. + static func frameSize(in frames: Data, at start: Int) -> Int { + frameOverhead + Int(readUInt32(frames, at: start + 2)) + } + + /// Where a run of whole frames from `start`, in frames this process wrote, ends if it is to take no more than + /// `room` bytes, and how many frames it takes. It takes the first frame whatever its size. + static func framesFitting(in frames: Data, from start: Int, room: Int) -> (end: Int, count: Int) { + var end = start + frameSize(in: frames, at: start) + var count = 1 + while end < frames.count, end + frameSize(in: frames, at: end) - start <= room { + end += frameSize(in: frames, at: end) + count += 1 + } + return (end, count) + } + + /// How many whole frames lie from `start` up to `end`, in frames this process wrote. + static func frameCount(in frames: Data, from start: Int, to end: Int) -> Int { + var count = 0 + var at = start + while at < end { + at += frameSize(in: frames, at: at) + count += 1 + } + return count + } + + static func readUInt16(_ data: Data, at offset: Int) -> UInt16 { + let start = data.startIndex + offset + return UInt16(data[start]) << 8 | UInt16(data[start + 1]) + } + + static func readUInt32(_ data: Data, at offset: Int) -> UInt32 { + let start = data.startIndex + offset + return UInt32(data[start]) << 24 | UInt32(data[start + 1]) << 16 | UInt32(data[start + 2]) << 8 | UInt32(data[start + 3]) + } +} diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift new file mode 100644 index 000000000..1dce1412a --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventProcessor.swift @@ -0,0 +1,125 @@ +import Foundation + +typealias EventSyncCompleteClosure = ((SynchronizingError?) -> Void) +/// Reports whether LaunchDarkly accepted every event a flush covered. +typealias FlushOutcomeClosure = (Bool) -> Void +// sourcery: autoMockable +protocol EventProcessor { + // sourcery: defaultMockValue = false + var isOnline: Bool { get set } + // sourcery: defaultMockValue = Date.distantPast + var lastEventResponseDate: Date { get } + + func record(_ event: Event) + // swiftlint:disable:next function_parameter_count + func recordFlagEvaluationEvents(flagKey: LDFlagKey, + value: LDValue, + defaultValue: LDValue, + featureFlag: FeatureFlag?, + context: LDContext, + includeReason: Bool) + func flush(completion: CompletionClosure?) + + /// Like `flush`. Reports `true` if LaunchDarkly accepted the pending batches, or there were none to send. Reports + /// `false` if the reporter is offline, a batch was refused for good, a retryable failure left batches on disk, or + /// an event the SDK had accepted could not be kept to send at all. + /// + /// Flushes arriving faster than a delivery completes join the one that is queued, since a delivery that has not + /// started yet will send their events too. A delivery that has already started is not joined, so an answer is + /// always about the batch the caller's own delivery took, never one an earlier delivery lost. + /// + /// The completion, like `flush`'s, runs on a background queue, and never on the reporter's own: a completion is + /// free to record events or flush again without waiting on the queue it is running on. It is not the caller's + /// thread either, and two completions may run at once, so anything it touches needs to expect that. + /// + /// Unlike `flush`, it writes nothing on the caller's thread: wherever events persist, it queues that write, so a + /// caller with a deadline is not held by the disk before it can start waiting. A `true` still means the events + /// were written, since the delivery writes them before it sends. + func flushReportingOutcome(completion: @escaping FlushOutcomeClosure) + + /// `flushReportingOutcome` for a caller about to lose the ability to send, `LDClient.flushAndWait`: it posts + /// whatever wait the backoff is in, as `flushAtClose()` does. After a refusal that wait is minutes to an hour, and + /// where nothing persists the events would not outlive it. + func flushBypassingBackoff(completion: @escaping FlushOutcomeClosure) + + /// The last delivery before the client closes: `flush`, except that it posts whatever wait the backoff is in, which + /// closing cuts short. After a refusal that wait is minutes to an hour, and where nothing persists the events would + /// not outlive it. + /// + /// Whether it posts is settled by whether the reporter is online when this is called, so the client going offline + /// as it closes, straight after, does not stop it. + func flushAtClose() + + /// Makes everything recorded so far outlive the process, without waiting for a delivery. + /// + /// The SDK does this itself at the points where an application is most likely to be about to die. It is worth + /// calling directly before deliberately terminating the process. + func commitRecordedEvents() + + /// Commits before returning wherever events persist, at `.deferred` as well as at `.immediate`, and not at all where + /// nothing is persisted, where a commit would only move the encode earlier. + /// + /// What a flush does, and what entering the background does: both are points the process may end soon after, where + /// a write that was only queued may never happen. + func commitWherePersisting() +} + +extension Array where Element == EventProcessor { + /// Starts every processor's delivery at once, and gives `completion` each one's outcome, in order, once all of them + /// have answered or `deadline` has passed, whichever is first. A processor that has not answered by the deadline + /// counts as not delivered, and its answer, should it come later, is ignored. With no processors it answers at + /// once, on the calling thread. + /// + /// The completion runs once, on the thread of the last answer, or a global queue at the deadline. A reporter + /// answering twice neither ends the wait before every processor has answered nor ends it again: its second answer + /// is ignored. + /// + /// - parameter deadline: where a caller cannot wait on a reporter that never answers. Without one, the completion + /// waits for every answer. + func flushReportingOutcomes(bypassingBackoff: Bool = false, + deadline: DispatchTime? = nil, + completion: @escaping (_ delivered: [Bool]) -> Void) { + guard !isEmpty + else { + completion([]) + return + } + let lock = UnfairLock() + /// Only to be used while holding `lock`: each processor answers from whichever thread its delivery ended on. + var outcomes = [Bool?](repeating: nil, count: count) + var outstanding = count + if let deadline { + DispatchQueue.global().asyncAfter(deadline: deadline) { + let ended: [Bool]? = lock.withLock { + guard outstanding > 0 + else { return nil } + outstanding = 0 + outcomes = outcomes.map { $0 ?? false } + return outcomes.compactMap { $0 } + } + if let ended { + completion(ended) + } + } + } + for (index, processor) in enumerated() { + let answer: FlushOutcomeClosure = { delivered in + let ended: [Bool]? = lock.withLock { + guard outcomes[index] == nil + else { return nil } + outcomes[index] = delivered + outstanding -= 1 + return outstanding == 0 ? outcomes.compactMap { $0 } : nil + } + if let ended { + completion(ended) + } + } + if bypassingBackoff { + processor.flushBypassingBackoff(completion: answer) + } else { + processor.flushReportingOutcome(completion: answer) + } + } + } +} diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift deleted file mode 100644 index b2e15ef04..000000000 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventReporter.swift +++ /dev/null @@ -1,426 +0,0 @@ -import Foundation -import OSLog - -typealias EventSyncCompleteClosure = ((SynchronizingError?) -> Void) -/// Reports whether LaunchDarkly accepted every event a flush covered. -typealias FlushOutcomeClosure = (Bool) -> Void -// sourcery: autoMockable -protocol EventReporting { - // sourcery: defaultMockValue = false - var isOnline: Bool { get set } - // sourcery: defaultMockValue = Date.distantPast - var lastEventResponseDate: Date { get } - - func record(_ event: Event) - // swiftlint:disable:next function_parameter_count - func recordFlagEvaluationEvents(flagKey: LDFlagKey, value: LDValue, defaultValue: LDValue, featureFlag: FeatureFlag?, context: LDContext, includeReason: Bool) - func flush(completion: CompletionClosure?) - - /// Like `flush`. Reports `true` if LaunchDarkly accepted the events, or there were none to send. Reports `false` if - /// the reporter is offline, or any of the events were lost: refused, still failing after the retry, or unable to - /// be serialized. - /// - /// Flushes arriving faster than a delivery completes join the one that is queued, since a delivery that has not - /// started yet will send their events too. A delivery that has already started is not joined, so an answer is - /// always about the batch the caller's own delivery took, never one an earlier delivery lost. - /// - /// The completion, like `flush`'s, runs on a background queue, and never on the reporter's own: a completion is - /// free to record events or flush again without waiting on the queue it is running on. It is not the caller's - /// thread either, and two completions may run at once, so anything it touches needs to expect that. - func flushReportingOutcome(completion: @escaping FlushOutcomeClosure) -} - -class NullEventReporter: EventReporting { - var isOnline: Bool = true - var lastEventResponseDate: Date = Date() - - func record(_ event: Event) { - } - - func recordFlagEvaluationEvents(flagKey: LDFlagKey, value: LDValue, defaultValue: LDValue, featureFlag: FeatureFlag?, context: LDContext, includeReason: Bool) { - } - - func flush(completion: CompletionClosure?) { - completion?() - } - - func flushReportingOutcome(completion: @escaping FlushOutcomeClosure) { - completion(true) - } -} - -class EventReporter: EventReporting { - var isOnline: Bool { - get { timerQueue.sync { eventReportTimer != nil } } - set { timerQueue.sync { newValue ? startReporting() : stopReporting() } } - } - - private(set) var lastEventResponseDate: Date - - let service: DarklyServiceProvider - - private let eventQueue = DispatchQueue(label: "com.launchdarkly.eventSyncQueue", qos: .userInitiated) - // These fields should only be used synchronized on the eventQueue - private(set) var eventStore: [Event] = [] - private(set) var contextSummarizer: ContextSummarizer - - /// Guards `queuedDelivery`. Taken on a caller's thread and on `eventQueue`, and never held across either. - private let deliveryLock = UnfairLock() - /// Callers waiting on a delivery that is queued and has not taken the event store yet, or nil where none is - /// queued. - private var queuedDelivery: [FlushOutcomeClosure]? - - /// How many deliveries have started, which is fewer than the flushes asked for where they were joined. Only to be - /// used while holding `deliveryLock`. - private var deliveryRunCount = 0 - - private var timerQueue = DispatchQueue(label: "com.launchdarkly.EventReporter.timerQueue") - private var eventReportTimer: TimeResponding? - var isReportingActive: Bool { eventReportTimer != nil } - - private let onSyncComplete: EventSyncCompleteClosure? - - /// Shared across concurrent flushes, so it must not be mutated after `init`. - private let encoder: JSONEncoder - - let encoding: Encoding - - init(service: DarklyServiceProvider, - onSyncComplete: EventSyncCompleteClosure?, - encoding: Encoding = .handWritten) { - self.service = service - self.onSyncComplete = onSyncComplete - self.lastEventResponseDate = Date() - self.encoding = encoding - self.encoder = EventReporter.makeEncoder(config: service.config) - self.contextSummarizer = ContextSummarizer(logger: service.config.logger, maxContexts: service.config.eventCapacity) - } - - func record(_ event: Event) { - // The eventReporter is created when the LDClient singleton is created, and kept for the app's lifetime. So while the use of self in the async block does setup a retain cycle, it's not going to cause a memory leak - eventQueue.sync { recordNoSync(event) } - } - - func recordNoSync(_ event: Event) { - if self.eventStore.count >= self.service.config.eventCapacity { - os_log("%s aborted. Event store is full", log: service.config.logger, type: .debug, typeName(and: #function)) - self.service.diagnosticCache?.incrementDroppedEventCount() - return - } - self.eventStore.append(event) - } - - // swiftlint:disable:next function_parameter_count - func recordFlagEvaluationEvents(flagKey: LDFlagKey, value: LDValue, defaultValue: LDValue, featureFlag: FeatureFlag?, context: LDContext, includeReason: Bool) { - let recordingFeatureEvent = featureFlag?.trackEvents == true - let recordingDebugEvent = featureFlag?.shouldCreateDebugEvents(lastEventReportResponseTime: lastEventResponseDate) ?? false - - eventQueue.sync { - // A refused evaluation is a loss like a refused event, since nothing later reconstructs its counter. Its - // full event is still decided by the event capacity alone. - if !contextSummarizer.trackRequest(flagKey: flagKey, reportedValue: value, featureFlag: featureFlag, defaultValue: defaultValue, context: context) { - service.diagnosticCache?.incrementDroppedEventCount() - } - if recordingFeatureEvent { - let featureEvent = FeatureEvent(key: flagKey, context: context, value: value, defaultValue: defaultValue, featureFlag: featureFlag, includeReason: includeReason, isDebug: false) - recordNoSync(featureEvent) - } - if recordingDebugEvent { - let debugEvent = FeatureEvent(key: flagKey, context: context, value: value, defaultValue: defaultValue, featureFlag: featureFlag, includeReason: includeReason, isDebug: true) - recordNoSync(debugEvent) - } - } - } - - private func startReporting() { - guard eventReportTimer == nil - else { return } - eventReportTimer = LDTimer(withTimeInterval: service.config.eventFlushInterval, fireQueue: eventQueue, execute: queueScheduledDelivery) - } - - private func stopReporting() { - eventReportTimer?.cancel() - eventReportTimer = nil - } - - func flush(completion: CompletionClosure?) { - // Nil stays nil, so a flush nobody waits on joins a queued delivery without adding to its waiters. - queueDelivery(completion.map { done in { _ in done() } }) - } - - func flushReportingOutcome(completion: @escaping FlushOutcomeClosure) { - queueDelivery(completion) - } - - private func queueScheduledDelivery() { - queueDelivery(nil) - } - - /// Queues a delivery, or joins one that is queued and has not started. - /// - /// A delivery that has not started yet will send everything recorded before it starts, which includes this - /// caller's events, so joining it is as good as queueing another. Without this, flushes arriving faster than a - /// post completes each queue their own, and the one that matters -- the flush at shutdown -- waits behind all of - /// them. - /// - /// A delivery that has already started is not joined. It has taken the event store, so it cannot speak for - /// anything recorded since, and holding later deliveries back until it answers would leave the store growing - /// through a retry while nothing drained it. - private func queueDelivery(_ completion: FlushOutcomeClosure?) { - deliveryLock.lock() - if queuedDelivery != nil { - if let completion { - queuedDelivery?.append(completion) - } - deliveryLock.unlock() - return - } - queuedDelivery = completion.map { [$0] } ?? [] - deliveryLock.unlock() - - eventQueue.async { self.runDelivery() } - } - - private func runDelivery() { - // Requests arriving from here on need a delivery of their own: this one is about to take the event store, and - // what it takes is all it can speak for. - deliveryLock.lock() - let waiters = queuedDelivery ?? [] - queuedDelivery = nil - deliveryRunCount += 1 - deliveryLock.unlock() - - reportEvents { delivered in - guard !waiters.isEmpty - else { return } - // Answered away from `eventQueue`, and from whichever thread the delivery ended on, so that every - // completion reaches its caller the same way. A completion is free to record or flush again: it is never - // running on the queue those wait for. - DispatchQueue.global().async { - for waiter in waiters { - waiter(delivered) - } - } - } - } - - private func reportEvents(completion: FlushOutcomeClosure?) { - guard isOnline - else { - os_log("%s aborted. EventReporter is offline", log: service.config.logger, type: .debug, typeName(and: #function)) - reportSyncComplete(.isOffline) - completion?(false) - return - } - - if contextSummarizer.hasLoggedRequests { - let summaries = contextSummarizer.getSummaries() - for summary in summaries { - let summaryEvent = SummaryEvent(flagRequestTracker: summary.tracker, context: summary.context) - self.eventStore.append(summaryEvent) - } - contextSummarizer.clear() - } - - guard !eventStore.isEmpty - else { - os_log("%s aborted. Event store is empty", log: service.config.logger, type: .debug, typeName(and: #function)) - reportSyncComplete(nil) - completion?(true) - return - } - - os_log("%s starting", log: service.config.logger, type: .debug, typeName(and: #function)) - - let toPublish = self.eventStore - self.eventStore = [] - - service.diagnosticCache?.recordEventsInLastBatch(eventsInLastBatch: toPublish.count) - - DispatchQueue.global().async { - self.publish(toPublish, UUID().uuidString, completion) - } - } - - private func publish(_ events: [Event], _ payloadId: String, _ completion: FlushOutcomeClosure?) { - guard let (eventData, isComplete) = encode(events) - else { - os_log("%s Failed to serialize event(s) for publication: %s", log: service.config.logger, type: .error, typeName(and: #function), String(describing: events)) - // Encoding is deterministic, so no retry would succeed; the events are lost. - completion?(false) - return - } - self.service.publishEventData(eventData, payloadId) { response in - switch self.processEventResponse(sentEvents: events.count, response: response.urlResponse as? HTTPURLResponse, error: response.error, isRetry: false) { - case .accepted: - completion?(isComplete) - case .refused, .dropped: - completion?(false) - case .retryable: - os_log("%s Retrying event post after delay.", log: self.service.config.logger, type: .debug, self.typeName(and: #function)) - DispatchQueue.global().asyncAfter(deadline: DispatchTime.now() + 1.0) { - self.service.publishEventData(eventData, payloadId) { response in - let outcome = self.processEventResponse(sentEvents: events.count, response: response.urlResponse as? HTTPURLResponse, error: response.error, isRetry: true) - completion?(outcome == .accepted && isComplete) - } - } - } - } - } - - private enum DeliveryOutcome { - case accepted - /// Refused with a status that no retry would change. The events are dropped. - case refused - case retryable - /// Failed with no retry left. - case dropped - } - - /// Encodes a run of events as the array the events endpoint takes. - /// - /// If the run cannot be encoded as a whole, each event is retried on its own. Events that fail are dropped rather - /// than put back, because encoding is deterministic and they would fail every later flush too. `isComplete` is - /// false when any were dropped. - private func encode(_ events: [Event]) -> (data: Data, isComplete: Bool)? { - guard encoding != .codable - else { - if let data = try? encoder.encode(events) { - return (data, true) - } - return encodeSkippingFailures(events, using: { try encoder.encode($0) }) - } - - let eventJSONWriter = EventJSONWriter(config: service.config) - return encodeSkippingFailures(events, using: { event in - guard let encoded = eventJSONWriter.encode(event) - else { throw EventEncodingError.handWritten } - return encoded - }) - } - - private func encodeSkippingFailures(_ events: [Event], using encodeOne: (Event) throws -> Data) -> (data: Data, isComplete: Bool)? { - var payload = Data([UInt8(ascii: "[")]) - var written = 0 - for event in events { - let encoded: Data - do { - encoded = try encodeOne(event) - } catch { - os_log("%s dropping unserializable event: %s", - log: service.config.logger, - type: .error, - typeName(and: #function), - String(describing: event)) - continue - } - if written > 0 { - payload.append(UInt8(ascii: ",")) - } - payload.append(encoded) - written += 1 - } - guard written > 0 - else { return nil } - payload.append(UInt8(ascii: "]")) - return (payload, written == events.count) - } - - private enum EventEncodingError: Error { - case handWritten - } - - private func processEventResponse(sentEvents: Int, response: HTTPURLResponse?, error: Error?, isRetry: Bool) -> DeliveryOutcome { - if error == nil && (200..<300).contains(response?.statusCode ?? 0) { - let serverTime = response?.headerDate ?? self.lastEventResponseDate - if serverTime > self.lastEventResponseDate { - self.lastEventResponseDate = serverTime - } - - os_log("%s Completed sending %d event(s)", log: service.config.logger, type: .debug, typeName(and: #function), sentEvents) - self.reportSyncComplete(nil) - return .accepted - } - - if let statusCode = response?.statusCode, HTTPURLResponse.StatusCodes.isTerminalStatusCode(statusCode) { - os_log("%s dropping events due to non-retriable response: %s", log: service.config.logger, type: .debug, typeName(and: #function), String(describing: response)) - isOnline = false - self.reportSyncComplete(.response(response)) - return .refused - } - - os_log("%s Sending events failed with error: %s response: %s", log: service.config.logger, type: .debug, typeName(and: #function), String(describing: error), String(describing: response)) - - if isRetry { - os_log("%s dropping events due to failed retry", log: service.config.logger, type: .debug, typeName(and: #function)) - if let error = error { - reportSyncComplete(.request(error)) - } else { - reportSyncComplete(.response(response)) - } - return .dropped - } - - return .retryable - } - - private func reportSyncComplete(_ result: SynchronizingError?) { - // The eventReporter is created when the LDClient singleton is created, and kept for the app's lifetime. So while the use of self in the async block does setup a retain cycle, it's not going to cause a memory leak - guard let onSyncComplete = onSyncComplete - else { return } - DispatchQueue.main.async { - onSyncComplete(result) - } - } -} - -extension EventReporter { - /// Which encoder recorded events go through. - /// - /// `.codable` is the reference `.handWritten` is tested against. The two produce equal JSON but not identical - /// bytes: key order differs, as do `/` against `\/` and `0` against `-0`. - enum Encoding { - case codable - case handWritten - } - - fileprivate static func makeEncoder(config: LDConfig) -> JSONEncoder { - let encoder = JSONEncoder() - encoder.userInfo = [ - LDContext.UserInfoKeys.allAttributesPrivate: config.allContextAttributesPrivate, - LDContext.UserInfoKeys.globalPrivateAttributes: config.privateContextAttributes.map { $0 } - ] - encoder.dateEncodingStrategy = .custom { date, encoder in - var container = encoder.singleValueContainer() - try container.encode(date.millisSince1970) - } - return encoder - } -} - -extension EventReporter: TypeIdentifying { } - -extension EventReporter { - func setLastEventResponseDate(_ date: Date) { - lastEventResponseDate = date - } - - /// Occupies the queue a delivery runs on, so a test can see what one that has not started yet collects. - func occupyQueue(until released: DispatchSemaphore) { - eventQueue.async { _ = released.wait(timeout: .now() + 10) } - } - - /// Callers waiting on the queued delivery. - var queuedDeliveryWaiterCount: Int { - deliveryLock.lock() - defer { deliveryLock.unlock() } - return queuedDelivery?.count ?? 0 - } - - /// Deliveries started, counting one for a run of flushes that joined each other. - var startedDeliveryCount: Int { - deliveryLock.lock() - defer { deliveryLock.unlock() } - return deliveryRunCount - } -} diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift new file mode 100644 index 000000000..a9bff26c1 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/EventStore.swift @@ -0,0 +1,969 @@ +import Foundation +import OSLog + +/// A group of encoded events that has been closed off for delivery. +/// +/// The identifier doubles as the payload ID sent to LaunchDarkly, so a batch that is retried after the process died +/// mid-delivery is recognized upstream as the same delivery rather than counted twice. +struct EventBatch: Equatable { + let payloadId: String + let eventCount: Int +} + +/// The batches awaiting delivery, and whether that is all of them. +struct BatchListing: Equatable { + let batches: [EventBatch] + /// False where the directory could not be listed. `batches` are then the ones this process last knew to be there, + /// which includes every batch it closed itself, but can miss one another process closed or a previous run left. + let isComplete: Bool + /// The events in batches this process counted that have left the directory where nothing could have delivered + /// them; see `VanishedBatch`. No longer counted from this listing on, so no later listing reports them again. + var vanishedEvents = 0 +} + +/// The request body for a batch, and how many events made it in. +struct BatchPayload: Equatable { + let body: Data + /// Fewer than the batch was closed holding where a frame after the first is torn or fails its checksum: the + /// reader stops there, and the events from it on are not in `body`. + let eventCount: Int +} + +protocol EventStoring { + /// Events counted against capacity: those recorded and not yet accepted by LaunchDarkly, whether they are reserved + /// while being encoded, staged in memory, written to the open log, or sitting in a closed batch -- except a batch + /// whose request is out. + /// + /// A batch being sent is left out so that a flush makes room as soon as it sends, as it did when events were only + /// held in memory; otherwise an event recorded while the request is out is refused for a batch LaunchDarkly may + /// already have. A send that fails puts the batch back, so a full store can briefly hold capacity plus that batch: + /// one at a time, since deliveries run one after another, and no more than `EventLogFormat.maxBatchSize`. + var pendingEventCount: Int { get } + + /// Whether a full event of any size could still be staged within the byte capacity. + /// + /// Asked before an event is accepted, when it has not been encoded and so has no size yet. An event accepted while + /// there is room can still be refused at `stageReserved(_:)` if it turns out not to fit. + var hasRoomForEventBytes: Bool { get } + + /// `pendingEventCount` and `hasRoomForEventBytes` in one acquisition of the lock, for the check every recorded event + /// makes, where the lock is contended by the commit staging events. + /// + /// - Parameters: + /// - heldElsewhere: events the caller holds that the store does not count yet. + /// - capacity: the most events to hold, counting `heldElsewhere`. + /// - Returns: whether one more full event can be accepted, by count and by bytes. + func admitsEvent(heldElsewhere: Int, capacity: Int) -> Bool + + /// Whether a commit writes anything, which is false where persistence is off and once a write has failed and the + /// store has given up on it. + var isPersisting: Bool { get } + + /// Stages an already encoded event, returning false if it was dropped because `capacity` or the byte capacity is + /// reached. + /// + /// Staging is a copy into a buffer, and never writes however much is staged; `commit()` is what makes the event + /// outlive the process, and deciding when to call it is the caller's. + /// + /// - Parameter bypassingCapacity: true for a summary, which is not refused for `capacity` and may use the part of + /// the byte capacity held back from full events, but is still refused past the byte capacity itself, whether or + /// not events are persisted. + func stage(_ encodedEvent: Data, bypassingCapacity: Bool) -> Bool + + /// Counts events that have been accepted but are still being encoded, so capacity sees them until they are staged. + func reserve(_ events: Int) + + /// Stages an event using up one reservation, rather than checking `capacity` again. Its bytes are checked, since + /// they were not known when it was accepted. + func stageReserved(_ encodedEvent: Data) -> Bool + + /// Gives back every reservation not used up by `stageReserved(_:)`, such as one for an event that failed to encode. + func releaseReservations() + + /// Hands every staged byte to the kernel, so that the events recorded so far survive the process dying. + func commit() + + /// Commits, then closes the open log into a batch to deliver, and whatever is still staged in memory into another. + /// + /// Returns the batch closed from the log, or the first closed from memory where the log held nothing, and nil when + /// there is nothing to send. A log a commit filled was closed into a batch of its own as it filled, and memory is + /// closed into as many batches as keep each within `EventLogFormat.maxBatchSize`; `pendingBatches()` lists every + /// one. + func closeBatch() -> EventBatch? + + /// Whether committed events are outside every batch. Right after `closeBatch()`, that means it could not close the + /// log holding them; they stay in that log for the next close to try again. + var holdsUnclosedEvents: Bool { get } + + /// The committed events lost since this was last asked because the open log was deleted unsent, which a close then + /// finds gone: they are in no batch, and no longer counted as unclosed either. + func takeVanishedLogEvents() -> Int + + /// Batches awaiting delivery, oldest first, including any recovered from a previous run of the application. + /// + /// A directory that could not be listed leaves the batches this process last counted, and what it counts for them, + /// as they were. It does not empty them. + func listPendingBatches() -> BatchListing + + /// The `[...]` request body for a batch, or nil if it holds no events, or is gone where another process or store + /// may have delivered it, neither of which loses anything. + /// + /// - throws: `UnreadableBatch` if the batch is there but holds nothing this version can send, which no later + /// attempt changes and which loses its events; `VanishedBatch` if it is gone and nothing could have delivered it; + /// anything else if it could not be read this time, which a later attempt may manage. + func payload(of batch: EventBatch) throws -> BatchPayload? + + /// Forgets a batch, which is only correct once LaunchDarkly has accepted it or permanently refused it. + func remove(_ batch: EventBatch) + + /// Stops counting a batch against capacity while its request is out, and keeps expiry, in this process or any + /// other, from deleting it meanwhile. Each call is ended by one `stoppedSending(_:)`, after `remove(_:)` where the + /// batch was delivered or refused. + func startedSending(_ batch: EventBatch) + + /// Ends one `startedSending(_:)`: counts a batch against capacity again after a send that will be retried later, + /// and lets expiry have it once nobody is sending it. + func stoppedSending(_ batch: EventBatch) + + /// Closes every log no live process holds, so its events join the batches to deliver: one a previous run left open, + /// or another process that has since ended. + func preparePayloadsFromInterruptedLogs() + + /// Deletes what has been stored for longer than the maximum age, except a batch whose request is out. + /// + /// The first call also empties and removes the directories of other environments of this application that nothing + /// has written to for as long, which is what becomes of a mobile key the application stopped using. + /// + /// - Returns: how many events were discarded. + func discardExpired() -> Int +} + +extension EventStoring { + /// The batches `listPendingBatches()` lists, whether or not the listing was complete. + func pendingBatches() -> [EventBatch] { + listPendingBatches().batches + } + + /// The body `payload(of:)` reads, however many events made it in. + func body(of batch: EventBatch) throws -> Data? { + try payload(of: batch)?.body + } +} + +/// A batch that is there to read and holds nothing this version can send: damaged, or written in a format it does not +/// read. Unlike a batch another process delivered, its events are lost. +struct UnreadableBatch: Error { + let payloadId: String +} + +/// A batch that held events and is gone where nothing could have delivered it: no other process uses the directory, +/// and no store in this one removed it. Something deleted it unsent, such as tvOS reclaiming its caches, so its events +/// are lost. +struct VanishedBatch: Error { + let payloadId: String +} + +/// A crash-durable append log of encoded analytics events. +/// +/// Events are appended to an open log, which is closed into a batch when the reporter is ready to deliver one. A batch +/// is deleted only once LaunchDarkly has accepted it, so a delivery interrupted by the process dying is retried on the +/// next run rather than lost. +/// +/// Bytes reach the kernel through `write(2)` and are never `fsync`ed. What this defends against is the process dying -- +/// which is what `fatalError`, an uncaught exception, and the OS killing a backgrounded application all are -- and a +/// process cannot take back bytes the kernel already holds; the kernel flushes them on its own schedule. Only losing +/// the kernel itself, to a panic or a power cut, can discard them, and covering that would mean `F_FULLFSYNC` on every +/// event, which costs milliseconds where this costs microseconds. The trade that buys is stated where the SDK records +/// events: a commit point is a point where staged bytes are committed, and the events an application cares most +/// about are recorded at one. +/// +/// What a log looks like on disk is `EventLogFormat`; the directory of them, and the log this process appends to, are +/// `EventDirectory` and `DiskEventLog`. This is what decides when events move between memory, that log, and the +/// batches closed off for delivery, and what counts them against capacity on the way. +final class EventStore: EventStoring { + /// Where the events are kept, or nil where the application asked for them to be kept off the disk. + /// + /// Nil rather than a directory this store is careful never to use, so that there is nowhere for it to write and + /// nothing for it to read: what a run with persistence turned on left behind waits for the next run that has it + /// turned on. A store that gave up on the disk partway through a session keeps its directory, since the batches + /// there are its own. + private let disk: EventDirectory? + private let capacity: Int + /// The most bytes summaries may take the store to. + private let byteCapacity: Int + /// The most bytes full events may take the store to: the byte capacity less a fifth of it, held back so that + /// summaries, which bypass `capacity`, still have room once full events are being refused. + private let eventByteCapacity: Int + /// How long a batch may go unwritten before it is discarded rather than delivered. + private let maxAge: TimeInterval + private let logger: OSLog + + /// Guards the buffer and the counters. It is held for a copy and never across a syscall, so an evaluation + /// recording an event waits on another thread's memcpy at worst, never on the disk. + private let bufferLock = UnfairLock() + /// Serializes commits so that two threads committing at once cannot interleave a frame or write frames out of + /// order. Taken before `bufferLock`, never after it. + private let ioLock: UnfairLock + + /// Only to be used while holding `bufferLock`. + /// + /// Full events staged, `bufferedEventCount` of them. + private var bufferData = Data() + private var bufferedEventCount = 0 + /// Summaries staged, `bufferedSummaryCount` of them, apart from the full events because capacity does not count + /// them. Everything staged is taken as the full events followed by the summaries, so that any part of it can say + /// how many of its frames are summaries from where the summaries start. + private var summaryData = Data() + private var bufferedSummaryCount = 0 + /// How many of the frames counted in each state below are summaries, state for state, so capacity can leave them + /// out. Changed in the same critical sections as the counts beside them. A batch this store did not close -- one a + /// previous run or another process left -- counts none: its name says how many frames it holds, not what they are, + /// so every one counts as an event, which errs towards refusing. + private var writingSummaries = 0 + private var committedSummaries = 0 + private var closedSummaries = 0 + private var sendingSummaries = 0 + private var reservedEvents = 0 + /// Taken out of the buffer to be written to the log, or assembled into a batch held in memory, and not yet + /// counted where they land. Both happen outside `bufferLock`, and for as long as they take the events are in + /// neither place; without this they would be counted nowhere, and capacity would admit that many more. + private var writingEvents = 0 + private var committedEvents = 0 + /// Events in closed batches: the sum of what the directory holds and `inMemoryBatches`. Changed only in the + /// critical section that changes one of those, so that it cannot drift from them; a total worked out from them and + /// written afterwards would overwrite a close or a removal in between. + private var closedEvents = 0 + /// Closed batches whose request is out, by payload ID, and what is in them as `closedEvents` counts it. Every one is + /// also counted there: one is only added while it is counted, and leaves in the same critical section that stops + /// counting it, so `sendingEvents` can never exceed `closedEvents`. + private var sendingBatches: [String: EventTally] = [:] + private var sendingEvents = 0 + /// Committed events that left the count with an open log deleted unsent, until `takeVanishedLogEvents()` takes them. + private var vanishedLogEvents = 0 + + /// The bytes of the events counted above, state for state. Staged bytes are `bufferData` itself. Changed in the + /// same critical sections as the event counts beside them, so that bytes and events are never counted in + /// different places. + private var writingBytes = 0 + private var committedBytes = 0 + private var closedBytes = 0 + private var sendingBytes = 0 + + /// Whether the directories of other environments have been looked at for ones nobody uses any more, which is + /// done once per process. Only to be used while holding `ioLock`. + private var hasDiscardedAbandonedEnvironments = false + /// Whether events are being written to disk, which is what the application asked for until a write fails and the + /// store gives up on persistence for the rest of the session. + private var persistEvents = true + + /// Batches held in memory because the filesystem would not take them, or because it was never asked. + /// + /// Persistence failing should cost durability and nothing else, so the store falls back to what the SDK did before + /// it kept a log: hold the events, deliver them, lose them only if the process dies. Kept in the order they were + /// closed, so delivery still goes oldest first. + /// + /// Only to be used while holding `heldLock`. + private var inMemoryBatches: [HeldBatch] = [] + + /// Guards `inMemoryBatches`. + /// + /// Its own lock rather than `ioLock`, so that a store the application told to keep off the disk -- where held + /// batches are the only ones there are -- never waits on the lock disk I/O is serialized under. Taken inside + /// `ioLock` where there is a disk, never the other way round, and taken before `bufferLock`, never after it. + private let heldLock = UnfairLock() + + /// Where the batches are kept, or nil for a store told to keep events off the disk. + var directory: URL? { + ioLock.withLock { disk?.url } + } + + /// Where the log this process appends to is, for a test or a diagnosis to look at. + var openLogUrl: URL? { + ioLock.withLock { disk?.openLogUrl } + } + + #if os(macOS) || targetEnvironment(macCatalyst) + /// Whether the open log is locked while it is open. Only to be changed before the store first touches the disk. + var locksOpenLog: Bool { + get { ioLock.withLock { disk?.locksOpenLog ?? false } } + set { ioLock.withLock { disk?.locksOpenLog = newValue } } + } + #endif + + /// Whether other processes of the application may use this directory, and so deliver its batches. + var sharesDirectoryWithOtherProcesses: Bool { + get { ioLock.withLock { disk?.sharesDirectoryWithOtherProcesses ?? false } } + set { ioLock.withLock { disk?.sharesDirectoryWithOtherProcesses = newValue } } + } + + convenience init(directory: URL, + capacity: Int, + byteCapacity: Int = LDConfig.Defaults.eventCapacityBytes, + maxAge: TimeInterval = LDConfig.Defaults.eventMaxAge, + persistEvents: Bool = true, + logger: OSLog, + readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }, + writeLog: @escaping (EventLogFile, Data) -> Int32? = { $0.writeAll($1) }) { + self.init(locatingDirectory: { directory }, capacity: capacity, byteCapacity: byteCapacity, maxAge: maxAge, + persistEvents: persistEvents, logger: logger, readFile: readFile, writeLog: writeLog) + } + + /// - Parameter locatingDirectory: answers where the batches go. Asked once, and only if the store reaches for the + /// disk, which one told not to persist events never does. + init(locatingDirectory: @escaping () -> URL, + capacity: Int, + byteCapacity: Int = LDConfig.Defaults.eventCapacityBytes, + maxAge: TimeInterval = LDConfig.Defaults.eventMaxAge, + persistEvents: Bool = true, + logger: OSLog, + readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }, + writeLog: @escaping (EventLogFile, Data) -> Int32? = { $0.writeAll($1) }) { + let ioLock = UnfairLock() + self.ioLock = ioLock + self.capacity = capacity + self.byteCapacity = byteCapacity > 0 ? byteCapacity : LDConfig.Defaults.eventCapacityBytes + self.eventByteCapacity = self.byteCapacity - self.byteCapacity / 5 + self.maxAge = maxAge > 0 ? maxAge : LDConfig.Defaults.eventMaxAge + self.logger = logger + // An application that has not asked for persistence gets the same store running the same way it runs once a + // write has failed: events are held, delivered, and lost only if the process dies. It gets no directory at + // all, so there is nothing for it to write to and nothing of a previous run's for it to read. + self.persistEvents = persistEvents + self.disk = persistEvents ? EventDirectory(locatingDirectory: locatingDirectory, + ioLock: ioLock, + logger: logger, + readFile: readFile, + writeLog: writeLog) + : nil + } + + // MARK: Recording + + var pendingEventCount: Int { + bufferLock.withLock { countedEvents } + } + + /// Every event counted against capacity. Only to be used while holding `bufferLock`. + private var countedEvents: Int { + // Summaries left out: they are not events but the record of evaluations already counted, and counting them + // would let the ones a long outage writes at every commit point refuse full events. + reservedEvents + bufferedEventCount + (writingEvents - writingSummaries) + (committedEvents - committedSummaries) + + ((closedEvents - closedSummaries) - (sendingEvents - sendingSummaries)) + } + + /// Every byte counted against the byte capacity, summaries included. Only to be used while holding `bufferLock`. + /// + /// Events reserved while being encoded have no bytes yet; they are counted when they are staged. + private var countedBytes: Int { + bufferData.count + summaryData.count + writingBytes + committedBytes + (closedBytes - sendingBytes) + } + + var hasRoomForEventBytes: Bool { + bufferLock.withLock { countedBytes < eventByteCapacity } + } + + func admitsEvent(heldElsewhere: Int, capacity: Int) -> Bool { + bufferLock.withLock { heldElsewhere + countedEvents < capacity && countedBytes < eventByteCapacity } + } + + var isPersisting: Bool { + bufferLock.withLock { persistEvents } + } + + func reserve(_ events: Int) { + bufferLock.withLock { reservedEvents += events } + } + + func releaseReservations() { + bufferLock.withLock { reservedEvents = 0 } + } + + func stage(_ encodedEvent: Data, bypassingCapacity: Bool = false) -> Bool { + stage(encodedEvent, bypassingCapacity: bypassingCapacity, usingReservation: false) + } + + func stageReserved(_ encodedEvent: Data) -> Bool { + stage(encodedEvent, bypassingCapacity: true, usingReservation: true) + } + + private func stage(_ encodedEvent: Data, bypassingCapacity: Bool, usingReservation: Bool) -> Bool { + guard encodedEvent.count <= EventLogFormat.maxFrameSize + else { + os_log("%s dropping an event larger than a log frame allows", log: logger, type: .debug, typeName(and: #function)) + if usingReservation { + bufferLock.withLock { reservedEvents = max(0, reservedEvents - 1) } + } + return false + } + + // Framed before the lock is taken, so that the critical section is the append. + let frame = EventLogFormat.frame(for: encodedEvent) + // Only a summary bypasses capacity without a reservation, and only a summary may use the bytes held back. + let isSummary = bypassingCapacity && !usingReservation + let staged = bufferLock.withLock { () -> Bool in + // In the same critical section as the append, so the event is never counted twice or not at all. + if usingReservation { + reservedEvents = max(0, reservedEvents - 1) + } + // Bytes bound events kept off the disk too: capacity counts events, so held in memory through an outage it + // would allow it that many times whatever size they are, and a summary, which capacity never refuses, + // would otherwise go on growing for the life of the process. + guard bypassingCapacity || countedEvents < capacity, + countedBytes + frame.count <= (isSummary ? byteCapacity : eventByteCapacity) + else { return false } + + if isSummary { + summaryData.append(frame) + bufferedSummaryCount += 1 + } else { + bufferData.append(frame) + bufferedEventCount += 1 + } + return true + } + if !staged { + os_log("%s dropping an event the store has no room for", log: logger, type: .debug, typeName(and: #function)) + } + return staged + } + + func commit() { + // Where the application asked for no disk there is nothing for a commit to reach, and so nothing to serialize. + guard disk != nil + else { return } + ioLock.withLock { writeStagedBytes() } + } + + func closeBatch() -> EventBatch? { + guard let disk + else { return closeInMemoryBatches().first } + + ioLock.lock() + defer { ioLock.unlock() } + + writeStagedBytes() + + let (committed, stillBuffered) = bufferLock.withLock { + (EventTally(events: committedEvents, bytes: committedBytes, summaries: committedSummaries), + bufferedEventCount + bufferedSummaryCount) + } + + let fromLog = committed.events > 0 ? closeOpenLog(of: disk, holding: committed) : nil + // Closed after the log, so the older events are delivered first. Both can hold events at once: a write that + // fails after earlier ones landed gives up on persistence with those earlier events in the log and the failed + // ones back in memory, and closing only the log would leave the rest to a delivery that reports success + // without them. A full log that could not be closed leaves events staged too. + let fromMemory = stillBuffered > 0 ? closeInMemoryBatches().first : nil + return fromLog ?? fromMemory + } + + /// Requires `ioLock`. + private func closeOpenLog(of disk: EventDirectory, holding committed: EventTally) -> EventBatch? { + ioLock.assertOwned() + // The file is the frames behind its header. A listing counts it from the file itself. + let closed = EventTally(events: committed.events, bytes: committed.bytes + EventLogFormat.fileHeader.count, + summaries: committed.summaries) + switch disk.closeLog(holding: closed) { + case .failed: + // A log that is still there is still the open one, and its events go on counting as committed until a + // later close manages it. + return nil + case .vanished(let deletedUnsent): + bufferLock.withLock { + if deletedUnsent { + vanishedLogEvents += committedEvents + } + committedEvents = 0 + committedSummaries = 0 + committedBytes = 0 + } + return nil + case .closed(let batch): + bufferLock.withLock { + committedEvents = 0 + committedSummaries = 0 + committedBytes = 0 + closedEvents += closed.events + closedSummaries += closed.summaries + closedBytes += closed.bytes + } + return batch + } + } + + // MARK: Delivery + + var holdsUnclosedEvents: Bool { + bufferLock.withLock { committedEvents > 0 } + } + + func takeVanishedLogEvents() -> Int { + bufferLock.withLock { + defer { vanishedLogEvents = 0 } + return vanishedLogEvents + } + } + + func listPendingBatches() -> BatchListing { + // Listed in one critical section where there is a disk, so that the files and what is held cannot be counted + // either side of a close that moves events between them. + guard let disk + else { + return BatchListing(batches: heldBatchesAndRecount(after: [], storedSummaries: 0, storedBytes: 0), isComplete: true) + } + return ioLock.withLock { + let listing = disk.listBatches() + return BatchListing(batches: heldBatchesAndRecount(after: listing.batches, storedSummaries: disk.storedSummaries, + storedBytes: disk.storedBytes), + isComplete: listing.isComplete, + vanishedEvents: listing.vanishedEvents) + } + } + + /// Adds the batches the filesystem would not take, after the files: anything held was closed by this run and so is + /// newer than whatever a previous run left on the disk. Then counts everything listed as closed, which is how + /// batches another process closed or delivered come to be counted or not. + /// + /// Requires `ioLock` where there is a disk, so that the count is of the listing as it stands. + /// + /// - Parameters: + /// - storedSummaries: the summaries this store knows of in the batches listed from the disk. + /// - storedBytes: the bytes in the batches listed from the disk. + private func heldBatchesAndRecount(after onDisk: [EventBatch], storedSummaries: Int, storedBytes: Int) -> [EventBatch] { + heldLock.withLock { + let batches = onDisk + inMemoryBatches.map { EventBatch(payloadId: $0.payloadId, eventCount: $0.eventCount) } + bufferLock.withLock { + closedEvents = batches.reduce(0) { $0 + $1.eventCount } + closedSummaries = inMemoryBatches.reduce(storedSummaries) { $0 + $1.summaries } + closedBytes = inMemoryBatches.reduce(storedBytes) { $0 + $1.body.count } + // One no longer listed is no longer counted, so it cannot be left out of the count either. + let listed = Set(batches.map { $0.payloadId }) + sendingBatches.keys.filter { !listed.contains($0) }.forEach(forgetSending) + } + return batches + } + } + + func payload(of batch: EventBatch) throws -> BatchPayload? { + if let held = heldLock.withLock({ inMemoryBatches.first { $0.payloadId == batch.payloadId } }) { + return BatchPayload(body: held.body, eventCount: held.eventCount) + } + + // A batch is either held or on the disk, never both and never moved between them, so a batch that is not held + // in a store with no disk does not exist. + guard let disk + else { return nil } + + return try ioLock.withLock { try disk.payload(of: batch) } + } + + func remove(_ batch: EventBatch) { + // Only a count this call takes away stops counting. Two deliveries can both get to the same batch, and a listing + // drops a batch another process delivered before this one gets to remove it. + let wasHeld = heldLock.withLock { () -> Bool in + guard let index = inMemoryBatches.firstIndex(where: { $0.payloadId == batch.payloadId }) + else { return false } + let held = inMemoryBatches.remove(at: index) + bufferLock.withLock { + closedEvents -= held.eventCount + closedSummaries -= held.summaries + closedBytes -= held.body.count + forgetSending(batch.payloadId) + } + return true + } + + guard !wasHeld + else { return } + guard let disk + else { + bufferLock.withLock { forgetSending(batch.payloadId) } + return + } + ioLock.withLock { + let counted = disk.remove(batch.payloadId, holding: batch.eventCount) + bufferLock.withLock { + closedEvents -= counted.events + closedSummaries -= counted.summaries + closedBytes -= counted.bytes + forgetSending(batch.payloadId) + } + } + } + + func preparePayloadsFromInterruptedLogs() { + guard let disk + else { return } + // Each log is read without `ioLock`, one at a time, and the lock taken only to move it and count it. A commit + // at a commit point then waits for at most one rename rather than for every log a crash left to be read. + var recovered = 0 + for leftOpen in ioLock.withLock({ disk.interruptedLogs() }) { + guard let claim = DiskEventLog.claimForRecovery(leftOpen) + else { continue } + let interrupted = disk.read(interruptedLog: leftOpen) + ioLock.withLock { + let prepared = disk.preparePayload(from: interrupted) + guard prepared.events > 0 + else { return } + recovered += prepared.events + bufferLock.withLock { + closedEvents += prepared.events + closedSummaries += prepared.summaries + closedBytes += prepared.bytes + } + } + claim.release() + } + if recovered > 0 { + os_log("%s prepared %d event(s) from a previous run of this application for delivery", log: logger, type: .debug, typeName(and: #function), recovered) + } + } + + func discardExpired() -> Int { + guard let disk + else { return 0 } + let cutoff = Date().addingTimeInterval(-maxAge) + + let (discarded, directory) = ioLock.withLock { () -> (Int, URL?) in + let sending = bufferLock.withLock { Set(sendingBatches.keys) } + let (events, counted) = disk.discard(lastWrittenBefore: cutoff, sparing: sending) + // In the same critical section as the delete, as for any other path that loses events without delivering + // them, so the store cannot go on counting what is gone. + bufferLock.withLock { + closedEvents -= counted.events + closedSummaries -= counted.summaries + closedBytes -= counted.bytes + } + defer { hasDiscardedAbandonedEnvironments = true } + return (events, hasDiscardedAbandonedEnvironments ? nil : disk.url) + } + + // Outside `ioLock`: none of it is this store's directory, and a commit point should not wait on it. + guard let directory + else { return discarded } + return discarded + EventDirectory.discardAbandonedEnvironments(besides: directory, lastWrittenBefore: cutoff) + } + + deinit { + // One acquisition: the lock is not recursive, and `commit()` would take it again. + ioLock.withLock { + writeStagedBytes() + disk?.closeOpenLog() + } + } +} + +// MARK: Sending + +extension EventStore { + /// Stops counting a batch against capacity while its request is out. Each call is ended by one + /// `stoppedSending(_:)`. + /// + /// Does nothing to the count for a batch this store no longer counts - one `remove(_:)` forgot, or a listing + /// dropped because another process delivered it - since leaving it out would take its events from the batches + /// still counted. + /// + /// A batch on the disk is also locked against expiry, by this process and any other, until `stoppedSending(_:)`: + /// one deleted while its request is out would leave nothing to retry from. + func startedSending(_ batch: EventBatch) { + guard let disk + else { + startedSending(batch.payloadId, disk: nil) + return + } + // Held across the check and the change, so that the batch cannot stop being counted in between. + ioLock.withLock { + startedSending(batch.payloadId, disk: disk) + if !heldLock.withLock({ inMemoryBatches.contains { $0.payloadId == batch.payloadId } }) { + disk.startedSending(batch.payloadId, holding: batch.eventCount) + } + } + } + + /// Requires `ioLock` where there is a disk. + private func startedSending(_ payloadId: String, disk: EventDirectory?) { + heldLock.withLock { + let counted = inMemoryBatches.first { $0.payloadId == payloadId } + .map { EventTally(events: $0.eventCount, bytes: $0.body.count, summaries: $0.summaries) } + ?? disk?.counted(payloadId) + guard let counted + else { return } + bufferLock.withLock { + guard sendingBatches[payloadId] == nil + else { return } + sendingBatches[payloadId] = counted + sendingEvents += counted.events + sendingSummaries += counted.summaries + sendingBytes += counted.bytes + } + } + } + + /// Counts a batch against capacity again, after a request that did not end with it removed. Does nothing to the + /// count for a batch `remove(_:)` already forgot. Ends every `startedSending(_:)`, removed or not, so that the lock + /// against expiry is released either way. + func stoppedSending(_ batch: EventBatch) { + if let disk { + ioLock.withLock { disk.stoppedSending(batch.payloadId) } + } + bufferLock.withLock { forgetSending(batch.payloadId) } + } + + /// Requires `bufferLock`. + fileprivate func forgetSending(_ payloadId: String) { + if let sending = sendingBatches.removeValue(forKey: payloadId) { + sendingEvents -= sending.events + sendingSummaries -= sending.summaries + sendingBytes -= sending.bytes + } + } +} + +// MARK: Writing + +extension EventStore { + /// Requires `ioLock`. + /// + /// Closes the open log into a batch wherever the next frame would take it past `EventLogFormat.maxBatchSize`, so a + /// commit can land in several batches. + private func writeStagedBytes() { + ioLock.assertOwned() + let taken = bufferLock.withLock { () -> StagedFrames? in + // Staged bytes are left where they are. With nowhere durable to put them, memory is better than dropping + // them: a delivery can still close them into a batch and send them. + guard persistEvents, bufferedEventCount + bufferedSummaryCount > 0 + else { return nil } + return takeStaged() + } + guard let taken + else { return } + + let bytes = taken.frames + let base = bytes.startIndex + var written = 0 + while written < bytes.count { + let room = EventLogFormat.maxBatchSize - EventLogFormat.fileHeaderSize - bufferLock.withLock { committedBytes } + if EventLogFormat.frameSize(in: bytes, at: written) > room { + guard closeFullLog() + else { break } + continue + } + let (end, landing) = EventLogFormat.framesFitting(in: bytes, from: written, room: room) + let landingSummaries = taken.summaries(from: written, to: end) + let isWhole = written == 0 && end == bytes.count + guard append(isWhole ? bytes : bytes.subdata(in: (base + written)..<(base + end))) + else { break } + // Counted where they landed in the same critical section that stops counting them on the way, so a + // capacity check never sees them in both places or in neither. + bufferLock.withLock { + writingEvents -= landing + writingSummaries -= landingSummaries + writingBytes -= end - written + committedEvents += landing + committedSummaries += landingSummaries + committedBytes += end - written + } + written = end + } + + if written < bytes.count { + bufferLock.withLock { putBack(taken, from: written) } + } + } + + /// Takes everything staged, counting it as being written until it is counted where it lands. Requires `bufferLock`. + private func takeStaged() -> StagedFrames { + let staged = StagedFrames(frames: summaryData.isEmpty ? bufferData : bufferData + summaryData, + summariesFrom: bufferData.count, + events: bufferedEventCount + bufferedSummaryCount, + summaries: bufferedSummaryCount) + bufferData = Data() + summaryData = Data() + bufferedEventCount = 0 + bufferedSummaryCount = 0 + writingEvents += staged.events + writingSummaries += staged.summaries + writingBytes += staged.frames.count + return staged + } + + /// Puts back what of `staged` from `start` on never landed, to be delivered from memory rather than lost. Ahead of + /// anything staged while the write was in flight, so the events keep the order they were recorded in. Requires + /// `bufferLock`. + private func putBack(_ staged: StagedFrames, from start: Int) { + let frames = staged.frames + let base = frames.startIndex + let summariesFrom = max(start, staged.summariesFrom) + let events = EventLogFormat.frameCount(in: frames, from: start, to: summariesFrom) + let summaries = EventLogFormat.frameCount(in: frames, from: summariesFrom, to: frames.count) + writingEvents -= events + summaries + writingSummaries -= summaries + writingBytes -= frames.count - start + bufferData = frames.subdata(in: (base + start)..<(base + summariesFrom)) + bufferData + summaryData = frames.subdata(in: (base + summariesFrom).. Bool { + ioLock.assertOwned() + guard let disk + else { return false } + let committed = bufferLock.withLock { + EventTally(events: committedEvents, bytes: committedBytes, summaries: committedSummaries) + } + _ = closeOpenLog(of: disk, holding: committed) + return bufferLock.withLock { committedBytes == 0 } + } + + /// Requires `ioLock`. + private func append(_ bytes: Data) -> Bool { + ioLock.assertOwned() + guard let disk + else { return false } + + guard disk.append(bytes) + else { + // Never trap on a full disk. The SDK gives up on persistence for the rest of the session rather than taking + // the application down with it, which is how other SDKs have crashed their hosts. + disablePersistence() + return false + } + return true + } +} + +/// A batch the filesystem would not take, kept where the SDK used to keep all of them. +private struct HeldBatch { + let payloadId: String + let body: Data + let eventCount: Int + /// How many of `eventCount` are summaries. + let summaries: Int +} + +/// Everything staged, taken out of the buffers in one step: the full events, then the summaries. +private struct StagedFrames { + let frames: Data + /// Where in `frames` the summaries start. + let summariesFrom: Int + /// Every frame in `frames`, summaries included. + let events: Int + let summaries: Int + + /// How many of the frames from `start` up to `end` are summaries. + func summaries(from start: Int, to end: Int) -> Int { + end <= summariesFrom ? 0 : EventLogFormat.frameCount(in: frames, from: max(start, summariesFrom), to: end) + } +} + +/// What the store does once the filesystem has refused it. +/// +/// Losing the disk should cost durability and nothing else, so the store falls back to what the SDK did before it kept +/// a log: hold the events in memory, deliver them from there, and lose them only if the process dies. That is a worse +/// guarantee than Tier 3 and a better one than dropping the events, which is what a full device used to mean here. +private extension EventStore { + /// Gives up on the disk for the rest of the session, leaving the events themselves alone. + /// + /// Requires `ioLock`. + func disablePersistence() { + ioLock.assertOwned() + disk?.closeOpenLog() + bufferLock.withLock { persistEvents = false } + } + + /// Closes whatever is staged into batches that never reach a file, as many as it takes to keep each within + /// `EventLogFormat.maxBatchSize`. Returns them oldest first. + /// + /// Called under `ioLock` where there is a disk, so that it and the log are closed as one step, and on its own + /// where there is not. + func closeInMemoryBatches() -> [EventBatch] { + let staged = bufferLock.withLock { takeStaged() } + let frames = staged.frames + + // Assembled outside `bufferLock`, because it copies every staged frame; `writingEvents` is what keeps them + // counted meanwhile. Through the same reader a batch on disk goes through, so a held batch and a written one + // produce byte-identical request bodies. + let base = frames.startIndex + var closed: [EventBatch] = [] + var held = 0 + var heldEvents = 0 + var heldSummaries = 0 + while held < frames.count { + let (end, batchEvents) = EventLogFormat.framesFitting(in: frames, from: held, + room: EventLogFormat.maxBatchSize - EventLogFormat.fileHeaderSize) + let batchSummaries = staged.summaries(from: held, to: end) + var body = Data(capacity: EventLogFormat.fileHeaderSize + end - held) + body.append(EventLogFormat.fileHeader) + body.append(frames[(base + held)..<(base + end)]) + guard EventLogFormat.assembleBody(inPlace: &body) > 0 + else { break } + + let payloadId = UUID().uuidString + heldLock.withLock { + inMemoryBatches.append(HeldBatch(payloadId: payloadId, body: body, eventCount: batchEvents, + summaries: batchSummaries)) + // Counted where they landed in the same critical section that stops counting them on the way, so a + // capacity check never sees them in both places or in neither. A held batch is counted as its body, + // which is what it keeps. + bufferLock.withLock { + writingEvents -= batchEvents + writingSummaries -= batchSummaries + writingBytes -= end - held + closedEvents += batchEvents + closedSummaries += batchSummaries + closedBytes += body.count + } + } + closed.append(EventBatch(payloadId: payloadId, eventCount: batchEvents)) + held = end + heldEvents += batchEvents + heldSummaries += batchSummaries + } + + if held < frames.count { + bufferLock.withLock { + writingEvents -= staged.events - heldEvents + writingSummaries -= staged.summaries - heldSummaries + writingBytes -= frames.count - held + } + } + return closed + } +} + +extension EventStore { + /// Every event the store is holding, encoded exactly as it will be sent. + /// + /// Reading the log means committing what is staged, so this is not free and is meant for tests and for diagnosing a + /// store rather than for the recording path. + func pendingEventPayloads() -> [Data] { + commit() + + ioLock.lock() + defer { ioLock.unlock() } + + var payloads = disk?.storedPayloads() ?? [] + // After the disk, which is older: a store that gave up on persistence holds what came later in memory. + let held = heldLock.withLock { inMemoryBatches.map { $0.body } } + for body in held { + let events = (try? JSONDecoder().decode([LDValue].self, from: body)) ?? [] + payloads.append(contentsOf: events.compactMap { try? JSONEncoder().encode($0) }) + } + var staged = EventLogFormat.fileHeader + staged.append(bufferLock.withLock { bufferData + summaryData }) + _ = EventLogFormat.forEachFrame(in: staged) { type, payload in + if type == EventLogFormat.eventFrame { + payloads.append(payload) + } + } + return payloads + } +} + +extension EventStore: TypeIdentifying { } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventProcessor.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventProcessor.swift new file mode 100644 index 000000000..aad1ca9fc --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/NullEventProcessor.swift @@ -0,0 +1,45 @@ +import Foundation + +class NullEventProcessor: EventProcessor { + var isOnline: Bool = true + var lastEventResponseDate: Date = Date() + + func record(_ event: Event) { + } + + // swiftlint:disable:next function_parameter_count + func recordFlagEvaluationEvents(flagKey: LDFlagKey, + value: LDValue, + defaultValue: LDValue, + featureFlag: FeatureFlag?, + context: LDContext, + includeReason: Bool) { + } + + // There is never anything to send, but a completion still answers on a background queue, as + // `DirectEventProcessor`'s does. Were it answered inline, the threading a caller sees would depend on whether events + // are turned on, and a caller doing what the contract allows -- taking a lock it already holds, say -- would + // deadlock only there. + func flush(completion: CompletionClosure?) { + guard let completion + else { return } + DispatchQueue.global().async { completion() } + } + + func flushReportingOutcome(completion: @escaping FlushOutcomeClosure) { + DispatchQueue.global().async { completion(true) } + } + + func flushBypassingBackoff(completion: @escaping FlushOutcomeClosure) { + flushReportingOutcome(completion: completion) + } + + func flushAtClose() { + } + + func commitRecordedEvents() { + } + + func commitWherePersisting() { + } +} diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift new file mode 100644 index 000000000..301fd93c5 --- /dev/null +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/OutboundEventBuffer.swift @@ -0,0 +1,99 @@ +import Foundation +import OSLog + +/// Turns what `DirectEventProcessor` records into what it stages: evaluations into summary counters, and events into the +/// JSON the events endpoint accepts. Android's `OutboundEventBuffer` plays the same part. +/// +/// It has no lock of its own. The summarizer is only to be used while holding the processor's `recordLock`, and the +/// encoders are only reached under its `commitLock`. +final class OutboundEventBuffer { + /// Which encoder recorded events go through. + /// + /// `.codable` is the reference `.handWritten` is tested against. The two produce equal JSON but not identical + /// bytes: key order differs, as do `/` against `\/` and `0` against `-0`. + enum Encoding { + case codable + case handWritten + } + + let encoding: Encoding + private let logger: OSLog + + /// Only to be used while holding the processor's `recordLock`. + private(set) var contextSummarizer: ContextSummarizer + + /// Only to be used while holding the processor's `commitLock`. + private let encoder: JSONEncoder + + /// Outlives a commit rather than being built per commit, so that its buffer and context cache span them. + /// + /// A commit point writes a single event, so a cache that lived for one commit would never be read. It needs no + /// lock of its own because it is only ever reached from `encode(_:)`, which runs under the processor's + /// `commitLock`. + private let eventJSONWriter: EventJSONWriter + + init(config: LDConfig, encoding: Encoding) { + self.encoding = encoding + self.logger = config.logger + self.contextSummarizer = ContextSummarizer(logger: config.logger, maxContexts: config.eventCapacity) + self.encoder = OutboundEventBuffer.makeEncoder(config: config) + self.eventJSONWriter = EventJSONWriter(config: config) + } + + /// Counts an evaluation into its context's summary. Requires the processor's `recordLock`. + /// + /// - Returns: false if the evaluation was not counted, because its context is not counted yet and as many contexts + /// as the summarizer allows already are. + func summarize(flagKey: LDFlagKey, + reportedValue: LDValue, + featureFlag: FeatureFlag?, + defaultValue: LDValue, + context: LDContext) -> Bool { + contextSummarizer.trackRequest(flagKey: flagKey, reportedValue: reportedValue, featureFlag: featureFlag, defaultValue: defaultValue, context: context) + } + + /// Takes the counters gathered so far, leaving the summarizer empty, or nothing if nothing was counted. Requires the + /// processor's `recordLock`. + func takeSummaries() -> [(tracker: FlagRequestTracker, context: LDContext)] { + let summaries = contextSummarizer.hasLoggedRequests ? contextSummarizer.getSummaries() : [] + if !summaries.isEmpty { + contextSummarizer.clear() + } + return summaries + } + + /// Requires the processor's `commitLock`. + func encode(_ event: Event) -> Data? { + let encoded: Data? = encoding == .codable ? try? encoder.encode(event) : eventJSONWriter.encode(event) + guard let encoded = encoded + else { + os_log("%s Failed to serialize event for publication: %s", log: logger, type: .error, typeName(and: #function), String(describing: event)) + return nil + } + return encoded + } + + /// Encodes the summary of one context's counters. Requires the processor's `commitLock`. + /// + /// A summary larger than one log frame is not split: the store refuses it whole, and its evaluations are counted + /// as lost. Reaching that takes one context evaluating dozens of the largest JSON flags LaunchDarkly allows + /// between two commits. + func encodeSummary(_ summary: (tracker: FlagRequestTracker, context: LDContext)) -> Data? { + encode(SummaryEvent(flagRequestTracker: summary.tracker, context: summary.context)) + } + + private static func makeEncoder(config: LDConfig) -> JSONEncoder { + let encoder = JSONEncoder() + encoder.userInfo = [ + LDContext.UserInfoKeys.allAttributesPrivate: config.allContextAttributesPrivate, + LDContext.UserInfoKeys.globalPrivateAttributes: config.privateContextAttributes.map { $0 } + ] + encoder.dateEncodingStrategy = .custom { date, encoder in + var container = encoder.singleValueContainer() + try container.encode(date.millisSince1970) + } + return encoder + } +} + +extension OutboundEventBuffer: TypeIdentifying { } diff --git a/LaunchDarkly/LaunchDarkly/ServiceObjects/RetryState.swift b/LaunchDarkly/LaunchDarkly/ServiceObjects/RetryState.swift index 749198b44..956a64de0 100644 --- a/LaunchDarkly/LaunchDarkly/ServiceObjects/RetryState.swift +++ b/LaunchDarkly/LaunchDarkly/ServiceObjects/RetryState.swift @@ -7,6 +7,10 @@ private struct Constants { static let extendedMaxDelay: TimeInterval = 60 * 60 // The consecutive successful polls that clear the backoff. static let pollingResetThreshold = 2 + static let deliveryInitialDelay: TimeInterval = 1 + // Where the attempt count goes at least to on an unexpected failure, so its first wait is 2^9 initial delays. + static let deliveryAttemptsAfterUnexpected = 10 + static let maxServerDirectedWait: TimeInterval = 60 * 60 } /// The delay bounds for one retry regime. @@ -98,3 +102,124 @@ final class PollingRetryState { return max(pollInterval, jitteredBackoff(extended, attempts: attempts)) } } + +/// When event delivery may post again after a failure, following the SDK Retry Behavior specification as event +/// durability §9 binds it. Android's `DeliveryRetryState` plays the same part. +/// +/// A failure never stops delivery, it only makes the next post wait. An ordinary one, such as a 503 or no network, +/// waits from a second up to the flush interval, so it is never later than the schedule already would be. A refusal +/// that is unlikely to pass soon, such as a 401, waits about eight and a half minutes, doubling up to an hour, rather +/// than deleting the batch: a valid key is sometimes refused for minutes during an incident, and every event the fleet +/// recorded meanwhile would be lost. A `Retry-After` sets the wait instead. One accepted delivery puts everything back. +/// +/// One per event processor, so each environment backs off on its own. Held only in memory, so a new run of the +/// application posts at once. Thread safe: a delivery's responses arrive on whichever thread the session answers on. +final class DeliveryRetryState { + /// The wait after a first failure, which a flush waiting on the delivery that failed sits out. + let initialDelay: TimeInterval + private let normalCeiling: TimeInterval + private let extendedCeiling: TimeInterval + private let random: () -> Double + + private let lock = UnfairLock() + // Guarded by lock. + private var attempts = 0 + private var maxDelay: TimeInterval + private var nextAttempt: DispatchTime? + + /// - Parameter flushInterval: the normal ceiling; never below the initial delay, so a short interval does not make + /// a failing service be posted to more often than once a second. + convenience init(flushInterval: TimeInterval) { + self.init(initialDelay: Constants.deliveryInitialDelay, + normalCeiling: max(flushInterval, Constants.deliveryInitialDelay), + extendedCeiling: Constants.extendedMaxDelay) + } + + init(initialDelay: TimeInterval, + normalCeiling: TimeInterval, + extendedCeiling: TimeInterval, + random: @escaping () -> Double = { Double.random(in: 0..<1) }) { + self.initialDelay = initialDelay + self.normalCeiling = normalCeiling + self.extendedCeiling = extendedCeiling + self.random = random + self.maxDelay = normalCeiling + } + + /// An accepted delivery, which is all it takes for the next failure to start from a second again. + func recordSuccess() { + lock.withLock { + attempts = 0 + maxDelay = normalCeiling + nextAttempt = nil + } + } + + /// - Parameters: + /// - unexpected: whether the failure is unlikely to pass soon: a 4xx other than 400, 408 and 429, or a TLS or + /// certificate failure. + /// - retryAfter: the wait the response asked for, or nil where it gave none. + /// - Returns: how long nothing may be posted. + @discardableResult + func recordFailure(unexpected: Bool, retryAfter: TimeInterval?) -> TimeInterval { + lock.withLock { + if unexpected { + attempts = max(attempts + 1, Constants.deliveryAttemptsAfterUnexpected) + maxDelay = extendedCeiling + } else { + attempts += 1 + } + let wait: TimeInterval + if let retryAfter, retryAfter >= 0 { + // Every instance hears the same value at the same moment, so it is jittered to keep them from posting + // together when it runs out. + let directed = min(retryAfter, Constants.maxServerDirectedWait) + wait = directed - random() * (directed / 2) + } else { + // Not jittered: each instance's failures follow its own history, which already spreads them. + wait = min(maxDelay, initialDelay * pow(2, Double(min(attempts - 1, 40)))) + } + nextAttempt = .now() + wait + return wait + } + } + + /// A failure of a post that went through a wait, as `flushAtClose()` and the blocking `flushAndWait` may: an extra + /// try, not a step of the backoff. It leaves the attempt count and the ceiling as they were, so a caller flushing + /// through a refusal several times does not push flush, the periodic delivery and the retry out to the hour. A + /// `Retry-After` can still make the wait longer, never shorter. Once the wait has run out it is an ordinary failure. + /// + /// - Parameters: as `recordFailure(unexpected:retryAfter:)`. + /// - Returns: how long nothing may be posted. + @discardableResult + func recordFailureDuringWait(unexpected: Bool, retryAfter: TimeInterval?) -> TimeInterval { + let stillWaiting: TimeInterval? = lock.withLock { + let now = DispatchTime.now() + guard let waitingUntil = nextAttempt, now < waitingUntil + else { return nil } + var until = waitingUntil + if let retryAfter, retryAfter >= 0 { + let directed = min(retryAfter, Constants.maxServerDirectedWait) + until = max(until, now + (directed - random() * (directed / 2))) + } + nextAttempt = until + return TimeInterval(until.uptimeNanoseconds - now.uptimeNanoseconds) / 1_000_000_000 + } + return stillWaiting ?? recordFailure(unexpected: unexpected, retryAfter: retryAfter) + } + + /// How long until a post is allowed; zero once it is. + var remainingWait: TimeInterval { + lock.withLock { + guard let nextAttempt + else { return 0 } + let now = DispatchTime.now() + guard now < nextAttempt + else { + self.nextAttempt = nil + return 0 + } + return TimeInterval(nextAttempt.uptimeNanoseconds - now.uptimeNanoseconds) / 1_000_000_000 + } + } +} diff --git a/LaunchDarkly/LaunchDarkly/UnfairLock.swift b/LaunchDarkly/LaunchDarkly/UnfairLock.swift index 99095606e..39adcc92d 100644 --- a/LaunchDarkly/LaunchDarkly/UnfairLock.swift +++ b/LaunchDarkly/LaunchDarkly/UnfairLock.swift @@ -31,6 +31,14 @@ final class UnfairLock { defer { unlock() } return try body() } + + /// Traps unless the calling thread holds the lock. Debug builds only, since it is called on recording paths. + @inline(__always) + func assertOwned() { + #if DEBUG + os_unfair_lock_assert_owner(unfairLock) + #endif + } } #else /// `os_unfair_lock` is Darwin only, so elsewhere this is Foundation's mutex. @@ -42,5 +50,8 @@ extension NSLock { defer { unlock() } return try body() } + + /// `NSLock` cannot tell which thread holds it, so there is nothing to check. + func assertOwned() {} } #endif diff --git a/LaunchDarkly/LaunchDarklyTests/EventJSONWriterTests.swift b/LaunchDarkly/LaunchDarklyTests/EventJSONWriterTests.swift index 93704f1c3..f7b58398c 100644 --- a/LaunchDarkly/LaunchDarklyTests/EventJSONWriterTests.swift +++ b/LaunchDarkly/LaunchDarklyTests/EventJSONWriterTests.swift @@ -28,13 +28,13 @@ final class EventJSONWriterTests: XCTestCase { _ = builder.trySetValue("firstName", "Ada") _ = builder.trySetValue("lastName", "Lovelace") _ = builder.trySetValue("email", "ada@example.com") - _ = builder.trySetValue("age", 36) + _ = builder.trySetValue("age", .number(36)) _ = builder.trySetValue("score", 12.5) - _ = builder.trySetValue("negative", -17) + _ = builder.trySetValue("negative", .number(-17)) _ = builder.trySetValue("verified", true) _ = builder.trySetValue("nothing", .null) _ = builder.trySetValue("tags", ["a", "b", "c"]) - _ = builder.trySetValue("address", ["street": "1 Main St", "city": "Springfield", "zip": 12345]) + _ = builder.trySetValue("address", ["street": "1 Main St", "city": "Springfield", "zip": .number(12345)]) _ = builder.trySetValue("deep", ["a": ["b": ["c": "d"]]]) return try! builder.build().get() } @@ -97,7 +97,7 @@ final class EventJSONWriterTests: XCTestCase { var device = LDContextBuilder(key: "device-key") device.kind("device") device.anonymous(true) - _ = device.trySetValue("os", ["name": "iOS", "version": 18]) + _ = device.trySetValue("os", ["name": "iOS", "version": .number(18)]) var builder = LDMultiContextBuilder() builder.addContext(try! user.build().get()) @@ -132,7 +132,7 @@ final class EventJSONWriterTests: XCTestCase { variation: 2, version: 11, trackEvents: true, - reason: ["kind": "RULE_MATCH", "ruleIndex": 1, "ruleId": "rule-id"], + reason: ["kind": "RULE_MATCH", "ruleIndex": .number(1), "ruleId": "rule-id"], trackReason: true) let unknownFlag = FeatureFlag(flagKey: "flag-key", value: true) @@ -144,11 +144,11 @@ final class EventJSONWriterTests: XCTestCase { return [ ("feature", FeatureEvent(key: "flag-key", context: context, value: true, defaultValue: false, featureFlag: plainFlag, includeReason: false, isDebug: false)), ("feature, reason", FeatureEvent(key: "flag-key", context: context, value: "v", defaultValue: "d", featureFlag: reasonFlag, includeReason: true, isDebug: false)), - ("feature, no flag", FeatureEvent(key: "flag-key", context: context, value: .null, defaultValue: [1, 2], featureFlag: nil, includeReason: false, isDebug: false)), + ("feature, no flag", FeatureEvent(key: "flag-key", context: context, value: .null, defaultValue: [.number(1), .number(2)], featureFlag: nil, includeReason: false, isDebug: false)), ("debug", FeatureEvent(key: "flag-key", context: context, value: true, defaultValue: false, featureFlag: plainFlag, includeReason: false, isDebug: true)), ("identify", IdentifyEvent(context: context)), ("custom, bare", CustomEvent(key: "custom-key", context: context)), - ("custom, data", CustomEvent(key: "custom \"key\"", context: context, data: ["a": 1, "b": [true, .null, "x"]])), + ("custom, data", CustomEvent(key: "custom \"key\"", context: context, data: ["a": .number(1), "b": [true, .null, "x"]])), ("custom, metric", CustomEvent(key: "custom-key", context: context, data: "payload", metricValue: 3.25)), ("summary", SummaryEvent(flagRequestTracker: tracker, context: context)) ] @@ -527,7 +527,7 @@ final class EventJSONWriterTests: XCTestCase { func testNumberFormatting() throws { let numbers: [Double] = [0, -0, 1, -1, 17, -17, 0.5, -0.5, 1.25, 3.141592653589793, - 1e10, 1e-10, 1e20, -1e20, Double(Int32.max), Double(Int64.max), 9007199254740993, + 1e10, 1e-10, 1e20, -1e20, Double(Int32.max), Double(Int64.max), 9007199254740992, 1e16, 1e17, -1e17, 1.5e17, 1e18, 9.2e18, -9.2e18, 9223372036854774784] for number in numbers { @@ -588,30 +588,27 @@ final class EventJSONWriterTests: XCTestCase { config.privateContextAttributes = [Reference("email")] for (name, event) in events(context: context) { - let expected = try XCTUnwrap(published(event, config: config, encoding: .codable), - "nothing published for \(name)") - let actual = try XCTUnwrap(published(event, config: config, encoding: .handWritten), - "nothing published for \(name)") + let codableStore = EventStore.temporary(capacity: .max) + defer { codableStore.deleteEverything() } + let handWrittenStore = EventStore.temporary(capacity: .max) + defer { handWrittenStore.deleteEverything() } - XCTAssertEqual(try canonical(expected), try canonical(actual), "mismatch through the reporter for \(name)") - } - } + let service = DarklyServiceMock() + service.config = config - /// Records one event and returns the payload the reporter handed to the service. - private func published(_ event: Event, config: LDConfig, encoding: EventReporter.Encoding) -> Data? { - let service = DarklyServiceMock() - service.config = config - service.stubEventResponse(success: true) + let codableReporter = DirectEventProcessor(service: service, onSyncComplete: nil, store: codableStore, encoding: .codable) + let handWrittenReporter = DirectEventProcessor(service: service, onSyncComplete: nil, store: handWrittenStore, encoding: .handWritten) - let reporter = EventReporter(service: service, onSyncComplete: nil, encoding: encoding) - reporter.isOnline = true - reporter.record(event) + codableReporter.record(event) + handWrittenReporter.record(event) + codableReporter.commitRecordedEvents() + handWrittenReporter.commitRecordedEvents() - let published = expectation(description: "published") - reporter.flush { published.fulfill() } - wait(for: [published], timeout: 5) + let expected = try XCTUnwrap(codableStore.pendingEventPayloads().first, "nothing recorded for \(name)") + let actual = try XCTUnwrap(handWrittenStore.pendingEventPayloads().first, "nothing recorded for \(name)") - return service.publishedEventData + XCTAssertEqual(try canonical(expected), try canonical(actual), "mismatch through the reporter for \(name)") + } } // MARK: Helpers diff --git a/LaunchDarkly/LaunchDarklyTests/LDClientEvaluationHookSpec.swift b/LaunchDarkly/LaunchDarklyTests/LDClientEvaluationHookSpec.swift index 95d7066d5..adf43e011 100644 --- a/LaunchDarkly/LaunchDarklyTests/LDClientEvaluationHookSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/LDClientEvaluationHookSpec.swift @@ -110,6 +110,21 @@ final class LDClientEvaluationHookSpec: XCTestCase { return testContext } + /// Identifies and then tracks, with `hook` holding only the stages those two produce. + /// + /// An identify's completion runs before its after-identify hooks, so waiting on the completion is not enough: + /// for the identify the client makes as it starts, and for this one. + private func identifyThenTrack(_ testContext: TestContext, _ hook: RecordingHook) { + expect(hook.recordedStages).toEventually(contain("afterIdentify")) + _ = hook.takeStages() + + waitUntil { done in + testContext.subject.identify(context: LDContext.stub()) { _ in done() } + } + expect(hook.recordedStages).toEventually(contain("afterIdentify")) + testContext.subject.track(key: "event-key") + } + // An evaluation reads the flag once, so every hook it reaches and the result it returns all describe that one read, // whatever the store does in the meantime. func testHooksAndTheResultDescribeOneReadOfTheFlag() { @@ -147,7 +162,7 @@ final class LDClientEvaluationHookSpec: XCTestCase { // The evaluation returns the default value, and the event it records reports that value under no variation and // no version, so that is the result the hook is told about too. - guard let key = keys.first ?? nil + guard let key = keys.first.flatMap({ $0 }) else { fail("The hook was not told about the evaluation.") return @@ -219,7 +234,7 @@ final class LDClientEvaluationHookSpec: XCTestCase { } // Deduplication applies to hooks only, so analytics events are unaffected. - XCTAssertEqual(testContext.eventReporterMock.recordFlagEvaluationEventsCallCount, 3) + XCTAssertEqual(testContext.eventProcessorMock.recordFlagEvaluationEventsCallCount, 3) } func testEvaluationsOfDifferentFlagsReachHooksSeparately() { @@ -400,12 +415,7 @@ final class LDClientEvaluationHookSpec: XCTestCase { func testADeduperForwardsTheStagesItDoesNotDeduplicate() { let hook = RecordingHook() let testContext = dedupeTestContext(hooks: [DedupingHook(hook, window: 60)]) - _ = hook.takeStages() - - waitUntil { done in - testContext.subject.identify(context: LDContext.stub()) { _ in done() } - } - testContext.subject.track(key: "event-key") + identifyThenTrack(testContext, hook) // Only evaluations are deduplicated; the identify and track stages are always forwarded. XCTAssertEqual(hook.takeStages(), ["beforeIdentify", "afterIdentify", "afterTrack"]) @@ -488,12 +498,7 @@ final class LDClientEvaluationHookSpec: XCTestCase { func testADecoratorForwardsTheStagesItDoesNotOverride() { let hook = RecordingHook() let testContext = dedupeTestContext(hooks: [CountingDecorator(hook)]) - _ = hook.takeStages() - - waitUntil { done in - testContext.subject.identify(context: LDContext.stub()) { _ in done() } - } - testContext.subject.track(key: "event-key") + identifyThenTrack(testContext, hook) // The decorator mentions neither identify nor track, and the hook it wraps is still told about both: a stage a // decorator leaves alone is forwarded rather than dropped. @@ -545,14 +550,27 @@ final class LDClientEvaluationHookSpec: XCTestCase { // Returned rather than built on demand, so that a test can recognize it by identity: `Metadata` exposes // nothing to compare it by. let hookMetadata = Metadata(name: "recording-hook") + /// Guards `stages`: the identify stages run on whichever thread the identify ended on. + private let lock = UnfairLock() private var stages: [String] = [] + /// The stages recorded since `takeStages` was last called, leaving them in place. + var recordedStages: [String] { + lock.withLock { stages } + } + /// The stages recorded since this was last called, so a test can ignore the identify the client makes as it /// starts. func takeStages() -> [String] { - let recorded = stages - stages = [] - return recorded + lock.withLock { + let recorded = stages + stages = [] + return recorded + } + } + + private func record(_ stage: String) { + lock.withLock { stages.append(stage) } } func metadata() -> Metadata { @@ -560,27 +578,27 @@ final class LDClientEvaluationHookSpec: XCTestCase { } func beforeEvaluation(seriesContext: EvaluationSeriesContext, seriesData: EvaluationSeriesData) -> EvaluationSeriesData { - stages.append("beforeEvaluation") + record("beforeEvaluation") return seriesData } func afterEvaluation(seriesContext: EvaluationSeriesContext, seriesData: EvaluationSeriesData, evaluationDetail: LDEvaluationDetail) -> EvaluationSeriesData { - stages.append("afterEvaluation") + record("afterEvaluation") return seriesData } func beforeIdentify(seriesContext: IdentifySeriesContext, seriesData: IdentifySeriesData) -> IdentifySeriesData { - stages.append("beforeIdentify") + record("beforeIdentify") return seriesData } func afterIdentify(seriesContext: IdentifySeriesContext, seriesData: IdentifySeriesData, result: IdentifyResult) -> IdentifySeriesData { - stages.append("afterIdentify") + record("afterIdentify") return seriesData } func afterTrack(seriesContext: TrackSeriesContext) { - stages.append("afterTrack") + record("afterTrack") } } diff --git a/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift b/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift index 2921f647d..6ff3c8146 100644 --- a/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/LDClientSpec.swift @@ -67,7 +67,7 @@ final class LDClientSpec: QuickSpec { let testContext = TestContext(startOnline: true) testContext.start() - expect(testContext.serviceFactoryMock.makeEventReporterCallCount) == 1 + expect(testContext.serviceFactoryMock.makeEventProcessorCallCount) == 1 expect(testContext.serviceFactoryMock.makeFlagSynchronizerCallCount) == 1 } @@ -86,19 +86,27 @@ final class LDClientSpec: QuickSpec { context("when configured to start online") { beforeEach { testContext = TestContext(startOnline: true) - withTimeout ? testContext.start(timeOut: 10.0) : testContext.start() + if withTimeout { + testContext.start(timeOut: 10.0) + } else { + testContext.start() + } } it("takes the client and service objects online") { expect(testContext.subject.isOnline) == true expect(testContext.subject.flagSynchronizer.isOnline) == testContext.subject.isOnline - expect(testContext.subject.eventReporter.isOnline) == testContext.subject.isOnline + expect(testContext.subject.eventProcessor.isOnline) == testContext.subject.isOnline + } + it("creates one event reporter, so one store owns the event directory") { + expect(testContext.serviceFactoryMock.makeEventProcessorCallCount) == 1 + expect(testContext.serviceFactoryMock.onEventSyncComplete).toNot(beNil()) } it("saves the config") { expect(testContext.subject.config) == testContext.config expect(testContext.subject.service.config) == testContext.config - expect(testContext.makeFlagSynchronizerStreamingMode) == testContext.config.streamingMode + expect(testContext.makeFlagSynchronizerStreamingMode) == testContext.config.effectiveStreamingMode expect(testContext.makeFlagSynchronizerPollingInterval) == testContext.config.flagPollingInterval(runMode: testContext.subject.runMode) - expect(testContext.serviceFactoryMock.makeEventReporterReceivedService?.config) == testContext.config + expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.config) == testContext.config } it("saves the context") { expect(testContext.subject.context) == testContext.context @@ -107,14 +115,14 @@ final class LDClientSpec: QuickSpec { if let makeFlagSynchronizerReceivedParameters = testContext.serviceFactoryMock.makeFlagSynchronizerReceivedParameters { expect(makeFlagSynchronizerReceivedParameters.service) === testContext.subject.service } - expect(testContext.serviceFactoryMock.makeEventReporterReceivedService?.context) == testContext.context + expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.context) == testContext.context } it("uncaches the new contexts flags") { expect(testContext.featureFlagCachingMock.getCachedDataCallCount) == 1 expect(testContext.featureFlagCachingMock.getCachedDataReceivedArguments?.cacheKey) == testContext.context.fullyQualifiedHashedKey() } it("records an identify event") { - expect(testContext.eventReporterMock.recordCallCount) == 1 + expect(testContext.eventProcessorMock.recordCallCount) == 1 expect((testContext.recordedEvent as? IdentifyEvent)?.context) == testContext.context } it("converts cached data") { @@ -129,19 +137,23 @@ final class LDClientSpec: QuickSpec { context("when configured to start offline") { beforeEach { testContext = TestContext() - withTimeout ? testContext.start(timeOut: 10.0) : testContext.start() + if withTimeout { + testContext.start(timeOut: 10.0) + } else { + testContext.start() + } } it("leaves the client and service objects offline") { expect(testContext.subject.isOnline) == false expect(testContext.subject.flagSynchronizer.isOnline) == testContext.subject.isOnline - expect(testContext.subject.eventReporter.isOnline) == testContext.subject.isOnline + expect(testContext.subject.eventProcessor.isOnline) == testContext.subject.isOnline } it("saves the config") { expect(testContext.subject.config) == testContext.config expect(testContext.subject.service.config) == testContext.config - expect(testContext.makeFlagSynchronizerStreamingMode) == testContext.config.streamingMode + expect(testContext.makeFlagSynchronizerStreamingMode) == testContext.config.effectiveStreamingMode expect(testContext.makeFlagSynchronizerPollingInterval) == testContext.config.flagPollingInterval(runMode: testContext.subject.runMode) - expect(testContext.serviceFactoryMock.makeEventReporterReceivedService?.config) == testContext.config + expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.config) == testContext.config } it("saves the context") { expect(testContext.subject.context) == testContext.context @@ -150,14 +162,14 @@ final class LDClientSpec: QuickSpec { if let makeFlagSynchronizerReceivedParameters = testContext.serviceFactoryMock.makeFlagSynchronizerReceivedParameters { expect(makeFlagSynchronizerReceivedParameters.service) === testContext.subject.service } - expect(testContext.serviceFactoryMock.makeEventReporterReceivedService?.context) == testContext.context + expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.context) == testContext.context } it("uncaches the new contexts flags") { expect(testContext.featureFlagCachingMock.getCachedDataCallCount) == 1 expect(testContext.featureFlagCachingMock.getCachedDataReceivedArguments?.cacheKey) == testContext.context.fullyQualifiedHashedKey() } it("records an identify event") { - expect(testContext.eventReporterMock.recordCallCount) == 1 + expect(testContext.eventProcessorMock.recordCallCount) == 1 expect((testContext.recordedEvent as? IdentifyEvent)?.context) == testContext.context } it("converts cached data") { @@ -173,7 +185,11 @@ final class LDClientSpec: QuickSpec { context("after setting context") { beforeEach { testContext = TestContext(startOnline: true).withContext(nil) - withTimeout ? testContext.start(timeOut: 10.0) : testContext.start() + if withTimeout { + testContext.start(timeOut: 10.0) + } else { + testContext.start() + } testContext.context = LDContext.stub() testContext.subject.internalIdentify(newContext: testContext.context, useCache: .yes) @@ -181,9 +197,9 @@ final class LDClientSpec: QuickSpec { it("saves the config") { expect(testContext.subject.config) == testContext.config expect(testContext.subject.service.config) == testContext.config - expect(testContext.makeFlagSynchronizerStreamingMode) == testContext.config.streamingMode + expect(testContext.makeFlagSynchronizerStreamingMode) == testContext.config.effectiveStreamingMode expect(testContext.makeFlagSynchronizerPollingInterval) == testContext.config.flagPollingInterval(runMode: testContext.subject.runMode) - expect(testContext.serviceFactoryMock.makeEventReporterReceivedService?.config) == testContext.config + expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.config) == testContext.config } it("saves the context") { expect(testContext.subject.context) == testContext.context @@ -192,14 +208,14 @@ final class LDClientSpec: QuickSpec { if let makeFlagSynchronizerReceivedParameters = testContext.serviceFactoryMock.makeFlagSynchronizerReceivedParameters { expect(makeFlagSynchronizerReceivedParameters.service.context) == testContext.context } - expect(testContext.serviceFactoryMock.makeEventReporterReceivedService?.context) == testContext.context + expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.context) == testContext.context } it("uncaches the new contexts flags") { expect(testContext.featureFlagCachingMock.getCachedDataCallCount) == 2 // called on init and subsequent identify expect(testContext.featureFlagCachingMock.getCachedDataReceivedArguments?.cacheKey) == testContext.context.fullyQualifiedHashedKey() } it("records an identify event") { - expect(testContext.eventReporterMock.recordCallCount) == 2 // both start and internalIdentify + expect(testContext.eventProcessorMock.recordCallCount) == 2 // both start and internalIdentify expect((testContext.recordedEvent as? IdentifyEvent)?.context) == testContext.context } it("converts cached data") { @@ -211,27 +227,31 @@ final class LDClientSpec: QuickSpec { context("without setting context") { beforeEach { testContext = TestContext(startOnline: true).withContext(nil) - withTimeout ? testContext.start(timeOut: 10.0) : testContext.start() + if withTimeout { + testContext.start(timeOut: 10.0) + } else { + testContext.start() + } } it("saves the config") { expect(testContext.subject.config) == testContext.config expect(testContext.subject.service.config) == testContext.config - expect(testContext.makeFlagSynchronizerStreamingMode) == testContext.config.streamingMode + expect(testContext.makeFlagSynchronizerStreamingMode) == testContext.config.effectiveStreamingMode expect(testContext.makeFlagSynchronizerPollingInterval) == testContext.config.flagPollingInterval(runMode: testContext.subject.runMode) - expect(testContext.serviceFactoryMock.makeEventReporterReceivedService?.config) == testContext.config + expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.config) == testContext.config } it("uses anonymous context") { expect(testContext.subject.context.fullyQualifiedKey()) == LDContext.defaultKey(kind: testContext.subject.context.kind) expect(testContext.subject.service.context) == testContext.subject.context expect(testContext.makeFlagSynchronizerService?.context) == testContext.subject.context - expect(testContext.serviceFactoryMock.makeEventReporterReceivedService?.context) == testContext.subject.context + expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.context) == testContext.subject.context } it("uncaches the new contexts flags") { expect(testContext.featureFlagCachingMock.getCachedDataCallCount) == 1 expect(testContext.featureFlagCachingMock.getCachedDataReceivedArguments?.cacheKey) == testContext.subject.context.fullyQualifiedHashedKey() } it("records an identify event") { - expect(testContext.eventReporterMock.recordCallCount) == 1 + expect(testContext.eventProcessorMock.recordCallCount) == 1 expect((testContext.recordedEvent as? IdentifyEvent)?.context) == testContext.subject.context } it("converts cached data") { @@ -244,7 +264,11 @@ final class LDClientSpec: QuickSpec { it("when called with cached flags for the context and environment") { let cachedFlags = ["test-flag": StorageItem.item(FeatureFlag(flagKey: "test-flag"))] let testContext = TestContext().withCached(flags: cachedFlags.featureFlags) - withTimeout ? testContext.start(timeOut: 10.0) : testContext.start() + if withTimeout { + testContext.start(timeOut: 10.0) + } else { + testContext.start() + } expect(testContext.featureFlagCachingMock.getCachedDataCallCount) == 1 expect(testContext.featureFlagCachingMock.getCachedDataReceivedArguments?.cacheKey) == testContext.context.fullyQualifiedHashedKey() @@ -257,7 +281,11 @@ final class LDClientSpec: QuickSpec { } it("when called without cached flags for the context") { let testContext = TestContext() - withTimeout ? testContext.start(timeOut: 10.0) : testContext.start() + if withTimeout { + testContext.start(timeOut: 10.0) + } else { + testContext.start() + } expect(testContext.featureFlagCachingMock.getCachedDataCallCount) == 1 expect(testContext.featureFlagCachingMock.getCachedDataReceivedArguments?.cacheKey) == testContext.context.fullyQualifiedHashedKey() @@ -379,7 +407,7 @@ final class LDClientSpec: QuickSpec { if os == .iOS && os == SystemCapabilities.operatingSystem { expect(testContext.subject.flagSynchronizer.isOnline) == os.isBackgroundEnabled } - expect(testContext.subject.eventReporter.isOnline) == testContext.subject.isOnline + expect(testContext.subject.eventProcessor.isOnline) == testContext.subject.isOnline } it("saves the config") { expect(testContext.subject.service.config) == testContext.config @@ -388,7 +416,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.makeFlagSynchronizerStreamingMode) == os.backgroundStreamingMode } expect(testContext.makeFlagSynchronizerPollingInterval) == testContext.config.flagPollingInterval(runMode: .background) - expect(testContext.serviceFactoryMock.makeEventReporterReceivedService?.config) == testContext.config + expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.config) == testContext.config } it("saves the context") { expect(testContext.subject.context) == testContext.context @@ -397,14 +425,14 @@ final class LDClientSpec: QuickSpec { if let makeFlagSynchronizerReceivedParameters = testContext.serviceFactoryMock.makeFlagSynchronizerReceivedParameters { expect(makeFlagSynchronizerReceivedParameters.service) === testContext.subject.service } - expect(testContext.serviceFactoryMock.makeEventReporterReceivedService?.context) == testContext.context + expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.context) == testContext.context } it("uncaches the new contexts flags") { expect(testContext.featureFlagCachingMock.getCachedDataCallCount) == 1 expect(testContext.featureFlagCachingMock.getCachedDataReceivedArguments?.cacheKey) == testContext.context.fullyQualifiedHashedKey() } it("records an identify event") { - expect(testContext.eventReporterMock.recordCallCount) == 1 + expect(testContext.eventProcessorMock.recordCallCount) == 1 expect((testContext.recordedEvent as? IdentifyEvent)?.context) == testContext.context } } @@ -421,13 +449,13 @@ final class LDClientSpec: QuickSpec { it("leaves the client and service objects offline") { expect(testContext.subject.isOnline) == true expect(testContext.subject.flagSynchronizer.isOnline) == false - expect(testContext.subject.eventReporter.isOnline) == true + expect(testContext.subject.eventProcessor.isOnline) == true } it("saves the config") { expect(testContext.subject.service.config) == testContext.config expect(testContext.makeFlagSynchronizerStreamingMode) == LDStreamingMode.polling expect(testContext.makeFlagSynchronizerPollingInterval) == testContext.config.flagPollingInterval(runMode: .background) - expect(testContext.serviceFactoryMock.makeEventReporterReceivedService?.config) == testContext.config + expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.config) == testContext.config } it("saves the context") { expect(testContext.subject.context) == testContext.context @@ -436,14 +464,14 @@ final class LDClientSpec: QuickSpec { if let makeFlagSynchronizerReceivedParameters = testContext.serviceFactoryMock.makeFlagSynchronizerReceivedParameters { expect(makeFlagSynchronizerReceivedParameters.service.context) == testContext.context } - expect(testContext.serviceFactoryMock.makeEventReporterReceivedService?.context) == testContext.context + expect(testContext.serviceFactoryMock.makeEventProcessorReceivedService?.context) == testContext.context } it("uncaches the new contexts flags") { expect(testContext.featureFlagCachingMock.getCachedDataCallCount) == 1 expect(testContext.featureFlagCachingMock.getCachedDataReceivedArguments?.cacheKey) == testContext.context.fullyQualifiedHashedKey() } it("records an identify event") { - expect(testContext.eventReporterMock.recordCallCount) == 1 + expect(testContext.eventProcessorMock.recordCallCount) == 1 expect((testContext.recordedEvent as? IdentifyEvent)?.context) == testContext.context } } @@ -468,13 +496,13 @@ final class LDClientSpec: QuickSpec { expect(testContext.makeFlagSynchronizerService?.context) == newContext expect(testContext.subject.isOnline) == true - expect(testContext.subject.eventReporter.isOnline) == true + expect(testContext.subject.eventProcessor.isOnline) == true expect(testContext.subject.flagSynchronizer.isOnline) == true expect(testContext.featureFlagCachingMock.getCachedDataCallCount) == 1 expect(testContext.featureFlagCachingMock.getCachedDataReceivedArguments?.cacheKey) == newContext.fullyQualifiedHashedKey() - expect(testContext.eventReporterMock.recordReceivedEvent?.kind == .identify).to(beTrue()) + expect(testContext.eventProcessorMock.recordReceivedEvent?.kind == .identify).to(beTrue()) } it("when the client is offline") { let testContext = TestContext() @@ -490,13 +518,13 @@ final class LDClientSpec: QuickSpec { expect(testContext.makeFlagSynchronizerService?.context) == newContext expect(testContext.subject.isOnline) == false - expect(testContext.subject.eventReporter.isOnline) == false + expect(testContext.subject.eventProcessor.isOnline) == false expect(testContext.subject.flagSynchronizer.isOnline) == false expect(testContext.featureFlagCachingMock.getCachedDataCallCount) == 1 expect(testContext.featureFlagCachingMock.getCachedDataReceivedArguments?.cacheKey) == newContext.fullyQualifiedHashedKey() - expect(testContext.eventReporterMock.recordReceivedEvent?.kind == .identify).to(beTrue()) + expect(testContext.eventProcessorMock.recordReceivedEvent?.kind == .identify).to(beTrue()) } it("when the new context has cached feature flags") { let stubFlags = FlagMaintainingMock.stubStoredItems() @@ -543,9 +571,9 @@ final class LDClientSpec: QuickSpec { expect(testContext.makeFlagSynchronizerService?.context) == testContext.context expect(testContext.subject.isOnline) == true - expect(testContext.subject.eventReporter.isOnline) == true + expect(testContext.subject.eventProcessor.isOnline) == true expect(testContext.subject.flagSynchronizer.isOnline) == true - expect(testContext.eventReporterMock.recordReceivedEvent?.kind == .identify).to(beTrue()) + expect(testContext.eventProcessorMock.recordReceivedEvent?.kind == .identify).to(beTrue()) } it("no cache requires no store interaction") { @@ -559,9 +587,9 @@ final class LDClientSpec: QuickSpec { expect(testContext.makeFlagSynchronizerService?.context) == testContext.context expect(testContext.subject.isOnline) == true - expect(testContext.subject.eventReporter.isOnline) == true + expect(testContext.subject.eventProcessor.isOnline) == true expect(testContext.subject.flagSynchronizer.isOnline) == true - expect(testContext.eventReporterMock.recordReceivedEvent?.kind == .identify).to(beTrue()) + expect(testContext.eventProcessorMock.recordReceivedEvent?.kind == .identify).to(beTrue()) } it("ifAvailable requires no store information on cache miss") { @@ -575,9 +603,9 @@ final class LDClientSpec: QuickSpec { expect(testContext.makeFlagSynchronizerService?.context) == testContext.context expect(testContext.subject.isOnline) == true - expect(testContext.subject.eventReporter.isOnline) == true + expect(testContext.subject.eventProcessor.isOnline) == true expect(testContext.subject.flagSynchronizer.isOnline) == true - expect(testContext.eventReporterMock.recordReceivedEvent?.kind == .identify).to(beTrue()) + expect(testContext.eventProcessorMock.recordReceivedEvent?.kind == .identify).to(beTrue()) } it("ifAvailable updates store when cache is present") { @@ -624,11 +652,11 @@ final class LDClientSpec: QuickSpec { expect(testContext.subject.isOnline) == true expect(testContext.flagSynchronizerMock.isOnline) == false - expect(testContext.subject.eventReporter.isOnline) == true + expect(testContext.subject.eventProcessor.isOnline) == true expect(testContext.subject.isInitialized) == false expect(completed).toEventually(beTrue()) } - it("takes the client back online at the foreground notification") { + it("takes the client back online at the foreground notification", onlyWhere: platformPostsLifecycleNotifications) { let testContext = backgroundedOnlineClient() testContext.subject.internalIdentify(newContext: LDContext.stub(), useCache: .no) @@ -637,7 +665,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.subject.isOnline).toEventually(beTrue()) expect(testContext.flagSynchronizerMock.isOnline).toEventually(beTrue()) } - it("recovers after repeated identifies while backgrounded") { + it("recovers after repeated identifies while backgrounded", onlyWhere: platformPostsLifecycleNotifications) { let testContext = backgroundedOnlineClient() testContext.subject.internalIdentify(newContext: LDContext.stub(), useCache: .no) testContext.subject.internalIdentify(newContext: LDContext.stub(), useCache: .no) @@ -688,19 +716,19 @@ final class LDClientSpec: QuickSpec { testContext.start() let original = testContext.featureFlagCachingMock.getCachedDataCallback var isOnlineDuringSwap: Bool? - var eventReporterOnlineDuringSwap: Bool? + var eventProcessorOnlineDuringSwap: Bool? testContext.featureFlagCachingMock.getCachedDataCallback = { try original?() isOnlineDuringSwap = testContext.subject.isOnline - eventReporterOnlineDuringSwap = testContext.subject.eventReporter.isOnline + eventProcessorOnlineDuringSwap = testContext.subject.eventProcessor.isOnline } testContext.subject.internalIdentify(newContext: LDContext.stub(), useCache: .no) expect(isOnlineDuringSwap) == true - expect(eventReporterOnlineDuringSwap) == true + expect(eventProcessorOnlineDuringSwap) == true } - it("recovers when the app backgrounds midway through an identify") { + it("recovers when the app backgrounds midway through an identify", onlyWhere: platformPostsLifecycleNotifications) { let testContext = TestContext(startOnline: true, enableBackgroundUpdates: false) testContext.start() let original = testContext.featureFlagCachingMock.getCachedDataCallback @@ -722,7 +750,7 @@ final class LDClientSpec: QuickSpec { // The mirror image of the background-then-identify cases above: identify then // background must end in the same state, so the two transitions compose in either // order. - it("stays online with the data source down when backgrounded after an identify") { + it("stays online with the data source down when backgrounded after an identify", onlyWhere: platformPostsLifecycleNotifications) { let testContext = TestContext(startOnline: true, enableBackgroundUpdates: false) testContext.start() testContext.subject.internalIdentify(newContext: LDContext.stub(), useCache: .no) @@ -772,7 +800,7 @@ final class LDClientSpec: QuickSpec { waitUntil { done in DispatchQueue.main.async { done() } } expect(count) == 1 } - it("leaves a client the application set offline offline") { + it("leaves a client the application set offline offline", onlyWhere: platformPostsLifecycleNotifications) { let testContext = TestContext(startOnline: false, enableBackgroundUpdates: false) testContext.start() testContext.subject.setRunMode(.background) @@ -803,7 +831,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.throttlerMock?.runThrottledCallCount) == 1 expect(testContext.subject.isOnline) == true expect(testContext.subject.flagSynchronizer.isOnline) == testContext.subject.isOnline - expect(testContext.subject.eventReporter.isOnline) == testContext.subject.isOnline + expect(testContext.subject.eventProcessor.isOnline) == testContext.subject.isOnline } it("set offline when the client is online") { let testContext = TestContext(startOnline: true) @@ -814,7 +842,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.throttlerMock?.runThrottledCallCount) == 0 expect(testContext.subject.isOnline) == false expect(testContext.subject.flagSynchronizer.isOnline) == testContext.subject.isOnline - expect(testContext.subject.eventReporter.isOnline) == testContext.subject.isOnline + expect(testContext.subject.eventProcessor.isOnline) == testContext.subject.isOnline } context("set online when the client runs in the background") { OperatingSystem.allOperatingSystems.forEach { os in @@ -834,7 +862,7 @@ final class LDClientSpec: QuickSpec { } expect(testContext.makeFlagSynchronizerPollingInterval) == testContext.config.flagPollingInterval(runMode: testContext.subject.runMode) - expect(testContext.subject.eventReporter.isOnline) == testContext.subject.isOnline + expect(testContext.subject.eventProcessor.isOnline) == testContext.subject.isOnline } it("while configured to disable background updates") { let testContext = TestContext(enableBackgroundUpdates: false) @@ -846,7 +874,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.subject.flagSynchronizer.isOnline) == testContext.subject.isOnline expect(testContext.makeFlagSynchronizerStreamingMode) == LDStreamingMode.polling expect(testContext.makeFlagSynchronizerPollingInterval) == testContext.config.flagPollingInterval(runMode: .background) - expect(testContext.subject.eventReporter.isOnline) == testContext.subject.isOnline + expect(testContext.subject.eventProcessor.isOnline) == testContext.subject.isOnline } } } @@ -859,7 +887,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.throttlerMock?.runThrottledCallCount) == 0 expect(testContext.subject.isOnline) == false expect(testContext.subject.flagSynchronizer.isOnline) == testContext.subject.isOnline - expect(testContext.subject.eventReporter.isOnline) == testContext.subject.isOnline + expect(testContext.subject.eventProcessor.isOnline) == testContext.subject.isOnline } } } @@ -872,7 +900,8 @@ final class LDClientSpec: QuickSpec { testContext.subject.close() expect(testContext.subject.isOnline) == false - expect(testContext.eventReporterMock.flushCallCount) == 1 + expect(testContext.eventProcessorMock.flushAtCloseCallCount) == 1 + expect(testContext.eventProcessorMock.flushCallCount) == 0 } it("when started and offline") { let testContext = TestContext() @@ -880,7 +909,8 @@ final class LDClientSpec: QuickSpec { testContext.subject.close() expect(testContext.subject.isOnline) == false - expect(testContext.eventReporterMock.flushCallCount) == 1 + expect(testContext.eventProcessorMock.flushAtCloseCallCount) == 1 + expect(testContext.eventProcessorMock.flushCallCount) == 0 } it("when already stopped") { let testContext = TestContext() @@ -889,7 +919,26 @@ final class LDClientSpec: QuickSpec { testContext.subject.close() expect(testContext.subject.isOnline) == false - expect(testContext.eventReporterMock.flushCallCount) == 1 + expect(testContext.eventProcessorMock.flushAtCloseCallCount) == 1 + expect(testContext.eventProcessorMock.flushCallCount) == 0 + } + it("answers other calls while an environment's final commit is under way") { + let testContext = TestContext() + testContext.start() + var answeredDuringClose = false + testContext.eventProcessorMock.flushAtCloseCallback = { + // Standing in for a commit stalled on the disk, which can hold the closing thread for the budget. + let answered = DispatchSemaphore(value: 0) + DispatchQueue.global().async { + _ = LDClient.get() + answered.signal() + } + answeredDuringClose = answered.wait(timeout: .now() + 1) == .success + } + testContext.subject.close() + + expect(answeredDuringClose) == true + expect(LDClient.get()).to(beNil()) } } } @@ -900,7 +949,7 @@ final class LDClientSpec: QuickSpec { let testContext = TestContext() testContext.start() testContext.subject.track(key: "customEvent", data: "abc", metricValue: 5.0) - let receivedEvent = testContext.eventReporterMock.recordReceivedEvent as? CustomEvent + let receivedEvent = testContext.eventProcessorMock.recordReceivedEvent as? CustomEvent expect(receivedEvent?.key) == "customEvent" expect(receivedEvent?.context) == testContext.context expect(receivedEvent?.data) == "abc" @@ -911,17 +960,17 @@ final class LDClientSpec: QuickSpec { config.sendEvents = false let testContext = TestContext(newConfig: config) testContext.start() - let priorRecordedEvents = testContext.eventReporterMock.recordCallCount + let priorRecordedEvents = testContext.eventProcessorMock.recordCallCount testContext.subject.track(key: "customEvent", data: "abc", metricValue: 5.0) - expect(testContext.eventReporterMock.recordCallCount) == priorRecordedEvents + expect(testContext.eventProcessorMock.recordCallCount) == priorRecordedEvents } context("does not record when client was stopped") { let testContext = TestContext() testContext.start() testContext.subject.close() - let priorRecordedEvents = testContext.eventReporterMock.recordCallCount + let priorRecordedEvents = testContext.eventProcessorMock.recordCallCount testContext.subject.track(key: "abc") - expect(testContext.eventReporterMock.recordCallCount) == priorRecordedEvents + expect(testContext.eventProcessorMock.recordCallCount) == priorRecordedEvents } } } @@ -950,12 +999,12 @@ final class LDClientSpec: QuickSpec { } it("records a flag evaluation event") { _ = testContext.subject.boolVariation(forKey: DarklyServiceMock.FlagKeys.bool, defaultValue: DefaultFlagValues.bool) - expect(testContext.eventReporterMock.recordFlagEvaluationEventsCallCount) == 1 - expect(testContext.eventReporterMock.recordFlagEvaluationEventsReceivedArguments?.flagKey) == DarklyServiceMock.FlagKeys.bool - expect(testContext.eventReporterMock.recordFlagEvaluationEventsReceivedArguments?.value) == DarklyServiceMock.FlagValues.bool - expect(testContext.eventReporterMock.recordFlagEvaluationEventsReceivedArguments?.defaultValue) == .bool(DefaultFlagValues.bool) - expect(testContext.eventReporterMock.recordFlagEvaluationEventsReceivedArguments?.featureFlag) == testContext.flagStoreMock.storedItems.featureFlags[DarklyServiceMock.FlagKeys.bool] - expect(testContext.eventReporterMock.recordFlagEvaluationEventsReceivedArguments?.context) == testContext.context + expect(testContext.eventProcessorMock.recordFlagEvaluationEventsCallCount) == 1 + expect(testContext.eventProcessorMock.recordFlagEvaluationEventsReceivedArguments?.flagKey) == DarklyServiceMock.FlagKeys.bool + expect(testContext.eventProcessorMock.recordFlagEvaluationEventsReceivedArguments?.value) == DarklyServiceMock.FlagValues.bool + expect(testContext.eventProcessorMock.recordFlagEvaluationEventsReceivedArguments?.defaultValue) == .bool(DefaultFlagValues.bool) + expect(testContext.eventProcessorMock.recordFlagEvaluationEventsReceivedArguments?.featureFlag) == testContext.flagStoreMock.storedItems.featureFlags[DarklyServiceMock.FlagKeys.bool] + expect(testContext.eventProcessorMock.recordFlagEvaluationEventsReceivedArguments?.context) == testContext.context } } } @@ -971,12 +1020,12 @@ final class LDClientSpec: QuickSpec { } it("records a flag evaluation event") { _ = testContext.subject.boolVariation(forKey: DarklyServiceMock.FlagKeys.bool, defaultValue: DefaultFlagValues.bool) - expect(testContext.eventReporterMock.recordFlagEvaluationEventsCallCount) == 1 - expect(testContext.eventReporterMock.recordFlagEvaluationEventsReceivedArguments?.flagKey) == DarklyServiceMock.FlagKeys.bool - expect(testContext.eventReporterMock.recordFlagEvaluationEventsReceivedArguments?.value) == .bool(DefaultFlagValues.bool) - expect(testContext.eventReporterMock.recordFlagEvaluationEventsReceivedArguments?.defaultValue) == .bool(DefaultFlagValues.bool) - expect(testContext.eventReporterMock.recordFlagEvaluationEventsReceivedArguments?.featureFlag).to(beNil()) - expect(testContext.eventReporterMock.recordFlagEvaluationEventsReceivedArguments?.context) == testContext.context + expect(testContext.eventProcessorMock.recordFlagEvaluationEventsCallCount) == 1 + expect(testContext.eventProcessorMock.recordFlagEvaluationEventsReceivedArguments?.flagKey) == DarklyServiceMock.FlagKeys.bool + expect(testContext.eventProcessorMock.recordFlagEvaluationEventsReceivedArguments?.value) == .bool(DefaultFlagValues.bool) + expect(testContext.eventProcessorMock.recordFlagEvaluationEventsReceivedArguments?.defaultValue) == .bool(DefaultFlagValues.bool) + expect(testContext.eventProcessorMock.recordFlagEvaluationEventsReceivedArguments?.featureFlag).to(beNil()) + expect(testContext.eventProcessorMock.recordFlagEvaluationEventsReceivedArguments?.context) == testContext.context } } } @@ -990,8 +1039,8 @@ final class LDClientSpec: QuickSpec { let storedItems = StoredItems(items: flags) testContext.flagStoreMock.replaceStore(newStoredItems: storedItems) var events = [FeatureEvent]() - testContext.eventReporterMock.recordFlagEvaluationEventsCallback = { - let args = testContext.eventReporterMock.recordFlagEvaluationEventsReceivedArguments! + testContext.eventProcessorMock.recordFlagEvaluationEventsCallback = { + let args = testContext.eventProcessorMock.recordFlagEvaluationEventsReceivedArguments! let event = FeatureEvent(key: args.flagKey, context: args.context, value: args.value, defaultValue: args.defaultValue, featureFlag: args.featureFlag, includeReason: args.includeReason, isDebug: false) events.append(event) } @@ -1020,8 +1069,8 @@ final class LDClientSpec: QuickSpec { var events = [FeatureEvent]() beforeEach { events = [] - testContext.eventReporterMock.recordFlagEvaluationEventsCallback = { - let args = testContext.eventReporterMock.recordFlagEvaluationEventsReceivedArguments! + testContext.eventProcessorMock.recordFlagEvaluationEventsCallback = { + let args = testContext.eventProcessorMock.recordFlagEvaluationEventsReceivedArguments! events.append(FeatureEvent(key: args.flagKey, context: args.context, value: args.value, defaultValue: args.defaultValue, featureFlag: args.featureFlag, includeReason: args.includeReason, isDebug: false)) } } @@ -1298,10 +1347,10 @@ final class LDClientSpec: QuickSpec { testContext.onSyncComplete?(.error(.response(forbiddenError))) expect(testContext.flagSynchronizerMock.isOnline).to(beTrue()) - expect(testContext.subject.getConnectionInformation().currentConnectionMode) == .establishingStreamingConnection + expect(testContext.subject.getConnectionInformation().currentConnectionMode) == (platformStreams ? .establishingStreamingConnection : .polling) expect(testContext.subject.isInitialized).to(beFalse()) expect(testContext.subject.isOnline).to(beTrue()) - expect(testContext.eventReporterMock.isOnline).to(beTrue()) + expect(testContext.eventProcessorMock.isOnline).to(beTrue()) } it("keeps the data source running on a recoverable flag error") { let testContext = TestContext(startOnline: true) @@ -1326,7 +1375,7 @@ final class LDClientSpec: QuickSpec { expect(after.lastConnectionFailureReason.description) == before.lastConnectionFailureReason.description expect(after.lastFailedConnection) == before.lastFailedConnection } - it("recovers the connection mode when a streaming sync succeeds after a terminal error") { + it("recovers the connection mode when a streaming sync succeeds after a terminal error", onlyWhere: platformStreams) { let testContext = TestContext(startOnline: true) testContext.start() testContext.subject.flagChangeNotifier = ClientServiceMockFactory(config: testContext.config).makeFlagChangeNotifier() @@ -1357,7 +1406,7 @@ final class LDClientSpec: QuickSpec { context("when online") { OperatingSystem.allOperatingSystems.forEach { os in context("on \(os)") { - it("background updates disabled") { + it("background updates disabled", onlyWhere: platformPostsLifecycleNotifications) { let testContext = TestContext(startOnline: true, enableBackgroundUpdates: false) testContext.start() NotificationCenter.default.post(name: SystemCapabilities.backgroundNotification!, object: self) @@ -1365,23 +1414,32 @@ final class LDClientSpec: QuickSpec { expect(testContext.subject.isOnline) == true expect(testContext.subject.runMode) == LDClientRunMode.background - expect(testContext.eventReporterMock.isOnline) == true + expect(testContext.eventProcessorMock.isOnline) == true expect(testContext.flagSynchronizerMock.isOnline) == false } - it("tries to deliver what it has before the process is suspended, where it is suspended") { + it("tries to deliver what it has before the process is suspended, where it is suspended", onlyWhere: platformPostsLifecycleNotifications) { let testContext = TestContext(startOnline: true, enableBackgroundUpdates: false) testContext.start() NotificationCenter.default.post(name: SystemCapabilities.backgroundNotification!, object: self) - #if os(iOS) || os(tvOS) - expect(testContext.eventReporterMock.flushReportingOutcomeCallCount).toEventually(equal(1)) - #else - // macOS posts this on every loss of focus, which suspends nothing. - expect(testContext.subject.runMode).toEventually(equal(LDClientRunMode.background)) - expect(testContext.eventReporterMock.flushReportingOutcomeCallCount) == 0 - #endif + if SystemCapabilities.suspendsWhenBackgrounded { + expect(testContext.eventProcessorMock.flushReportingOutcomeCallCount).toEventually(equal(1)) + } else { + // macOS posts this on every loss of focus, which suspends nothing. + expect(testContext.subject.runMode).toEventually(equal(LDClientRunMode.background)) + expect(testContext.eventProcessorMock.flushReportingOutcomeCallCount) == 0 + } + } + it("writes what it has before the notification returns, on every platform", onlyWhere: platformPostsLifecycleNotifications) { + let testContext = TestContext(startOnline: true, enableBackgroundUpdates: false) + testContext.start() + NotificationCenter.default.post(name: SystemCapabilities.backgroundNotification!, object: self) + + // Checked without waiting: a write only queued is one a process killed in the + // background may never get to, macOS included, where nothing is delivered here. + expect(testContext.eventProcessorMock.commitWherePersistingCallCount) == 1 } - it("background updates enabled") { + it("background updates enabled", onlyWhere: platformPostsLifecycleNotifications) { let testContext = TestContext(startOnline: true) testContext.start() @@ -1392,7 +1450,7 @@ final class LDClientSpec: QuickSpec { expect(testContext.subject.isOnline) == true expect(testContext.subject.runMode) == LDClientRunMode.background - expect(testContext.eventReporterMock.isOnline) == true + expect(testContext.eventProcessorMock.isOnline) == true // TODO(os-tests): We need to expand this to the other OSs if os == .iOS && os == SystemCapabilities.operatingSystem { @@ -1403,14 +1461,14 @@ final class LDClientSpec: QuickSpec { } } } - it("when offline") { + it("when offline", onlyWhere: platformPostsLifecycleNotifications) { let testContext = TestContext() testContext.start() NotificationCenter.default.post(name: SystemCapabilities.backgroundNotification!, object: self) expect(testContext.subject.isOnline) == false expect(testContext.subject.runMode) == LDClientRunMode.background - expect(testContext.eventReporterMock.isOnline) == false + expect(testContext.eventProcessorMock.isOnline) == false expect(testContext.flagSynchronizerMock.isOnline) == false } } @@ -1420,24 +1478,24 @@ final class LDClientSpec: QuickSpec { context("after starting client") { OperatingSystem.allOperatingSystems.forEach { os in context("on \(os)") { - it("when online at foreground notification") { + it("when online at foreground notification", onlyWhere: platformPostsLifecycleNotifications) { let testContext = TestContext(startOnline: true) testContext.start(runMode: .background) NotificationCenter.default.post(name: SystemCapabilities.foregroundNotification!, object: self) expect(testContext.subject.isOnline) == true expect(testContext.subject.runMode) == LDClientRunMode.foreground - expect(testContext.eventReporterMock.isOnline) == true + expect(testContext.eventProcessorMock.isOnline) == true expect(testContext.flagSynchronizerMock.isOnline) == true } - it("when offline at foreground notification") { + it("when offline at foreground notification", onlyWhere: platformPostsLifecycleNotifications) { let testContext = TestContext() testContext.start(runMode: .background) NotificationCenter.default.post(name: SystemCapabilities.foregroundNotification!, object: self) expect(testContext.subject.isOnline) == false expect(testContext.subject.runMode) == LDClientRunMode.foreground - expect(testContext.eventReporterMock.isOnline) == false + expect(testContext.eventProcessorMock.isOnline) == false expect(testContext.flagSynchronizerMock.isOnline) == false } } @@ -1457,7 +1515,7 @@ final class LDClientSpec: QuickSpec { testContext.start() testContext.subject.setRunMode(.background) - expect(testContext.eventReporterMock.isOnline) == true + expect(testContext.eventProcessorMock.isOnline) == true expect(testContext.flagSynchronizerMock.isOnline) == true expect(testContext.flagSynchronizerMock.streamingMode) == LDStreamingMode.streaming } @@ -1466,7 +1524,7 @@ final class LDClientSpec: QuickSpec { testContext.start() testContext.subject.setRunMode(.background) - expect(testContext.eventReporterMock.isOnline) == true + expect(testContext.eventProcessorMock.isOnline) == true expect(testContext.flagSynchronizerMock.isOnline) == true expect(testContext.flagSynchronizerMock.streamingMode) == LDStreamingMode.polling expect(testContext.flagSynchronizerMock.pollingInterval) == testContext.config.flagPollingInterval(runMode: .background) @@ -1477,7 +1535,7 @@ final class LDClientSpec: QuickSpec { testContext.start() testContext.subject.setRunMode(.background) - expect(testContext.eventReporterMock.isOnline) == true + expect(testContext.eventProcessorMock.isOnline) == true expect(testContext.flagSynchronizerMock.isOnline) == false expect(testContext.flagSynchronizerMock.streamingMode) == LDStreamingMode.polling expect(testContext.flagSynchronizerMock.pollingInterval) == testContext.config.flagPollingInterval(runMode: .background) @@ -1486,13 +1544,13 @@ final class LDClientSpec: QuickSpec { it("set foreground") { let testContext = TestContext(startOnline: true) testContext.start() - let eventReporterIsOnlineSetCount = testContext.eventReporterMock.isOnlineSetCount + let eventProcessorIsOnlineSetCount = testContext.eventProcessorMock.isOnlineSetCount let flagSynchronizerIsOnlineSetCount = testContext.flagSynchronizerMock.isOnlineSetCount let makeFlagSynchronizerCallCount = testContext.serviceFactoryMock.makeFlagSynchronizerCallCount testContext.subject.setRunMode(.foreground) - expect(testContext.eventReporterMock.isOnline) == true - expect(testContext.eventReporterMock.isOnlineSetCount) == eventReporterIsOnlineSetCount + expect(testContext.eventProcessorMock.isOnline) == true + expect(testContext.eventProcessorMock.isOnlineSetCount) == eventProcessorIsOnlineSetCount expect(testContext.flagSynchronizerMock.isOnline) == true expect(testContext.flagSynchronizerMock.isOnlineSetCount) == flagSynchronizerIsOnlineSetCount expect(testContext.serviceFactoryMock.makeFlagSynchronizerCallCount) == makeFlagSynchronizerCallCount @@ -1503,13 +1561,13 @@ final class LDClientSpec: QuickSpec { let testContext = TestContext(startOnline: true) testContext.start() testContext.subject.setRunMode(.background) - let eventReporterIsOnlineSetCount = testContext.eventReporterMock.isOnlineSetCount + let eventProcessorIsOnlineSetCount = testContext.eventProcessorMock.isOnlineSetCount let flagSynchronizerIsOnlineSetCount = testContext.flagSynchronizerMock.isOnlineSetCount let makeFlagSynchronizerCallCount = testContext.serviceFactoryMock.makeFlagSynchronizerCallCount testContext.subject.setRunMode(.background) - expect(testContext.eventReporterMock.isOnline) == true - expect(testContext.eventReporterMock.isOnlineSetCount) == eventReporterIsOnlineSetCount + expect(testContext.eventProcessorMock.isOnline) == true + expect(testContext.eventProcessorMock.isOnlineSetCount) == eventProcessorIsOnlineSetCount expect(testContext.flagSynchronizerMock.isOnline) == true expect(testContext.flagSynchronizerMock.isOnlineSetCount) == flagSynchronizerIsOnlineSetCount expect(testContext.serviceFactoryMock.makeFlagSynchronizerCallCount) == makeFlagSynchronizerCallCount @@ -1520,7 +1578,7 @@ final class LDClientSpec: QuickSpec { testContext.start(runMode: .background) testContext.subject.setRunMode(.foreground) - expect(testContext.eventReporterMock.isOnline) == true + expect(testContext.eventProcessorMock.isOnline) == true expect(testContext.flagSynchronizerMock.isOnline) == true expect(testContext.flagSynchronizerMock.streamingMode) == LDStreamingMode.streaming } @@ -1529,7 +1587,7 @@ final class LDClientSpec: QuickSpec { testContext.start(runMode: .background) testContext.subject.setRunMode(.foreground) - expect(testContext.eventReporterMock.isOnline) == true + expect(testContext.eventProcessorMock.isOnline) == true expect(testContext.flagSynchronizerMock.isOnline) == true expect(testContext.flagSynchronizerMock.streamingMode) == LDStreamingMode.polling expect(testContext.flagSynchronizerMock.pollingInterval) == testContext.config.flagPollingInterval(runMode: .foreground) @@ -1546,7 +1604,7 @@ final class LDClientSpec: QuickSpec { testContext.subject.setRunMode(.background) - expect(testContext.eventReporterMock.isOnline) == false + expect(testContext.eventProcessorMock.isOnline) == false expect(testContext.flagSynchronizerMock.isOnline) == false expect(testContext.flagSynchronizerMock.streamingMode) == LDStreamingMode.streaming } @@ -1556,7 +1614,7 @@ final class LDClientSpec: QuickSpec { testContext.subject.setRunMode(.background) - expect(testContext.eventReporterMock.isOnline) == false + expect(testContext.eventProcessorMock.isOnline) == false expect(testContext.flagSynchronizerMock.isOnline) == false expect(testContext.flagSynchronizerMock.streamingMode) == LDStreamingMode.polling expect(testContext.flagSynchronizerMock.pollingInterval) == testContext.config.flagPollingInterval(runMode: .background) @@ -1566,13 +1624,13 @@ final class LDClientSpec: QuickSpec { let testContext = TestContext() waitUntil { testContext.start(completion: $0) } - let eventReporterIsOnlineSetCount = testContext.eventReporterMock.isOnlineSetCount + let eventProcessorIsOnlineSetCount = testContext.eventProcessorMock.isOnlineSetCount let flagSynchronizerIsOnlineSetCount = testContext.flagSynchronizerMock.isOnlineSetCount let makeFlagSynchronizerCallCount = testContext.serviceFactoryMock.makeFlagSynchronizerCallCount testContext.subject.setRunMode(.foreground) - expect(testContext.eventReporterMock.isOnline) == false - expect(testContext.eventReporterMock.isOnlineSetCount) == eventReporterIsOnlineSetCount + expect(testContext.eventProcessorMock.isOnline) == false + expect(testContext.eventProcessorMock.isOnlineSetCount) == eventProcessorIsOnlineSetCount expect(testContext.flagSynchronizerMock.isOnline) == false expect(testContext.flagSynchronizerMock.isOnlineSetCount) == flagSynchronizerIsOnlineSetCount expect(testContext.serviceFactoryMock.makeFlagSynchronizerCallCount) == makeFlagSynchronizerCallCount @@ -1583,13 +1641,13 @@ final class LDClientSpec: QuickSpec { let testContext = TestContext() waitUntil { testContext.start(runMode: .background, completion: $0) } - let eventReporterIsOnlineSetCount = testContext.eventReporterMock.isOnlineSetCount + let eventProcessorIsOnlineSetCount = testContext.eventProcessorMock.isOnlineSetCount let flagSynchronizerIsOnlineSetCount = testContext.flagSynchronizerMock.isOnlineSetCount let makeFlagSynchronizerCallCount = testContext.serviceFactoryMock.makeFlagSynchronizerCallCount testContext.subject.setRunMode(.background) - expect(testContext.eventReporterMock.isOnline) == false - expect(testContext.eventReporterMock.isOnlineSetCount) == eventReporterIsOnlineSetCount + expect(testContext.eventProcessorMock.isOnline) == false + expect(testContext.eventProcessorMock.isOnlineSetCount) == eventProcessorIsOnlineSetCount expect(testContext.flagSynchronizerMock.isOnline) == false expect(testContext.flagSynchronizerMock.isOnlineSetCount) == flagSynchronizerIsOnlineSetCount expect(testContext.serviceFactoryMock.makeFlagSynchronizerCallCount) == makeFlagSynchronizerCallCount @@ -1601,7 +1659,7 @@ final class LDClientSpec: QuickSpec { testContext.subject.setRunMode(.foreground) - expect(testContext.eventReporterMock.isOnline) == false + expect(testContext.eventProcessorMock.isOnline) == false expect(testContext.flagSynchronizerMock.isOnline) == false expect(testContext.flagSynchronizerMock.streamingMode) == LDStreamingMode.streaming } @@ -1611,7 +1669,7 @@ final class LDClientSpec: QuickSpec { testContext.subject.setRunMode(.foreground) - expect(testContext.eventReporterMock.isOnline) == false + expect(testContext.eventProcessorMock.isOnline) == false expect(testContext.flagSynchronizerMock.isOnline) == false expect(testContext.flagSynchronizerMock.streamingMode) == LDStreamingMode.polling expect(testContext.flagSynchronizerMock.pollingInterval) == testContext.config.flagPollingInterval(runMode: .foreground) @@ -1629,8 +1687,10 @@ final class LDClientSpec: QuickSpec { describe("flag synchronizer streaming mode") { OperatingSystem.allOperatingSystems.forEach { os in it("on \(os) sets the flag synchronizer streaming mode") { - // TODO(os-tests): We need to expand this to the other OSs - if os == .watchOS { + // TODO(os-tests): We need to expand this to the other OSs. The client runs as the platform the + // tests do, whatever `os` says. + guard os == SystemCapabilities.operatingSystem + else { return } @@ -1648,13 +1708,13 @@ final class LDClientSpec: QuickSpec { let testContext = TestContext() testContext.start() testContext.subject.flush() - expect(testContext.eventReporterMock.flushCallCount) == 1 + expect(testContext.eventProcessorMock.flushCallCount) == 1 } } describe("flush with a completion") { /// Answers from another thread, because this call does not block and must not be answered inside it. - func answer(_ mock: EventReportingMock, with delivered: Bool, after delay: TimeInterval = 0.05) { + func answer(_ mock: EventProcessorMock, with delivered: Bool, after delay: TimeInterval = 0.05) { mock.flushReportingOutcomeCallback = { [weak mock] in let completion = mock?.flushReportingOutcomeReceivedCompletion DispatchQueue.global().asyncAfter(deadline: .now() + delay) { completion?(delivered) } @@ -1664,7 +1724,7 @@ final class LDClientSpec: QuickSpec { it("reports true when delivery succeeds") { let testContext = TestContext() testContext.start() - answer(testContext.eventReporterMock, with: true) + answer(testContext.eventProcessorMock, with: true) var delivered: Bool? testContext.subject.flush { delivered = $0 } @@ -1676,7 +1736,7 @@ final class LDClientSpec: QuickSpec { it("reports false when delivery cannot finish") { let testContext = TestContext() testContext.start() - answer(testContext.eventReporterMock, with: false) + answer(testContext.eventProcessorMock, with: false) var delivered: Bool? testContext.subject.flush { delivered = $0 } @@ -1689,26 +1749,67 @@ final class LDClientSpec: QuickSpec { stale.subject.close() let current = TestContext() current.start() - answer(current.eventReporterMock, with: true) + answer(current.eventProcessorMock, with: true) var delivered: Bool? stale.subject.flush { delivered = $0 } - expect(delivered) == false - expect(current.eventReporterMock.flushReportingOutcomeCallCount) == 0 + expect(delivered).toEventually(beFalse()) + expect(current.eventProcessorMock.flushReportingOutcomeCallCount) == 0 + } + + it("answers a closed client off the caller's thread, as it does one with a delivery to wait for") { + let testContext = TestContext() + testContext.start() + testContext.subject.close() + + // The contract holds whether or not the client turned out to be closed: a caller told its completion + // is elsewhere may take a lock there that it is holding now. + let caller = Thread.current + var answeredOnCaller: Bool? + waitUntil { done in + testContext.subject.flush { _ in + answeredOnCaller = Thread.current === caller + done() + } + } + expect(answeredOnCaller) == false } it("reports once every environment has answered, and only once") { let testContext = TestContext() try testContext.config.setSecondaryMobileKeys(["alternate": Constants.alternateMockMobileKey]) testContext.start() - let answering = LDClient.get(environment: "alternate")?.eventReporter as? EventReportingMock + let answering = LDClient.get(environment: "alternate")?.eventProcessor as? EventProcessorMock expect(answering).toNot(beNil()) // The one that fails answers last, so reporting before every environment is in would say true. - answer(testContext.eventReporterMock, with: true, after: 0.05) + answer(testContext.eventProcessorMock, with: true, after: 0.05) + answer(answering!, with: false, after: 0.2) + + var answers: [Bool] = [] + testContext.subject.flush { answers.append($0) } + expect(answers).toEventually(equal([false])) + Thread.sleep(forTimeInterval: 0.2) + expect(answers) == [false] + } + + it("waits for every environment where one answers twice") { + let testContext = TestContext() + try testContext.config.setSecondaryMobileKeys(["alternate": Constants.alternateMockMobileKey]) + testContext.start() + testContext.eventProcessorMock.flushReportingOutcomeCallback = { [weak testContext] in + let completion = testContext?.eventProcessorMock.flushReportingOutcomeReceivedCompletion + DispatchQueue.global().async { + completion?(true) + completion?(true) + } + } + let answering = LDClient.get(environment: "alternate")?.eventProcessor as? EventProcessorMock + expect(answering).toNot(beNil()) answer(answering!, with: false, after: 0.2) var answers: [Bool] = [] testContext.subject.flush { answers.append($0) } + // Two answers from the primary are not one from each, so a `true` here would be about events still unsent. expect(answers).toEventually(equal([false])) Thread.sleep(forTimeInterval: 0.2) expect(answers) == [false] @@ -1717,25 +1818,26 @@ final class LDClientSpec: QuickSpec { describe("flushAndWait") { /// Answers inside the call, because `flushAndWait` blocks the test thread. - func answer(_ mock: EventReportingMock, with delivered: Bool) { - mock.flushReportingOutcomeCallback = { [weak mock] in - mock?.flushReportingOutcomeReceivedCompletion?(delivered) + func answer(_ mock: EventProcessorMock, with delivered: Bool) { + mock.flushBypassingBackoffCallback = { [weak mock] in + mock?.flushBypassingBackoffReceivedCompletion?(delivered) } } it("reports true when delivery succeeds") { let testContext = TestContext() testContext.start() - answer(testContext.eventReporterMock, with: true) + answer(testContext.eventProcessorMock, with: true) expect(testContext.subject.flushAndWait(timeout: 1.0)) == true - expect(testContext.eventReporterMock.flushReportingOutcomeCallCount) == 1 + expect(testContext.eventProcessorMock.flushBypassingBackoffCallCount) == 1 + expect(testContext.eventProcessorMock.flushReportingOutcomeCallCount) == 0 } it("reports false when delivery cannot finish") { let testContext = TestContext() testContext.start() - answer(testContext.eventReporterMock, with: false) + answer(testContext.eventProcessorMock, with: false) expect(testContext.subject.flushAndWait(timeout: 1.0)) == false } @@ -1743,15 +1845,26 @@ final class LDClientSpec: QuickSpec { it("reports false when the budget expires first") { let testContext = TestContext() testContext.start() - testContext.eventReporterMock.flushReportingOutcomeCallback = nil + testContext.eventProcessorMock.flushBypassingBackoffCallback = nil expect(testContext.subject.flushAndWait(timeout: 0.05)) == false } + it("does not wait without a budget, though it still starts the delivery") { + let testContext = TestContext() + testContext.start() + // Answers inside the call, so a zero budget that waited at all would report true. + answer(testContext.eventProcessorMock, with: true) + + expect(testContext.subject.flushAndWait(timeout: 0)) == false + // Still worth calling: at `.immediate` the events are on disk by the time the delivery is queued. + expect(testContext.eventProcessorMock.flushBypassingBackoffCallCount) == 1 + } + it("reports false once closed") { let testContext = TestContext() testContext.start() - answer(testContext.eventReporterMock, with: true) + answer(testContext.eventProcessorMock, with: true) testContext.subject.close() expect(testContext.subject.flushAndWait(timeout: 1.0)) == false @@ -1763,11 +1876,11 @@ final class LDClientSpec: QuickSpec { stale.subject.close() let current = TestContext() current.start() - answer(current.eventReporterMock, with: true) + answer(current.eventProcessorMock, with: true) // Its own events were never attempted, so the outcome of another client's delivery says nothing of them. expect(stale.subject.flushAndWait(timeout: 1.0)) == false - expect(current.eventReporterMock.flushReportingOutcomeCallCount) == 0 + expect(current.eventProcessorMock.flushBypassingBackoffCallCount) == 0 } it("gives every environment the whole budget rather than what the one before it left") { @@ -1778,10 +1891,10 @@ final class LDClientSpec: QuickSpec { // Each takes most of the budget, so delivering them one after another would leave the second // environment past the deadline before its own delivery had begun. for environment in ["alternate", LDConfig.Constants.primaryEnvironmentName] { - let reporter = LDClient.get(environment: environment)?.eventReporter as? EventReportingMock + let reporter = LDClient.get(environment: environment)?.eventProcessor as? EventProcessorMock expect(reporter).toNot(beNil()) - reporter?.flushReportingOutcomeCallback = { [weak reporter] in - let completion = reporter?.flushReportingOutcomeReceivedCompletion + reporter?.flushBypassingBackoffCallback = { [weak reporter] in + let completion = reporter?.flushBypassingBackoffReceivedCompletion DispatchQueue.global().asyncAfter(deadline: .now() + 0.3) { completion?(true) } } } @@ -1795,13 +1908,13 @@ final class LDClientSpec: QuickSpec { testContext.start() // One answers most of the way through the budget and the other never does, so the budget the first // spent is the budget the second has left. Giving each its own copy of it would take twice as long. - let answering = LDClient.get(environment: "alternate")?.eventReporter as? EventReportingMock + let answering = LDClient.get(environment: "alternate")?.eventProcessor as? EventProcessorMock expect(answering).toNot(beNil()) - answering?.flushReportingOutcomeCallback = { [weak answering] in - let completion = answering?.flushReportingOutcomeReceivedCompletion + answering?.flushBypassingBackoffCallback = { [weak answering] in + let completion = answering?.flushBypassingBackoffReceivedCompletion DispatchQueue.global().asyncAfter(deadline: .now() + 0.5) { completion?(true) } } - testContext.eventReporterMock.flushReportingOutcomeCallback = nil + testContext.eventProcessorMock.flushBypassingBackoffCallback = nil // Answering at 0.5s into a 0.6s budget leaves the second wait 0.1s, against the 0.6s a fresh copy // would give it, so the two outcomes are 0.6s and 1.1s with room on either side of the threshold. @@ -1809,6 +1922,23 @@ final class LDClientSpec: QuickSpec { expect(testContext.subject.flushAndWait(timeout: 0.6)) == false expect(Date().timeIntervalSince(started)) < 0.85 } + + it("waits for every environment where one answers twice") { + let testContext = TestContext() + try testContext.config.setSecondaryMobileKeys(["alternate": Constants.alternateMockMobileKey]) + testContext.start() + testContext.eventProcessorMock.flushBypassingBackoffCallback = { [weak testContext] in + let completion = testContext?.eventProcessorMock.flushBypassingBackoffReceivedCompletion + completion?(true) + completion?(true) + } + let silent = LDClient.get(environment: "alternate")?.eventProcessor as? EventProcessorMock + expect(silent).toNot(beNil()) + silent?.flushBypassingBackoffCallback = nil + + // Two answers from one environment are not one from each, and the other's events are still unsent. + expect(testContext.subject.flushAndWait(timeout: 0.2)) == false + } } } @@ -1835,7 +1965,7 @@ final class LDClientSpec: QuickSpec { let testContext = TestContext(startOnline: true) testContext.start() expect(testContext.subject.isOnline) == true - expect(testContext.subject.connectionInformation.currentConnectionMode).to(equal(.establishingStreamingConnection)) + expect(testContext.subject.connectionInformation.currentConnectionMode) == (platformStreams ? .establishingStreamingConnection : .polling) } it("when client was started in background") { let testContext = TestContext(startOnline: true, enableBackgroundUpdates: false) @@ -1953,7 +2083,7 @@ final class LDClientSpec: QuickSpec { testContext.subject.internalIdentify(newContext: LDContext.stub(), useCache: .yes) expect(testContext.subject.isInitialized) == true } - it("becomes true when setOnline(true) is refused in the background") { + it("becomes true when setOnline(true) is refused in the background", onlyWhere: platformPostsLifecycleNotifications) { let testContext = TestContext(startOnline: true, enableBackgroundUpdates: false) testContext.start() testContext.subject.setRunMode(.background) @@ -1965,7 +2095,7 @@ final class LDClientSpec: QuickSpec { NotificationCenter.default.post(name: SystemCapabilities.foregroundNotification!, object: self) expect(testContext.subject.isInitialized) == true } - it("remains false across a background and foreground cycle without flags") { + it("remains false across a background and foreground cycle without flags", onlyWhere: platformPostsLifecycleNotifications) { let testContext = TestContext(startOnline: true, enableBackgroundUpdates: false) testContext.start() expect(testContext.subject.isInitialized) == false @@ -2016,4 +2146,31 @@ extension OperatingSystem { } } +private extension LDConfig { + /// The mode a foreground client connects in. Platforms that cannot stream, watchOS among them, poll whatever the + /// config asks for. + var effectiveStreamingMode: LDStreamingMode { + allowStreamingMode ? streamingMode : .polling + } +} + +/// watchOS posts no lifecycle notifications, and does not stream. +private let platformPostsLifecycleNotifications = SystemCapabilities.foregroundNotification != nil + && SystemCapabilities.backgroundNotification != nil +private let platformStreams = SystemCapabilities.operatingSystem.isStreamingEnabled + +/// `it`, for a spec that only applies where `condition` holds. Elsewhere the spec is reported pending rather than run. +private func it(_ description: String, + onlyWhere condition: Bool, + file: FileString = #file, + line: UInt = #line, + closure: @escaping () throws -> Void) { + guard condition + else { + pending(description) { } + return + } + it(description, file: file, line: line, closure: closure) +} + private class ErrorMock: Error { } diff --git a/LaunchDarkly/LaunchDarklyTests/Mocks/ClientServiceMockFactory.swift b/LaunchDarkly/LaunchDarklyTests/Mocks/ClientServiceMockFactory.swift index c74e7132e..047789419 100644 --- a/LaunchDarkly/LaunchDarklyTests/Mocks/ClientServiceMockFactory.swift +++ b/LaunchDarkly/LaunchDarklyTests/Mocks/ClientServiceMockFactory.swift @@ -72,17 +72,19 @@ final class ClientServiceMockFactory: ClientServiceCreating { return makeFlagChangeNotifierReturnValue } - var makeEventReporterCallCount = 0 - var makeEventReporterReceivedConfig: LDConfig? = nil - var makeEventReporterReceivedService: DarklyServiceProvider? = nil + var makeEventProcessorCallCount = 0 + var makeEventProcessorReceivedConfig: LDConfig? = nil + var makeEventProcessorReceivedService: DarklyServiceProvider? = nil var onEventSyncComplete: EventSyncCompleteClosure? = nil - func makeEventReporter(config: LDConfig, service: DarklyServiceProvider, onSyncComplete: EventSyncCompleteClosure?) -> EventReporting { - makeEventReporterCallCount += 1 - makeEventReporterReceivedConfig = config - makeEventReporterReceivedService = service + func makeEventProcessor(config: LDConfig, + service: DarklyServiceProvider, + onSyncComplete: EventSyncCompleteClosure?) -> EventProcessor { + makeEventProcessorCallCount += 1 + makeEventProcessorReceivedConfig = config + makeEventProcessorReceivedService = service onEventSyncComplete = onSyncComplete - let mock = EventReportingMock() + let mock = EventProcessorMock() // Answer by default so backgrounding does not hold its activity assertion until timeout. mock.flushReportingOutcomeCallback = { [weak mock] in mock?.flushReportingOutcomeReceivedCompletion?(true) @@ -90,8 +92,8 @@ final class ClientServiceMockFactory: ClientServiceCreating { return mock } - func makeEventReporter(config: LDConfig, service: DarklyServiceProvider) -> EventReporting { - return makeEventReporter(config: config, service: service, onSyncComplete: nil) + func makeEventProcessor(config: LDConfig, service: DarklyServiceProvider) -> EventProcessor { + return makeEventProcessor(config: config, service: service, onSyncComplete: nil) } var makeStreamingProviderCallCount = 0 diff --git a/LaunchDarkly/LaunchDarklyTests/Mocks/DarklyServiceMock.swift b/LaunchDarkly/LaunchDarklyTests/Mocks/DarklyServiceMock.swift index 94dca6865..0d4bce729 100644 --- a/LaunchDarkly/LaunchDarklyTests/Mocks/DarklyServiceMock.swift +++ b/LaunchDarkly/LaunchDarklyTests/Mocks/DarklyServiceMock.swift @@ -46,7 +46,7 @@ final class DarklyServiceMock: DarklyServiceProvider { } struct Constants { - static let error = NSError(domain: NSURLErrorDomain, code: Int(CFNetworkErrors.cfurlErrorResourceUnavailable.rawValue), userInfo: nil) + static let error = NSError(domain: NSURLErrorDomain, code: NSURLErrorResourceUnavailable, userInfo: nil) static let jsonErrorString = "Bad json data" static let errorData = jsonErrorString.data(using: .utf8)! @@ -139,8 +139,23 @@ final class DarklyServiceMock: DarklyServiceProvider { private var _stubbedEventResponse: ServiceResponse? private var _publishEventDataCallCount = 0 private var _publishedEventData: Data? + private var _publishedEventPayloads: [Data] = [] + private var _publishedPayloadIds: [String] = [] private var _holdsEventCompletions = false private var heldEventCompletions: [ServiceCompletionHandler] = [] + private var _eventStatusForPost: ((Int) -> Int)? + private var _publishedEventStatuses: [Int?] = [] + + /// The status to answer the nth post with, counting from 1, in place of `stubbedEventResponse`; 0 for a network + /// error with no response. + var eventStatusForPost: ((Int) -> Int)? { + get { withEventLock { _eventStatusForPost } } + set { withEventLock { _eventStatusForPost = newValue } } + } + /// The status each post was answered with, in the order of `publishedEventPayloads`; nil where it had none. + var publishedEventStatuses: [Int?] { + withEventLock { _publishedEventStatuses } + } var stubbedEventResponse: ServiceResponse? { get { withEventLock { _stubbedEventResponse } } @@ -154,6 +169,14 @@ final class DarklyServiceMock: DarklyServiceProvider { get { withEventLock { _publishedEventData } } set { withEventLock { _publishedEventData = newValue } } } + /// Every payload published, in order, so a test can see what a delivery of several batches sent. + var publishedEventPayloads: [Data] { + withEventLock { _publishedEventPayloads } + } + /// Every payload ID used, in order, so a test can tell a retry of a delivery from a new one. + var publishedPayloadIds: [String] { + withEventLock { _publishedPayloadIds } + } /// Holds responses so a test can act while a delivery is in flight. var holdsEventCompletions: Bool { get { withEventLock { _holdsEventCompletions } } @@ -165,14 +188,28 @@ final class DarklyServiceMock: DarklyServiceProvider { let response: ServiceResponse? = withEventLock { _publishEventDataCallCount += 1 _publishedEventData = eventData + _publishedEventPayloads.append(eventData) + _publishedPayloadIds.append(payloadId) guard !_holdsEventCompletions else { + _publishedEventStatuses.append(nil) if let completion { heldEventCompletions.append(completion) } return nil } - return _stubbedEventResponse ?? (nil, nil, nil, nil) + var response = _stubbedEventResponse ?? (nil, nil, nil, nil) + if let statusForPost = _eventStatusForPost { + let status = statusForPost(_publishEventDataCallCount) + response = status == 0 + ? (nil, nil, Constants.error, nil) + : (nil, + HTTPURLResponse(url: config.eventsUrl, statusCode: status, httpVersion: Constants.httpVersion, headerFields: nil), + nil, + nil) + } + _publishedEventStatuses.append((response.urlResponse as? HTTPURLResponse)?.statusCode) + return response } if let response { completion?(response) diff --git a/LaunchDarkly/LaunchDarklyTests/Mocks/DiagnosticCachingMock.swift b/LaunchDarkly/LaunchDarklyTests/Mocks/DiagnosticCachingMock.swift new file mode 100644 index 000000000..cc818d990 --- /dev/null +++ b/LaunchDarkly/LaunchDarklyTests/Mocks/DiagnosticCachingMock.swift @@ -0,0 +1,134 @@ +import Foundation + +@testable import LaunchDarkly + +/// Written by hand rather than generated, because the SDK calls it from whichever thread records, commits or delivers +/// events while a test reads the counts. Every property is guarded by `lock`; callbacks run outside it. +final class DiagnosticCachingMock: DiagnosticCaching { + private let lock = NSLock() + + private var _getDiagnosticIdCallCount = 0 + private var _getDiagnosticIdCallback: (() throws -> Void)? + private var _getDiagnosticIdReturnValue: DiagnosticId! + + private var _getCurrentStatsAndResetCallCount = 0 + private var _getCurrentStatsAndResetCallback: (() throws -> Void)? + private var _getCurrentStatsAndResetReturnValue: DiagnosticStats! + + private var _incrementDroppedEventCountCallCount = 0 + private var _incrementDroppedEventCountCallback: (() throws -> Void)? + + private var _recordEventsInLastBatchCallCount = 0 + private var _recordEventsInLastBatchCallback: (() throws -> Void)? + private var _recordEventsInLastBatchReceivedEventsInLastBatch: Int? + + private var _addStreamInitCallCount = 0 + private var _addStreamInitCallback: (() throws -> Void)? + private var _addStreamInitReceivedStreamInit: DiagnosticStreamInit? + + private func synchronized(_ body: () -> T) -> T { + lock.lock() + defer { lock.unlock() } + return body() + } + + var getDiagnosticIdCallCount: Int { + get { synchronized { _getDiagnosticIdCallCount } } + set { synchronized { _getDiagnosticIdCallCount = newValue } } + } + var getDiagnosticIdCallback: (() throws -> Void)? { + get { synchronized { _getDiagnosticIdCallback } } + set { synchronized { _getDiagnosticIdCallback = newValue } } + } + var getDiagnosticIdReturnValue: DiagnosticId! { + get { synchronized { _getDiagnosticIdReturnValue } } + set { synchronized { _getDiagnosticIdReturnValue = newValue } } + } + func getDiagnosticId() -> DiagnosticId { + let (callback, returnValue) = synchronized { () -> ((() throws -> Void)?, DiagnosticId?) in + _getDiagnosticIdCallCount += 1 + return (_getDiagnosticIdCallback, _getDiagnosticIdReturnValue) + } + try! callback?() + return returnValue! + } + + var getCurrentStatsAndResetCallCount: Int { + get { synchronized { _getCurrentStatsAndResetCallCount } } + set { synchronized { _getCurrentStatsAndResetCallCount = newValue } } + } + var getCurrentStatsAndResetCallback: (() throws -> Void)? { + get { synchronized { _getCurrentStatsAndResetCallback } } + set { synchronized { _getCurrentStatsAndResetCallback = newValue } } + } + var getCurrentStatsAndResetReturnValue: DiagnosticStats! { + get { synchronized { _getCurrentStatsAndResetReturnValue } } + set { synchronized { _getCurrentStatsAndResetReturnValue = newValue } } + } + func getCurrentStatsAndReset() -> DiagnosticStats { + let (callback, returnValue) = synchronized { () -> ((() throws -> Void)?, DiagnosticStats?) in + _getCurrentStatsAndResetCallCount += 1 + return (_getCurrentStatsAndResetCallback, _getCurrentStatsAndResetReturnValue) + } + try! callback?() + return returnValue! + } + + var incrementDroppedEventCountCallCount: Int { + get { synchronized { _incrementDroppedEventCountCallCount } } + set { synchronized { _incrementDroppedEventCountCallCount = newValue } } + } + var incrementDroppedEventCountCallback: (() throws -> Void)? { + get { synchronized { _incrementDroppedEventCountCallback } } + set { synchronized { _incrementDroppedEventCountCallback = newValue } } + } + func incrementDroppedEventCount() { + let callback = synchronized { () -> (() throws -> Void)? in + _incrementDroppedEventCountCallCount += 1 + return _incrementDroppedEventCountCallback + } + try! callback?() + } + + var recordEventsInLastBatchCallCount: Int { + get { synchronized { _recordEventsInLastBatchCallCount } } + set { synchronized { _recordEventsInLastBatchCallCount = newValue } } + } + var recordEventsInLastBatchCallback: (() throws -> Void)? { + get { synchronized { _recordEventsInLastBatchCallback } } + set { synchronized { _recordEventsInLastBatchCallback = newValue } } + } + var recordEventsInLastBatchReceivedEventsInLastBatch: Int? { + get { synchronized { _recordEventsInLastBatchReceivedEventsInLastBatch } } + set { synchronized { _recordEventsInLastBatchReceivedEventsInLastBatch = newValue } } + } + func recordEventsInLastBatch(eventsInLastBatch: Int) { + let callback = synchronized { () -> (() throws -> Void)? in + _recordEventsInLastBatchCallCount += 1 + _recordEventsInLastBatchReceivedEventsInLastBatch = eventsInLastBatch + return _recordEventsInLastBatchCallback + } + try! callback?() + } + + var addStreamInitCallCount: Int { + get { synchronized { _addStreamInitCallCount } } + set { synchronized { _addStreamInitCallCount = newValue } } + } + var addStreamInitCallback: (() throws -> Void)? { + get { synchronized { _addStreamInitCallback } } + set { synchronized { _addStreamInitCallback = newValue } } + } + var addStreamInitReceivedStreamInit: DiagnosticStreamInit? { + get { synchronized { _addStreamInitReceivedStreamInit } } + set { synchronized { _addStreamInitReceivedStreamInit = newValue } } + } + func addStreamInit(streamInit: DiagnosticStreamInit) { + let callback = synchronized { () -> (() throws -> Void)? in + _addStreamInitCallCount += 1 + _addStreamInitReceivedStreamInit = streamInit + return _addStreamInitCallback + } + try! callback?() + } +} diff --git a/LaunchDarkly/LaunchDarklyTests/Models/DiagnosticEventSpec.swift b/LaunchDarkly/LaunchDarklyTests/Models/DiagnosticEventSpec.swift index b76e4186e..3a6089d33 100644 --- a/LaunchDarkly/LaunchDarklyTests/Models/DiagnosticEventSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/Models/DiagnosticEventSpec.swift @@ -174,7 +174,7 @@ final class DiagnosticEventSpec: QuickSpec { customConfig.baseUrl = URL(string: "https://clientstream.launchdarkly.com")! customConfig.eventsUrl = URL(string: "https://app.launchdarkly.com")! customConfig.streamUrl = URL(string: "https://mobile.launchdarkly.com")! - customConfig.eventCapacity = 1_000 + customConfig.eventCapacity = 3_000 customConfig.connectionTimeout = 30.0 customConfig.eventFlushInterval = 60.0 customConfig.streamingMode = .polling @@ -202,7 +202,7 @@ final class DiagnosticEventSpec: QuickSpec { expect(diagnosticConfig.customBaseURI) == false expect(diagnosticConfig.customEventsURI) == false expect(diagnosticConfig.customStreamURI) == false - expect(diagnosticConfig.eventsCapacity) == 100 + expect(diagnosticConfig.eventsCapacity) == 2_000 expect(diagnosticConfig.connectTimeoutMillis) == 10_000 expect(diagnosticConfig.eventsFlushIntervalMillis) == 30_000 expect(diagnosticConfig.streamingDisabled) == false @@ -224,7 +224,7 @@ final class DiagnosticEventSpec: QuickSpec { expect(diagnosticConfig.customBaseURI) == true expect(diagnosticConfig.customEventsURI) == true expect(diagnosticConfig.customStreamURI) == true - expect(diagnosticConfig.eventsCapacity) == 1_000 + expect(diagnosticConfig.eventsCapacity) == 3_000 expect(diagnosticConfig.connectTimeoutMillis) == 30_000 expect(diagnosticConfig.eventsFlushIntervalMillis) == 60_000 expect(diagnosticConfig.streamingDisabled) == true diff --git a/LaunchDarkly/LaunchDarklyTests/Models/LDConfigSpec.swift b/LaunchDarkly/LaunchDarklyTests/Models/LDConfigSpec.swift index db24862ca..db179191a 100644 --- a/LaunchDarkly/LaunchDarklyTests/Models/LDConfigSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/Models/LDConfigSpec.swift @@ -7,6 +7,9 @@ final class LDConfigSpec: XCTestCase { struct Constants { fileprivate static let alternateMockUrl = URL(string: "https://dummy.alternate.com")! fileprivate static let eventCapacity = 10 + fileprivate static let eventPersistence = EventPersistence.immediate + fileprivate static let eventCapacityBytes = 4096 + fileprivate static let eventMaxAge: TimeInterval = 60 fileprivate static let connectionTimeout: TimeInterval = 0.01 fileprivate static let eventFlushInterval: TimeInterval = 0.01 fileprivate static let flagPollingInterval: TimeInterval = 0.01 @@ -26,6 +29,7 @@ final class LDConfigSpec: XCTestCase { fileprivate static let maxCachedContexts = -1 fileprivate static let diagnosticOptOut = true fileprivate static let diagnosticRecordingInterval: TimeInterval = 600.0 + fileprivate static let eventDeliveryDelay: TimeInterval = 5 fileprivate static let wrapperName = "ReactNative" fileprivate static let wrapperVersion = "0.1.0" fileprivate static let additionalHeaders = ["Proxy-Authorization": "creds"] @@ -37,6 +41,10 @@ final class LDConfigSpec: XCTestCase { ("event URL", Constants.alternateMockUrl, { c, v in c.eventsUrl = v as! URL }), ("stream URL", Constants.alternateMockUrl, { c, v in c.streamUrl = v as! URL }), ("event capacity", Constants.eventCapacity, { c, v in c.eventCapacity = v as! Int }), + ("event persistence", Constants.eventPersistence, { c, v in c.eventPersistence = v as! EventPersistence }), + ("event capacity bytes", Constants.eventCapacityBytes, { c, v in c.eventCapacityBytes = v as! Int }), + ("event max age", Constants.eventMaxAge, { c, v in c.eventMaxAge = v as! TimeInterval }), + ("event delivery delay", Constants.eventDeliveryDelay, { c, v in c.eventDeliveryDelay = v as! TimeInterval }), ("connection timeout", Constants.connectionTimeout, { c, v in c.connectionTimeout = v as! TimeInterval }), ("event flush interval", Constants.eventFlushInterval, { c, v in c.eventFlushInterval = v as! TimeInterval }), ("poll interval", Constants.flagPollingInterval, { c, v in c.flagPollingInterval = v as! TimeInterval }), @@ -55,6 +63,10 @@ final class LDConfigSpec: XCTestCase { ("wrapper version", Constants.wrapperVersion, { c, v in c.wrapperVersion = v as! String? }), ("additional headers", Constants.additionalHeaders, { c, v in c.additionalHeaders = v as! [String: String]})] + func testEventPersistenceIsOffUnlessTheApplicationAsksForIt() { + XCTAssertEqual(LDConfig(mobileKey: LDConfig.Constants.mockMobileKey, autoEnvAttributes: .disabled).eventPersistence, .disabled) + } + func testInitDefault() { let config = LDConfig(mobileKey: LDConfig.Constants.mockMobileKey, autoEnvAttributes: .disabled) XCTAssertEqual(config.mobileKey, LDConfig.Constants.mockMobileKey) @@ -62,6 +74,10 @@ final class LDConfigSpec: XCTestCase { XCTAssertEqual(config.eventsUrl, LDConfig.Defaults.eventsUrl) XCTAssertEqual(config.streamUrl, LDConfig.Defaults.streamUrl) XCTAssertEqual(config.eventCapacity, LDConfig.Defaults.eventCapacity) + XCTAssertEqual(config.eventPersistence, LDConfig.Defaults.eventPersistence) + XCTAssertEqual(config.eventCapacityBytes, 20 * 1024 * 1024) + XCTAssertEqual(config.eventMaxAge, 7 * 24 * 60 * 60) + XCTAssertEqual(config.eventDeliveryDelay, 2) XCTAssertEqual(config.connectionTimeout, LDConfig.Defaults.connectionTimeout) XCTAssertEqual(config.eventFlushInterval, LDConfig.Defaults.eventFlushInterval) XCTAssertEqual(config.flagPollingInterval, LDConfig.Defaults.flagPollingInterval) @@ -103,6 +119,9 @@ final class LDConfigSpec: XCTestCase { XCTAssertEqual(config.eventsUrl, Constants.alternateMockUrl, "\(os)") XCTAssertEqual(config.streamUrl, Constants.alternateMockUrl, "\(os)") XCTAssertEqual(config.eventCapacity, Constants.eventCapacity, "\(os)") + XCTAssertEqual(config.eventCapacityBytes, Constants.eventCapacityBytes, "\(os)") + XCTAssertEqual(config.eventMaxAge, Constants.eventMaxAge, "\(os)") + XCTAssertEqual(config.eventDeliveryDelay, Constants.eventDeliveryDelay, "\(os)") XCTAssertEqual(config.connectionTimeout, Constants.connectionTimeout, "\(os)") XCTAssertEqual(config.eventFlushInterval, Constants.eventFlushInterval, "\(os)") XCTAssertEqual(config.flagPollingInterval, Constants.flagPollingInterval, "\(os)") @@ -181,6 +200,20 @@ final class LDConfigSpec: XCTestCase { XCTAssertEqual(config.diagnosticRecordingInterval, config.minima.diagnosticRecordingInterval + 0.001) } + func testEventDeliveryDelayIsKeptBetweenZeroAndAMinute() { + var config = LDConfig(mobileKey: LDConfig.Constants.mockMobileKey, autoEnvAttributes: .disabled) + config.eventDeliveryDelay = 0 + XCTAssertEqual(config.eventDeliveryDelay, 0) + config.eventDeliveryDelay = -1 + XCTAssertEqual(config.eventDeliveryDelay, 0) + config.eventDeliveryDelay = 61 + XCTAssertEqual(config.eventDeliveryDelay, 60) + config.eventDeliveryDelay = .infinity + XCTAssertEqual(config.eventDeliveryDelay, 60) + config.eventDeliveryDelay = .nan + XCTAssertEqual(config.eventDeliveryDelay, 2) + } + func testEquals() { let defaultConfig = LDConfig(mobileKey: LDConfig.Constants.mockMobileKey, autoEnvAttributes: .disabled) // same config diff --git a/LaunchDarkly/LaunchDarklyTests/Networking/DarklyServiceSpec.swift b/LaunchDarkly/LaunchDarklyTests/Networking/DarklyServiceSpec.swift index 1fe2209ab..6693c9832 100644 --- a/LaunchDarkly/LaunchDarklyTests/Networking/DarklyServiceSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/Networking/DarklyServiceSpec.swift @@ -53,14 +53,18 @@ final class DarklyServiceSpec: QuickSpec { } override func spec() { - getFeatureFlagsSpec() - flagRequestEtagSpec() clearFlagRequestCacheSpec() createEventSourceSpec() - publishEventDataSpec() diagnosticCacheSpec() sessionSpec() + // On the watchOS simulator these requests reach the network even with the OHHTTPStubs protocol installed in + // the session, so nothing answers them. + #if !os(watchOS) + getFeatureFlagsSpec() + flagRequestEtagSpec() + publishEventDataSpec() publishDiagnosticSpec() + #endif afterEach { HTTPStubs.removeAllStubs() diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundActivitySpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundActivitySpec.swift index f3de67f69..c44ff50d5 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundActivitySpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundActivitySpec.swift @@ -77,7 +77,6 @@ final class BackgroundActivitySpec: QuickSpec { } } - #if os(iOS) || os(tvOS) || os(watchOS) describe("an assertion from the process") { /// Stands in for `performExpiringActivity`, which answers on a system queue rather than the caller's. /// @@ -140,14 +139,9 @@ final class BackgroundActivitySpec: QuickSpec { // Ended by the cap, rather than by the work or by never having been held. expect(Date().timeIntervalSince(started)) > 0.2 } - } - #else - describe("an assertion from the process") { - it("runs the work on the calling thread without asking the system, which has no assertions here") { - var asked = false - let assertion = BackgroundActivity.processAssertion(logger: .disabled, - perform: { _, _ in asked = true }, - maximumDuration: 30) + + it("runs the work on the calling thread where the platform has no assertions") { + let assertion = BackgroundActivity.processAssertion(logger: .disabled, perform: nil, maximumDuration: 30) var ran = false assertion("test") { finished in @@ -156,9 +150,11 @@ final class BackgroundActivitySpec: QuickSpec { } expect(ran) == true - expect(asked) == false + } + + it("asks the system for one only where the platform has them") { + expect(SystemCapabilities.expiringActivity != nil) == SystemCapabilities.suspendsWhenBackgrounded } } - #endif } } diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundEventDeliverySpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundEventDeliverySpec.swift new file mode 100644 index 000000000..6f245345e --- /dev/null +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/BackgroundEventDeliverySpec.swift @@ -0,0 +1,129 @@ +import Foundation +import Quick +import Nimble +@testable import LaunchDarkly + +final class BackgroundEventDeliverySpec: QuickSpec { + override func spec() { + describe("going to the background") { + var primary: EventProcessorMock! + var secondary: EventProcessorMock! + var environments: [BackgroundEventDelivery.Environment] = [] + var assertions = 0 + var released = 0 + var activity: BackgroundActivity! + beforeEach { + primary = EventProcessorMock() + secondary = EventProcessorMock() + environments = [BackgroundEventDelivery.Environment(name: "primary", eventProcessor: primary, logger: .disabled), + BackgroundEventDelivery.Environment(name: "secondary", eventProcessor: secondary, logger: .disabled)] + assertions = 0 + released = 0 + activity = BackgroundActivity(reason: "test") { _, work in + assertions += 1 + work { released += 1 } + } + } + + func delivery(suspendsWhenBackgrounded: Bool) -> BackgroundEventDelivery { + BackgroundEventDelivery(environments: { environments }, + suspendsWhenBackgrounded: suspendsWhenBackgrounded, + activity: activity) + } + + it("commits every environment, and delivers nothing, where the platform does not suspend") { + delivery(suspendsWhenBackgrounded: false).applicationDidEnterBackground() + + expect(primary.commitWherePersistingCallCount) == 1 + expect(secondary.commitWherePersistingCallCount) == 1 + expect(primary.flushReportingOutcomeCallCount) == 0 + expect(secondary.flushReportingOutcomeCallCount) == 0 + expect(assertions) == 0 + } + + it("commits every environment before it delivers any, where the platform suspends") { + var committedBeforeDelivering = false + primary.flushReportingOutcomeCallback = { + committedBeforeDelivering = primary.commitWherePersistingCallCount == 1 + && secondary.commitWherePersistingCallCount == 1 + } + + delivery(suspendsWhenBackgrounded: true).applicationDidEnterBackground() + + expect(primary.flushReportingOutcomeCallCount) == 1 + expect(secondary.flushReportingOutcomeCallCount) == 1 + expect(committedBeforeDelivering) == true + } + + it("takes one assertion for every environment, and holds it until all of them answer") { + let subject = delivery(suspendsWhenBackgrounded: true) + subject.applicationDidEnterBackground() + + expect(assertions) == 1 + expect(released) == 0 + + primary.flushReportingOutcomeReceivedCompletion?(true) + expect(released) == 0 + + secondary.flushReportingOutcomeReceivedCompletion?(true) + expect(released) == 1 + } + + it("releases the assertion where deliveries fail too") { + let subject = delivery(suspendsWhenBackgrounded: true) + subject.applicationDidEnterBackground() + + primary.flushReportingOutcomeReceivedCompletion?(false) + secondary.flushReportingOutcomeReceivedCompletion?(true) + expect(released) == 1 + } + + it("waits for every environment where one answers twice") { + let subject = delivery(suspendsWhenBackgrounded: true) + subject.applicationDidEnterBackground() + + primary.flushReportingOutcomeReceivedCompletion?(true) + primary.flushReportingOutcomeReceivedCompletion?(true) + expect(released) == 0 + + secondary.flushReportingOutcomeReceivedCompletion?(true) + secondary.flushReportingOutcomeReceivedCompletion?(true) + expect(released) == 1 + } + + it("releases the assertion where an environment never answers, so a later backgrounding still delivers") { + var releases = 0 + let lock = UnfairLock() + let counted = BackgroundActivity(reason: "test") { _, work in + work { lock.withLock { releases += 1 } } + } + let subject = BackgroundEventDelivery(environments: { environments }, + suspendsWhenBackgrounded: true, + activity: counted, + answerTimeout: 0.1) + subject.applicationDidEnterBackground() + primary.flushReportingOutcomeReceivedCompletion?(true) + expect(lock.withLock { releases }).toEventually(equal(1)) + + subject.applicationDidEnterBackground() + expect(primary.flushReportingOutcomeCallCount) == 2 + } + + it("takes no assertion where no client is started") { + environments = [] + delivery(suspendsWhenBackgrounded: true).applicationDidEnterBackground() + + expect(assertions) == 0 + } + + it("reads the environments at each backgrounding") { + let subject = delivery(suspendsWhenBackgrounded: false) + environments.removeLast() + subject.applicationDidEnterBackground() + + expect(primary.commitWherePersistingCallCount) == 1 + expect(secondary.commitWherePersistingCallCount) == 0 + } + } + } +} diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/ContextSummarizerSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/ContextSummarizerSpec.swift index 94dd06683..eea0da02d 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/ContextSummarizerSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/ContextSummarizerSpec.swift @@ -176,10 +176,10 @@ final class ContextSummarizerSpec: QuickSpec { privateAttributes.forEach { builder.addPrivateAttribute(Reference($0)) } return try! builder.build().get() } - let nested: LDValue = ["x": 1, "y": ["z", true]] - let forward = context([("email", "a@example.com"), ("plan", nested), ("score", 2)], + let nested: LDValue = ["x": .number(1), "y": ["z", true]] + let forward = context([("email", "a@example.com"), ("plan", nested), ("score", .number(2))], privateAttributes: ["email", "plan"]) - let backward = context([("score", 2), ("plan", nested), ("email", "a@example.com")], + let backward = context([("score", .number(2)), ("plan", nested), ("email", "a@example.com")], privateAttributes: ["/plan", "email"]) expect(forward) == backward diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DiagnosticReporterSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DiagnosticReporterSpec.swift index 78dcf71ea..9d4b6f1d7 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DiagnosticReporterSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DiagnosticReporterSpec.swift @@ -112,6 +112,22 @@ final class DiagnosticReporterSpec: XCTestCase { tst.stop() } + func testInitEventWaitsForTheLaunchGateWhereEventsDoNotPersist() { + let tst = TestContext() + tst.queueResponse(status: 202) + let events = DirectEventProcessor(service: tst.service, onSyncComplete: nil, + store: EventStore.temporary(capacity: 10, persistEvents: false), + pacing: DirectEventProcessor.Pacing(launchDelay: 0.5, gap: 0)) + tst.subject.sendsInitAfter = events.whenLaunchGateOpens + events.isOnline = true + tst.subject.setMode(.foreground, online: true) + + tst.expectNoEvent() + XCTAssertEqual(tst.takeEvent().kind, .diagnosticInit) + events.isOnline = false + tst.stop() + } + func testRetries() { let tst = TestContext() tst.queueResponse(status: 500) diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift new file mode 100644 index 000000000..294b14643 --- /dev/null +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/DirectEventProcessorSpec.swift @@ -0,0 +1,4063 @@ +import Foundation +import Quick +import Nimble +@testable import LaunchDarkly + +class DirectEventProcessorSpec: QuickSpec { + /// The persistence a `TestContext` gets where it names neither one nor a store. Immediate here, as the durable + /// behaviour is what most of these are about; `DirectEventProcessorWithoutPersistenceSpec` runs them all again with + /// it off, which is what an application gets. + class var persistence: EventPersistence { .immediate } + + /// `persistence` of the spec whose examples are running, for the `TestContext` default, which cannot see the class. + fileprivate static var runningPersistence: EventPersistence = .immediate + + /// Whether the spec runs with a disk, for examples whose subject is the disk. + var persists: Bool { type(of: self).persistence != .disabled } + + struct Constants { + static let eventFlushInterval: TimeInterval = 10.0 + static let eventFlushIntervalHalfSecond: TimeInterval = 0.5 + /// How long a flush or delivery may take before a test gives up on it. A wait ends as soon as what it waits + /// for happens, so this costs a passing test nothing, and a loaded CI simulator can take over a second. + static let deliveryTimeout: DispatchTimeInterval = .seconds(10) + } + + /// Retries at once and never waits, so every flush posts: for tests about what is kept through failures rather than + /// when it is posted again. + static func withoutBackoff() -> DeliveryRetryState { + DeliveryRetryState(initialDelay: 0, normalCeiling: 0, extendedCeiling: 0) + } + + struct TestContext { + var eventProcessor: DirectEventProcessor! + var config: LDConfig! + var context: LDContext! + var serviceMock: DarklyServiceMock! + var store: EventStore! + var events: [Event] = [] + var lastEventResponseDate: Date + var eventStubResponseDate: Date? + var syncResult: SynchronizingError? = nil + var diagnosticCache: DiagnosticCachingMock + + init(eventCount: Int = 0, + eventFlushInterval: TimeInterval? = nil, + lastEventResponseDate: Date = Date.distantPast, + stubResponseSuccess: Bool = true, + stubResponseOnly: Bool = false, + stubResponseErrorOnly: Bool = false, + eventStubResponseDate: Date? = nil, + onSyncComplete: EventSyncCompleteClosure? = nil, + store: EventStore? = nil, + eventCapacity: Int = Event.Kind.allKinds.count, + eventPersistence: EventPersistence? = nil, + commitQueue: DispatchQueue = DispatchQueue(label: "com.launchdarkly.tests.DirectEventProcessor.commitQueue"), + retryState: DeliveryRetryState? = nil) { + + config = LDConfig.stub + config.eventCapacity = eventCapacity + // A store a test brings decides, so the processor is never configured for a disk its store does not have. + let persistence = eventPersistence + ?? store.map { $0.isPersisting ? .immediate : .disabled } + ?? DirectEventProcessorSpec.runningPersistence + config.eventPersistence = persistence + config.eventFlushInterval = eventFlushInterval ?? Constants.eventFlushInterval + + context = LDContext.stub() + + self.eventStubResponseDate = eventStubResponseDate?.adjustedForHttpUrlHeaderUse + serviceMock = DarklyServiceMock() + serviceMock.config = config + serviceMock.stubEventResponse(success: stubResponseSuccess, responseOnly: stubResponseOnly, errorOnly: stubResponseErrorOnly, responseDate: self.eventStubResponseDate) + + diagnosticCache = DiagnosticCachingMock() + serviceMock.diagnosticCache = diagnosticCache + + self.lastEventResponseDate = lastEventResponseDate.adjustedForHttpUrlHeaderUse + self.store = store ?? EventStore.temporary(capacity: config.eventCapacity, persistEvents: persistence != .disabled) + eventProcessor = DirectEventProcessor(service: serviceMock, onSyncComplete: onSyncComplete, + store: self.store, commitQueue: commitQueue, retryState: retryState, + pacing: .noWaits) + (0.. [LDValue] { + eventProcessor.commitRecordedEvents() + return store.pendingEventPayloads().compactMap { try? JSONDecoder().decode(LDValue.self, from: $0) } + } + + /// The events recorded so far, in the JSON they encode to, for comparison with `committedEvents()`. + func recordedEventsAsJSON() -> [LDValue] { + events.compactMap { encodeToLDValue($0) } + } + + /// Flushes, and answers whether a flush made from the completion got anywhere. + /// + /// A completion is free to flush again, which only works where it is not running on the queue deliveries go + /// through: from there the delivery it asks for would be queued behind the completion asking for it. Waited + /// on against a deadline, so a reporter answering on its own queue fails this rather than hanging the test + /// process on a deadlock it can never leave. + func flushedFromACompletion() -> Bool { + var flushed = false + waitUntil(timeout: .seconds(5)) { done in + eventProcessor.flushReportingOutcome { _ in + let finished = DispatchSemaphore(value: 0) + eventProcessor.flush { finished.signal() } + flushed = finished.wait(timeout: .now() + 1) == .success + done() + } + } + return flushed + } + + /// Flushes and returns what the flush answered. + func flushOutcome() -> Bool? { + var delivered: Bool? + // Long enough for a refused post and its retry a second later. + waitUntil(timeout: .seconds(10)) { done in + eventProcessor.flushReportingOutcome { result in + delivered = result + done() + } + } + return delivered + } + + func flushBypassingBackoffOutcome() -> Bool? { + var delivered: Bool? + waitUntil(timeout: .seconds(10)) { done in + eventProcessor.flushBypassingBackoff { result in + delivered = result + done() + } + } + return delivered + } + + func cleanUp() { + eventProcessor.isOnline = false + store.deleteEverything() + } + } + + override func spec() { + let persistence = type(of: self).persistence + beforeEach { DirectEventProcessorSpec.runningPersistence = persistence } + + initSpec() + isOnlineSpec() + recordEventSpec() + testRecordFlagEvaluationEvents() + reportEventsSpec() + unserializableEventSpec() + privacySpec() + reportTimerSpec() + durabilitySpec() + commitSpec() + flushReportingOutcomeSpec() + outageSpec() + flagTrafficSpec() + nullEventProcessorSpec() + if !persists { + withoutPersistenceSpec() + } + } + + /// The situations where holding events only in memory is hardest, which the examples shared with persistence do + /// not set out to reach. + private func withoutPersistenceSpec() { + describe("without persistence") { + var testContext: TestContext! + afterEach { + testContext?.cleanUp() + testContext = nil + } + + /// The custom event keys LaunchDarkly took, once per payload ID however often it was sent, in the order taken. + func acceptedKeys() -> [String] { + let mock = testContext.serviceMock! + var payloadIds = Set() + var keys: [String] = [] + for (index, status) in mock.publishedEventStatuses.enumerated() where status == 202 { + if payloadIds.insert(mock.publishedPayloadIds[index]).inserted { + keys += customKeys(mock.publishedEventPayloads[index]) + } + } + return keys + } + + func record(_ key: String) { + testContext.eventProcessor.record(CustomEvent(key: key, context: testContext.context)) + } + + it("never works out where its directory is through refusals, a retry, deliveries and a refusal that makes it wait") { + let lock = UnfairLock() + var located = 0 + let store = EventStore(locatingDirectory: { + lock.withLock { located += 1 } + return FileManager.default.temporaryDirectory + .appendingPathComponent("com.launchdarkly.tests.events", isDirectory: true) + .appendingPathComponent(UUID().uuidString, isDirectory: true) + }, + capacity: Event.Kind.allKinds.count, + persistEvents: false, + logger: .disabled) + // Refused and refused again on the retry; then the batch held from that and the next one are accepted; + // then the service says the key is no good, after which nothing is posted for a while. The waits are + // cut to milliseconds, which makes that while about four seconds. + testContext = TestContext(store: store, + retryState: DeliveryRetryState(initialDelay: 0.008, normalCeiling: 0.008, extendedCeiling: 3600)) + testContext.serviceMock.eventStatusForPost = { post in post <= 2 ? 503 : post <= 4 ? 202 : 401 } + testContext.eventProcessor.isOnline = true + + record("refused") + expect(testContext.flushOutcome()) == false + record("accepted") + expect(testContext.flushOutcome()) == true + record("refused-for-good") + expect(testContext.flushOutcome()) == false + expect(testContext.eventProcessor.isOnline) == true + record("while-waiting") + testContext.eventProcessor.flush(completion: nil) + waitLettingTimersFire(1.5) + + expect(testContext.serviceMock.publishEventDataCallCount) == 5 + expect(acceptedKeys()) == ["refused", "accepted"] + testContext.eventProcessor.isOnline = false + expect(lock.withLock { located }) == 0 + } + + it("keeps the oldest events up to capacity through an outage, and delivers each once in order") { + // A batch refused twice used to be dropped. Now it waits in memory, still counted against capacity, so + // an outage costs the newest events rather than the oldest and memory stays bounded however long it lasts. + let capacity = 20 + testContext = TestContext(eventCapacity: capacity, retryState: DirectEventProcessorSpec.withoutBackoff()) + testContext.serviceMock.respondToEvents(with: 503) + testContext.eventProcessor.isOnline = true + + (0..<10).forEach { record("e-\($0)") } + expect(testContext.flushOutcome()) == false + (10..<100).forEach { record("e-\($0)") } + expect(testContext.flushOutcome()) == false + + testContext.serviceMock.stubEventResponse(success: true) + testContext.serviceMock.eventStatusForPost = { _ in 202 } + expect(testContext.flushOutcome()) == true + + expect(acceptedKeys()) == (0.. [LDValue] in + guard case .array(let events)? = try? JSONDecoder().decode(LDValue.self, from: body) + else { return [] } + return events.filter { $0.kindField == "summary" } + } + expect(summaries.count).to(equal(capacity), description: "round \(round)") + } + } + } + } + + private func flagTrafficSpec() { + describe("giving way to flag values") { + var testContext: TestContext! + var nextRun: DirectEventProcessor? + beforeEach { + testContext = TestContext() + } + afterEach { + nextRun?.isOnline = false + nextRun = nil + testContext.cleanUp() + } + + /// A reporter on what a previous run left: `batches` batches of one event each. + func nextRunLeft(_ batches: Int, pacing: DirectEventProcessor.Pacing, retryState: DeliveryRetryState? = nil) -> DirectEventProcessor { + for _ in 0.. [Date] { + var times: [Date] = [] + let deadline = Date().addingTimeInterval(10) + while times.count < count && Date() < deadline { + let posted = testContext.serviceMock.publishEventDataCallCount + while times.count < posted { + times.append(Date()) + } + Thread.sleep(forTimeInterval: 0.005) + } + return times + } + + itPersisting("sends what a previous run left only once the launch delay has passed after going online") { + let reporter = nextRunLeft(1, pacing: DirectEventProcessor.Pacing(launchDelay: 0.8, gap: 0)) + let wentOnline = Date() + + reporter.isOnline = true + + Thread.sleep(forTimeInterval: 0.5) + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + let times = postTimes(1) + expect(times.count) == 1 + expect(times.first?.timeIntervalSince(wentOnline)) >= 0.75 + } + + itPersisting("counts the launch delay from going online rather than from the reporter starting") { + let reporter = nextRunLeft(1, pacing: DirectEventProcessor.Pacing(launchDelay: 0.6, gap: 0)) + Thread.sleep(forTimeInterval: 0.8) + + reporter.isOnline = true + + Thread.sleep(forTimeInterval: 0.3) + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + } + + itPersisting("sends what a previous run left on a flush, without waiting for the launch gate") { + let reporter = nextRunLeft(1, pacing: DirectEventProcessor.Pacing(launchDelay: 5, gap: 0)) + reporter.isOnline = true + + waitUntil(timeout: .seconds(3)) { done in + reporter.flush(completion: done) + } + + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + } + + itPersisting("sends what a previous run left a batch at a time, with a gap after each answer") { + let reporter = nextRunLeft(3, pacing: DirectEventProcessor.Pacing(launchDelay: 0, gap: 0.4)) + + reporter.isOnline = true + + let times = postTimes(3) + expect(times.count) == 3 + for (earlier, later) in zip(times, times.dropFirst()) { + expect(later.timeIntervalSince(earlier)) >= 0.3 + } + } + + itPersisting("sends the rest at once when a flush is asked for while pacing") { + let reporter = nextRunLeft(3, pacing: DirectEventProcessor.Pacing(launchDelay: 0, gap: 10)) + reporter.isOnline = true + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + + waitUntil(timeout: .seconds(3)) { done in + reporter.flush(completion: done) + } + + expect(testContext.serviceMock.publishEventDataCallCount) == 3 + } + + itPersisting("recovers what a previous run left at utility, though it is started from the main thread") { + let store = RecoveryObservingStore(testContext.store.nextRun()) + nextRun = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: store, pacing: .noWaits) + + expect(store.recoveryQoS).toEventually(equal(QOS_CLASS_UTILITY), timeout: Constants.deliveryTimeout) + } + + itPersisting("holds a delivery asked for during the recovery until the recovery is done") { + testContext.recordEvents(1) + testContext.eventProcessor.commitRecordedEvents() + _ = testContext.store.closeBatch() + let store = RecoveryObservingStore(testContext.store.nextRun(), holdingRecovery: true) + defer { store.release.signal() } + let reporter = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: store, pacing: .noWaits) + nextRun = reporter + reporter.isOnline = true + + var delivered: Bool? + reporter.flushReportingOutcome { delivered = $0 } + Thread.sleep(forTimeInterval: 0.3) + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + + store.release.signal() + expect(delivered).toEventually(beTrue(), timeout: Constants.deliveryTimeout) + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + } + + itPersisting("keeps the gap for the delivery a backoff's wait sets when it ends") { + // A refusal waits the extended ceiling, longer than a pass sits out, so the pass ends and the delivery + // is set for when the wait ends. + let reporter = nextRunLeft(3, pacing: DirectEventProcessor.Pacing(launchDelay: 0, gap: 10), + retryState: DeliveryRetryState(initialDelay: 0.1, normalCeiling: 0.1, extendedCeiling: 0.5)) + testContext.serviceMock.eventStatusForPost = { post in post == 1 ? HTTPURLResponse.StatusCodes.unauthorized : 202 } + reporter.isOnline = true + + // The refused batch again, then the gap before the next. + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(2), timeout: Constants.deliveryTimeout) + Thread.sleep(forTimeInterval: 1) + expect(testContext.serviceMock.publishEventDataCallCount) == 2 + } + + itPersisting("stops a paced delivery on going offline, and keeps the rest") { + let reporter = nextRunLeft(3, pacing: DirectEventProcessor.Pacing(launchDelay: 0, gap: 0.5)) + reporter.isOnline = true + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + + reporter.isOnline = false + Thread.sleep(forTimeInterval: 1.5) + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + + reporter.isOnline = true + waitUntil(timeout: .seconds(3)) { done in + reporter.flush(completion: done) + } + expect(testContext.serviceMock.publishEventDataCallCount) == 3 + } + + /// Whether what waits on `reporter`'s launch gate has run, `after` it went online. + func gateOpened(_ reporter: DirectEventProcessor, after: TimeInterval) -> Bool { + let opened = UnfairLock() + var hasOpened = false + reporter.whenLaunchGateOpens { opened.withLock { hasOpened = true } } + reporter.isOnline = true + Thread.sleep(forTimeInterval: after) + return opened.withLock { hasOpened } + } + + it("holds what waits for the launch gate until the launch delay has passed, whether or not events persist") { + let reporter = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: testContext.store, + pacing: DirectEventProcessor.Pacing(launchDelay: 1, gap: 0)) + nextRun = reporter + + expect(gateOpened(reporter, after: 0.6)) == false + expect(gateOpened(reporter, after: 0.8)) == true + } + + it("holds nothing at the launch gate where the launch delay is zero") { + let reporter = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: testContext.store, + pacing: DirectEventProcessor.Pacing(launchDelay: 0, gap: 0)) + nextRun = reporter + + expect(gateOpened(reporter, after: 0.2)) == true + } + + it("takes the launch delay from the configuration") { + var config = testContext.serviceMock.config + config.eventDeliveryDelay = 1 + testContext.serviceMock.config = config + let reporter = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: nil, store: testContext.store) + nextRun = reporter + + expect(gateOpened(reporter, after: 0.6)) == false + expect(gateOpened(reporter, after: 0.8)) == true + } + + itPersisting("does not let a report interval shorter than the launch delay send what a previous run left") { + testContext = TestContext(eventFlushInterval: 0.1) + let reporter = nextRunLeft(1, pacing: DirectEventProcessor.Pacing(launchDelay: 0.8, gap: 0)) + let wentOnline = Date() + + reporter.isOnline = true + + // Waited for on the run loop, which the report timer fires on. + waitLettingTimersFire(0.5) + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + expect(Date().timeIntervalSince(wentOnline)) >= 0.75 + } + + itPersisting("keeps pacing a backlog through the report intervals that end during it") { + testContext = TestContext(eventFlushInterval: 0.2) + let reporter = nextRunLeft(2, pacing: DirectEventProcessor.Pacing(launchDelay: 0.2, gap: 1)) + + reporter.isOnline = true + + // Waited for on the run loop, which the report timer fires on. + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + let first = Date() + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(2), timeout: .seconds(3)) + expect(Date().timeIntervalSince(first)) >= 0.8 + } + } + } + + /// An example whose subject is the disk, left out where the spec runs without one. + private func itPersisting(_ description: String, file: FileString = #file, line: UInt = #line, closure: @escaping () throws -> Void) { + guard persists + else { return } + it(description, file: file, line: line, closure: closure) + } + + private func nullEventProcessorSpec() { + describe("NullEventProcessor") { + it("answers every flush off the caller's thread") { + let reporter = NullEventProcessor() + let caller = Thread.current + let lock = UnfairLock() + var answers: [(delivered: Bool, onCaller: Bool)] = [] + let record: (Bool) -> Void = { delivered in + lock.withLock { answers.append((delivered, Thread.current === caller)) } + } + + // Events are off, so there is never anything to wait for. The threading a caller sees still must not + // depend on that: one holding a lock its completion takes would deadlock only with events off. + reporter.flush { record(true) } + reporter.flushReportingOutcome(completion: record) + + expect(lock.withLock { answers.count }).toEventually(equal(2), timeout: Constants.deliveryTimeout) + lock.withLock { + expect(answers.map { $0.delivered }) == [true, true] + expect(answers.contains { $0.onCaller }) == false + } + } + } + } + + private func initSpec() { + describe("init") { + var testContext: TestContext! + beforeEach { + testContext = TestContext() + testContext.eventProcessor = DirectEventProcessor(service: testContext.serviceMock, onSyncComplete: { _ in }, store: testContext.store, pacing: .noWaits) + } + it("starts offline without reporting events") { + expect(testContext.eventProcessor.service) === testContext.serviceMock + expect(testContext.eventProcessor.isOnline) == false + expect(testContext.eventProcessor.isReportingActive) == false + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + } + } + describe("the byte capacity") { + var config: LDConfig! + beforeEach { + config = LDConfig(mobileKey: LDConfig.Constants.mockMobileKey, autoEnvAttributes: .disabled) + } + it("is the whole of it for a single environment") { + config.eventCapacityBytes = 2_400 + expect(DirectEventProcessor.shareOfCapacityBytes(config: config)) == 2_400 + } + it("is shared equally by the configured environments") { + config.eventCapacityBytes = 2_400 + try? config.setSecondaryMobileKeys(["second": "mob-second", "third": "mob-third", "fourth": "mob-fourth"]) + expect(DirectEventProcessor.shareOfCapacityBytes(config: config)) == 600 + } + it("shares the default where none is set") { + config.eventCapacityBytes = 0 + try? config.setSecondaryMobileKeys(["second": "mob-second"]) + expect(DirectEventProcessor.shareOfCapacityBytes(config: config)) == LDConfig.Defaults.eventCapacityBytes / 2 + } + it("leaves each environment at least a byte") { + config.eventCapacityBytes = 2 + try? config.setSecondaryMobileKeys(["second": "mob-second", "third": "mob-third"]) + expect(DirectEventProcessor.shareOfCapacityBytes(config: config)) == 1 + } + } + } + + private func isOnlineSpec() { + describe("isOnline") { + var testContext: TestContext! + beforeEach { + testContext = TestContext() + } + afterEach { + testContext.cleanUp() + } + context("online to offline") { + beforeEach { + testContext.eventProcessor.isOnline = true + + testContext.eventProcessor.isOnline = false + } + it("goes offline and stops reporting") { + expect(testContext.eventProcessor.isOnline) == false + expect(testContext.eventProcessor.isReportingActive) == false + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + } + } + context("offline to online") { + context("with events") { + beforeEach { + testContext = TestContext(eventCount: Event.Kind.allKinds.count) + + testContext.eventProcessor.isOnline = true + } + it("goes online and starts reporting") { + expect(testContext.eventProcessor.isOnline) == true + expect(testContext.eventProcessor.isReportingActive) == true + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + } + } + context("without events") { + beforeEach { + testContext = TestContext() + + testContext.eventProcessor.isOnline = true + } + it("goes online and starts reporting") { + expect(testContext.eventProcessor.isOnline) == true + expect(testContext.eventProcessor.isReportingActive) == true + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + } + } + } + context("online to online") { + beforeEach { + testContext = TestContext() + testContext.eventProcessor.isOnline = true + + testContext.eventProcessor.isOnline = true + } + it("stays online and continues reporting") { + expect(testContext.eventProcessor.isOnline) == true + expect(testContext.eventProcessor.isReportingActive) == true + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + } + } + context("offline to offline") { + beforeEach { + testContext = TestContext(eventCount: Event.Kind.allKinds.count) + + testContext.eventProcessor.isOnline = false + } + it("stays offline and does not start reporting") { + expect(testContext.eventProcessor.isOnline) == false + expect(testContext.eventProcessor.isReportingActive) == false + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + expect(testContext.committedEvents()) == testContext.recordedEventsAsJSON() + } + } + } + } + + private func recordEventSpec() { + describe("recordEvent") { + var testContext: TestContext! + context("event store empty") { + beforeEach { + testContext = TestContext() + testContext.recordEvents(Event.Kind.allKinds.count) // Stub events, call testContext.eventProcessor.recordEvent, and keeps them in testContext.events + } + it("records events up to event capacity") { + expect(testContext.eventProcessor.isOnline) == false + expect(testContext.eventProcessor.isReportingActive) == false + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + expect(testContext.committedEvents()) == testContext.recordedEventsAsJSON() + } + it("does not record a dropped event to diagnosticCache") { + expect(testContext.diagnosticCache.incrementDroppedEventCountCallCount) == 0 + } + } + context("event store full") { + var extraEvent: Event! + beforeEach { + testContext = TestContext(eventCount: Event.Kind.allKinds.count) + extraEvent = Event.stub(.feature, with: testContext.context) + + testContext.eventProcessor.record(extraEvent) + } + it("doesn't record any more events") { + expect(testContext.eventProcessor.isOnline) == false + expect(testContext.eventProcessor.isReportingActive) == false + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + expect(testContext.committedEvents()) == testContext.recordedEventsAsJSON() + } + it("records a dropped event to diagnosticCache") { + expect(testContext.diagnosticCache.incrementDroppedEventCountCallCount) == 1 + } + } + } + } + + /// An event holding a NaN or infinite number cannot be serialized by either encoder. It is dropped, and + /// the events beside it and after it are still delivered. + private func unserializableEventSpec() { + describe("unserializable events") { + var serviceMock: DarklyServiceMock! + var ldContext: LDContext! + var reporter: DirectEventProcessor! + var store: EventStore! + + func makeReporter(encoding: OutboundEventBuffer.Encoding) -> DirectEventProcessor { + var config = LDConfig.stub + config.eventCapacity = 8 + ldContext = LDContext.stub() + serviceMock = DarklyServiceMock() + serviceMock.config = config + serviceMock.stubEventResponse(success: true) + store = EventStore.temporary(capacity: config.eventCapacity) + return DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: encoding, pacing: .noWaits) + } + + afterEach { + reporter?.isOnline = false + } + + for encoding in [OutboundEventBuffer.Encoding.codable, .handWritten] { + context("\(encoding): when an event holds a non-finite metric") { + beforeEach { + reporter = makeReporter(encoding: encoding) + reporter.isOnline = true + } + it("drops the poisoned batch and delivers a later event") { + waitUntil(timeout: Constants.deliveryTimeout) { done in + reporter.record(CustomEvent(key: "poison", context: ldContext, metricValue: .nan)) + reporter.flush(completion: done) + } + expect(serviceMock.publishEventDataCallCount) == 0 + expect(store.pendingEventCount) == 0 + + waitUntil(timeout: Constants.deliveryTimeout) { done in + reporter.record(CustomEvent(key: "after-poison", context: ldContext, metricValue: 1.0)) + reporter.flush(completion: done) + } + expect(serviceMock.publishEventDataCallCount) == 1 + let published = try JSONDecoder().decode(LDValue.self, from: serviceMock.publishedEventData!) + valueIsArray(published) { events in + expect(events.count) == 1 + valueIsObject(events[0]) { body in + expect(body["key"]) == "after-poison" + } + } + } + } + + context("\(encoding): when an event holds a non-finite metric beside a valid event") { + beforeEach { + reporter = makeReporter(encoding: encoding) + reporter.isOnline = true + } + it("drops only the poisoned event") { + waitUntil(timeout: Constants.deliveryTimeout) { done in + reporter.record(CustomEvent(key: "poison", context: ldContext, metricValue: .nan)) + reporter.record(CustomEvent(key: "kept", context: ldContext, metricValue: 1.0)) + reporter.flush(completion: done) + } + expect(serviceMock.publishEventDataCallCount) == 1 + let published = try JSONDecoder().decode(LDValue.self, from: serviceMock.publishedEventData!) + valueIsArray(published) { events in + expect(events.count) == 1 + valueIsObject(events[0]) { body in + expect(body["key"]) == "kept" + } + } + } + } + + context("\(encoding): when an event holds a non-finite summary beside a valid event") { + beforeEach { + reporter = makeReporter(encoding: encoding) + reporter.isOnline = true + } + it("drops only the poisoned summary") { + waitUntil(timeout: Constants.deliveryTimeout) { done in + reporter.recordFlagEvaluationEvents(flagKey: "poison-flag", + value: .number(.nan), + defaultValue: .number(0), + featureFlag: nil, + context: ldContext, + includeReason: false) + reporter.record(IdentifyEvent(context: ldContext)) + reporter.flush(completion: done) + } + expect(serviceMock.publishEventDataCallCount) == 1 + let published = try JSONDecoder().decode(LDValue.self, from: serviceMock.publishedEventData!) + valueIsArray(published) { events in + expect(events.count) == 1 + valueIsObject(events[0]) { body in + expect(body["kind"]) == "identify" + } + } + } + } + + context("\(encoding): when an event holds a non-finite summary value") { + beforeEach { + reporter = makeReporter(encoding: encoding) + reporter.isOnline = true + } + it("drops the poisoned summary and delivers a later one") { + waitUntil(timeout: Constants.deliveryTimeout) { done in + reporter.recordFlagEvaluationEvents(flagKey: "poison-flag", + value: .number(.nan), + defaultValue: .number(0), + featureFlag: nil, + context: ldContext, + includeReason: false) + reporter.flush(completion: done) + } + expect(serviceMock.publishEventDataCallCount) == 0 + expect(store.pendingEventCount) == 0 + expect(reporter.withContextSummarizer { $0.hasLoggedRequests }) == false + + waitUntil(timeout: Constants.deliveryTimeout) { done in + reporter.recordFlagEvaluationEvents(flagKey: "after-poison", + value: .bool(true), + defaultValue: .bool(false), + featureFlag: nil, + context: ldContext, + includeReason: false) + reporter.flush(completion: done) + } + expect(serviceMock.publishEventDataCallCount) == 1 + let published = try JSONDecoder().decode(LDValue.self, from: serviceMock.publishedEventData!) + valueIsArray(published) { events in + expect(events.count) == 1 + valueIsObject(events[0]) { body in + expect(body["kind"]) == "summary" + valueIsObject(body["features"]) { features in + expect(features["after-poison"]).toNot(beNil()) + } + } + } + } + } + } + } + } + + /// Context redaction as it reaches the payload the processor posts, configured through `LDConfig` as an application + /// configures it, with each encoder. + /// + /// The encoder tests pin what redaction produces. These pin that the configuration reaches the encoder, which + /// events inline a context and which of those also redact an anonymous context's attributes, and that a summary, + /// which inlines the context it counted, is redacted like any other event. Android's `EventProcessorPrivacyTest` + /// covers the same ground. + private func privacySpec() { + describe("context privacy") { + var serviceMock: DarklyServiceMock! + var reporter: DirectEventProcessor! + var store: EventStore! + + let trackedFlag = FeatureFlag(flagKey: "flag-key", value: true, variation: 1, flagVersion: 2, trackEvents: true) + + func person() -> LDContext { + var builder = LDContextBuilder(key: "user-key") + builder.name("Sandy") + _ = builder.trySetValue("email", "sandy@example.com") + _ = builder.trySetValue("address", ["city": "Oakland", "street": "123 Main St"]) + return try! builder.build().get() + } + + func makeReporter(_ encoding: OutboundEventBuffer.Encoding, + configure: (inout LDConfig) -> Void = { _ in }) -> DirectEventProcessor { + var config = LDConfig.stub + config.eventCapacity = 100 + configure(&config) + serviceMock = DarklyServiceMock() + serviceMock.config = config + serviceMock.stubEventResponse(success: true) + store = EventStore.temporary(capacity: config.eventCapacity) + reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: encoding, pacing: .noWaits) + reporter.isOnline = true + return reporter + } + + /// Records, flushes, and answers the events in the one request the flush posted. + func posted(_ record: (DirectEventProcessor) -> Void) -> [[String: LDValue]] { + record(reporter) + waitUntil(timeout: Constants.deliveryTimeout) { done in reporter.flush(completion: done) } + expect(serviceMock.publishEventDataCallCount) == 1 + guard let data = serviceMock.publishedEventData, + case .array(let events)? = try? JSONDecoder().decode(LDValue.self, from: data) + else { return [] } + return events.map(object) + } + + func object(_ value: LDValue?) -> [String: LDValue] { + guard case .object(let members)? = value + else { return [:] } + return members + } + + func inlinedContext(ofKind kind: String, in events: [[String: LDValue]]) -> [String: LDValue] { + object(events.first { $0["kind"] == .string(kind) }?["context"]) + } + + /// What a context reports as redacted, sorted so the comparison is stable. + func redacted(_ context: [String: LDValue]) -> [String] { + guard case .array(let names)? = object(context["_meta"])["redactedAttributes"] + else { return [] } + return names.compactMap { name -> String? in + guard case .string(let string) = name + else { return nil } + return string + }.sorted() + } + + afterEach { + reporter?.isOnline = false + store?.deleteEverything() + } + + for encoding in [OutboundEventBuffer.Encoding.codable, .handWritten] { + context("\(encoding)") { + it("redacts and reports an attribute the configuration names private") { + _ = makeReporter(encoding) { $0.privateContextAttributes = [Reference("email")] } + let delivered = inlinedContext(ofKind: "identify", in: posted { $0.record(IdentifyEvent(context: person())) }) + + expect(delivered["key"]) == "user-key" + expect(delivered["email"]).to(beNil()) + expect(delivered["name"]) == "Sandy" + expect(redacted(delivered)) == ["email"] + } + + it("leaves only the key and kind where every attribute is private") { + _ = makeReporter(encoding) { $0.allContextAttributesPrivate = true } + let delivered = inlinedContext(ofKind: "identify", in: posted { $0.record(IdentifyEvent(context: person())) }) + + expect(delivered["key"]) == "user-key" + expect(delivered["kind"]) == "user" + expect(delivered["name"]).to(beNil()) + expect(delivered["email"]).to(beNil()) + expect(delivered["address"]).to(beNil()) + expect(redacted(delivered)) == ["address", "email", "name"] + } + + it("reports each name as the reference that names it where every attribute is private") { + var builder = LDContextBuilder(key: "user-key") + _ = builder.trySetValue("/ssn", "123-45-6789") + _ = builder.trySetValue("/a~b", "secret") + _ = builder.trySetValue("c/d", "value") + let slashed = try! builder.build().get() + _ = makeReporter(encoding) { $0.allContextAttributesPrivate = true } + let delivered = inlinedContext(ofKind: "identify", in: posted { $0.record(IdentifyEvent(context: slashed)) }) + + expect(delivered["/ssn"]).to(beNil()) + expect(redacted(delivered)) == ["/~1a~0b", "/~1ssn", "c/d"] + } + + it("redacts only the property a private sub-attribute names") { + _ = makeReporter(encoding) { $0.privateContextAttributes = [Reference("/address/street")] } + let delivered = inlinedContext(ofKind: "identify", in: posted { $0.record(IdentifyEvent(context: person())) }) + + expect(object(delivered["address"])["city"]) == "Oakland" + expect(object(delivered["address"])["street"]).to(beNil()) + expect(redacted(delivered)) == ["/address/street"] + } + + it("redacts an attribute the context itself marks private") { + var builder = LDContextBuilder(key: "user-key") + builder.name("Sandy") + _ = builder.trySetValue("email", "sandy@example.com") + builder.addPrivateAttribute(Reference("email")) + let marked = try! builder.build().get() + _ = makeReporter(encoding) + let delivered = inlinedContext(ofKind: "identify", in: posted { $0.record(IdentifyEvent(context: marked)) }) + + expect(delivered["email"]).to(beNil()) + expect(delivered["name"]) == "Sandy" + expect(redacted(delivered)) == ["email"] + } + + it("redacts an anonymous context's attributes from a feature event but not an identify event") { + var builder = LDContextBuilder(key: "anon-key") + builder.anonymous(true) + _ = builder.trySetValue("email", "sandy@example.com") + let anonymous = try! builder.build().get() + _ = makeReporter(encoding) + let events = posted { + $0.record(IdentifyEvent(context: anonymous)) + $0.recordFlagEvaluationEvents(flagKey: "flag-key", value: true, defaultValue: false, + featureFlag: trackedFlag, context: anonymous, includeReason: false) + } + + // An identify event is the application deliberately registering this context. + expect(inlinedContext(ofKind: "identify", in: events)["email"]) == "sandy@example.com" + // A feature event redacts every attribute of an anonymous context unasked. + let feature = inlinedContext(ofKind: "feature", in: events) + expect(feature["key"]) == "anon-key" + expect(feature["email"]).to(beNil()) + expect(redacted(feature)) == ["email"] + } + + it("redacts the private attributes of the context a summary inlines") { + _ = makeReporter(encoding) { $0.privateContextAttributes = [Reference("email")] } + let events = posted { + $0.recordFlagEvaluationEvents(flagKey: "flag-key", value: true, defaultValue: false, + featureFlag: nil, context: person(), includeReason: false) + } + + let summarized = inlinedContext(ofKind: "summary", in: events) + expect(summarized["key"]) == "user-key" + expect(summarized["email"]).to(beNil()) + expect(redacted(summarized)) == ["email"] + expect(object(summarized["address"])["city"]) == "Oakland" + } + + it("redacts every kind of a multi-context") { + var multi = LDMultiContextBuilder() + for (kind, email) in [("user", "sandy@example.com"), ("org", "billing@example.com")] { + var part = LDContextBuilder(key: "\(kind)-key") + part.kind(kind) + _ = part.trySetValue("email", .string(email)) + multi.addContext(try! part.build().get()) + } + let both = try! multi.build().get() + _ = makeReporter(encoding) { $0.privateContextAttributes = [Reference("email")] } + let delivered = inlinedContext(ofKind: "identify", in: posted { $0.record(IdentifyEvent(context: both)) }) + + expect(delivered["kind"]) == "multi" + for kind in ["user", "org"] { + expect(object(delivered[kind])["email"]).to(beNil(), description: kind) + expect(redacted(object(delivered[kind]))).to(equal(["email"]), description: kind) + } + } + + it("posts no private value in any event") { + // Over the whole request body rather than one field, so that a private value cannot slip + // through in an event kind the tests above do not name. + _ = makeReporter(encoding) { $0.privateContextAttributes = [Reference("email")] } + let events = posted { + $0.record(IdentifyEvent(context: person())) + $0.record(CustomEvent(key: "an-event", context: person())) + $0.recordFlagEvaluationEvents(flagKey: "flag-key", value: true, defaultValue: false, + featureFlag: trackedFlag, context: person(), includeReason: false) + } + + let kinds = events.compactMap { event -> String? in + guard case .string(let kind)? = event["kind"] + else { return nil } + return kind + } + expect(Set(kinds)) == ["identify", "custom", "feature", "summary"] + let body = String(data: serviceMock.publishedEventData ?? Data(), encoding: .utf8) ?? "" + expect(body.contains("sandy@example.com")) == false + } + } + } + } + } + + private func reportEventsSpec() { + describe("reportEvents") { + var testContext: TestContext! + var eventStubResponseDate: Date! + beforeEach { + eventStubResponseDate = Date().addingTimeInterval(-TimeInterval(3)) + } + afterEach { + testContext.cleanUp() + } + let erOnline = { + expect(testContext.eventProcessor.isOnline) == true + expect(testContext.eventProcessor.isReportingActive) == true + } + context("online") { + context("success") { + context("with events and tracked requests") { + beforeEach { + waitUntil(timeout: Constants.deliveryTimeout) { syncComplete in + testContext = TestContext(eventStubResponseDate: eventStubResponseDate, onSyncComplete: { result in + testContext.syncResult = result + syncComplete() + }) + testContext.eventProcessor.isOnline = true + testContext.recordEvents(Event.Kind.nonSummaryKinds.count) + // Track some flag evaluations to populate the contextSummarizer + testContext.eventProcessor.recordFlagEvaluationEvents( + flagKey: "test-flag", + value: .bool(true), + defaultValue: .bool(false), + featureFlag: nil, + context: testContext.context, + includeReason: false + ) + testContext.eventProcessor.flush(completion: nil) + } + } + it("reports events and a summary event") { + erOnline() + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) + valueIsArray(published) { valueArray in + expect(valueArray.count) == testContext.events.count + 1 + expect(Array(valueArray.prefix(testContext.events.count))) == testContext.events.map { encodeToLDValue($0) } + valueIsObject(valueArray[testContext.events.count]) { summaryObject in + expect(summaryObject["kind"]) == "summary" + } + } + expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 + expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 + expect(testContext.store.pendingEventCount) == 0 + expect(testContext.eventProcessor.lastEventResponseDate) == testContext.eventStubResponseDate + expect(testContext.eventProcessor.withContextSummarizer { $0.hasLoggedRequests }) == false + expect(testContext.syncResult).to(beNil()) + } + } + context("with events only") { + beforeEach { + waitUntil(timeout: Constants.deliveryTimeout) { syncComplete in + testContext = TestContext(eventStubResponseDate: eventStubResponseDate, onSyncComplete: { result in + testContext.syncResult = result + syncComplete() + }) + testContext.eventProcessor.isOnline = true + testContext.recordEvents(Event.Kind.nonSummaryKinds.count) + testContext.eventProcessor.flush(completion: nil) + } + } + it("reports events without a summary event") { + erOnline() + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) + expect(published) == encodeToLDValue(testContext.events) + expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 + expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == testContext.events.count + expect(testContext.store.pendingEventCount) == 0 + expect(testContext.eventProcessor.lastEventResponseDate) == testContext.eventStubResponseDate + expect(testContext.eventProcessor.withContextSummarizer { $0.hasLoggedRequests }) == false + expect(testContext.syncResult).to(beNil()) + } + } + context("with tracked requests only") { + beforeEach { + waitUntil(timeout: Constants.deliveryTimeout) { syncComplete in + testContext = TestContext(eventStubResponseDate: eventStubResponseDate, onSyncComplete: { result in + testContext.syncResult = result + syncComplete() + }) + testContext.eventProcessor.isOnline = true + // Track some flag evaluations to populate the contextSummarizer + testContext.eventProcessor.recordFlagEvaluationEvents( + flagKey: "test-flag", + value: .bool(true), + defaultValue: .bool(false), + featureFlag: nil, + context: testContext.context, + includeReason: false + ) + testContext.eventProcessor.flush(completion: nil) + } + } + it("reports only a summary event") { + erOnline() + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) + valueIsArray(published) { valueArray in + expect(valueArray.count) == 1 + valueIsObject(valueArray[0]) { summaryObject in + expect(summaryObject["kind"]) == "summary" + } + } + expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 + expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == 1 + expect(testContext.store.pendingEventCount) == 0 + expect(testContext.eventProcessor.lastEventResponseDate) == testContext.eventStubResponseDate + expect(testContext.eventProcessor.withContextSummarizer { $0.hasLoggedRequests }) == false + expect(testContext.syncResult).to(beNil()) + } + } + context("without events or tracked requests") { + beforeEach { + waitUntil(timeout: Constants.deliveryTimeout) { syncComplete in + testContext = TestContext(eventStubResponseDate: eventStubResponseDate, onSyncComplete: { result in + testContext.syncResult = result + syncComplete() + }) + testContext.eventProcessor.isOnline = true + testContext.eventProcessor.flush(completion: nil) + } + } + it("does not report events") { + erOnline() + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 0 + expect(testContext.store.pendingEventCount) == 0 + expect(testContext.eventProcessor.lastEventResponseDate) == Date.distantPast + expect(testContext.eventProcessor.withContextSummarizer { $0.hasLoggedRequests }) == false + expect(testContext.syncResult).to(beNil()) + } + } + context("while a commit is stuck writing") { + it("takes the delivered batch out of the store before telling observers") { + let writeStarted = DispatchSemaphore(value: 0) + let finishWrite = DispatchSemaphore(value: 0) + let blockLock = NSLock() + var blockNextWrite = false + // A slow disk, holding the store's I/O, which removing a delivered batch also needs. + let store = EventStore.temporary(writeLog: { file, bytes in + blockLock.lock() + let blocks = blockNextWrite + blockNextWrite = false + blockLock.unlock() + if blocks { + writeStarted.signal() + finishWrite.wait() + } + return file.writeAll(bytes) + }) + var pendingWhenTold: Int? + testContext = TestContext(onSyncComplete: { _ in + pendingWhenTold = store.pendingEventCount + }, store: store) + let reporter = testContext.eventProcessor! + let context = testContext.context! + reporter.isOnline = true + testContext.serviceMock.holdsEventCompletions = true + reporter.record(CustomEvent(key: "delivered", context: context)) + reporter.flush(completion: nil) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + + blockLock.lock() + blockNextWrite = true + blockLock.unlock() + DispatchQueue.global().async { + reporter.record(CustomEvent(key: "written", context: context)) + } + expect(writeStarted.wait(timeout: .now() + 2)) == .success + DispatchQueue.global().async { + testContext.serviceMock.releaseHeldEventCompletions() + } + // Long enough for an announcement made ahead of the removal to reach the main thread. + waitUntil(timeout: Constants.deliveryTimeout) { done in + DispatchQueue.main.asyncAfter(deadline: .now() + 0.3, execute: done) + } + finishWrite.signal() + + expect(pendingWhenTold).toEventuallyNot(beNil(), timeout: Constants.deliveryTimeout) + // Only the event still being written. The delivered one was gone before anyone heard. + expect(pendingWhenTold) == 1 + } + } + } + context("failure") { + context("server error") { + beforeEach { + waitUntil(timeout: .seconds(10)) { syncComplete in + testContext = TestContext(stubResponseSuccess: false, eventStubResponseDate: eventStubResponseDate, onSyncComplete: { result in + testContext.syncResult = result + syncComplete() + }) + testContext.eventProcessor.isOnline = true + testContext.recordEvents(Event.Kind.nonSummaryKinds.count) + // Track some flag evaluations to populate the contextSummarizer + testContext.eventProcessor.recordFlagEvaluationEvents( + flagKey: "test-flag", + value: .bool(true), + defaultValue: .bool(false), + featureFlag: nil, + context: testContext.context, + includeReason: false + ) + testContext.eventProcessor.flush(completion: nil) + } + } + it("keeps the events to retry after the failure") { + erOnline() + expect(testContext.serviceMock.publishEventDataCallCount) == 2 // 1 retry attempt + // The events are not lost by a failed delivery; they stay in the log, summary included, + // though only the full events count against capacity. + expect(testContext.store.pendingEventCount) == testContext.events.count + let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) + valueIsArray(published) { valueArray in + expect(valueArray.count) == testContext.events.count + 1 + expect(Array(valueArray.prefix(testContext.events.count))) == testContext.events.map { encodeToLDValue($0) } + valueIsObject(valueArray[testContext.events.count]) { summaryObject in + expect(summaryObject["kind"]) == "summary" + } + } + expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 + expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 + expect(testContext.eventProcessor.lastEventResponseDate) == Date.distantPast + expect(testContext.eventProcessor.withContextSummarizer { $0.hasLoggedRequests }) == false + guard case let .request(error) = testContext.syncResult + else { + fail("Expected error result for event send") + return + } + expect(error as NSError?) == DarklyServiceMock.Constants.error + } + } + context("response only") { + beforeEach { + waitUntil(timeout: .seconds(10)) { syncComplete in + testContext = TestContext(stubResponseSuccess: false, stubResponseOnly: true, eventStubResponseDate: eventStubResponseDate, onSyncComplete: { result in + testContext.syncResult = result + syncComplete() + }) + testContext.eventProcessor.isOnline = true + testContext.recordEvents(Event.Kind.nonSummaryKinds.count) + // Track some flag evaluations to populate the contextSummarizer + testContext.eventProcessor.recordFlagEvaluationEvents( + flagKey: "test-flag", + value: .bool(true), + defaultValue: .bool(false), + featureFlag: nil, + context: testContext.context, + includeReason: false + ) + testContext.eventProcessor.flush(completion: nil) + } + } + it("keeps the events to retry after the failure") { + erOnline() + expect(testContext.serviceMock.publishEventDataCallCount) == 2 // 1 retry attempt + // The events are not lost by a failed delivery; they stay in the log, summary included, + // though only the full events count against capacity. + expect(testContext.store.pendingEventCount) == testContext.events.count + let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) + valueIsArray(published) { valueArray in + expect(valueArray.count) == testContext.events.count + 1 + expect(Array(valueArray.prefix(testContext.events.count))) == testContext.events.map { encodeToLDValue($0) } + valueIsObject(valueArray[testContext.events.count]) { summaryObject in + expect(summaryObject["kind"]) == "summary" + } + } + expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 + expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 + expect(testContext.eventProcessor.lastEventResponseDate) == Date.distantPast + expect(testContext.eventProcessor.withContextSummarizer { $0.hasLoggedRequests }) == false + let expectedError = testContext.serviceMock.errorEventHTTPURLResponse + guard case let .response(error) = testContext.syncResult + else { + fail("Expected error result for event send") + return + } + let httpError = error as? HTTPURLResponse + expect(httpError?.url) == expectedError?.url + expect(httpError?.statusCode) == expectedError?.statusCode + } + } + context("error only") { + beforeEach { + waitUntil(timeout: .seconds(10)) { syncComplete in + testContext = TestContext(stubResponseSuccess: false, stubResponseErrorOnly: true, eventStubResponseDate: eventStubResponseDate, onSyncComplete: { result in + testContext.syncResult = result + syncComplete() + }) + testContext.eventProcessor.isOnline = true + testContext.recordEvents(Event.Kind.nonSummaryKinds.count) + // Track some flag evaluations to populate the contextSummarizer + testContext.eventProcessor.recordFlagEvaluationEvents( + flagKey: "test-flag", + value: .bool(true), + defaultValue: .bool(false), + featureFlag: nil, + context: testContext.context, + includeReason: false + ) + testContext.eventProcessor.flush(completion: nil) + } + } + it("keeps the events to retry after the failure") { + erOnline() + expect(testContext.serviceMock.publishEventDataCallCount) == 2 // 1 retry attempt + // The events are not lost by a failed delivery; they stay in the log, summary included, + // though only the full events count against capacity. + expect(testContext.store.pendingEventCount) == testContext.events.count + let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) + valueIsArray(published) { valueArray in + expect(valueArray.count) == testContext.events.count + 1 + expect(Array(valueArray.prefix(testContext.events.count))) == testContext.events.map { encodeToLDValue($0) } + valueIsObject(valueArray[testContext.events.count]) { summaryObject in + expect(summaryObject["kind"]) == "summary" + } + } + expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 + expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 + expect(testContext.eventProcessor.lastEventResponseDate) == Date.distantPast + expect(testContext.eventProcessor.withContextSummarizer { $0.hasLoggedRequests }) == false + guard case let .request(error) = testContext.syncResult + else { + fail("Expected error result for event send") + return + } + expect(error as NSError?) == DarklyServiceMock.Constants.error + } + } + context("unexpected response") { + beforeEach { + waitUntil(timeout: .seconds(10)) { syncComplete in + testContext = TestContext(eventStubResponseDate: eventStubResponseDate, onSyncComplete: { result in + testContext.syncResult = result + syncComplete() + }) + let unauthorized = HTTPURLResponse(url: testContext.serviceMock.config.eventsUrl, + statusCode: HTTPURLResponse.StatusCodes.unauthorized, + httpVersion: DarklyServiceMock.Constants.httpVersion, + headerFields: nil) + testContext.serviceMock.stubbedEventResponse = (nil, unauthorized, nil, nil) + testContext.eventProcessor.isOnline = true + testContext.recordEvents(Event.Kind.nonSummaryKinds.count) + testContext.eventProcessor.flush(completion: nil) + } + } + it("keeps the events and keeps reporting, without retrying soon") { + // A valid key is sometimes refused for minutes during an incident, so the events wait for + // the longer backoff rather than being deleted with the SDK going quiet. + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + expect(testContext.store.pendingEventCount) == testContext.events.count + expect(testContext.eventProcessor.isOnline).to(beTrue()) + expect(testContext.eventProcessor.isReportingActive).to(beTrue()) + if case let .response(error) = testContext.syncResult { + expect((error as? HTTPURLResponse)?.statusCode) == HTTPURLResponse.StatusCodes.unauthorized + } else { + fail("Expected response error result for event send") + } + waitLettingTimersFire(1.5) + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + } + } + } + } + context("offline") { + beforeEach { + waitUntil(timeout: Constants.deliveryTimeout) { syncComplete in + testContext = TestContext(eventStubResponseDate: eventStubResponseDate, onSyncComplete: { result in + testContext.syncResult = result + syncComplete() + }) + testContext.recordEvents(Event.Kind.nonSummaryKinds.count) + // Track some flag evaluations to populate the contextSummarizer + testContext.eventProcessor.recordFlagEvaluationEvents( + flagKey: "test-flag", + value: .bool(true), + defaultValue: .bool(false), + featureFlag: nil, + context: testContext.context, + includeReason: false + ) + testContext.eventProcessor.flush(completion: nil) + } + } + it("doesn't report events, but persists them") { + expect(testContext.eventProcessor.isOnline) == false + expect(testContext.eventProcessor.isReportingActive) == false + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 0 + expect(testContext.eventProcessor.lastEventResponseDate) == Date.distantPast + + // Being offline delays delivery, not durability: a flush is a commit point, so the recorded events and + // the evaluations counted so far are on disk even though nothing can be sent. + let pending = testContext.committedEvents() + expect(pending.dropLast().map { $0 }) == testContext.recordedEventsAsJSON() + expect(pending.last?.kindField) == "summary" + expect(testContext.eventProcessor.withContextSummarizer { $0.hasLoggedRequests }) == false + guard case .isOffline = testContext.syncResult + else { + fail("Expected error .isOffline result for event send") + return + } + } + } + } + } + + func testRecordFlagEvaluationEvents() { + let context = LDContext.stub() + let serviceMock = DarklyServiceMock() + + /// A reporter writing to a store of its own, and that store, so a test can read back what was recorded. + func makeReporter() -> (DirectEventProcessor, EventStore) { + let store = EventStore.temporary() + return (DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, pacing: .noWaits), store) + } + + describe("recordFlagEvaluationEvents") { + it("counts an evaluation the summarizer turns away as dropped, and still keeps its full event") { + var config = LDConfig.stub + config.eventCapacity = 2 + let service = DarklyServiceMock() + service.config = config + let diagnosticCache = DiagnosticCachingMock() + service.diagnosticCache = diagnosticCache + let store = EventStore.temporary(capacity: config.eventCapacity) + defer { store.deleteEverything() } + let reporter = DirectEventProcessor(service: service, onSyncComplete: nil, store: store, pacing: .noWaits) + let untracked = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: false) + let tracked = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: true) + + // Capacity bounds the number of contexts counted as well as the number of events held. + for key in ["a", "b"] { + reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: untracked, context: LDContext.stub(key: key), includeReason: false) + } + expect(diagnosticCache.incrementDroppedEventCountCallCount) == 0 + + reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: tracked, context: LDContext.stub(key: "c"), includeReason: false) + + expect(diagnosticCache.incrementDroppedEventCountCallCount) == 1 + expect(reporter.withContextSummarizer { $0.getSummaries() }.count) == 2 + reporter.commitRecordedEvents() + expect(recordedEvents(store).compactMap { $0.kindField }) == ["feature", "summary", "summary"] + } + it("unknown flag") { + let (reporter, store) = makeReporter() + defer { store.deleteEverything() } + reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: nil, context: context, includeReason: true) + expect(recordedEvents(store)).to(beEmpty()) + expect(reporter.withContextSummarizer { $0.hasLoggedRequests }) == true + let summaries = reporter.withContextSummarizer { $0.getSummaries() } + expect(summaries.count) == 1 + let tracker = summaries[0].tracker + expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" + expect(tracker.flagCounters["flag-key"]?.flagValueCounters.count) == 1 + expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: nil, version: nil)]?.count) == 1 + expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: nil, version: nil)]?.value) == "a" + } + it("untracked flag") { + let (reporter, store) = makeReporter() + defer { store.deleteEverything() } + let flag = FeatureFlag(flagKey: "unused", value: nil, variation: 1, flagVersion: 2, trackEvents: false) + reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: true) + expect(recordedEvents(store)).to(beEmpty()) + expect(reporter.withContextSummarizer { $0.hasLoggedRequests }) == true + let summaries = reporter.withContextSummarizer { $0.getSummaries() } + expect(summaries.count) == 1 + let tracker = summaries[0].tracker + expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" + expect(tracker.flagCounters["flag-key"]?.flagValueCounters.count) == 1 + expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.count) == 1 + expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.value) == "a" + } + it("tracked flag") { + let (reporter, store) = makeReporter() + defer { store.deleteEverything() } + let flag = FeatureFlag(flagKey: "unused", value: nil, variation: 1, flagVersion: 2, trackEvents: true) + reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: true) + let expected = FeatureEvent(key: "flag-key", context: context, value: "a", defaultValue: "b", featureFlag: flag, includeReason: true, isDebug: false) + expect(expectedEvents(reporter.pendingEventsForTesting)) == expectedEvents([expected]) + expect(recordedEvents(store)).to(beEmpty()) + expect(reporter.withContextSummarizer { $0.hasLoggedRequests }) == true + let summaries = reporter.withContextSummarizer { $0.getSummaries() } + expect(summaries.count) == 1 + let tracker = summaries[0].tracker + expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" + expect(tracker.flagCounters["flag-key"]?.flagValueCounters.count) == 1 + expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.count) == 1 + expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.value) == "a" + } + it("debug until past date") { + let (reporter, store) = makeReporter() + defer { store.deleteEverything() } + let flag = FeatureFlag(flagKey: "unused", value: nil, variation: 1, flagVersion: 2, trackEvents: false, debugEventsUntilDate: Date().addingTimeInterval(-1.0)) + reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: true) + expect(recordedEvents(store)).to(beEmpty()) + expect(reporter.withContextSummarizer { $0.hasLoggedRequests }) == true + let summaries = reporter.withContextSummarizer { $0.getSummaries() } + expect(summaries.count) == 1 + let tracker = summaries[0].tracker + expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" + expect(tracker.flagCounters["flag-key"]?.flagValueCounters.count) == 1 + expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.count) == 1 + expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.value) == "a" + } + it("debug until future date") { + let (reporter, store) = makeReporter() + defer { store.deleteEverything() } + let flag = FeatureFlag(flagKey: "unused", value: nil, variation: 1, flagVersion: 2, trackEvents: false, debugEventsUntilDate: Date().addingTimeInterval(3.0)) + reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: false) + let expected = FeatureEvent(key: "flag-key", context: context, value: "a", defaultValue: "b", featureFlag: flag, includeReason: false, isDebug: true) + expect(expectedEvents(reporter.pendingEventsForTesting)) == expectedEvents([expected]) + expect(recordedEvents(store)).to(beEmpty()) + expect(reporter.withContextSummarizer { $0.hasLoggedRequests }) == true + let summaries = reporter.withContextSummarizer { $0.getSummaries() } + expect(summaries.count) == 1 + let tracker = summaries[0].tracker + expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" + expect(tracker.flagCounters["flag-key"]?.flagValueCounters.count) == 1 + expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.count) == 1 + expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.value) == "a" + } + it("debug until future date earlier than service date") { + let (reporter, store) = makeReporter() + defer { store.deleteEverything() } + reporter.setLastEventResponseDate(Date().addingTimeInterval(10.0)) + let flag = FeatureFlag(flagKey: "unused", value: nil, variation: 1, flagVersion: 2, trackEvents: false, debugEventsUntilDate: Date().addingTimeInterval(3.0)) + reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: true) + expect(recordedEvents(store)).to(beEmpty()) + expect(reporter.withContextSummarizer { $0.hasLoggedRequests }) == true + let summaries = reporter.withContextSummarizer { $0.getSummaries() } + expect(summaries.count) == 1 + let tracker = summaries[0].tracker + expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" + expect(tracker.flagCounters["flag-key"]?.flagValueCounters.count) == 1 + expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.count) == 1 + expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.value) == "a" + } + it("tracked flag and debug date in future") { + let (reporter, store) = makeReporter() + defer { store.deleteEverything() } + reporter.setLastEventResponseDate(Date().addingTimeInterval(-3.0)) + let flag = FeatureFlag(flagKey: "unused", value: nil, variation: 1, flagVersion: 2, trackEvents: true, debugEventsUntilDate: Date()) + reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: false) + let expectedFeature = FeatureEvent(key: "flag-key", context: context, value: "a", defaultValue: "b", featureFlag: flag, includeReason: false, isDebug: false) + let expectedDebug = FeatureEvent(key: "flag-key", context: context, value: "a", defaultValue: "b", featureFlag: flag, includeReason: false, isDebug: true) + expect(expectedEvents(reporter.pendingEventsForTesting)) == expectedEvents([expectedFeature, expectedDebug]) + expect(recordedEvents(store)).to(beEmpty()) + expect(reporter.withContextSummarizer { $0.hasLoggedRequests }) == true + let summaries = reporter.withContextSummarizer { $0.getSummaries() } + expect(summaries.count) == 1 + let tracker = summaries[0].tracker + expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" + expect(tracker.flagCounters["flag-key"]?.flagValueCounters.count) == 1 + expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.count) == 1 + expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.value) == "a" + } + it("records events concurrently") { + let (reporter, store) = makeReporter() + defer { store.deleteEverything() } + reporter.setLastEventResponseDate(Date()) + let flag = FeatureFlag(flagKey: "unused", trackEvents: true, debugEventsUntilDate: Date().addingTimeInterval(3.0)) + + let counter = DispatchSemaphore(value: 0) + DispatchQueue.concurrentPerform(iterations: 10) { _ in + reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: false) + counter.signal() + } + (0..<10).forEach { _ in counter.wait() } + + let held = expectedEvents(reporter.pendingEventsForTesting) + expect(held.count) == 20 + expect(held.filter { $0.kindField == "feature" }.count) == 10 + expect(held.filter { $0.kindField == "debug" }.count) == 10 + expect(recordedEvents(store)).to(beEmpty()) + expect(reporter.withContextSummarizer { $0.hasLoggedRequests }) == true + let summaries = reporter.withContextSummarizer { $0.getSummaries() } + expect(summaries.count) == 1 + let tracker = summaries[0].tracker + expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" + expect(tracker.flagCounters["flag-key"]?.flagValueCounters.count) == 1 + expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: nil, version: nil)]?.count) == 10 + expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: nil, version: nil)]?.value) == "a" + } + } + } + + private func reportTimerSpec() { + describe("report timer fires") { + var testContext: TestContext! + afterEach { + testContext.cleanUp() + } + context("with events") { + beforeEach { + testContext = TestContext(eventFlushInterval: Constants.eventFlushIntervalHalfSecond) + testContext.eventProcessor.isOnline = true + testContext.recordEvents(Event.Kind.allKinds.count) + } + it("reports events") { + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + expect(testContext.store.pendingEventCount).toEventually(equal(0), timeout: Constants.deliveryTimeout) + expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch).toEventually(equal(testContext.events.count), timeout: Constants.deliveryTimeout) + let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) + valueIsArray(published) { valueArray in + expect(valueArray.count) == testContext.events.count + expect(valueArray) == testContext.events.map { encodeToLDValue($0) } + } + } + } + context("on later intervals") { + beforeEach { + testContext = TestContext(eventFlushInterval: Constants.eventFlushIntervalHalfSecond) + testContext.eventProcessor.isOnline = true + testContext.recordEvents(Event.Kind.nonSummaryKinds.count) + } + it("publishes events recorded after the first fire") { + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + testContext.recordEvents(Event.Kind.nonSummaryKinds.count) + expect(testContext.serviceMock.publishEventDataCallCount) + .toEventually(equal(2), timeout: .seconds(2)) + } + } + it("without events") { + testContext = TestContext(eventFlushInterval: Constants.eventFlushIntervalHalfSecond) + testContext.eventProcessor.isOnline = true + + waitUntil(timeout: Constants.deliveryTimeout) { done in + DispatchQueue.main.asyncAfter(deadline: DispatchTime.now() + Constants.eventFlushIntervalHalfSecond) { + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + done() + } + } + } + } + } +} + +extension DirectEventProcessorSpec { + /// What the SDK is able to report after the process it was recording in is gone. + /// + /// These are written against what is in the directory, and a store on a copy of it, which is what the next run of + /// the application amounts to: it sees the bytes that reached the disk and nothing that was still in memory. + private func durabilitySpec() { + describe("surviving the process") { + var testContext: TestContext! + afterEach { + testContext?.cleanUp() + testContext = nil + } + + /// The events the next run of the application would find. + func eventsLeftOnDisk() -> [LDValue] { + testContext.store.payloadsOnDisk.compactMap { try? JSONDecoder().decode(LDValue.self, from: $0) } + } + + itPersisting("has written a tracked event by the time recording it returns") { + testContext = TestContext() + + // The sequence that loses events today: something is tracked and the process ends immediately after. + testContext.eventProcessor.record(CustomEvent(key: "fatal-error", context: testContext.context, data: ["message": "boom"])) + + let onDisk = eventsLeftOnDisk() + expect(onDisk.count) == 1 + expect(onDisk.first?.kindField) == "custom" + } + + itPersisting("writes nothing for a tracked event at .deferred, and writes it with the rest at the pending threshold") { + let commits = DispatchQueue(label: "com.launchdarkly.tests.commits") + testContext = TestContext(eventCapacity: 100, eventPersistence: .deferred, commitQueue: commits) + + testContext.eventProcessor.record(CustomEvent(key: "fatal-error", context: testContext.context, data: ["message": "boom"])) + // Behind any write the track queued, so that write has run by now. + commits.sync { } + expect(eventsLeftOnDisk()).to(beEmpty()) + + for index in 1.. [String] { + inner.payloadsOnDisk.compactMap { try? JSONDecoder().decode(LDValue.self, from: $0).keyField } + } + + let start = DispatchSemaphore(value: 0) + let finished = DispatchGroup() + let missingLock = NSLock() + var missingOnReturn: [String] = [] + for index in 0.. 1 + } + + it("counts a run toward capacity while it is being encoded") { + var config = LDConfig.stub + config.eventCapacity = 1 + config.eventPersistence = .deferred + let serviceMock = DarklyServiceMock() + serviceMock.config = config + let diagnosticCache = DiagnosticCachingMock() + serviceMock.diagnosticCache = diagnosticCache + let store = EventStore.temporary(capacity: config.eventCapacity) + defer { store.deleteEverything() } + let commits = DispatchQueue(label: "com.launchdarkly.tests.suspended") + commits.suspend() + defer { commits.resume() } + let reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: .codable, commitQueue: commits, pacing: .noWaits) + + let slow = EncodingGate(key: "slow", context: LDContext.stub()) + reporter.record(slow) + let committed = DispatchSemaphore(value: 0) + DispatchQueue.global().async { + reporter.commitRecordedEvents() + committed.signal() + } + expect(slow.started.wait(timeout: .now() + 5)) == .success + + // Neither held nor staged at this moment, the slow event is only in the commit encoding it. + reporter.record(CustomEvent(key: "late", context: LDContext.stub())) + + slow.proceed.signal() + expect(committed.wait(timeout: .now() + 5)) == .success + reporter.commitRecordedEvents() + + expect(store.pendingEventCount) == 1 + expect(diagnosticCache.incrementDroppedEventCountCallCount) == 1 + } + + it("does not close a batch partway through another commit's run") { + var config = LDConfig.stub + config.eventPersistence = .deferred + let serviceMock = DarklyServiceMock() + serviceMock.config = config + serviceMock.stubEventResponse(success: true) + let inner = EventStore.temporary(capacity: config.eventCapacity) + defer { inner.deleteEverything() } + let store = ClosingHookStore(inner) + let commits = DispatchQueue(label: "com.launchdarkly.tests.suspended") + commits.suspend() + defer { commits.resume() } + let reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: .codable, commitQueue: commits, pacing: .noWaits) + + let slow = EncodingGate(key: "slow", context: LDContext.stub()) + let competingCommit = DispatchSemaphore(value: 0) + store.beforeClosing = { + reporter.record(CustomEvent(key: "first", context: LDContext.stub())) + reporter.record(slow) + DispatchQueue.global().async { + reporter.commitRecordedEvents() + competingCommit.signal() + } + // Long enough for an unguarded commit to stage "first" and block encoding "slow". + _ = slow.started.wait(timeout: .now() + 0.5) + } + store.afterClosing = { slow.proceed.signal() } + + reporter.isOnline = true + waitUntil(timeout: .seconds(10)) { done in + reporter.flushReportingOutcome { _ in done() } + } + expect(competingCommit.wait(timeout: .now() + 5)) == .success + + for keys in store.closedKeys { + expect(keys.contains("first")) == keys.contains("slow") + } + let rest = try inner.closeBatch().flatMap { try inner.body(of: $0) }.map(customKeys) ?? [] + expect(rest) == ["first", "slow"] + } + + it("counts an event once while the commit that staged it is still running") { + var config = LDConfig.stub + config.eventCapacity = 3 + config.eventPersistence = .deferred + let serviceMock = DarklyServiceMock() + serviceMock.config = config + let diagnosticCache = DiagnosticCachingMock() + serviceMock.diagnosticCache = diagnosticCache + let store = EventStore.temporary(capacity: config.eventCapacity) + defer { store.deleteEverything() } + let commits = DispatchQueue(label: "com.launchdarkly.tests.suspended") + commits.suspend() + defer { commits.resume() } + let reporter = DirectEventProcessor(service: serviceMock, onSyncComplete: nil, store: store, encoding: .codable, commitQueue: commits, pacing: .noWaits) + + let slow = EncodingGate(key: "slow", context: LDContext.stub()) + reporter.record(CustomEvent(key: "first", context: LDContext.stub())) + reporter.record(slow) + let committed = DispatchSemaphore(value: 0) + DispatchQueue.global().async { + reporter.commitRecordedEvents() + committed.signal() + } + expect(slow.started.wait(timeout: .now() + 5)) == .success + + // Paused with "first" staged and "slow" still encoding. Counted once each, they leave room for one more + // under a capacity of three; "first" counted both as staged and as reserved would not. + reporter.record(CustomEvent(key: "late", context: LDContext.stub())) + + slow.proceed.signal() + expect(committed.wait(timeout: .now() + 5)) == .success + reporter.commitRecordedEvents() + + expect(diagnosticCache.incrementDroppedEventCountCallCount) == 0 + expect(store.pendingEventCount) == 3 + } + + itPersisting("makes room in the summarizer at every commit point") { + // Counters become summary events and are reset at every commit point, so the summarizer only holds the + // contexts seen since the last one. Without the reset, an application identifying its way through more + // contexts than the capacity would start losing counts partway through. + testContext = TestContext(lastEventResponseDate: Date(), eventCapacity: 2) + let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: false) + let contexts = 6 + for index in 0.. 2 ? ENOSPC : file.writeAll(bytes) + }) + testContext = TestContext(store: store) + testContext.eventProcessor.record(CustomEvent(key: "written", context: LDContext.stub())) + // Its write fails, so persistence is given up with "written" in the log and this in memory. + testContext.eventProcessor.record(CustomEvent(key: "held", context: LDContext.stub())) + expect(store.isPersisting) == false + + testContext.eventProcessor.isOnline = true + var delivered: Bool? + waitUntil(timeout: Constants.deliveryTimeout) { done in + testContext.eventProcessor.flushReportingOutcome { result in + delivered = result + done() + } + } + expect(delivered) == true + // Neither left behind, and oldest first. + expect(testContext.serviceMock.publishedEventPayloads.flatMap(customKeys)) == ["written", "held"] + expect(store.pendingEventCount) == 0 + } + + it("reports false when an event could not be serialized, though the rest were accepted") { + testContext = TestContext() + testContext.eventProcessor.record(CustomEvent(key: "poison", context: LDContext.stub(), metricValue: .nan)) + testContext.recordEvents(1) + testContext.eventProcessor.isOnline = true + var delivered: Bool? + waitUntil(timeout: Constants.deliveryTimeout) { done in + testContext.eventProcessor.flushReportingOutcome { result in + delivered = result + done() + } + } + expect(delivered) == false + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + + // Told once, to the caller who was there to hear it. + waitUntil(timeout: Constants.deliveryTimeout) { done in + testContext.eventProcessor.flushReportingOutcome { result in + delivered = result + done() + } + } + expect(delivered) == true + } + + it("reports false when an event serialized too large to store") { + testContext = TestContext() + // Larger than a log frame holds, so the store refuses it however much room is left. + let oversized = String(repeating: "x", count: EventLogFormat.maxFrameSize) + testContext.eventProcessor.record(CustomEvent(key: oversized, context: LDContext.stub())) + testContext.eventProcessor.record(CustomEvent(key: "fine", context: LDContext.stub())) + testContext.eventProcessor.isOnline = true + var delivered: Bool? + waitUntil(timeout: Constants.deliveryTimeout) { done in + testContext.eventProcessor.flushReportingOutcome { result in + delivered = result + done() + } + } + expect(delivered) == false + // The event beside it still went. + expect(testContext.serviceMock.publishedEventPayloads.flatMap(customKeys)) == ["fine"] + } + + it("sends a flush of more than a batch as requests no larger than a batch, every event once") { + // Small enough that a batch closed at the limit is within one event of it, so a body that is short of + // the limit by more than that shows the split happened somewhere else. + let dataCharacters = EventLogFormat.maxBatchSize / 32 + let eventCount = 100 + testContext = TestContext(eventCapacity: eventCount) + let padding = LDValue.string(String(repeating: "x", count: dataCharacters)) + for index in 1...eventCount { + testContext.eventProcessor.record(CustomEvent(key: "event-\(index)", context: LDContext.stub(), + data: padding)) + } + testContext.eventProcessor.isOnline = true + var delivered: Bool? + waitUntil(timeout: .seconds(30)) { done in + testContext.eventProcessor.flushReportingOutcome { result in + delivered = result + done() + } + } + expect(delivered) == true + + // Before the service compresses it, so the larger of the two sizes. + let bodies = testContext.serviceMock.publishedEventPayloads + let sizes = bodies.map(\.count).sorted() + expect(sizes.count) > 1 + expect(sizes.last ?? .max) <= EventLogFormat.maxBatchSize + // Only the batch the events ran out in may be short of full. + expect(sizes.dropFirst().allSatisfy { $0 > EventLogFormat.maxBatchSize - dataCharacters - 4096 }) == true + expect(Set(testContext.serviceMock.publishedPayloadIds).count) == bodies.count + expect(bodies.flatMap(customKeys).sorted()) == (1...eventCount).map { "event-\($0)" }.sorted() + } + + it("drops a summary larger than a frame whole and delivers the events beside it") { + // Long keys, so that a few thousand flags take one context's summary past what a frame holds. + let keyPrefix = String(repeating: "k", count: 1000) + let flagCount = EventLogFormat.maxFrameSize / 1000 * 3 / 2 + testContext = TestContext() + let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: false) + let context = LDContext.stub() + // First, ahead of the debug events the evaluations fill the capacity with. + testContext.eventProcessor.record(CustomEvent(key: "fine", context: context)) + for index in 0.. [LDValue] in + guard case .array(let events)? = try? JSONDecoder().decode(LDValue.self, from: body) + else { return [] } + return events + } + expect(events.filter { $0.kindField == "summary" }).to(beEmpty()) + expect(testContext.serviceMock.publishedEventPayloads.flatMap(customKeys)) == ["fine"] + } + + it("reports false when a retryable failure leaves the batch on disk") { + testContext = TestContext(stubResponseSuccess: false) + testContext.recordEvents(1) + testContext.eventProcessor.isOnline = true + var delivered: Bool? + waitUntil(timeout: .seconds(10)) { done in + testContext.eventProcessor.flushReportingOutcome { result in + delivered = result + done() + } + } + expect(delivered) == false + expect(testContext.store.pendingBatches().count) == 1 + } + + it("reports false when LaunchDarkly refuses the batch with a 401, and keeps it") { + testContext = TestContext() + let unauthorized = HTTPURLResponse(url: testContext.serviceMock.config.eventsUrl, + statusCode: HTTPURLResponse.StatusCodes.unauthorized, + httpVersion: DarklyServiceMock.Constants.httpVersion, + headerFields: nil) + testContext.serviceMock.stubbedEventResponse = (nil, unauthorized, nil, nil) + testContext.recordEvents(1) + testContext.eventProcessor.isOnline = true + var delivered: Bool? + waitUntil(timeout: Constants.deliveryTimeout) { done in + testContext.eventProcessor.flushReportingOutcome { result in + delivered = result + done() + } + } + expect(delivered) == false + expect(testContext.eventProcessor.isOnline) == true + expect(testContext.store.pendingEventCount) == 1 + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + } + + it("starts no second request while a delivery is in flight, and sends what was recorded meanwhile after it") { + testContext = TestContext() + testContext.recordEvents(1) + testContext.eventProcessor.isOnline = true + testContext.serviceMock.holdsEventCompletions = true + + testContext.eventProcessor.flushReportingOutcome { _ in } + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + + testContext.recordEvents(1) + let second = Answer() + testContext.eventProcessor.flushReportingOutcome(completion: second.record) + Thread.sleep(forTimeInterval: 0.3) + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + expect(second.value()).to(beNil()) + + testContext.serviceMock.holdsEventCompletions = false + testContext.serviceMock.releaseHeldEventCompletions() + expect(second.value()).toEventually(beTrue(), timeout: Constants.deliveryTimeout) + expect(testContext.serviceMock.publishEventDataCallCount) == 2 + expect(Set(testContext.serviceMock.publishedPayloadIds).count) == 2 + } + + // Every batch whose request is out stops counting against capacity. With a request per flush, a run of + // flushes each held up by a slow response would take the store a batch past capacity per flush. + it("holds the store to its capacity through flushes that each find a delivery in flight") { + testContext = TestContext(eventCapacity: 2) + testContext.eventProcessor.isOnline = true + testContext.serviceMock.holdsEventCompletions = true + testContext.eventProcessor.record(CustomEvent(key: "a-0", context: testContext.context)) + testContext.eventProcessor.record(CustomEvent(key: "a-1", context: testContext.context)) + testContext.eventProcessor.flush(completion: nil) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + + for round in 0..<3 { + testContext.eventProcessor.record(CustomEvent(key: "b\(round)-0", context: testContext.context)) + testContext.eventProcessor.record(CustomEvent(key: "b\(round)-1", context: testContext.context)) + testContext.eventProcessor.flush(completion: nil) + } + Thread.sleep(forTimeInterval: 0.3) + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + + testContext.serviceMock.holdsEventCompletions = false + testContext.serviceMock.releaseHeldEventCompletions() + waitUntil(timeout: Constants.deliveryTimeout) { done in + testContext.eventProcessor.flushReportingOutcome { _ in done() } + } + // The batch out made room for one store's worth, and nothing past it was taken. + expect(testContext.serviceMock.publishedEventPayloads.flatMap(customKeys)) == ["a-0", "a-1", "b0-0", "b0-1"] + } + + it("inherits the outcome of a delivery holding the batch it would have sent") { + testContext = TestContext(stubResponseSuccess: false) + testContext.recordEvents(1) + testContext.eventProcessor.isOnline = true + testContext.serviceMock.holdsEventCompletions = true + + testContext.eventProcessor.flushReportingOutcome { _ in } + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + + // Everything on disk belongs to the delivery already running, and that may include this caller's + // events: a commit only stages them, and whichever delivery runs next is what closes them into a + // batch. So this caller is answered by that delivery rather than told its events arrived. + let answer = Answer() + testContext.eventProcessor.flushReportingOutcome(completion: answer.record) + Thread.sleep(forTimeInterval: 0.3) + expect(answer.value()).to(beNil()) + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + + testContext.serviceMock.holdsEventCompletions = false + testContext.serviceMock.releaseHeldEventCompletions() + // Allows for the one-second retry, after which the batch stays on disk and nobody was told it went. + expect(answer.value()).toEventually(beFalse(), timeout: .seconds(10)) + } + + it("tells the caller whose delivery the loss happened before, not whichever answers first") { + testContext = TestContext() + testContext.recordEvents(1) + testContext.eventProcessor.isOnline = true + testContext.serviceMock.holdsEventCompletions = true + + // A closes a batch of its own and waits on it. Nothing has been lost at the point it closed. + let first = Answer() + testContext.eventProcessor.flushReportingOutcome(completion: first.record) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + + // B is asked for while A's post is out, with an event the store will never take among what it will + // close. A's batch closed before that event was dropped, so the loss is B's. + testContext.eventProcessor.record(CustomEvent(key: "poison", context: LDContext.stub(), metricValue: .nan)) + testContext.recordEvents(1) + let second = Answer() + testContext.eventProcessor.flushReportingOutcome(completion: second.record) + + // With one shared flag, whichever delivery answered first would take the loss, and B would report + // success for the event it dropped. + testContext.serviceMock.holdsEventCompletions = false + testContext.serviceMock.releaseHeldEventCompletions() + + expect(first.value()).toEventually(beTrue(), timeout: Constants.deliveryTimeout) + expect(second.value()).toEventually(beFalse(), timeout: Constants.deliveryTimeout) + expect(testContext.serviceMock.publishEventDataCallCount) == 2 + } + + it("tells the caller whose delivery lost an event, not a later one with nothing of its own to send") { + testContext = TestContext() + testContext.eventProcessor.record(CustomEvent(key: "poison", context: LDContext.stub(), metricValue: .nan)) + testContext.recordEvents(1) + testContext.eventProcessor.isOnline = true + testContext.serviceMock.holdsEventCompletions = true + + // A drops the event the store will never take as it closes its batch, so the loss is A's. + let first = Answer() + testContext.eventProcessor.flushReportingOutcome(completion: first.record) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + + // B has nothing of its own. Answered as soon as A's batch is, it would take A's loss and leave A + // reporting success for the event A dropped. + let second = Answer() + testContext.eventProcessor.flushReportingOutcome(completion: second.record) + testContext.serviceMock.holdsEventCompletions = false + testContext.serviceMock.releaseHeldEventCompletions() + + expect(first.value()).toEventually(beFalse(), timeout: Constants.deliveryTimeout) + expect(second.value()).toEventually(beTrue(), timeout: Constants.deliveryTimeout) + } + + it("answers away from the queue a delivery runs on, after a delivery") { + testContext = TestContext() + testContext.recordEvents(1) + testContext.eventProcessor.isOnline = true + + expect(testContext.flushedFromACompletion()) == true + } + + it("answers away from the queue a delivery runs on, with nothing to send") { + testContext = TestContext() + testContext.eventProcessor.isOnline = true + + // The path that answers without ever leaving the reporter's queue, so it is the one that would + // deadlock while the delivery path, answering from the response, did not. + expect(testContext.flushedFromACompletion()) == true + } + + it("sends one delivery for the flushes that arrive before it starts, and answers them all") { + testContext = TestContext(stubResponseSuccess: false) + testContext.recordEvents(1) + testContext.eventProcessor.isOnline = true + let released = DispatchSemaphore(value: 0) + testContext.eventProcessor.occupyQueue(until: released) + + // An application flushing in a loop would otherwise queue a delivery each time, leaving the flush + // that matters waiting behind all of them, and growing the waiter list without bound. + for _ in 0..<100 { + testContext.eventProcessor.flush(completion: nil) + } + expect(testContext.eventProcessor.queuedDeliveryWaiterCount) == 0 + + // Both completions run at once, and the poll below reads from a third thread. + let answers = Answers() + for _ in 0..<2 { + testContext.eventProcessor.flushReportingOutcome(completion: answers.record) + } + expect(testContext.eventProcessor.queuedDeliveryWaiterCount) == 2 + + released.signal() + // Allows for the one-second retry, after which the batch stays on disk. + expect(answers.value()).toEventually(equal([false, false]), timeout: .seconds(10)) + expect(testContext.eventProcessor.startedDeliveryCount) == 1 + expect(testContext.serviceMock.publishEventDataCallCount) == 2 // the first attempt and its retry + } + + it("answers every flush that arrived during a refused delivery with its outcome") { + testContext = TestContext(stubResponseSuccess: false) + testContext.recordEvents(1) + testContext.eventProcessor.isOnline = true + testContext.serviceMock.holdsEventCompletions = true + + let answers = Answers() + testContext.eventProcessor.flushReportingOutcome(completion: answers.record) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + + // Held on the first post, each of these finds the only batch claimed by it. None of them is told + // its events arrived on the strength of having nothing left of its own to send. + for _ in 0..<20 { + testContext.eventProcessor.flushReportingOutcome(completion: answers.record) + } + + testContext.serviceMock.holdsEventCompletions = false + testContext.serviceMock.releaseHeldEventCompletions() + + // Allows for the one-second retry the second delivery makes. + expect(answers.value().count).toEventually(equal(21), timeout: .seconds(10)) + expect(answers.value().contains(true)) == false + } + } + } + + private func outageSpec() { + describe("delivering through an outage") { + var testContext: TestContext! + afterEach { + testContext?.cleanUp() + testContext = nil + } + + /// Refuses a batch with `status` for several flushes in a row, each of which attempts it and retries it once, + /// then lets the service recover and checks that the batch arrives exactly once, under the payload ID it was + /// first sent with. + func expectKeptThroughRefusals(status: Int, persistence: EventPersistence, refusedFlushes: Int) { + testContext = TestContext(store: EventStore.temporary(persistEvents: persistence != .disabled), + eventPersistence: persistence, + retryState: DirectEventProcessorSpec.withoutBackoff()) + testContext.serviceMock.respondToEvents(with: status) + testContext.eventProcessor.record(CustomEvent(key: "kept", context: testContext.context)) + testContext.eventProcessor.isOnline = true + + for flush in 0.. 0 + + // An application flushing on every move to the background during a refusal. + for _ in 0..<4 { + expect(testContext.flushBypassingBackoffOutcome()) == false + } + + expect(testContext.serviceMock.publishEventDataCallCount) == 5 + expect(retryState.remainingWait) <= waitAfterTheRefusal + } + + it("posts nothing for a flush bypassing the backoff while offline") { + testContext = TestContext() + testContext.eventProcessor.record(CustomEvent(key: "an-event", context: testContext.context)) + + expect(testContext.flushBypassingBackoffOutcome()) == false + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + } + + it("posts nothing at close when the client was already offline") { + testContext = TestContext() + testContext.eventProcessor.record(CustomEvent(key: "an-event", context: testContext.context)) + + testContext.eventProcessor.flushAtClose() + waitLettingTimersFire(0.5) + + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + } + + it("waits as long as a Retry-After asks before posting again") { + testContext = TestContext() + let busy = HTTPURLResponse(url: testContext.serviceMock.config.eventsUrl, + statusCode: 503, + httpVersion: DarklyServiceMock.Constants.httpVersion, + headerFields: [HTTPURLResponse.HeaderKeys.retryAfter: "4"]) + testContext.serviceMock.stubbedEventResponse = (nil, busy, nil, nil) + testContext.eventProcessor.record(CustomEvent(key: "an-event", context: testContext.context)) + testContext.eventProcessor.isOnline = true + + // Four seconds, jittered down to no less than two, which is past the second a flush waits out. + expect(testContext.flushOutcome()) == false + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + testContext.serviceMock.stubEventResponse(success: true) + waitLettingTimersFire(1.8) + expect(testContext.serviceMock.publishEventDataCallCount) == 1 + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(2), timeout: .seconds(5)) + expect(testContext.serviceMock.publishedEventPayloads.map(customKeys).last) == ["an-event"] + } + + it("delivers events recorded during an outage after the batch refused before them, oldest first and once each") { + testContext = TestContext(retryState: DirectEventProcessorSpec.withoutBackoff()) + testContext.serviceMock.respondToEvents(with: 503) + testContext.eventProcessor.isOnline = true + testContext.eventProcessor.record(CustomEvent(key: "first", context: testContext.context)) + expect(testContext.flushOutcome()) == false + testContext.eventProcessor.record(CustomEvent(key: "second", context: testContext.context)) + expect(testContext.flushOutcome()) == false + + // Two flushes, each an attempt and a retry, all at the first batch: delivery stops at the oldest batch + // that fails in a way that may pass. + expect(testContext.serviceMock.publishedEventPayloads.map(customKeys)) == Array(repeating: ["first"], count: 4) + + testContext.serviceMock.stubEventResponse(success: true) + expect(testContext.flushOutcome()) == true + + expect(testContext.serviceMock.publishedEventPayloads.dropFirst(4).map(customKeys)) == [["first"], ["second"]] + } + + // A batch per flush would grow without bound for as long as an outage lasts: files on the disk, or batches + // held in memory, each with a summary of its own. + for persistence in [EventPersistence.immediate, .disabled] { + it("closes no batch behind the one blocked during an outage, persistence: \(persistence)") { + testContext = TestContext(store: EventStore.temporary(persistEvents: persistence != .disabled), + eventPersistence: persistence, + retryState: DirectEventProcessorSpec.withoutBackoff()) + testContext.serviceMock.respondToEvents(with: 503) + testContext.eventProcessor.isOnline = true + testContext.eventProcessor.record(CustomEvent(key: "blocked", context: testContext.context)) + for flush in 0..<3 { + expect(testContext.flushOutcome()) == false + testContext.eventProcessor.record(CustomEvent(key: "during-\(flush)", context: testContext.context)) + } + // The attempt and its retry for each flush, all of the batch at the head. + expect(testContext.serviceMock.publishedEventPayloads.map(customKeys)) == Array(repeating: ["blocked"], count: 6) + expect(testContext.store.listPendingBatches().batches.count) == 1 + + testContext.serviceMock.stubEventResponse(success: true) + expect(testContext.flushOutcome()) == true + + expect(testContext.serviceMock.publishedEventPayloads.dropFirst(6).map(customKeys)) + == [["blocked"], ["during-0", "during-1", "during-2"]] + expect(testContext.store.listPendingBatches().batches).to(beEmpty()) + } + + // Every flush of an outage is a commit point, and each stages a summary of the evaluations since the + // last. Counted as events, they would leave a long outage refusing every track. + it("leaves room for events beside the summaries an outage writes, persistence: \(persistence)") { + testContext = TestContext(lastEventResponseDate: Date(), + store: EventStore.temporary(capacity: 2, persistEvents: persistence != .disabled), + eventCapacity: 2, eventPersistence: persistence, + retryState: DirectEventProcessorSpec.withoutBackoff()) + testContext.serviceMock.respondToEvents(with: 503) + testContext.eventProcessor.isOnline = true + testContext.eventProcessor.record(CustomEvent(key: "blocked", context: testContext.context)) + expect(testContext.flushOutcome()) == false + let flag = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: false) + for _ in 0..<3 { + testContext.eventProcessor.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: testContext.context, includeReason: false) + expect(testContext.flushOutcome()) == false + } + + testContext.eventProcessor.record(CustomEvent(key: "after-the-summaries", context: testContext.context)) + testContext.serviceMock.stubEventResponse(success: true) + expect(testContext.flushOutcome()) == true + expect(testContext.serviceMock.publishedEventPayloads.flatMap(customKeys)) + .to(contain("after-the-summaries"), description: "refused for room the summaries took") + } + } + + it("does not tell a flush its events arrived when an earlier delivery nobody waited on failed to send them") { + testContext = TestContext() + testContext.serviceMock.respondToEvents(with: 503) + testContext.eventProcessor.record(CustomEvent(key: "kept", context: testContext.context)) + testContext.eventProcessor.isOnline = true + + // Stands in for the scheduled delivery, or the one at backgrounding: it closes the event into a batch, + // fails to send it, and keeps it. + waitUntil(timeout: .seconds(10)) { done in + testContext.eventProcessor.flush(completion: done) + } + expect(testContext.serviceMock.publishEventDataCallCount) == 2 + + // There is nothing left to close, but the event is still waiting in its batch, not delivered. + expect(testContext.flushOutcome()) == false + } + + it("starts a delivery asked for while a refused post waits to retry once that retry is answered") { + testContext = TestContext() + testContext.serviceMock.respondToEvents(with: 503) + testContext.eventProcessor.record(CustomEvent(key: "kept", context: testContext.context)) + testContext.eventProcessor.isOnline = true + testContext.eventProcessor.flush(completion: nil) + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: Constants.deliveryTimeout) + + // Well inside the one-second retry delay. The retry belongs to the delivery still running, so this + // one waits for it rather than posting beside it. + let answer = Answer() + testContext.eventProcessor.flushReportingOutcome(completion: answer.record) + Thread.sleep(forTimeInterval: 0.3) + expect(testContext.eventProcessor.startedDeliveryCount) == 1 + + // One retry and no more; the delivery behind it finds the backoff still waiting, and the batch then + // waits for the delivery that wait sets. + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(2), timeout: .seconds(3)) + expect(testContext.eventProcessor.startedDeliveryCount).toEventually(equal(2), timeout: Constants.deliveryTimeout) + expect(answer.value()).toEventually(beFalse(), timeout: Constants.deliveryTimeout) + waitLettingTimersFire(1.5) + expect(testContext.serviceMock.publishEventDataCallCount) == 2 + } + + it("sends events recorded while offline with the next scheduled delivery") { + testContext = TestContext(eventFlushInterval: Constants.eventFlushIntervalHalfSecond) + testContext.eventProcessor.record(CustomEvent(key: "held", context: testContext.context)) + waitLettingTimersFire(2 * Constants.eventFlushIntervalHalfSecond) + expect(testContext.serviceMock.publishEventDataCallCount) == 0 + + testContext.eventProcessor.isOnline = true + + expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1), timeout: .seconds(3)) + expect(testContext.serviceMock.publishedEventPayloads.map(customKeys)) == [["held"]] + } + + it("posts a refused batch no more often than the backoff allows, and not at all once offline") { + let interval = 0.2 + let window = 2.5 + testContext = TestContext(eventFlushInterval: interval) + testContext.serviceMock.respondToEvents(with: 503) + testContext.eventProcessor.record(CustomEvent(key: "refused", context: testContext.context)) + testContext.eventProcessor.isOnline = true + waitLettingTimersFire(window) + + // At most one post a second, the backoff's ceiling where the interval is shorter; a loop would be in the + // hundreds by now. + let posts = testContext.serviceMock.publishEventDataCallCount + expect(posts) >= 2 + expect(posts) <= Int(window / 1.0) + 1 + + // Offline stops it entirely, kept batch or not. Waits out a retry already scheduled before counting. + testContext.eventProcessor.isOnline = false + waitLettingTimersFire(1.0 + interval) + let whenOffline = testContext.serviceMock.publishEventDataCallCount + waitLettingTimersFire(interval * 3) + expect(testContext.serviceMock.publishEventDataCallCount) == whenOffline + } + } + } +} + +/// The events a store is holding, as JSON with `creationDate` dropped. +/// +/// Dropping the timestamp is what lets an expectation be written as the event the SDK should have produced, rather than +/// as a list of whichever fields the test remembered to check: the recorded event and the expected one are constructed +/// moments apart and differ only in when they were made. +private func recordedEvents(_ store: EventStore) -> [LDValue] { + store.pendingEventPayloads() + .compactMap { try? JSONDecoder().decode(LDValue.self, from: $0) } + .map(withoutCreationDate) +} + +/// How many evaluations a summary event counts, across all of its flags and variations. +private func summaryCount(_ summary: LDValue) -> Int { + guard case .object(let fields) = summary, case .object(let features)? = fields["features"] + else { return 0 } + var total = 0 + for case .object(let flag) in features.values { + guard case .array(let counters)? = flag["counters"] else { continue } + for case .object(let counter) in counters { + if case .number(let count)? = counter["count"] { + total += Int(count) + } + } + } + return total +} + +/// A custom event whose encoding waits to be let through, so a test can act while a commit is encoding it. +private final class EncodingGate: CustomEvent { + let started = DispatchSemaphore(value: 0) + let proceed = DispatchSemaphore(value: 0) + + override func encode(to encoder: Encoder) throws { + started.signal() + _ = proceed.wait(timeout: .now() + 5) + try super.encode(to: encoder) + } +} + +/// A store that runs a hook on either side of closing a batch, and remembers the custom event keys of each batch closed. +private final class ClosingHookStore: EventStoring { + let inner: EventStoring + var beforeClosing: (() -> Void)? + var afterClosing: (() -> Void)? + /// Runs at the start of every commit. + var beforeCommitting: (() -> Void)? + private(set) var closedKeys: [[String]] = [] + + init(_ inner: EventStoring) { + self.inner = inner + } + + var pendingEventCount: Int { inner.pendingEventCount } + var hasRoomForEventBytes: Bool { inner.hasRoomForEventBytes } + func admitsEvent(heldElsewhere: Int, capacity: Int) -> Bool { inner.admitsEvent(heldElsewhere: heldElsewhere, capacity: capacity) } + var isPersisting: Bool { inner.isPersisting } + func stage(_ encodedEvent: Data, bypassingCapacity: Bool) -> Bool { inner.stage(encodedEvent, bypassingCapacity: bypassingCapacity) } + func reserve(_ events: Int) { inner.reserve(events) } + func stageReserved(_ encodedEvent: Data) -> Bool { inner.stageReserved(encodedEvent) } + func releaseReservations() { inner.releaseReservations() } + func commit() { + beforeCommitting?() + inner.commit() + } + var holdsUnclosedEvents: Bool { inner.holdsUnclosedEvents } + func takeVanishedLogEvents() -> Int { inner.takeVanishedLogEvents() } + func listPendingBatches() -> BatchListing { inner.listPendingBatches() } + func payload(of batch: EventBatch) throws -> BatchPayload? { try inner.payload(of: batch) } + func remove(_ batch: EventBatch) { inner.remove(batch) } + func startedSending(_ batch: EventBatch) { inner.startedSending(batch) } + func stoppedSending(_ batch: EventBatch) { inner.stoppedSending(batch) } + func preparePayloadsFromInterruptedLogs() { inner.preparePayloadsFromInterruptedLogs() } + func discardExpired() -> Int { inner.discardExpired() } + + func closeBatch() -> EventBatch? { + let before = beforeClosing + beforeClosing = nil + before?() + let batch = inner.closeBatch() + if let batch, let body = try? inner.body(of: batch) { + closedKeys.append(customKeys(body)) + } + let after = afterClosing + afterClosing = nil + after?() + return batch + } +} + +/// What a flush was told, written from whichever thread answered it and read from the test's. +private final class Answer { + private let lock = UnfairLock() + private var answered: Bool? + + /// Pass as a `FlushOutcomeClosure`. + func record(_ delivered: Bool) { + lock.withLock { answered = delivered } + } + + /// Nil until the flush has been answered. + func value() -> Bool? { + lock.withLock { answered } + } +} + +/// Reads that fail for one batch, as they do once a permission is taken away, counted. Set from the test's thread and +/// read on the reporter's. +private final class FailingReads { + private let lock = UnfairLock() + private var payloadId: String? + private var failed = 0 + + /// Fails every read of the batch from here on, or none where nil. + func fail(_ payloadId: String?) { + lock.withLock { self.payloadId = payloadId } + } + + var failures: Int { + lock.withLock { failed } + } + + /// Pass as the store's `readFile`. + func read(_ url: URL) throws -> Data { + let fails = lock.withLock { () -> Bool in + guard let payloadId, url.lastPathComponent.contains(payloadId) + else { return false } + failed += 1 + return true + } + if fails { + throw NSError(domain: NSPOSIXErrorDomain, code: Int(EACCES)) + } + return try Data(contentsOf: url) + } +} + +/// What a run of flushes were told, written from whichever threads answered them and read from the test's. +private final class Answers { + private let lock = UnfairLock() + private var answered: [Bool] = [] + + /// Pass as a `FlushOutcomeClosure`. + func record(_ delivered: Bool) { + lock.withLock { answered.append(delivered) } + } + + /// In the order the flushes were answered, which is not the order they were asked in. + func value() -> [Bool] { + lock.withLock { answered } + } +} + +/// A store that holds a listing open, so a test can see what else is waiting while one is under way. +private final class SlowListingStore: EventStoring { + let inner: EventStoring + /// Signalled once a listing is being held. + let listing = DispatchSemaphore(value: 0) + /// Signal to let it finish. + let release = DispatchSemaphore(value: 0) + + private let lock = UnfairLock() + private var listings = 0 + + init(_ inner: EventStoring) { + self.inner = inner + } + + var pendingEventCount: Int { inner.pendingEventCount } + var hasRoomForEventBytes: Bool { inner.hasRoomForEventBytes } + func admitsEvent(heldElsewhere: Int, capacity: Int) -> Bool { inner.admitsEvent(heldElsewhere: heldElsewhere, capacity: capacity) } + var isPersisting: Bool { inner.isPersisting } + func stage(_ encodedEvent: Data, bypassingCapacity: Bool) -> Bool { inner.stage(encodedEvent, bypassingCapacity: bypassingCapacity) } + func reserve(_ events: Int) { inner.reserve(events) } + func stageReserved(_ encodedEvent: Data) -> Bool { inner.stageReserved(encodedEvent) } + func releaseReservations() { inner.releaseReservations() } + func commit() { inner.commit() } + func closeBatch() -> EventBatch? { inner.closeBatch() } + var holdsUnclosedEvents: Bool { inner.holdsUnclosedEvents } + func takeVanishedLogEvents() -> Int { inner.takeVanishedLogEvents() } + func payload(of batch: EventBatch) throws -> BatchPayload? { try inner.payload(of: batch) } + func remove(_ batch: EventBatch) { inner.remove(batch) } + func startedSending(_ batch: EventBatch) { inner.startedSending(batch) } + func stoppedSending(_ batch: EventBatch) { inner.stoppedSending(batch) } + func preparePayloadsFromInterruptedLogs() { inner.preparePayloadsFromInterruptedLogs() } + func discardExpired() -> Int { inner.discardExpired() } + + /// Every listing but the first is held, the first being the one the reporter makes as it starts up. + func listPendingBatches() -> BatchListing { + let held = lock.withLock { () -> Bool in + listings += 1 + return listings > 1 + } + if held { + listing.signal() + _ = release.wait(timeout: .now() + 5) + // Once opened the gate stays open. + release.signal() + } + return inner.listPendingBatches() + } +} + +/// A store that notes the QoS the previous run's events are recovered at, and can hold the recovery open. +private final class RecoveryObservingStore: EventStoring { + let inner: EventStoring + /// Signal to let a held recovery finish. + let release = DispatchSemaphore(value: 0) + private let holds: Bool + private let lock = UnfairLock() + private var observedQoS: qos_class_t? + + init(_ inner: EventStoring, holdingRecovery holds: Bool = false) { + self.inner = inner + self.holds = holds + } + + var recoveryQoS: qos_class_t? { lock.withLock { observedQoS } } + + var pendingEventCount: Int { inner.pendingEventCount } + var hasRoomForEventBytes: Bool { inner.hasRoomForEventBytes } + func admitsEvent(heldElsewhere: Int, capacity: Int) -> Bool { inner.admitsEvent(heldElsewhere: heldElsewhere, capacity: capacity) } + var isPersisting: Bool { inner.isPersisting } + func stage(_ encodedEvent: Data, bypassingCapacity: Bool) -> Bool { inner.stage(encodedEvent, bypassingCapacity: bypassingCapacity) } + func reserve(_ events: Int) { inner.reserve(events) } + func stageReserved(_ encodedEvent: Data) -> Bool { inner.stageReserved(encodedEvent) } + func releaseReservations() { inner.releaseReservations() } + func commit() { inner.commit() } + func closeBatch() -> EventBatch? { inner.closeBatch() } + var holdsUnclosedEvents: Bool { inner.holdsUnclosedEvents } + func takeVanishedLogEvents() -> Int { inner.takeVanishedLogEvents() } + func payload(of batch: EventBatch) throws -> BatchPayload? { try inner.payload(of: batch) } + func remove(_ batch: EventBatch) { inner.remove(batch) } + func startedSending(_ batch: EventBatch) { inner.startedSending(batch) } + func stoppedSending(_ batch: EventBatch) { inner.stoppedSending(batch) } + func discardExpired() -> Int { inner.discardExpired() } + func listPendingBatches() -> BatchListing { inner.listPendingBatches() } + + func preparePayloadsFromInterruptedLogs() { + lock.withLock { observedQoS = qos_class_self() } + if holds { + _ = release.wait(timeout: .now() + 5) + } + inner.preparePayloadsFromInterruptedLogs() + } +} + +/// Waits while letting the main run loop turn, since that is where the report timer fires. +private func waitLettingTimersFire(_ seconds: TimeInterval) { + RunLoop.current.run(until: Date().addingTimeInterval(seconds)) +} + +private func customKeys(_ body: Data) -> [String] { + guard case .array(let events)? = try? JSONDecoder().decode(LDValue.self, from: body) + else { return [] } + return events.compactMap { event in + guard event.kindField == "custom", case .object(let fields) = event, case .string(let key) = fields["key"] + else { return nil } + return key + } +} + +private func expectedEvents(_ events: [Event]) -> [LDValue] { + events.compactMap { encodeToLDValue($0) }.map(withoutCreationDate) +} + +private func withoutCreationDate(_ value: LDValue) -> LDValue { + guard case .object(var fields) = value + else { return value } + fields["creationDate"] = nil + return .object(fields) +} + +private extension LDValue { + var kindField: String? { + guard case .object(let fields) = self, case .string(let kind) = fields["kind"] + else { return nil } + return kind + } + + var keyField: String? { + guard case .object(let fields) = self, case .string(let key) = fields["key"] + else { return nil } + return key + } +} + +private extension DarklyServiceMock { + /// Answers every event post with this status and no error, the way a service that is reachable answers. + func respondToEvents(with status: Int) { + stubbedEventResponse = (nil, + HTTPURLResponse(url: config.eventsUrl, statusCode: status, httpVersion: Constants.httpVersion, headerFields: nil), + nil, + nil) + } +} + +private extension Date { + var adjustedForHttpUrlHeaderUse: Date { + let headerDateFormatter = DateFormatter.httpUrlHeaderFormatter + let dateString = headerDateFormatter.string(from: self) + return headerDateFormatter.date(from: dateString) ?? self + } +} + +extension Event.Kind { + static var nonSummaryKinds: [Event.Kind] { + [feature, debug, identify, custom] + } +} + +/// `DirectEventProcessorSpec` with persistence off, which is what an application gets by default. +final class DirectEventProcessorWithoutPersistenceSpec: DirectEventProcessorSpec { + override class var persistence: EventPersistence { .disabled } +} diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EnvironmentReporting/WatchOSEnvironmentReporterSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EnvironmentReporting/WatchOSEnvironmentReporterSpec.swift index 9026d8c98..8373efd2f 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EnvironmentReporting/WatchOSEnvironmentReporterSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EnvironmentReporting/WatchOSEnvironmentReporterSpec.swift @@ -1,29 +1,26 @@ #if os(watchOS) import Foundation +import WatchKit import XCTest @testable import LaunchDarkly -final class WatchOSEnvironmentReporterSpec: XCTest { +final class WatchOSEnvironmentReporterSpec: XCTestCase { func testDefaultReporterBehavior() { let chain = EnvironmentReporterChainBase() chain.setNext(WatchOSEnvironmentReporter()) - XCTAssert(!chain.applicationInfo.isEmpty()) + XCTAssertNotNil(chain.applicationInfo.applicationId) XCTAssertNotEqual(chain.deviceModel, "UNKNOWN") XCTAssertNotEqual(chain.systemVersion, "UNKNOWN") - XCTAssertNotEqual(chain.systemName, "UNKNOWN") - - XCTAssertEqual(chain.operatingSystem, .watchOS) XCTAssertNil(chain.vendorUUID) } func testBuilderDoesNotIncludeWatchInfoWithoutExplicitOptIn() { - let builder = EnvironmentReporterBuilder() - let reporting = builder.build() + let reporting = EnvironmentReporterBuilder().build() - XCTAssertEqual(reporting.operatingSystem, .watchOS) + XCTAssertEqual(reporting.deviceModel, EnvironmentReporterChainBase().deviceModel) } func testEnsureBuilderUsesCorrectReporter() { @@ -32,7 +29,7 @@ final class WatchOSEnvironmentReporterSpec: XCTest { let reporting = builder.build() - XCTAssertEqual(reporting.operatingSystem, .watchOS) + XCTAssertEqual(reporting.deviceModel, WKInterfaceDevice.current().model) } } #endif diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventProcessorOutcomesSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventProcessorOutcomesSpec.swift new file mode 100644 index 000000000..abd8be8e4 --- /dev/null +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventProcessorOutcomesSpec.swift @@ -0,0 +1,89 @@ +import Foundation +import Quick +import Nimble +@testable import LaunchDarkly + +final class EventProcessorOutcomesSpec: QuickSpec { + override func spec() { + describe("flushing several processors") { + var first: EventProcessorMock! + var second: EventProcessorMock! + var answers: [[Bool]] = [] + var processors: [EventProcessor] { [first!, second!] } + beforeEach { + first = EventProcessorMock() + second = EventProcessorMock() + answers = [] + } + + it("answers with each processor's outcome, in order, once all of them have answered") { + processors.flushReportingOutcomes { answers.append($0) } + second.flushReportingOutcomeReceivedCompletion?(false) + expect(answers) == [] + + first.flushReportingOutcomeReceivedCompletion?(true) + expect(answers) == [[true, false]] + } + + it("answers at once where there are no processors") { + [EventProcessor]().flushReportingOutcomes { answers.append($0) } + expect(answers) == [[]] + } + + it("waits for every processor where one answers twice, and answers once") { + processors.flushReportingOutcomes { answers.append($0) } + first.flushReportingOutcomeReceivedCompletion?(true) + first.flushReportingOutcomeReceivedCompletion?(false) + expect(answers) == [] + + second.flushReportingOutcomeReceivedCompletion?(true) + second.flushReportingOutcomeReceivedCompletion?(true) + expect(answers) == [[true, true]] + } + + it("waits for a processor that never answers, where there is no deadline") { + processors.flushReportingOutcomes { answers.append($0) } + first.flushReportingOutcomeReceivedCompletion?(true) + Thread.sleep(forTimeInterval: 0.1) + expect(answers) == [] + } + + it("counts a processor that has not answered by the deadline as undelivered, and ignores its late answer") { + var answered: [[Bool]] = [] + let lock = UnfairLock() + processors.flushReportingOutcomes(deadline: .now() + 0.1) { outcome in + lock.withLock { answered.append(outcome) } + } + first.flushReportingOutcomeReceivedCompletion?(true) + + expect(lock.withLock { answered }).toEventually(equal([[true, false]])) + second.flushReportingOutcomeReceivedCompletion?(true) + expect(lock.withLock { answered }) == [[true, false]] + } + + it("answers once where every processor answers before the deadline") { + var answered: [[Bool]] = [] + let lock = UnfairLock() + processors.flushReportingOutcomes(deadline: .now() + 0.1) { outcome in + lock.withLock { answered.append(outcome) } + } + first.flushReportingOutcomeReceivedCompletion?(true) + second.flushReportingOutcomeReceivedCompletion?(true) + + Thread.sleep(forTimeInterval: 0.2) + expect(lock.withLock { answered }) == [[true, true]] + } + + it("posts through the backoff only where asked to") { + processors.flushReportingOutcomes(bypassingBackoff: true) { answers.append($0) } + expect(first.flushBypassingBackoffCallCount) == 1 + expect(second.flushBypassingBackoffCallCount) == 1 + expect(first.flushReportingOutcomeCallCount) == 0 + + first.flushBypassingBackoffReceivedCompletion?(true) + second.flushBypassingBackoffReceivedCompletion?(true) + expect(answers) == [[true, true]] + } + } + } +} diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift deleted file mode 100644 index 4062a7ba5..000000000 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventReporterSpec.swift +++ /dev/null @@ -1,1158 +0,0 @@ -import Foundation -import Quick -import Nimble -@testable import LaunchDarkly - -final class EventReporterSpec: QuickSpec { - struct Constants { - static let eventFlushInterval: TimeInterval = 10.0 - static let eventFlushIntervalHalfSecond: TimeInterval = 0.5 - /// How long a flush or delivery may take before a test gives up on it. A wait ends as soon as what it waits - /// for happens, so this costs a passing test nothing, and a loaded CI simulator can take over a second. - static let deliveryTimeout: DispatchTimeInterval = .seconds(10) - } - - struct TestContext { - var eventReporter: EventReporter! - var config: LDConfig! - var context: LDContext! - var serviceMock: DarklyServiceMock! - var events: [Event] = [] - var lastEventResponseDate: Date - var eventStubResponseDate: Date? - var syncResult: SynchronizingError? = nil - var diagnosticCache: DiagnosticCachingMock - - init(eventCount: Int = 0, - eventFlushInterval: TimeInterval? = nil, - lastEventResponseDate: Date = Date.distantPast, - stubResponseSuccess: Bool = true, - stubResponseOnly: Bool = false, - stubResponseErrorOnly: Bool = false, - eventStubResponseDate: Date? = nil, - onSyncComplete: EventSyncCompleteClosure? = nil) { - - config = LDConfig.stub - config.eventCapacity = Event.Kind.allKinds.count - config.eventFlushInterval = eventFlushInterval ?? Constants.eventFlushInterval - - context = LDContext.stub() - - self.eventStubResponseDate = eventStubResponseDate?.adjustedForHttpUrlHeaderUse - serviceMock = DarklyServiceMock() - serviceMock.config = config - serviceMock.stubEventResponse(success: stubResponseSuccess, responseOnly: stubResponseOnly, errorOnly: stubResponseErrorOnly, responseDate: self.eventStubResponseDate) - - diagnosticCache = DiagnosticCachingMock() - serviceMock.diagnosticCache = diagnosticCache - - self.lastEventResponseDate = lastEventResponseDate.adjustedForHttpUrlHeaderUse - eventReporter = EventReporter(service: serviceMock, onSyncComplete: onSyncComplete) - (0.. Bool { - var recorded = false - waitUntil(timeout: .seconds(5)) { done in - eventReporter.flushReportingOutcome { _ in - let finished = DispatchSemaphore(value: 0) - DispatchQueue.global().async { - eventReporter.record(CustomEvent(key: "recorded from a completion", context: LDContext.stub())) - finished.signal() - } - recorded = finished.wait(timeout: .now() + 1) == .success - done() - } - } - return recorded - } - } - - override func spec() { - initSpec() - isOnlineSpec() - recordEventSpec() - testRecordFlagEvaluationEvents() - reportEventsSpec() - flushReportingOutcomeSpec() - unserializableEventSpec() - reportTimerSpec() - } - - private func flushReportingOutcomeSpec() { - describe("flushReportingOutcome") { - var testContext: TestContext! - afterEach { - testContext.eventReporter.isOnline = false - } - - it("reports true when LaunchDarkly accepts the events") { - testContext = TestContext() - testContext.recordEvents(1) - testContext.eventReporter.isOnline = true - - var delivered: Bool? - waitUntil { done in - testContext.eventReporter.flushReportingOutcome { result in - delivered = result - done() - } - } - expect(delivered) == true - } - - it("reports true when there is nothing to deliver") { - testContext = TestContext() - testContext.eventReporter.isOnline = true - - var delivered: Bool? - waitUntil { done in - testContext.eventReporter.flushReportingOutcome { result in - delivered = result - done() - } - } - expect(delivered) == true - } - - it("reports false while offline") { - testContext = TestContext() - testContext.recordEvents(1) - - var delivered: Bool? - waitUntil { done in - testContext.eventReporter.flushReportingOutcome { result in - delivered = result - done() - } - } - expect(delivered) == false - expect(testContext.serviceMock.publishEventDataCallCount) == 0 - } - - it("keeps draining the event store while a delivery is in flight") { - testContext = TestContext() - testContext.recordEvents(1) - testContext.eventReporter.isOnline = true - testContext.serviceMock.holdsEventCompletions = true - - testContext.eventReporter.flushReportingOutcome { _ in } - expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) - - // Holding this back until the first delivery answers would let the store fill through a retry, which - // can run for twice the connection timeout, and events recorded past the capacity are lost. - testContext.recordEvents(1) - testContext.eventReporter.flushReportingOutcome { _ in } - expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(2)) - - testContext.serviceMock.holdsEventCompletions = false - testContext.serviceMock.releaseHeldEventCompletions() - } - - it("reports false when LaunchDarkly permanently refuses the events, and goes offline") { - testContext = TestContext() - let unauthorized = HTTPURLResponse(url: testContext.serviceMock.config.eventsUrl, - statusCode: HTTPURLResponse.StatusCodes.unauthorized, - httpVersion: DarklyServiceMock.Constants.httpVersion, - headerFields: nil) - testContext.serviceMock.stubbedEventResponse = (nil, unauthorized, nil, nil) - testContext.recordEvents(1) - testContext.eventReporter.isOnline = true - - var delivered: Bool? - waitUntil { done in - testContext.eventReporter.flushReportingOutcome { result in - delivered = result - done() - } - } - expect(delivered) == false - expect(testContext.eventReporter.isOnline) == false - } - - it("reports false when no event could be serialized") { - testContext = TestContext() - testContext.eventReporter.record(CustomEvent(key: "poison", context: LDContext.stub(), metricValue: .nan)) - testContext.eventReporter.isOnline = true - - var delivered: Bool? - waitUntil { done in - testContext.eventReporter.flushReportingOutcome { result in - delivered = result - done() - } - } - expect(delivered) == false - expect(testContext.serviceMock.publishEventDataCallCount) == 0 - } - - it("reports false when some events could not be serialized, though the rest were accepted") { - testContext = TestContext() - testContext.eventReporter.record(CustomEvent(key: "poison", context: LDContext.stub(), metricValue: .nan)) - testContext.eventReporter.record(CustomEvent(key: "kept", context: LDContext.stub(), metricValue: 1.0)) - testContext.eventReporter.isOnline = true - - var delivered: Bool? - waitUntil { done in - testContext.eventReporter.flushReportingOutcome { result in - delivered = result - done() - } - } - expect(delivered) == false - expect(testContext.serviceMock.publishEventDataCallCount) == 1 - } - - it("answers away from the queue a recorded event waits on, after a delivery") { - testContext = TestContext() - testContext.recordEvents(1) - testContext.eventReporter.isOnline = true - - expect(testContext.recordedFromACompletion()) == true - } - - it("answers away from the queue a recorded event waits on, with nothing to send") { - testContext = TestContext() - testContext.eventReporter.isOnline = true - - // The path that answers without ever leaving the reporter's queue, so it is the one that would - // deadlock while the delivery path, answering from the response, did not. - expect(testContext.recordedFromACompletion()) == true - } - - it("sends one delivery for the flushes that arrive before it starts, and answers them all") { - testContext = TestContext(stubResponseSuccess: false) - testContext.recordEvents(1) - testContext.eventReporter.isOnline = true - let released = DispatchSemaphore(value: 0) - testContext.eventReporter.occupyQueue(until: released) - - // An application flushing in a loop would otherwise queue a delivery each time, leaving the flush - // that matters waiting behind all of them, and growing the waiter list without bound. - for _ in 0..<100 { - testContext.eventReporter.flush(completion: nil) - } - expect(testContext.eventReporter.queuedDeliveryWaiterCount) == 0 - - var delivered: [Bool] = [] - let answers = UnfairLock() - // Both completions run at once, and the poll below reads from a third thread. - let answered: () -> [Bool] = { - answers.lock() - defer { answers.unlock() } - return delivered - } - for _ in 0..<2 { - testContext.eventReporter.flushReportingOutcome { result in - answers.lock() - delivered.append(result) - answers.unlock() - } - } - expect(testContext.eventReporter.queuedDeliveryWaiterCount) == 2 - - released.signal() - // Allows for the one-second retry; the failed events are dropped. - expect(answered()).toEventually(equal([false, false]), timeout: .seconds(10)) - expect(testContext.eventReporter.startedDeliveryCount) == 1 - expect(testContext.serviceMock.publishEventDataCallCount) == 2 // the first attempt and its retry - } - } - } - - private func initSpec() { - describe("init") { - var testContext: TestContext! - beforeEach { - testContext = TestContext() - testContext.eventReporter = EventReporter(service: testContext.serviceMock) { _ in } - } - it("starts offline without reporting events") { - expect(testContext.eventReporter.service) === testContext.serviceMock - expect(testContext.eventReporter.isOnline) == false - expect(testContext.eventReporter.isReportingActive) == false - expect(testContext.serviceMock.publishEventDataCallCount) == 0 - } - } - } - - private func isOnlineSpec() { - describe("isOnline") { - var testContext: TestContext! - beforeEach { - testContext = TestContext() - } - afterEach { - testContext.eventReporter.isOnline = false - } - context("online to offline") { - beforeEach { - testContext.eventReporter.isOnline = true - - testContext.eventReporter.isOnline = false - } - it("goes offline and stops reporting") { - expect(testContext.eventReporter.isOnline) == false - expect(testContext.eventReporter.isReportingActive) == false - expect(testContext.serviceMock.publishEventDataCallCount) == 0 - } - } - context("offline to online") { - context("with events") { - beforeEach { - testContext = TestContext(eventCount: Event.Kind.allKinds.count) - - testContext.eventReporter.isOnline = true - } - it("goes online and starts reporting") { - expect(testContext.eventReporter.isOnline) == true - expect(testContext.eventReporter.isReportingActive) == true - expect(testContext.serviceMock.publishEventDataCallCount) == 0 - } - } - context("without events") { - beforeEach { - testContext = TestContext() - - testContext.eventReporter.isOnline = true - } - it("goes online and starts reporting") { - expect(testContext.eventReporter.isOnline) == true - expect(testContext.eventReporter.isReportingActive) == true - expect(testContext.serviceMock.publishEventDataCallCount) == 0 - } - } - } - context("online to online") { - beforeEach { - testContext = TestContext() - testContext.eventReporter.isOnline = true - - testContext.eventReporter.isOnline = true - } - it("stays online and continues reporting") { - expect(testContext.eventReporter.isOnline) == true - expect(testContext.eventReporter.isReportingActive) == true - expect(testContext.serviceMock.publishEventDataCallCount) == 0 - } - } - context("offline to offline") { - beforeEach { - testContext = TestContext(eventCount: Event.Kind.allKinds.count) - - testContext.eventReporter.isOnline = false - } - it("stays offline and does not start reporting") { - expect(testContext.eventReporter.isOnline) == false - expect(testContext.eventReporter.isReportingActive) == false - expect(testContext.serviceMock.publishEventDataCallCount) == 0 - expect(testContext.eventReporter.eventStore) == testContext.events - } - } - } - } - - private func recordEventSpec() { - describe("recordEvent") { - var testContext: TestContext! - context("event store empty") { - beforeEach { - testContext = TestContext() - testContext.recordEvents(Event.Kind.allKinds.count) // Stub events, call testContext.eventReporter.recordEvent, and keeps them in testContext.events - } - it("records events up to event capacity") { - expect(testContext.eventReporter.isOnline) == false - expect(testContext.eventReporter.isReportingActive) == false - expect(testContext.serviceMock.publishEventDataCallCount) == 0 - expect(testContext.eventReporter.eventStore) == testContext.events - } - it("does not record a dropped event to diagnosticCache") { - expect(testContext.diagnosticCache.incrementDroppedEventCountCallCount) == 0 - } - } - context("event store full") { - var extraEvent: Event! - beforeEach { - testContext = TestContext(eventCount: Event.Kind.allKinds.count) - extraEvent = Event.stub(.feature, with: testContext.context) - - testContext.eventReporter.record(extraEvent) - } - it("doesn't record any more events") { - expect(testContext.eventReporter.isOnline) == false - expect(testContext.eventReporter.isReportingActive) == false - expect(testContext.serviceMock.publishEventDataCallCount) == 0 - expect(testContext.eventReporter.eventStore) == testContext.events - } - it("records a dropped event to diagnosticCache") { - expect(testContext.diagnosticCache.incrementDroppedEventCountCallCount) == 1 - } - } - } - } - - /// An event holding a NaN or infinite number cannot be serialized by either encoder. It is dropped, and - /// the events beside it and after it are still delivered. - private func unserializableEventSpec() { - describe("unserializable events") { - var serviceMock: DarklyServiceMock! - var ldContext: LDContext! - var reporter: EventReporter! - - func makeReporter(encoding: EventReporter.Encoding) -> EventReporter { - var config = LDConfig.stub - config.eventCapacity = 8 - ldContext = LDContext.stub() - serviceMock = DarklyServiceMock() - serviceMock.config = config - serviceMock.stubEventResponse(success: true) - return EventReporter(service: serviceMock, onSyncComplete: nil, encoding: encoding) - } - - afterEach { - reporter?.isOnline = false - } - - for encoding in [EventReporter.Encoding.codable, .handWritten] { - context("\(encoding): when an event holds a non-finite metric") { - beforeEach { - reporter = makeReporter(encoding: encoding) - reporter.isOnline = true - } - it("drops the poisoned batch and delivers a later event") { - waitUntil(timeout: Constants.deliveryTimeout) { done in - reporter.record(CustomEvent(key: "poison", context: ldContext, metricValue: .nan)) - reporter.flush(completion: done) - } - expect(serviceMock.publishEventDataCallCount) == 0 - expect(reporter.eventStore.isEmpty) == true - - waitUntil(timeout: Constants.deliveryTimeout) { done in - reporter.record(CustomEvent(key: "after-poison", context: ldContext, metricValue: 1.0)) - reporter.flush(completion: done) - } - expect(serviceMock.publishEventDataCallCount) == 1 - let published = try JSONDecoder().decode(LDValue.self, from: serviceMock.publishedEventData!) - valueIsArray(published) { events in - expect(events.count) == 1 - valueIsObject(events[0]) { body in - expect(body["key"]) == "after-poison" - } - } - } - } - - context("\(encoding): when an event holds a non-finite metric beside a valid event") { - beforeEach { - reporter = makeReporter(encoding: encoding) - reporter.isOnline = true - } - it("drops only the poisoned event") { - waitUntil(timeout: Constants.deliveryTimeout) { done in - reporter.record(CustomEvent(key: "poison", context: ldContext, metricValue: .nan)) - reporter.record(CustomEvent(key: "kept", context: ldContext, metricValue: 1.0)) - reporter.flush(completion: done) - } - expect(serviceMock.publishEventDataCallCount) == 1 - let published = try JSONDecoder().decode(LDValue.self, from: serviceMock.publishedEventData!) - valueIsArray(published) { events in - expect(events.count) == 1 - valueIsObject(events[0]) { body in - expect(body["key"]) == "kept" - } - } - } - } - - context("\(encoding): when an event holds a non-finite summary beside a valid event") { - beforeEach { - reporter = makeReporter(encoding: encoding) - reporter.isOnline = true - } - it("drops only the poisoned summary") { - waitUntil(timeout: Constants.deliveryTimeout) { done in - reporter.recordFlagEvaluationEvents(flagKey: "poison-flag", - value: .number(.nan), - defaultValue: .number(0), - featureFlag: nil, - context: ldContext, - includeReason: false) - reporter.record(IdentifyEvent(context: ldContext)) - reporter.flush(completion: done) - } - expect(serviceMock.publishEventDataCallCount) == 1 - let published = try JSONDecoder().decode(LDValue.self, from: serviceMock.publishedEventData!) - valueIsArray(published) { events in - expect(events.count) == 1 - valueIsObject(events[0]) { body in - expect(body["kind"]) == "identify" - } - } - } - } - - context("\(encoding): when an event holds a non-finite summary value") { - beforeEach { - reporter = makeReporter(encoding: encoding) - reporter.isOnline = true - } - it("drops the poisoned summary and delivers a later one") { - waitUntil(timeout: Constants.deliveryTimeout) { done in - reporter.recordFlagEvaluationEvents(flagKey: "poison-flag", - value: .number(.nan), - defaultValue: .number(0), - featureFlag: nil, - context: ldContext, - includeReason: false) - reporter.flush(completion: done) - } - expect(serviceMock.publishEventDataCallCount) == 0 - expect(reporter.eventStore.isEmpty) == true - expect(reporter.contextSummarizer.hasLoggedRequests) == false - - waitUntil(timeout: Constants.deliveryTimeout) { done in - reporter.recordFlagEvaluationEvents(flagKey: "after-poison", - value: .bool(true), - defaultValue: .bool(false), - featureFlag: nil, - context: ldContext, - includeReason: false) - reporter.flush(completion: done) - } - expect(serviceMock.publishEventDataCallCount) == 1 - let published = try JSONDecoder().decode(LDValue.self, from: serviceMock.publishedEventData!) - valueIsArray(published) { events in - expect(events.count) == 1 - valueIsObject(events[0]) { body in - expect(body["kind"]) == "summary" - valueIsObject(body["features"]) { features in - expect(features["after-poison"]).toNot(beNil()) - } - } - } - } - } - } - } - } - - private func reportEventsSpec() { - describe("reportEvents") { - var testContext: TestContext! - var eventStubResponseDate: Date! - beforeEach { - eventStubResponseDate = Date().addingTimeInterval(-TimeInterval(3)) - } - afterEach { - testContext.eventReporter.isOnline = false - } - let erOnline = { - expect(testContext.eventReporter.isOnline) == true - expect(testContext.eventReporter.isReportingActive) == true - } - context("online") { - context("success") { - context("with events and tracked requests") { - beforeEach { - waitUntil { syncComplete in - testContext = TestContext(eventStubResponseDate: eventStubResponseDate, onSyncComplete: { result in - testContext.syncResult = result - syncComplete() - }) - testContext.eventReporter.isOnline = true - testContext.recordEvents(Event.Kind.nonSummaryKinds.count) - // Track some flag evaluations to populate the contextSummarizer - testContext.eventReporter.recordFlagEvaluationEvents( - flagKey: "test-flag", - value: .bool(true), - defaultValue: .bool(false), - featureFlag: nil, - context: testContext.context, - includeReason: false - ) - testContext.eventReporter.flush(completion: nil) - } - } - it("reports events and a summary event") { - erOnline() - expect(testContext.serviceMock.publishEventDataCallCount) == 1 - let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) - valueIsArray(published) { valueArray in - expect(valueArray.count) == testContext.events.count + 1 - expect(Array(valueArray.prefix(testContext.events.count))) == testContext.events.map { encodeToLDValue($0) } - valueIsObject(valueArray[testContext.events.count]) { summaryObject in - expect(summaryObject["kind"]) == "summary" - } - } - expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 - expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 - expect(testContext.eventReporter.eventStore.isEmpty) == true - expect(testContext.eventReporter.lastEventResponseDate) == testContext.eventStubResponseDate - expect(testContext.eventReporter.contextSummarizer.hasLoggedRequests) == false - expect(testContext.syncResult).to(beNil()) - } - } - context("with events only") { - beforeEach { - waitUntil { syncComplete in - testContext = TestContext(eventStubResponseDate: eventStubResponseDate, onSyncComplete: { result in - testContext.syncResult = result - syncComplete() - }) - testContext.eventReporter.isOnline = true - testContext.recordEvents(Event.Kind.nonSummaryKinds.count) - testContext.eventReporter.flush(completion: nil) - } - } - it("reports events without a summary event") { - erOnline() - expect(testContext.serviceMock.publishEventDataCallCount) == 1 - let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) - expect(published) == encodeToLDValue(testContext.events) - expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 - expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == testContext.events.count - expect(testContext.eventReporter.eventStore.isEmpty) == true - expect(testContext.eventReporter.lastEventResponseDate) == testContext.eventStubResponseDate - expect(testContext.eventReporter.contextSummarizer.hasLoggedRequests) == false - expect(testContext.syncResult).to(beNil()) - } - } - context("with tracked requests only") { - beforeEach { - waitUntil { syncComplete in - testContext = TestContext(eventStubResponseDate: eventStubResponseDate, onSyncComplete: { result in - testContext.syncResult = result - syncComplete() - }) - testContext.eventReporter.isOnline = true - // Track some flag evaluations to populate the contextSummarizer - testContext.eventReporter.recordFlagEvaluationEvents( - flagKey: "test-flag", - value: .bool(true), - defaultValue: .bool(false), - featureFlag: nil, - context: testContext.context, - includeReason: false - ) - testContext.eventReporter.flush(completion: nil) - } - } - it("reports only a summary event") { - erOnline() - expect(testContext.serviceMock.publishEventDataCallCount) == 1 - let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) - valueIsArray(published) { valueArray in - expect(valueArray.count) == 1 - valueIsObject(valueArray[0]) { summaryObject in - expect(summaryObject["kind"]) == "summary" - } - } - expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 - expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == 1 - expect(testContext.eventReporter.eventStore.isEmpty) == true - expect(testContext.eventReporter.lastEventResponseDate) == testContext.eventStubResponseDate - expect(testContext.eventReporter.contextSummarizer.hasLoggedRequests) == false - expect(testContext.syncResult).to(beNil()) - } - } - context("without events or tracked requests") { - beforeEach { - waitUntil { syncComplete in - testContext = TestContext(eventStubResponseDate: eventStubResponseDate, onSyncComplete: { result in - testContext.syncResult = result - syncComplete() - }) - testContext.eventReporter.isOnline = true - testContext.eventReporter.flush(completion: nil) - } - } - it("does not report events") { - erOnline() - expect(testContext.serviceMock.publishEventDataCallCount) == 0 - expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 0 - expect(testContext.eventReporter.eventStore.isEmpty) == true - expect(testContext.eventReporter.lastEventResponseDate) == Date.distantPast - expect(testContext.eventReporter.contextSummarizer.hasLoggedRequests) == false - expect(testContext.syncResult).to(beNil()) - } - } - } - context("failure") { - context("server error") { - beforeEach { - waitUntil(timeout: .seconds(10)) { syncComplete in - testContext = TestContext(stubResponseSuccess: false, eventStubResponseDate: eventStubResponseDate, onSyncComplete: { result in - testContext.syncResult = result - syncComplete() - }) - testContext.eventReporter.isOnline = true - testContext.recordEvents(Event.Kind.nonSummaryKinds.count) - // Track some flag evaluations to populate the contextSummarizer - testContext.eventReporter.recordFlagEvaluationEvents( - flagKey: "test-flag", - value: .bool(true), - defaultValue: .bool(false), - featureFlag: nil, - context: testContext.context, - includeReason: false - ) - testContext.eventReporter.flush(completion: nil) - } - } - it("drops events after the failure") { - erOnline() - expect(testContext.serviceMock.publishEventDataCallCount) == 2 // 1 retry attempt - expect(testContext.eventReporter.eventStore.isEmpty) == true - let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) - valueIsArray(published) { valueArray in - expect(valueArray.count) == testContext.events.count + 1 - expect(Array(valueArray.prefix(testContext.events.count))) == testContext.events.map { encodeToLDValue($0) } - valueIsObject(valueArray[testContext.events.count]) { summaryObject in - expect(summaryObject["kind"]) == "summary" - } - } - expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 - expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 - expect(testContext.eventReporter.lastEventResponseDate) == Date.distantPast - expect(testContext.eventReporter.contextSummarizer.hasLoggedRequests) == false - guard case let .request(error) = testContext.syncResult - else { - fail("Expected error result for event send") - return - } - expect(error as NSError?) == DarklyServiceMock.Constants.error - } - } - context("response only") { - beforeEach { - waitUntil(timeout: .seconds(10)) { syncComplete in - testContext = TestContext(stubResponseSuccess: false, stubResponseOnly: true, eventStubResponseDate: eventStubResponseDate, onSyncComplete: { result in - testContext.syncResult = result - syncComplete() - }) - testContext.eventReporter.isOnline = true - testContext.recordEvents(Event.Kind.nonSummaryKinds.count) - // Track some flag evaluations to populate the contextSummarizer - testContext.eventReporter.recordFlagEvaluationEvents( - flagKey: "test-flag", - value: .bool(true), - defaultValue: .bool(false), - featureFlag: nil, - context: testContext.context, - includeReason: false - ) - testContext.eventReporter.flush(completion: nil) - } - } - it("drops events after the failure") { - erOnline() - expect(testContext.serviceMock.publishEventDataCallCount) == 2 // 1 retry attempt - expect(testContext.eventReporter.eventStore.isEmpty) == true - let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) - valueIsArray(published) { valueArray in - expect(valueArray.count) == testContext.events.count + 1 - expect(Array(valueArray.prefix(testContext.events.count))) == testContext.events.map { encodeToLDValue($0) } - valueIsObject(valueArray[testContext.events.count]) { summaryObject in - expect(summaryObject["kind"]) == "summary" - } - } - expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 - expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 - expect(testContext.eventReporter.lastEventResponseDate) == Date.distantPast - expect(testContext.eventReporter.contextSummarizer.hasLoggedRequests) == false - let expectedError = testContext.serviceMock.errorEventHTTPURLResponse - guard case let .response(error) = testContext.syncResult - else { - fail("Expected error result for event send") - return - } - let httpError = error as? HTTPURLResponse - expect(httpError?.url) == expectedError?.url - expect(httpError?.statusCode) == expectedError?.statusCode - } - } - context("error only") { - beforeEach { - waitUntil(timeout: .seconds(10)) { syncComplete in - testContext = TestContext(stubResponseSuccess: false, stubResponseErrorOnly: true, eventStubResponseDate: eventStubResponseDate, onSyncComplete: { result in - testContext.syncResult = result - syncComplete() - }) - testContext.eventReporter.isOnline = true - testContext.recordEvents(Event.Kind.nonSummaryKinds.count) - // Track some flag evaluations to populate the contextSummarizer - testContext.eventReporter.recordFlagEvaluationEvents( - flagKey: "test-flag", - value: .bool(true), - defaultValue: .bool(false), - featureFlag: nil, - context: testContext.context, - includeReason: false - ) - testContext.eventReporter.flush(completion: nil) - } - } - it("drops events events after the failure") { - erOnline() - expect(testContext.serviceMock.publishEventDataCallCount) == 2 // 1 retry attempt - expect(testContext.eventReporter.eventStore.isEmpty) == true - let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) - valueIsArray(published) { valueArray in - expect(valueArray.count) == testContext.events.count + 1 - expect(Array(valueArray.prefix(testContext.events.count))) == testContext.events.map { encodeToLDValue($0) } - valueIsObject(valueArray[testContext.events.count]) { summaryObject in - expect(summaryObject["kind"]) == "summary" - } - } - expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 1 - expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch) == Event.Kind.nonSummaryKinds.count + 1 - expect(testContext.eventReporter.lastEventResponseDate) == Date.distantPast - expect(testContext.eventReporter.contextSummarizer.hasLoggedRequests) == false - guard case let .request(error) = testContext.syncResult - else { - fail("Expected error result for event send") - return - } - expect(error as NSError?) == DarklyServiceMock.Constants.error - } - } - context("non-retriable response") { - beforeEach { - waitUntil(timeout: .seconds(10)) { syncComplete in - testContext = TestContext(eventStubResponseDate: eventStubResponseDate, onSyncComplete: { result in - testContext.syncResult = result - syncComplete() - }) - let unauthorized = HTTPURLResponse(url: testContext.serviceMock.config.eventsUrl, - statusCode: HTTPURLResponse.StatusCodes.unauthorized, - httpVersion: DarklyServiceMock.Constants.httpVersion, - headerFields: nil) - testContext.serviceMock.stubbedEventResponse = (nil, unauthorized, nil, nil) - testContext.eventReporter.isOnline = true - testContext.recordEvents(Event.Kind.nonSummaryKinds.count) - testContext.eventReporter.flush(completion: nil) - } - } - it("drops events and stops reporting") { - expect(testContext.serviceMock.publishEventDataCallCount) == 1 - expect(testContext.eventReporter.eventStore).to(beEmpty()) - expect(testContext.eventReporter.isOnline).to(beFalse()) - expect(testContext.eventReporter.isReportingActive).to(beFalse()) - if case let .response(error) = testContext.syncResult { - expect((error as? HTTPURLResponse)?.statusCode) == HTTPURLResponse.StatusCodes.unauthorized - } else { - fail("Expected response error result for event send") - } - } - it("can be brought back online") { - testContext.eventReporter.isOnline = true - expect(testContext.eventReporter.isOnline).to(beTrue()) - expect(testContext.eventReporter.isReportingActive).to(beTrue()) - } - } - } - } - context("offline") { - beforeEach { - waitUntil { syncComplete in - testContext = TestContext(eventStubResponseDate: eventStubResponseDate, onSyncComplete: { result in - testContext.syncResult = result - syncComplete() - }) - testContext.recordEvents(Event.Kind.nonSummaryKinds.count) - // Track some flag evaluations to populate the contextSummarizer - testContext.eventReporter.recordFlagEvaluationEvents( - flagKey: "test-flag", - value: .bool(true), - defaultValue: .bool(false), - featureFlag: nil, - context: testContext.context, - includeReason: false - ) - testContext.eventReporter.flush(completion: nil) - } - } - it("doesn't report events") { - expect(testContext.eventReporter.isOnline) == false - expect(testContext.eventReporter.isReportingActive) == false - expect(testContext.serviceMock.publishEventDataCallCount) == 0 - expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount) == 0 - expect(testContext.eventReporter.eventStore) == testContext.events - expect(testContext.eventReporter.lastEventResponseDate) == Date.distantPast - expect(testContext.eventReporter.contextSummarizer.hasLoggedRequests) == true - guard case .isOffline = testContext.syncResult - else { - fail("Expected error .isOffline result for event send") - return - } - } - } - } - } - - func testRecordFlagEvaluationEvents() { - let context = LDContext.stub() - let serviceMock = DarklyServiceMock() - describe("recordFlagEvaluationEvents") { - it("counts an evaluation the summarizer turns away as dropped, and still keeps its full event") { - var config = LDConfig.stub - config.eventCapacity = 2 - let service = DarklyServiceMock() - service.config = config - let diagnosticCache = DiagnosticCachingMock() - service.diagnosticCache = diagnosticCache - let reporter = EventReporter(service: service, onSyncComplete: nil) - let untracked = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: false) - let tracked = FeatureFlag(flagKey: "flag-key", value: nil, variation: 1, flagVersion: 2, trackEvents: true) - - // Capacity bounds the number of contexts counted as well as the number of events held. - for key in ["a", "b"] { - reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: untracked, context: LDContext.stub(key: key), includeReason: false) - } - expect(diagnosticCache.incrementDroppedEventCountCallCount) == 0 - - reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: tracked, context: LDContext.stub(key: "c"), includeReason: false) - - expect(diagnosticCache.incrementDroppedEventCountCallCount) == 1 - expect(reporter.contextSummarizer.getSummaries().count) == 2 - expect(reporter.eventStore.map { $0.kind }) == [.feature] - } - it("unknown flag") { - let reporter = EventReporter(service: serviceMock, onSyncComplete: nil) - reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: nil, context: context, includeReason: true) - expect(reporter.eventStore.count) == 0 - expect(reporter.contextSummarizer.hasLoggedRequests) == true - let summaries = reporter.contextSummarizer.getSummaries() - expect(summaries.count) == 1 - let tracker = summaries[0].tracker - expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" - expect(tracker.flagCounters["flag-key"]?.flagValueCounters.count) == 1 - expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: nil, version: nil)]?.count) == 1 - expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: nil, version: nil)]?.value) == "a" - } - it("untracked flag") { - let reporter = EventReporter(service: serviceMock, onSyncComplete: nil) - let flag = FeatureFlag(flagKey: "unused", value: nil, variation: 1, flagVersion: 2, trackEvents: false) - reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: true) - expect(reporter.eventStore.count) == 0 - expect(reporter.contextSummarizer.hasLoggedRequests) == true - let summaries = reporter.contextSummarizer.getSummaries() - expect(summaries.count) == 1 - let tracker = summaries[0].tracker - expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" - expect(tracker.flagCounters["flag-key"]?.flagValueCounters.count) == 1 - expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.count) == 1 - expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.value) == "a" - } - it("tracked flag") { - let reporter = EventReporter(service: serviceMock, onSyncComplete: nil) - let flag = FeatureFlag(flagKey: "unused", value: nil, variation: 1, flagVersion: 2, trackEvents: true) - reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: true) - expect(reporter.eventStore.count) == 1 - expect((reporter.eventStore[0] as? FeatureEvent)?.kind) == .feature - expect((reporter.eventStore[0] as? FeatureEvent)?.key) == "flag-key" - expect((reporter.eventStore[0] as? FeatureEvent)?.context.contextHash()) == context.contextHash() - expect((reporter.eventStore[0] as? FeatureEvent)?.value) == "a" - expect((reporter.eventStore[0] as? FeatureEvent)?.defaultValue) == "b" - expect((reporter.eventStore[0] as? FeatureEvent)?.featureFlag) == flag - expect((reporter.eventStore[0] as? FeatureEvent)?.includeReason) == true - expect(reporter.contextSummarizer.hasLoggedRequests) == true - let summaries = reporter.contextSummarizer.getSummaries() - expect(summaries.count) == 1 - let tracker = summaries[0].tracker - expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" - expect(tracker.flagCounters["flag-key"]?.flagValueCounters.count) == 1 - expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.count) == 1 - expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.value) == "a" - } - it("debug until past date") { - let reporter = EventReporter(service: serviceMock, onSyncComplete: nil) - let flag = FeatureFlag(flagKey: "unused", value: nil, variation: 1, flagVersion: 2, trackEvents: false, debugEventsUntilDate: Date().addingTimeInterval(-1.0)) - reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: true) - expect(reporter.eventStore.count) == 0 - expect(reporter.contextSummarizer.hasLoggedRequests) == true - let summaries = reporter.contextSummarizer.getSummaries() - expect(summaries.count) == 1 - let tracker = summaries[0].tracker - expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" - expect(tracker.flagCounters["flag-key"]?.flagValueCounters.count) == 1 - expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.count) == 1 - expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.value) == "a" - } - it("debug until future date") { - let reporter = EventReporter(service: serviceMock, onSyncComplete: nil) - let flag = FeatureFlag(flagKey: "unused", value: nil, variation: 1, flagVersion: 2, trackEvents: false, debugEventsUntilDate: Date().addingTimeInterval(3.0)) - reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: false) - expect(reporter.eventStore.count) == 1 - expect((reporter.eventStore[0] as? FeatureEvent)?.kind) == .debug - expect((reporter.eventStore[0] as? FeatureEvent)?.key) == "flag-key" - expect((reporter.eventStore[0] as? FeatureEvent)?.context) == context - expect((reporter.eventStore[0] as? FeatureEvent)?.value) == "a" - expect((reporter.eventStore[0] as? FeatureEvent)?.defaultValue) == "b" - expect((reporter.eventStore[0] as? FeatureEvent)?.featureFlag) == flag - expect((reporter.eventStore[0] as? FeatureEvent)?.includeReason) == false - expect(reporter.contextSummarizer.hasLoggedRequests) == true - let summaries = reporter.contextSummarizer.getSummaries() - expect(summaries.count) == 1 - let tracker = summaries[0].tracker - expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" - expect(tracker.flagCounters["flag-key"]?.flagValueCounters.count) == 1 - expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.count) == 1 - expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.value) == "a" - } - it("debug until future date earlier than service date") { - let reporter = EventReporter(service: serviceMock, onSyncComplete: nil) - reporter.setLastEventResponseDate(Date().addingTimeInterval(10.0)) - let flag = FeatureFlag(flagKey: "unused", value: nil, variation: 1, flagVersion: 2, trackEvents: false, debugEventsUntilDate: Date().addingTimeInterval(3.0)) - reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: true) - expect(reporter.eventStore.count) == 0 - expect(reporter.contextSummarizer.hasLoggedRequests) == true - let summaries = reporter.contextSummarizer.getSummaries() - expect(summaries.count) == 1 - let tracker = summaries[0].tracker - expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" - expect(tracker.flagCounters["flag-key"]?.flagValueCounters.count) == 1 - expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.count) == 1 - expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.value) == "a" - } - it("tracked flag and debug date in future") { - let reporter = EventReporter(service: serviceMock, onSyncComplete: nil) - reporter.setLastEventResponseDate(Date().addingTimeInterval(-3.0)) - let flag = FeatureFlag(flagKey: "unused", value: nil, variation: 1, flagVersion: 2, trackEvents: true, debugEventsUntilDate: Date()) - reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: false) - expect(reporter.eventStore.count) == 2 - let featureEvent = reporter.eventStore.first { $0.kind == .feature } as? FeatureEvent - let debugEvent = reporter.eventStore.first { $0.kind == .debug } as? FeatureEvent - expect(featureEvent?.kind) == .feature - expect(featureEvent?.key) == "flag-key" - expect(featureEvent?.context.contextHash()) == context.contextHash() - expect(featureEvent?.value) == "a" - expect(featureEvent?.defaultValue) == "b" - expect(featureEvent?.featureFlag) == flag - expect(featureEvent?.includeReason) == false - expect(debugEvent?.kind) == .debug - expect(debugEvent?.key) == "flag-key" - expect(debugEvent?.context) == context - expect(debugEvent?.value) == "a" - expect(debugEvent?.defaultValue) == "b" - expect(debugEvent?.featureFlag) == flag - expect(debugEvent?.includeReason) == false - expect(reporter.contextSummarizer.hasLoggedRequests) == true - let summaries = reporter.contextSummarizer.getSummaries() - expect(summaries.count) == 1 - let tracker = summaries[0].tracker - expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" - expect(tracker.flagCounters["flag-key"]?.flagValueCounters.count) == 1 - expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.count) == 1 - expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: 1, version: 2)]?.value) == "a" - } - it("records events concurrently") { - let reporter = EventReporter(service: serviceMock, onSyncComplete: nil) - reporter.setLastEventResponseDate(Date()) - let flag = FeatureFlag(flagKey: "unused", trackEvents: true, debugEventsUntilDate: Date().addingTimeInterval(3.0)) - - let counter = DispatchSemaphore(value: 0) - DispatchQueue.concurrentPerform(iterations: 10) { _ in - reporter.recordFlagEvaluationEvents(flagKey: "flag-key", value: "a", defaultValue: "b", featureFlag: flag, context: context, includeReason: false) - counter.signal() - } - (0..<10).forEach { _ in counter.wait() } - - expect(reporter.eventStore.count) == 20 - expect(reporter.eventStore.filter { $0.kind == .feature }.count) == 10 - expect(reporter.eventStore.filter { $0.kind == .debug }.count) == 10 - expect(reporter.contextSummarizer.hasLoggedRequests) == true - let summaries = reporter.contextSummarizer.getSummaries() - expect(summaries.count) == 1 - let tracker = summaries[0].tracker - expect(tracker.flagCounters["flag-key"]?.defaultValue) == "b" - expect(tracker.flagCounters["flag-key"]?.flagValueCounters.count) == 1 - expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: nil, version: nil)]?.count) == 10 - expect(tracker.flagCounters["flag-key"]?.flagValueCounters[CounterKey(variation: nil, version: nil)]?.value) == "a" - } - } - } - - private func reportTimerSpec() { - describe("report timer fires") { - var testContext: TestContext! - afterEach { - testContext.eventReporter.isOnline = false - } - context("with events") { - beforeEach { - testContext = TestContext(eventFlushInterval: Constants.eventFlushIntervalHalfSecond) - testContext.eventReporter.isOnline = true - testContext.recordEvents(Event.Kind.allKinds.count) - } - it("reports events") { - expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) - expect(testContext.eventReporter.eventStore.isEmpty).toEventually(beTrue()) - expect(testContext.diagnosticCache.recordEventsInLastBatchCallCount).toEventually(equal(1)) - expect(testContext.diagnosticCache.recordEventsInLastBatchReceivedEventsInLastBatch).toEventually(equal(testContext.events.count)) - let published = try JSONDecoder().decode(LDValue.self, from: testContext.serviceMock.publishedEventData!) - valueIsArray(published) { valueArray in - expect(valueArray.count) == testContext.events.count - expect(valueArray) == testContext.events.map { encodeToLDValue($0) } - } - } - } - context("on later intervals") { - beforeEach { - testContext = TestContext(eventFlushInterval: Constants.eventFlushIntervalHalfSecond) - testContext.eventReporter.isOnline = true - testContext.recordEvents(Event.Kind.nonSummaryKinds.count) - } - it("publishes events recorded after the first fire") { - expect(testContext.serviceMock.publishEventDataCallCount).toEventually(equal(1)) - testContext.recordEvents(Event.Kind.nonSummaryKinds.count) - expect(testContext.serviceMock.publishEventDataCallCount) - .toEventually(equal(2), timeout: .seconds(2)) - } - } - it("without events") { - testContext = TestContext(eventFlushInterval: Constants.eventFlushIntervalHalfSecond) - testContext.eventReporter.isOnline = true - - waitUntil { done in - DispatchQueue.main.asyncAfter(deadline: DispatchTime.now() + Constants.eventFlushIntervalHalfSecond) { - expect(testContext.serviceMock.publishEventDataCallCount) == 0 - done() - } - } - } - } - } -} - -private extension Date { - var adjustedForHttpUrlHeaderUse: Date { - let headerDateFormatter = DateFormatter.httpUrlHeaderFormatter - let dateString = headerDateFormatter.string(from: self) - return headerDateFormatter.date(from: dateString) ?? self - } -} - -extension Event.Kind { - static var nonSummaryKinds: [Event.Kind] { - [feature, debug, identify, custom] - } -} diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift new file mode 100644 index 000000000..ecaf0ecfe --- /dev/null +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/EventStoreSpec.swift @@ -0,0 +1,2343 @@ +import Foundation +import Quick +import Nimble +@testable import LaunchDarkly + +final class EventStoreSpec: QuickSpec { + override func spec() { + stagingSpec() + recoverySpec() + corruptionSpec() + batchSpec() + capacitySpec() + byteCapacitySpec() + batchSizeSpec() + expirySpec() + fallbackSpec() + optOutSpec() + multiProcessSpec() + withoutPersistenceSpec() + #if os(macOS) || targetEnvironment(macCatalyst) + lockedInstancesSpec() + #endif + } + + #if os(macOS) || targetEnvironment(macCatalyst) + private func lockedInstancesSpec() { + // A Mac runs a second instance of an application over the same directory on request. + describe("live instances holding their logs locked") { + var first: EventStore! + afterEach { + first.deleteEverything() + } + + func instance() -> EventStore { + EventStore(directory: first.directory!, capacity: 100, logger: .disabled) + } + + beforeEach { + first = EventStore(directory: EventStore.temporary().directory!, capacity: 100, logger: .disabled) + } + + it("gives each a log of its own, and leaves the first's alone") { + _ = first.stage(EventStoreSpec.payload("first")) + first.commit() + + let second = instance() + second.preparePayloadsFromInterruptedLogs() + expect(second.pendingBatches()).to(beEmpty()) + + _ = second.stage(EventStoreSpec.payload("second")) + second.commit() + expect(second.openLogUrl!) != first.openLogUrl! + + _ = first.stage(EventStoreSpec.payload("first-again")) + first.commit() + let fromFirst = first.closeBatch() + let fromSecond = second.closeBatch() + expect(EventStoreSpec.keys(ofBody: fromFirst.flatMap { try? first.body(of: $0) })) == ["first", "first-again"] + expect(EventStoreSpec.keys(ofBody: fromSecond.flatMap { try? second.body(of: $0) })) == ["second"] + } + + it("leaves a log another process holds locked alone until the lock is let go") { + var ended: EventStore? = instance() + _ = ended?.stage(EventStoreSpec.payload("held")) + ended?.commit() + let log = ended!.openLogUrl! + ended = nil + // A descriptor of its own, as another live instance's would be, so nothing in this process knows of it. + let held = EventLogFile.openLocked(log, flags: O_RDONLY)! + + first.preparePayloadsFromInterruptedLogs() + expect(first.pendingBatches()).to(beEmpty()) + + held.close() + first.preparePayloadsFromInterruptedLogs() + expect(EventStoreSpec.keys(ofBody: first.pendingBatches().first.flatMap { try? first.body(of: $0) })) == ["held"] + } + + it("recovers the log an instance that has ended left open") { + _ = first.stage(EventStoreSpec.payload("first")) + first.commit() + + var ended: EventStore? = instance() + _ = ended?.stage(EventStoreSpec.payload("from-the-ended-instance")) + ended?.commit() + // Going away closes its descriptor, which releases the lock, as the process dying would. + ended = nil + + let next = instance() + next.preparePayloadsFromInterruptedLogs() + let batches = next.pendingBatches() + expect(batches.count) == 1 + expect(EventStoreSpec.keys(ofBody: batches.first.flatMap { try? next.body(of: $0) })) == ["from-the-ended-instance"] + } + + it("recovers the log a previous run left open, once nothing holds it") { + _ = first.stage(EventStoreSpec.payload("left-open")) + first.commit() + let directory = first.directory! + first = nil + first = EventStore(directory: directory, capacity: 100, logger: .disabled) + + first.preparePayloadsFromInterruptedLogs() + let batches = first.pendingBatches() + expect(batches.count) == 1 + expect(EventStoreSpec.keys(ofBody: batches.first.flatMap { try? first.body(of: $0) })) == ["left-open"] + } + } + } + #endif + + private func multiProcessSpec() { + describe("several processes sharing a directory") { + var app: EventStore! + afterEach { + app.deleteEverything() + } + + /// Another process of the same application, using the same directory. Unlocked, as on an iPhone or iPad, + /// where the logs a live store has open are known without one. + func otherProcess() -> EventStore { + EventStore.sharing(app.directory!) + } + + beforeEach { + app = EventStore.sharing(EventStore.temporary().directory!) + } + + it("leaves a log another process still has open where it is, so that process's later events reach it") { + _ = app.stage(EventStoreSpec.payload("before")) + app.commit() + + // Starting up, the other process recovers what no live process holds, and nothing else. + let widget = otherProcess() + widget.preparePayloadsFromInterruptedLogs() + expect(widget.pendingBatches()).to(beEmpty()) + + _ = app.stage(EventStoreSpec.payload("after")) + app.commit() + let batch = app.closeBatch() + expect(batch?.eventCount) == 2 + expect(EventStoreSpec.keys(ofBody: batch.flatMap { try? app.body(of: $0) })) == ["before", "after"] + } + + it("keeps each process's events in a log of its own") { + let widget = otherProcess() + _ = app.stage(EventStoreSpec.payload("from-app")) + app.commit() + _ = widget.stage(EventStoreSpec.payload("from-widget")) + widget.commit() + + expect(app.openLogUrl!) != widget.openLogUrl! + let fromApp = app.closeBatch() + let fromWidget = widget.closeBatch() + expect(EventStoreSpec.keys(ofBody: fromApp.flatMap { try? app.body(of: $0) })) == ["from-app"] + expect(EventStoreSpec.keys(ofBody: fromWidget.flatMap { try? widget.body(of: $0) })) == ["from-widget"] + } + + it("delivers a batch another process closed, since that process may never run again") { + _ = app.stage(EventStoreSpec.payload("closed-by-app")) + let closed = app.closeBatch() + + let widget = otherProcess() + widget.preparePayloadsFromInterruptedLogs() + expect(closed).toNot(beNil()) + expect(widget.pendingBatches().map(\.payloadId)) == [closed?.payloadId].compactMap { $0 } + } + + it("recovers the log a process that has ended left open, whichever process it was") { + var widget: EventStore? = otherProcess() + _ = widget?.stage(EventStoreSpec.payload("left-open")) + widget?.commit() + // Going away closes its log, as the process ending would. + widget = nil + + app.preparePayloadsFromInterruptedLogs() + let batches = app.pendingBatches() + expect(batches.count) == 1 + expect(EventStoreSpec.keys(ofBody: batches.first.flatMap { try? app.body(of: $0) })) == ["left-open"] + } + + it("names a log for a UUID rather than for the process") { + _ = app.stage(EventStoreSpec.payload("named")) + app.commit() + + let logs = try FileManager.default.contentsOfDirectory(atPath: app.directory!.path) + .filter { $0.hasPrefix(EventDirectory.logPrefix) } + expect(logs.count) == 1 + expect(logs.first.flatMap { UUID(uuidString: String($0.dropFirst(EventDirectory.logPrefix.count))) }).toNot(beNil()) + expect(try FileManager.default.contentsOfDirectory(atPath: app.directory!.path) + .filter { $0.hasPrefix(EventDirectory.openingLogPrefix) }).to(beEmpty()) + } + } + } + + private func optOutSpec() { + describe("a store the application did not ask to persist") { + var store: EventStore! + beforeEach { + store = EventStore.temporary(persistEvents: false) + } + afterEach { + store.deleteEverything() + } + + it("writes nothing to the disk, so the directory is never even created") { + // Given the directory rather than asked for it, because this store has none to give: it is told to + // keep events off the disk, and so is built without anywhere to put them. + let directory = EventStore.temporaryDirectory() + let store = EventStore(directory: directory, capacity: 100, persistEvents: false, logger: .disabled) + defer { try? FileManager.default.removeItem(at: directory) } + + _ = store.stage(EventStoreSpec.payload("held")) + store.commit() + + expect(store.directory).to(beNil()) + expect(FileManager.default.fileExists(atPath: directory.path)) == false + expect(EventStoreSpec.keys(of: EventStore.sharing(directory))).to(beEmpty()) + } + + it("delivers the events from memory, so opting out costs durability and nothing else") { + _ = store.stage(EventStoreSpec.payload("held")) + let batch = store.closeBatch()! + + expect(batch.eventCount) == 1 + expect(store.pendingEventCount) == 1 + expect(try store.body(of: batch)) == Data("[{\"kind\":\"custom\",\"key\":\"held\"}]".utf8) + } + + it("asks the filesystem nothing, even about a batch it does not hold") { + var reads = 0 + let directory = EventStore.temporaryDirectory() + let store = EventStore(directory: directory, + capacity: 100, + persistEvents: false, + logger: .disabled, + readFile: { _ in + reads += 1 + return Data() + }) + defer { try? FileManager.default.removeItem(at: directory) } + + _ = store.stage(EventStoreSpec.payload("held")) + let batch = store.closeBatch() + expect(store.pendingBatches().map(\.payloadId)) == [batch?.payloadId] + expect(EventStoreSpec.keys(ofBody: batch.flatMap { try? store.body(of: $0) })) == ["held"] + batch.map(store.remove) + + // A batch this store never closed, which is what sends the paths that form a file URL down the branch + // that would otherwise ask the filesystem about it. + let stranger = EventBatch(payloadId: UUID().uuidString, eventCount: 1) + expect(try store.body(of: stranger)).to(beNil()) + store.remove(stranger) + + expect(reads) == 0 + expect(store.directory).to(beNil()) + expect(FileManager.default.fileExists(atPath: directory.path)) == false + } + + it("leaves alone what a run that did persist left behind, for the next run that persists") { + let persisting = EventStore.temporary() + defer { persisting.deleteEverything() } + _ = persisting.stage(EventStoreSpec.payload("closed")) + persisting.commit() + expect(persisting.closeBatch()).toNot(beNil()) + _ = persisting.stage(EventStoreSpec.payload("open")) + persisting.commit() + + let withoutPersistence = EventStore.sharing(persisting.directory!, persistEvents: false) + withoutPersistence.preparePayloadsFromInterruptedLogs() + expect(withoutPersistence.pendingBatches()).to(beEmpty()) + expect(withoutPersistence.pendingEventCount) == 0 + + _ = withoutPersistence.stage(EventStoreSpec.payload("in-memory")) + let own = withoutPersistence.closeBatch() + expect(withoutPersistence.pendingBatches().map(\.payloadId)) == [own?.payloadId] + + let persistingAgain = persisting.nextRun() + persistingAgain.preparePayloadsFromInterruptedLogs() + let bodies = persistingAgain.pendingBatches().map { EventStoreSpec.keys(ofBody: try? persistingAgain.body(of: $0)) } + expect(bodies) == [["closed"], ["open"]] + } + } + } + + /// A store that can never write, because the directory it was given sits underneath a regular file and so cannot + /// be created. Standing in for the full disk and the revoked permission, neither of which a test can arrange. + private static func unwritableStore(capacity: Int = 100) -> EventStore { + let blocker = FileManager.default.temporaryDirectory + .appendingPathComponent("com.launchdarkly.tests.events", isDirectory: true) + .appendingPathComponent(UUID().uuidString) + try? FileManager.default.createDirectory(at: blocker.deletingLastPathComponent(), + withIntermediateDirectories: true) + FileManager.default.createFile(atPath: blocker.path, contents: Data("not a directory".utf8)) + + return EventStore(directory: blocker.appendingPathComponent("store", isDirectory: true), + capacity: capacity, + logger: .disabled) + } + + /// A store reading the same directory as another, standing in for another process: it sees the batches, and leaves + /// alone the log the live store appends to. `nextRun()` stands in for the next run of the application. + private static func reader(sharing store: EventStore, + readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }) -> EventStore { + EventStore.sharing(store.directory!, readFile: readFile) + } + + /// The one `ready-` file in a store's directory. + private static func onlyBatchFile(in store: EventStore) throws -> URL { + try FileManager.default + .contentsOfDirectory(at: store.directory!, includingPropertiesForKeys: nil) + .first { $0.lastPathComponent.hasPrefix("ready-") }! + } + + /// Renames a batch to the name a version that did not count them in the name gave it, which is what such a + /// version leaves behind for this one to deliver. + @discardableResult + private static func asUncountedName(_ batch: URL) throws -> URL { + let name = batch.lastPathComponent + let renamed = batch.deletingLastPathComponent() + .appendingPathComponent(String(name[.. URL { + // Fields this version would take for a payload ID and a count, were the name its own. + let renamed = batch.deletingLastPathComponent().appendingPathComponent("ready-v2-7.\(UUID().uuidString)") + try FileManager.default.moveItem(at: batch, to: renamed) + return renamed + } + + /// What a process out of file descriptors gets for a file that is perfectly intact. + private static func readFailing(while failing: @escaping () -> Bool) -> (URL) throws -> Data { + { url in + guard !failing() + else { throw NSError(domain: NSPOSIXErrorDomain, code: Int(EMFILE)) } + return try Data(contentsOf: url) + } + } + + /// Puts a read-only open of the same file behind the log's descriptor, so that cutting it back fails while the file + /// itself can still be renamed or removed. + private static func refuseTruncating(_ file: EventLogFile) { + var path = [CChar](repeating: 0, count: Int(MAXPATHLEN)) + guard fcntl(file.descriptor, F_GETPATH, &path) == 0 + else { return fail("could not find the log's path") } + let readOnly = open(path, O_RDONLY) + guard readOnly >= 0 + else { return fail("could not reopen the log") } + _ = dup2(readOnly, file.descriptor) + _ = close(readOnly) + } + + private static func payload(_ key: String) -> Data { + Data("{\"kind\":\"custom\",\"key\":\"\(key)\"}".utf8) + } + + /// Where each frame of a log written by this version starts. + private static func frameOffsets(in log: Data) -> [Int] { + var offsets: [Int] = [] + var cursor = EventLogFormat.fileHeaderSize + while cursor + EventLogFormat.frameOverhead <= log.count { + offsets.append(cursor) + cursor += EventLogFormat.frameOverhead + Int(EventLogFormat.readUInt32(log, at: cursor + 2)) + } + return offsets + } + + /// An event whose frame is exactly `frameBytes` long, padded out in a `data` field. + private static func payload(framedIn frameBytes: Int, key: String) -> Data { + let bare = "{\"kind\":\"custom\",\"key\":\"\(key)\",\"data\":\"\"}" + let padding = frameBytes - EventLogFormat.frameOverhead - bare.utf8.count + precondition(padding >= 0, "\(key) does not fit in \(frameBytes) bytes") + return Data("{\"kind\":\"custom\",\"key\":\"\(key)\",\"data\":\"\(String(repeating: "x", count: padding))\"}".utf8) + } + + private static func keys(of store: EventStore) -> [String] { + keys(in: store.pendingEventPayloads()) + } + + private static func keys(in payloads: [Data]) -> [String] { + payloads.compactMap { + guard case .object(let fields) = try? JSONDecoder().decode(LDValue.self, from: $0), + case .string(let key) = fields["key"] + else { return nil } + return key + } + } + + private func stagingSpec() { + describe("staging") { + var store: EventStore! + beforeEach { + store = EventStore.temporary() + } + afterEach { + store.deleteEverything() + } + + it("keeps a staged event out of the log until it is committed") { + _ = store.stage(EventStoreSpec.payload("staged-only")) + + expect(EventStoreSpec.keys(in: store.payloadsOnDisk)).to(beEmpty()) + expect(store.pendingEventCount) == 1 + } + + it("puts a committed event where another process would find it") { + _ = store.stage(EventStoreSpec.payload("committed")) + store.commit() + + expect(EventStoreSpec.keys(in: store.payloadsOnDisk)) == ["committed"] + } + + it("never writes on its own, however much is staged") { + // When to write is the caller's decision. 64 KB is well past what a store that wrote on its own would + // let build up. + let event = EventStoreSpec.payload(String(repeating: "p", count: 1_024)) + for _ in 0..<64 { + _ = store.stage(event) + } + + expect(store.payloadsOnDisk).to(beEmpty()) + expect(store.pendingEventCount) == 64 + + store.commit() + + expect(store.payloadsOnDisk.count) == 64 + } + + it("keeps events in the order they were recorded") { + for index in 0..<5 { + _ = store.stage(EventStoreSpec.payload("event-\(index)")) + } + store.commit() + + expect(EventStoreSpec.keys(in: store.payloadsOnDisk)) == ["event-0", "event-1", "event-2", "event-3", "event-4"] + } + } + } + + private func recoverySpec() { + describe("recovering a log a previous run left open") { + var store: EventStore! + afterEach { + store.deleteEverything() + } + + it("delivers the events as a batch") { + store = EventStore.temporary() + _ = store.stage(EventStoreSpec.payload("survived")) + store.commit() + + // Nothing closed the log, as nothing would if the process had died here. + let next = store.nextRun() + next.preparePayloadsFromInterruptedLogs() + + let batches = next.pendingBatches() + expect(batches.count) == 1 + expect(batches.first?.eventCount) == 1 + expect(try next.body(of: batches[0])) == Data("[{\"kind\":\"custom\",\"key\":\"survived\"}]".utf8) + } + + it("leaves nothing behind when the log held no events") { + store = EventStore.temporary() + _ = store.stage(EventStoreSpec.payload("only")) + store.commit() + let batch = store.closeBatch() + store.remove(batch!) + + let next = store.nextRun() + next.preparePayloadsFromInterruptedLogs() + + expect(next.pendingBatches()).to(beEmpty()) + expect(next.pendingEventCount) == 0 + } + + it("keeps this run's events out of it, even when this run commits before recovery is asked for") { + store = EventStore.temporary() + for index in 0..<2 { + _ = store.stage(EventStoreSpec.payload("previous-\(index)")) + } + store.commit() + // The previous run died partway through writing its last frame. + let log = store.openLogUrl! + try Data(contentsOf: log).dropLast(3).write(to: log) + + let next = store.nextRun() + _ = next.stage(EventStoreSpec.payload("this-run")) + next.commit() + next.preparePayloadsFromInterruptedLogs() + _ = next.stage(EventStoreSpec.payload("later")) + next.commit() + _ = next.closeBatch() + + let bodies = next.pendingBatches().map { EventStoreSpec.keys(ofBody: try? next.body(of: $0)) } + expect(bodies) == [["previous-0"], ["this-run", "later"]] + expect(next.pendingEventCount) == 3 + } + + it("reads the log without holding up a commit") { + store = EventStore.temporary() + _ = store.stage(EventStoreSpec.payload("from-the-previous-run")) + store.commit() + + let reading = DispatchSemaphore(value: 0) + let release = DispatchSemaphore(value: 0) + let next = store.nextRun(readFile: { url in + if url.lastPathComponent.hasPrefix(EventDirectory.logPrefix) { + reading.signal() + _ = release.wait(timeout: .now() + 5) + } + return try Data(contentsOf: url) + }) + let recovered = DispatchSemaphore(value: 0) + DispatchQueue.global().async { + next.preparePayloadsFromInterruptedLogs() + recovered.signal() + } + expect(reading.wait(timeout: .now() + 5)) == .success + + // Recovery is partway through reading the log, which a commit does not wait for. + let committed = DispatchSemaphore(value: 0) + DispatchQueue.global().async { + _ = next.stage(EventStoreSpec.payload("this-run")) + next.commit() + committed.signal() + } + expect(committed.wait(timeout: .now() + 2)) == .success + release.signal() + expect(recovered.wait(timeout: .now() + 5)) == .success + + expect(next.pendingEventCount) == 2 + let batches = next.pendingBatches() + expect(batches.count) == 1 + expect(EventStoreSpec.keys(ofBody: try next.body(of: batches[0]))) == ["from-the-previous-run"] + expect(EventStoreSpec.keys(in: next.payloadsOnDisk)) == ["from-the-previous-run", "this-run"] + } + } + } + + private static func keys(ofBody body: Data?) -> [String] { + guard let body, case .array(let events)? = try? JSONDecoder().decode(LDValue.self, from: body) + else { return [] } + return events.compactMap { + guard case .object(let fields) = $0, case .string(let key) = fields["key"] + else { return nil } + return key + } + } + + private func corruptionSpec() { + describe("a damaged log") { + var store: EventStore! + afterEach { + store.deleteEverything() + } + + it("keeps every event before a frame the writer did not finish") { + store = EventStore.temporary() + for index in 0..<3 { + _ = store.stage(EventStoreSpec.payload("event-\(index)")) + } + store.commit() + + // A process killed partway through a write leaves the last frame short of its declared length. + let log = store.openLogUrl! + let whole = try Data(contentsOf: log) + try whole.dropLast(8).write(to: log) + + expect(EventStoreSpec.keys(in: store.payloadsOnDisk)) == ["event-0", "event-1"] + } + + context("where the storage changed a frame after it was written") { + var log: URL! + var second: Int! + beforeEach { + store = EventStore.temporary() + for index in 0..<3 { + _ = store.stage(EventStoreSpec.payload("event-\(index)")) + } + store.commit() + log = store.openLogUrl! + second = EventStoreSpec.frameOffsets(in: try! Data(contentsOf: log))[1] + } + + it("keeps every event before a payload with a flipped bit") { + var damaged = try Data(contentsOf: log) + damaged[second + EventLogFormat.frameHeaderSize + 3] ^= 0x04 + try damaged.write(to: log) + + expect(EventStoreSpec.keys(in: store.payloadsOnDisk)) == ["event-0"] + } + + it("keeps every event before a zero-filled payload") { + var damaged = try Data(contentsOf: log) + let start = second + EventLogFormat.frameHeaderSize + let size = Int(EventLogFormat.readUInt32(damaged, at: second + 2)) + damaged.replaceSubrange(start..<(start + size), with: Data(count: size)) + try damaged.write(to: log) + + expect(EventStoreSpec.keys(in: store.payloadsOnDisk)) == ["event-0"] + } + + it("keeps every event before a damaged checksum") { + var damaged = try Data(contentsOf: log) + let size = Int(EventLogFormat.readUInt32(damaged, at: second + 2)) + damaged[second + EventLogFormat.frameHeaderSize + size + 1] ^= 0x10 + try damaged.write(to: log) + + expect(EventStoreSpec.keys(in: store.payloadsOnDisk)) == ["event-0"] + } + + it("keeps every event before a size that still fits in the file") { + // A size that is merely wrong rather than impossible passes every bound a torn tail is caught by, + // and would have the reader start the next frame partway through this one's payload. + var damaged = try Data(contentsOf: log) + damaged[second + 5] -= 1 + try damaged.write(to: log) + + expect(EventStoreSpec.keys(in: store.payloadsOnDisk)) == ["event-0"] + } + } + + it("loses a batch whose first frame is damaged, rather than taking it for one with nothing to send") { + store = EventStore.temporary() + _ = store.stage(EventStoreSpec.payload("damaged")) + let batch = store.closeBatch()! + let file = try EventStoreSpec.onlyBatchFile(in: store) + var damaged = try Data(contentsOf: file) + damaged[EventLogFormat.fileHeaderSize + EventLogFormat.frameHeaderSize] ^= 0x01 + try damaged.write(to: file) + + expect { try store.body(of: batch) }.to(throwError(errorType: UnreadableBatch.self)) + } + + it("lists a batch named without a count whose first frame is damaged as one event, and loses it") { + store = EventStore.temporary() + _ = store.stage(EventStoreSpec.payload("damaged")) + _ = store.closeBatch() + let file = try EventStoreSpec.asUncountedName(EventStoreSpec.onlyBatchFile(in: store)) + var damaged = try Data(contentsOf: file) + damaged[EventLogFormat.fileHeaderSize + EventLogFormat.frameHeaderSize] ^= 0x01 + try damaged.write(to: file) + + // Counted as no events, it would be dropped as a loss of none, which no flush hears about. + let next = EventStoreSpec.reader(sharing: store) + let batches = next.pendingBatches() + expect(batches.map { $0.eventCount }) == [1] + guard let batch = batches.first + else { return } + expect { try next.body(of: batch) }.to(throwError(errorType: UnreadableBatch.self)) + } + + it("says how many events made it into the body of a batch damaged after its first frame") { + store = EventStore.temporary() + ["first", "second", "third"].forEach { _ = store.stage(EventStoreSpec.payload($0)) } + let batch = store.closeBatch()! + let file = try EventStoreSpec.onlyBatchFile(in: store) + var damaged = try Data(contentsOf: file) + damaged[damaged.count - 1] ^= 0x01 + try damaged.write(to: file) + + let payload = try store.payload(of: batch) + expect(batch.eventCount) == 3 + expect(payload?.eventCount) == 2 + expect(EventStoreSpec.keys(ofBody: payload?.body)) == ["first", "second"] + } + + it("sends no frames a failed write left behind where it could not be cut back") { + store = EventStore.temporary() + _ = store.stage(EventStoreSpec.payload("written")) + let batch = store.closeBatch()! + let file = try EventStoreSpec.onlyBatchFile(in: store) + // Whole frames past the one the batch was closed holding, which memory delivers instead. + var leftBehind = try Data(contentsOf: file) + leftBehind.append(EventLogFormat.frame(for: EventStoreSpec.payload("from-memory"))) + try leftBehind.write(to: file) + + expect(EventStoreSpec.keys(ofBody: try? store.body(of: batch))) == ["written"] + let next = store.nextRun() + let listed = next.pendingBatches() + expect(listed.map { $0.eventCount }) == [1] + expect(EventStoreSpec.keys(ofBody: listed.first.flatMap { try? next.body(of: $0) })) == ["written"] + } + + it("is kept for a later listing when a batch named without a count cannot be read") { + store = EventStore.temporary() + _ = store.stage(EventStoreSpec.payload("kept")) + store.commit() + _ = store.closeBatch() + try EventStoreSpec.asUncountedName(EventStoreSpec.onlyBatchFile(in: store)) + + var failing = true + let next = EventStoreSpec.reader(sharing: store, + readFile: EventStoreSpec.readFailing(while: { failing })) + expect(next.pendingBatches()).to(beEmpty()) + + // A read can fail on an intact file, as it does in a process out of file descriptors, so the batch has + // to still be there once reads work again. + failing = false + let batches = next.pendingBatches() + expect(batches.count) == 1 + expect(EventStoreSpec.keys(ofBody: try next.body(of: batches[0]))) == ["kept"] + } + + it("is listed even while reads are failing when its name counts it") { + store = EventStore.temporary() + _ = store.stage(EventStoreSpec.payload("kept")) + _ = store.closeBatch() + + // The read a listing used to need is the one a process out of file descriptors cannot make. With the + // count in the name there is nothing to read until the batch is sent, and a body that cannot be read + // is a delivery to retry rather than a batch that is invisible until it can be. + let next = EventStoreSpec.reader(sharing: store, + readFile: EventStoreSpec.readFailing(while: { true })) + let batches = next.pendingBatches() + expect(batches.count) == 1 + expect(batches[0].eventCount) == 1 + expect { try next.body(of: batches[0]) }.to(throwError()) + } + + it("is a failure to retry, not nothing to send, when a body cannot be read") { + var failing = false + store = EventStore.temporary(readFile: EventStoreSpec.readFailing(while: { failing })) + _ = store.stage(EventStoreSpec.payload("kept")) + let batch = store.closeBatch()! + + failing = true + expect { try store.body(of: batch) }.to(throwError()) + // Still the store's to deliver: reporting nothing to send would have it dropped. + expect(store.pendingBatches()) == [batch] + + failing = false + expect(EventStoreSpec.keys(ofBody: try store.body(of: batch))) == ["kept"] + } + + it("is closed off uncounted when a log a previous run left open cannot be read") { + store = EventStore.temporary() + _ = store.stage(EventStoreSpec.payload("from-the-previous-run")) + store.commit() + + var failing = true + let next = store.nextRun(readFile: EventStoreSpec.readFailing(while: { failing })) + let leftOpen = next.directory!.appendingPathComponent(store.openLogUrl!.lastPathComponent) + next.preparePayloadsFromInterruptedLogs() + // Moved out from under the name recovery looks for, and not deleted. + expect(FileManager.default.fileExists(atPath: leftOpen.path)) == false + + // The events that are intact are only waiting for a read that works to be counted and delivered. + _ = next.stage(EventStoreSpec.payload("this-run")) + next.commit() + failing = false + let batches = next.pendingBatches() + expect(batches.count) == 1 + expect(EventStoreSpec.keys(ofBody: try next.body(of: batches[0]))) == ["from-the-previous-run"] + expect(EventStoreSpec.keys(in: next.payloadsOnDisk)) == ["from-the-previous-run", "this-run"] + } + + it("is listed as one event and unreadable when a batch named without a count is in a format this version does not read") { + store = EventStore.temporary() + _ = store.stage(EventStoreSpec.payload("unreadable")) + store.commit() + _ = store.closeBatch() + + let batchUrl = try EventStoreSpec.asUncountedName(EventStoreSpec.onlyBatchFile(in: store)) + var corrupted = try Data(contentsOf: batchUrl) + corrupted[4] = 0xFF + corrupted[5] = 0xFF + try corrupted.write(to: batchUrl) + + // Left for the delivery that reaches it, which removes it and counts the loss, rather than deleted + // here uncounted. + let next = EventStoreSpec.reader(sharing: store) + let batches = next.pendingBatches() + expect(batches.map { $0.eventCount }) == [1] + guard let batch = batches.first + else { return } + expect { try next.body(of: batch) }.to(throwError(errorType: UnreadableBatch.self)) + next.remove(batch) + expect(FileManager.default.fileExists(atPath: batchUrl.path)) == false + } + } + + describe("a file larger than this version reads") { + var store: EventStore! + beforeEach { + store = EventStore.temporary() + } + afterEach { + store.deleteEverything() + } + + it("is deleted where a previous run left it as a log") { + let log = try EventStoreSpec.tooLargeFile(EventDirectory.logPrefix + UUID().uuidString, in: store) + + store.preparePayloadsFromInterruptedLogs() + + expect(store.pendingBatches()).to(beEmpty()) + expect(FileManager.default.fileExists(atPath: log.path)) == false + } + + it("is listed by the count in its name, and unreadable") { + try EventStoreSpec.tooLargeFile("ready-v1-\(UUID().uuidString).3", in: store) + + let batches = store.pendingBatches() + expect(batches.map { $0.eventCount }) == [3] + expect { try store.body(of: batches[0]) }.to(throwError(errorType: UnreadableBatch.self)) + } + + it("is listed as one event, the fewest it can hold, where its name has no count, and unreadable") { + try EventStoreSpec.tooLargeFile("ready-v1-\(UUID().uuidString)", in: store) + + let batches = store.pendingBatches() + expect(batches.map { $0.eventCount }) == [1] + expect { try store.body(of: batches[0]) }.to(throwError(errorType: UnreadableBatch.self)) + } + + it("is discarded as one event once it expires, where nothing counts it") { + let uncounted = try EventStoreSpec.tooLargeFile("ready-v1-\(UUID().uuidString)", in: store) + let log = try EventStoreSpec.tooLargeFile(EventDirectory.logPrefix + UUID().uuidString, in: store) + try EventStoreSpec.age(uncounted, byDays: 8) + try EventStoreSpec.age(log, byDays: 8) + + expect(store.discardExpired()) == 2 + expect(FileManager.default.fileExists(atPath: uncounted.path)) == false + expect(FileManager.default.fileExists(atPath: log.path)) == false + } + } + } + + /// A file in the store's directory one byte past the largest this version reads, starting as a log does. + @discardableResult + private static func tooLargeFile(_ name: String, in store: EventStore) throws -> URL { + try FileManager.default.createDirectory(at: store.directory!, withIntermediateDirectories: true) + let file = store.directory!.appendingPathComponent(name) + var contents = EventLogFormat.fileHeader + contents.append(Data(count: EventDirectory.maxReadSize + 1 - contents.count)) + try contents.write(to: file) + return file + } + + private func batchSpec() { + describe("batches") { + var store: EventStore! + beforeEach { + store = EventStore.temporary() + } + afterEach { + store.deleteEverything() + } + + it("assembles the events into one request body") { + _ = store.stage(EventStoreSpec.payload("first")) + _ = store.stage(EventStoreSpec.payload("second")) + + let batch = store.closeBatch() + expect(batch?.eventCount) == 2 + + let body = try store.body(of: batch!) + expect(body) == Data("[{\"kind\":\"custom\",\"key\":\"first\"},{\"kind\":\"custom\",\"key\":\"second\"}]".utf8) + } + + it("assembles one event, and events held in memory, into the same body") { + _ = store.stage(EventStoreSpec.payload("only")) + expect(try store.body(of: store.closeBatch()!)) == Data("[{\"kind\":\"custom\",\"key\":\"only\"}]".utf8) + + let inMemory = EventStore.temporary(persistEvents: false) + defer { inMemory.deleteEverything() } + _ = inMemory.stage(EventStoreSpec.payload("first")) + _ = inMemory.stage(EventStoreSpec.payload("second")) + expect(try inMemory.body(of: inMemory.closeBatch()!)) + == Data("[{\"kind\":\"custom\",\"key\":\"first\"},{\"kind\":\"custom\",\"key\":\"second\"}]".utf8) + } + + it("frames an event as the Android SDK does") { + // The checksum is zlib's CRC-32, so these bytes are what the Android SDK writes for the same event too. + let event = Data("{\"kind\":\"custom\",\"key\":\"k\"}".utf8) + let frame = EventLogFormat.frame(for: event) + expect(frame) == Data([0x00, 0x01, 0x00, 0x00, 0x00, 0x1B]) + event + Data([0xCB, 0x2B, 0x5E, 0x6F]) + } + + it("has nothing to close when nothing was recorded") { + expect(store.closeBatch()).to(beNil()) + } + + it("holds a batch until it is removed, so a failed delivery can be retried") { + _ = store.stage(EventStoreSpec.payload("retry-me")) + let batch = store.closeBatch()! + + expect(store.pendingBatches()) == [batch] + expect(store.pendingEventCount) == 1 + expect(try store.body(of: batch)).toNot(beNil()) + + store.remove(batch) + + expect(store.pendingBatches()).to(beEmpty()) + expect(store.pendingEventCount) == 0 + } + + it("counts a closed batch from its name without reading the file back") { + _ = store.stage(EventStoreSpec.payload("first")) + _ = store.stage(EventStoreSpec.payload("second")) + let batch = store.closeBatch()! + + // Nothing appends to a batch once it is closed, so the count its name carries still holds and + // listing it has no reason to go to the disk. Damaging the file is how the test tells the two apart: + // a listing that read would refuse this batch. + let batchUrl = store.directory!.appendingPathComponent("ready-v1-\(batch.payloadId).2") + try Data("not an event log".utf8).write(to: batchUrl) + + // The store that closed it and one that did not both count it from the name alone. + expect(store.pendingBatches()) == [batch] + expect(EventStoreSpec.reader(sharing: store).pendingBatches()) == [batch] + + // The damage is found where the batch is read to be sent, and reads as a loss rather than as a failure + // to retry, so the delivery removes it instead of trying it forever, and says it lost events. + expect { try store.body(of: batch) }.to(throwError(errorType: UnreadableBatch.self)) + } + + it("reads a batch named without a count once and then remembers it") { + _ = store.stage(EventStoreSpec.payload("left-by-a-previous-version")) + let batch = store.closeBatch()! + + // A name without a count, which is what a recovery that could not read a log leaves behind for a + // listing to count. + let counted = store.directory!.appendingPathComponent("ready-v1-\(batch.payloadId).1") + let uncounted = store.directory!.appendingPathComponent("ready-v1-\(batch.payloadId)") + try FileManager.default.moveItem(at: counted, to: uncounted) + + let reader = EventStoreSpec.reader(sharing: store) + expect(reader.pendingBatches()) == [batch] + // Still deliverable under the name it has, which is not the one its count would give it. + expect(try reader.body(of: batch)).toNot(beNil()) + reader.remove(batch) + expect(FileManager.default.fileExists(atPath: uncounted.path)) == false + } + + it("does not take a payload id ending in zero-led digits for a count") { + _ = store.stage(EventStoreSpec.payload("left-by-a-previous-version")) + let batch = store.closeBatch()! + + // Its last group fits an Int32 once the leading zeros are dropped. + let payloadId = "6F9619FF-8B86-D011-B42D-000012345678" + let counted = store.directory!.appendingPathComponent("ready-v1-\(batch.payloadId).1") + let uncounted = store.directory!.appendingPathComponent("ready-v1-\(payloadId)") + try FileManager.default.moveItem(at: counted, to: uncounted) + + expect(EventStoreSpec.reader(sharing: store).pendingBatches()) == [EventBatch(payloadId: payloadId, eventCount: 1)] + } + + it("reads a name with a field after the count by position") { + _ = store.stage(EventStoreSpec.payload("first")) + _ = store.stage(EventStoreSpec.payload("second")) + let batch = store.closeBatch()! + + // What a later version might add, which this one has no use for. + let counted = store.directory!.appendingPathComponent("ready-v1-\(batch.payloadId).2") + let extended = store.directory!.appendingPathComponent("ready-v1-\(batch.payloadId).2.v2-something") + try FileManager.default.moveItem(at: counted, to: extended) + + expect(EventStoreSpec.reader(sharing: store).pendingBatches()) == [batch] + } + + it("counts a batch whose count does not parse by reading it") { + _ = store.stage(EventStoreSpec.payload("only")) + let batch = store.closeBatch()! + + let counted = store.directory!.appendingPathComponent("ready-v1-\(batch.payloadId).1") + let unparseable = store.directory!.appendingPathComponent("ready-v1-\(batch.payloadId).not-a-count") + try FileManager.default.moveItem(at: counted, to: unparseable) + + expect(EventStoreSpec.reader(sharing: store).pendingBatches()) == [batch] + } + + it("names a batch with the version of how its name reads") { + _ = store.stage(EventStoreSpec.payload("only")) + let batch = store.closeBatch()! + + let named = store.directory!.appendingPathComponent("ready-v1-\(batch.payloadId).1") + expect(FileManager.default.fileExists(atPath: named.path)) == true + } + + it("neither counts nor sends a batch another version named") { + _ = store.stage(EventStoreSpec.payload("first")) + _ = store.stage(EventStoreSpec.payload("second")) + _ = store.closeBatch() + let batch = try EventStoreSpec.asAnotherVersionsBatch(try EventStoreSpec.onlyBatchFile(in: store)) + + let reader = EventStoreSpec.reader(sharing: store) + expect(reader.pendingBatches()).to(beEmpty()) + expect(reader.pendingEventCount) == 0 + expect(reader.pendingEventPayloads()).to(beEmpty()) + expect(FileManager.default.fileExists(atPath: batch.path)) == true + } + + it("gives a batch its own payload id, so a retry is not read as a new delivery") { + _ = store.stage(EventStoreSpec.payload("first")) + let first = store.closeBatch()! + _ = store.stage(EventStoreSpec.payload("second")) + let second = store.closeBatch()! + + expect(first.payloadId).toNot(equal(second.payloadId)) + } + + it("keeps recording into a fresh log after one is closed") { + _ = store.stage(EventStoreSpec.payload("before")) + _ = store.closeBatch() + _ = store.stage(EventStoreSpec.payload("after")) + store.commit() + + expect(EventStoreSpec.keys(of: store)) == ["before", "after"] + expect(store.pendingEventCount) == 2 + } + + it("treats a batch something else already removed as nothing to send, and removing it again as harmless") { + _ = store.stage(EventStoreSpec.payload("delivered")) + let batch = store.closeBatch()! + _ = store.stage(EventStoreSpec.payload("still-pending")) + let other = store.closeBatch()! + + // Delivered through a store reading the same directory, as the next run would. + EventStoreSpec.reader(sharing: store).remove(batch) + + // Nothing, which a delivery takes as already delivered rather than as a failure to retry. + expect(try store.body(of: batch)).to(beNil()) + store.remove(batch) + + expect(store.pendingBatches()) == [other] + expect(store.pendingEventCount) == 1 + } + + it("keeps every batch it counted when the directory cannot be listed, and says the listing is incomplete") { + _ = store.stage(EventStoreSpec.payload("first")) + let first = store.closeBatch()! + _ = store.stage(EventStoreSpec.payload("second")) + let second = store.closeBatch()! + _ = store.pendingBatches() + + let directory = store.directory! + // Writable and searchable but not readable: its files can be created, renamed and read by name, and + // the directory cannot be listed. + let unlistable: [FileAttributeKey: Any] = [.posixPermissions: 0o300] + let listable: [FileAttributeKey: Any] = [.posixPermissions: 0o755] + try FileManager.default.setAttributes(unlistable, ofItemAtPath: directory.path) + defer { try? FileManager.default.setAttributes(listable, ofItemAtPath: directory.path) } + + _ = store.stage(EventStoreSpec.payload("third")) + let third = store.closeBatch()! + + // Not empty: the batch just closed is still there to send, and nothing stops counting. + let listing = store.listPendingBatches() + expect(listing.isComplete) == false + expect(listing.batches) == [first, second, third] + expect(store.pendingEventCount) == 3 + expect(EventStoreSpec.keys(ofBody: try? store.body(of: third))) == ["third"] + + try FileManager.default.setAttributes(listable, ofItemAtPath: directory.path) + expect(store.listPendingBatches()) == BatchListing(batches: [first, second, third], isComplete: true) + } + + it("keeps a log it could not close open, and closes all of it the next time") { + _ = store.stage(EventStoreSpec.payload("first")) + store.commit() + + let directory = store.directory! + // A directory with no write permission is one nothing in it can be renamed in. + let readOnly: [FileAttributeKey: Any] = [.posixPermissions: 0o555] + let writable: [FileAttributeKey: Any] = [.posixPermissions: 0o755] + try FileManager.default.setAttributes(readOnly, ofItemAtPath: directory.path) + defer { try? FileManager.default.setAttributes(writable, ofItemAtPath: directory.path) } + + expect(store.closeBatch()).to(beNil()) + expect(store.holdsUnclosedEvents) == true + expect(store.pendingEventCount) == 1 + + try FileManager.default.setAttributes(writable, ofItemAtPath: directory.path) + _ = store.stage(EventStoreSpec.payload("second")) + store.commit() + + // One batch holding both, named for what it holds, and no log left behind for a later launch. + let batch = store.closeBatch()! + expect(batch.eventCount) == 2 + expect(EventStoreSpec.keys(ofBody: try? store.body(of: batch))) == ["first", "second"] + expect(store.holdsUnclosedEvents) == false + expect(store.pendingBatches()) == [batch] + let logs = try FileManager.default.contentsOfDirectory(atPath: directory.path) + .filter { $0.hasPrefix(EventDirectory.logPrefix) } + expect(logs).to(beEmpty()) + } + } + } + + private func withoutPersistenceSpec() { + describe("finding its directory") { + for persistEvents in [false, true] { + it(persistEvents ? "works out where its directory is once, when it first reaches for the disk" + : "never works out where its directory is") { + let lock = UnfairLock() + var located = 0 + let directory = FileManager.default.temporaryDirectory + .appendingPathComponent("com.launchdarkly.tests.events", isDirectory: true) + .appendingPathComponent(UUID().uuidString, isDirectory: true) + let store = EventStore(locatingDirectory: { lock.withLock { located += 1 }; return directory }, + capacity: 100, + persistEvents: persistEvents, + logger: .disabled) + defer { try? FileManager.default.removeItem(at: directory) } + expect(lock.withLock { located }) == 0 + #if os(macOS) || targetEnvironment(macCatalyst) + if persistEvents { + store.locksOpenLog = false + } + #endif + + store.preparePayloadsFromInterruptedLogs() + _ = store.stage(EventStoreSpec.payload("first")) + store.commit() + let batch = store.closeBatch()! + _ = store.stage(EventStoreSpec.payload("second")) + expect(store.pendingBatches().map(\.payloadId)) == [batch.payloadId] + expect(EventStoreSpec.keys(ofBody: try store.body(of: batch))) == ["first"] + store.startedSending(batch) + store.stoppedSending(batch) + store.remove(batch) + expect(store.pendingEventPayloads().count) == 1 + expect(store.pendingEventCount) == 1 + + expect(lock.withLock { located }) == (persistEvents ? 1 : 0) + } + } + } + + describe("the store without persistence") { + it("lists its batches oldest first, as the store with a disk does") { + let store = EventStore.temporary(persistEvents: false) + var closed: [String] = [] + // Enough that hash order coming out right by chance is not a worry. + for index in 0..<20 { + _ = store.stage(EventStoreSpec.payload("event-\(index)"), bypassingCapacity: false) + if let batch = store.closeBatch() { + closed.append(batch.payloadId) + } + } + + expect(store.pendingBatches().map(\.payloadId)) == closed + + store.remove(EventBatch(payloadId: closed[3], eventCount: 1)) + closed.remove(at: 3) + expect(store.pendingBatches().map(\.payloadId)) == closed + expect(EventStoreSpec.keys(ofBody: try store.body(of: EventBatch(payloadId: closed[3], eventCount: 1)))) == ["event-4"] + } + } + } + + private func capacitySpec() { + describe("capacity") { + var store: EventStore! + afterEach { + store.deleteEverything() + } + + it("refuses events once it is reached") { + store = EventStore.temporary(capacity: 2) + + expect(store.stage(EventStoreSpec.payload("first"))) == true + expect(store.stage(EventStoreSpec.payload("second"))) == true + expect(store.stage(EventStoreSpec.payload("third"))) == false + expect(store.pendingEventCount) == 2 + } + + it("counts the events in a batch that has not been delivered yet") { + store = EventStore.temporary(capacity: 1) + + expect(store.stage(EventStoreSpec.payload("first"))) == true + _ = store.closeBatch() + + expect(store.stage(EventStoreSpec.payload("second"))) == false + } + + it("does not count a batch while its request is out, and counts it again if the send is to be retried") { + store = EventStore.temporary(capacity: 1) + expect(store.stage(EventStoreSpec.payload("first"))) == true + let batch = store.closeBatch()! + + store.startedSending(batch) + expect(store.pendingEventCount) == 0 + expect(store.stage(EventStoreSpec.payload("while-sending"))) == true + + store.stoppedSending(batch) + expect(store.pendingEventCount) == 2 + expect(store.stage(EventStoreSpec.payload("after-retryable"))) == false + } + + it("takes a sent batch out of the count only once when it is delivered") { + store = EventStore.temporary(capacity: 2) + expect(store.stage(EventStoreSpec.payload("first"))) == true + let batch = store.closeBatch()! + expect(store.stage(EventStoreSpec.payload("second"))) == true + + store.startedSending(batch) + store.remove(batch) + expect(store.pendingEventCount) == 1 + store.stoppedSending(batch) + expect(store.pendingEventCount) == 1 + _ = store.pendingBatches() + expect(store.pendingEventCount) == 1 + } + + it("does the same without persistence") { + store = EventStore.temporary(capacity: 1, persistEvents: false) + expect(store.stage(EventStoreSpec.payload("first"))) == true + let batch = store.closeBatch()! + + store.startedSending(batch) + expect(store.pendingEventCount) == 0 + expect(store.stage(EventStoreSpec.payload("while-sending"))) == true + store.stoppedSending(batch) + expect(store.pendingEventCount) == 2 + expect(store.stage(EventStoreSpec.payload("after-retryable"))) == false + store.startedSending(batch) + store.remove(batch) + expect(store.pendingEventCount) == 1 + store.stoppedSending(batch) + expect(store.pendingEventCount) == 1 + } + + // As when another process delivered it and a listing dropped it before this one got to it: leaving it out + // would take its events from the batches still counted, and admit that many more. + for persistEvents in [true, false] { + it("does not leave a batch already removed out of the count when its send starts, persisting: \(persistEvents)") { + store = EventStore.temporary(capacity: 2, persistEvents: persistEvents) + expect(store.stage(EventStoreSpec.payload("first"))) == true + let removed = store.closeBatch()! + expect(store.stage(EventStoreSpec.payload("second"))) == true + _ = store.closeBatch() + store.remove(removed) + + store.startedSending(removed) + expect(store.pendingEventCount) == 1 + expect(store.stage(EventStoreSpec.payload("third"))) == true + expect(store.stage(EventStoreSpec.payload("fourth"))) == false + + store.stoppedSending(removed) + expect(store.pendingEventCount) == 2 + } + } + + it("does not leave a batch another process delivered out of the count when its send starts") { + store = EventStore.temporary(capacity: 2) + expect(store.stage(EventStoreSpec.payload("first"))) == true + let delivered = store.closeBatch()! + expect(store.stage(EventStoreSpec.payload("second"))) == true + _ = store.closeBatch() + let other = EventStore.sharing(store.directory!) + _ = other.pendingBatches() + other.remove(delivered) + _ = store.pendingBatches() + + store.startedSending(delivered) + expect(store.pendingEventCount) == 1 + expect(store.stage(EventStoreSpec.payload("third"))) == true + expect(store.stage(EventStoreSpec.payload("fourth"))) == false + } + + // The bound `LDConfig.eventCapacity` documents: the reporter reads the disk in the background after starting, + // so what an earlier run left only counts from that listing on. + it("counts what an earlier run left undelivered from the first listing on") { + let earlier = EventStore.temporary(capacity: 2) + defer { earlier.deleteEverything() } + _ = earlier.stage(EventStoreSpec.payload("earlier-1")) + _ = earlier.stage(EventStoreSpec.payload("earlier-2")) + expect(earlier.closeBatch()).toNot(beNil()) + store = EventStore(directory: earlier.directory!, capacity: 2, logger: .disabled) + + expect(store.stage(EventStoreSpec.payload("before-listing"))) == true + + store.preparePayloadsFromInterruptedLogs() + expect(store.pendingBatches().count) == 1 + expect(store.pendingEventCount) == 3 + expect(store.stage(EventStoreSpec.payload("after-listing"))) == false + } + + it("counts the events a commit is still writing, so a slow write does not make room") { + let writing = DispatchSemaphore(value: 0) + let resume = DispatchSemaphore(value: 0) + var blocked = false + store = EventStore.temporary(capacity: 2, writeLog: { file, bytes in + if !blocked { + blocked = true + writing.signal() + resume.wait() + } + return file.writeAll(bytes) + }) + expect(store.stage(EventStoreSpec.payload("first"))) == true + expect(store.stage(EventStoreSpec.payload("second"))) == true + + let committed = DispatchSemaphore(value: 0) + DispatchQueue.global().async { + store.commit() + committed.signal() + } + writing.wait() + + expect(store.pendingEventCount) == 2 + expect(store.stage(EventStoreSpec.payload("third"))) == false + + resume.signal() + committed.wait() + + expect(store.pendingEventCount) == 2 + expect(store.stage(EventStoreSpec.payload("fourth"))) == false + } + + it("takes a summary even when it is full, since those evaluations were already counted") { + store = EventStore.temporary(capacity: 1) + + expect(store.stage(EventStoreSpec.payload("first"))) == true + expect(store.stage(EventStoreSpec.payload("summary"), bypassingCapacity: true)) == true + // Kept, but not counted as an event. + expect(store.pendingEventCount) == 1 + } + + // Written at every commit point, summaries would otherwise fill capacity during an outage, and full events + // would be refused for the summaries of evaluations already counted. + for persistEvents in [true, false] { + it("leaves summaries out of the event count in every state, persisting: \(persistEvents)") { + store = EventStore.temporary(capacity: 2, persistEvents: persistEvents) + expect(store.stage(EventStoreSpec.payload("summary-1"), bypassingCapacity: true)) == true + expect(store.stage(EventStoreSpec.payload("event-1"))) == true + expect(store.pendingEventCount).to(equal(1), description: "staged") + store.commit() + expect(store.pendingEventCount).to(equal(1), description: "committed") + + let batch = store.closeBatch()! + expect(batch.eventCount).to(equal(2), description: "the batch holds both frames") + expect(store.pendingEventCount).to(equal(1), description: "closed") + expect(store.pendingBatches()) == [batch] + expect(store.pendingEventCount).to(equal(1), description: "listed") + + expect(store.stage(EventStoreSpec.payload("summary-2"), bypassingCapacity: true)) == true + expect(store.stage(EventStoreSpec.payload("event-2"))) == true + expect(store.stage(EventStoreSpec.payload("event-3"))).to(equal(false), description: "capacity still counts the events") + + store.startedSending(batch) + expect(store.pendingEventCount).to(equal(1), description: "sending") + store.stoppedSending(batch) + expect(store.pendingEventCount).to(equal(2), description: "sent and failed") + store.remove(batch) + expect(store.pendingEventCount).to(equal(1), description: "delivered") + } + } + + it("does not count as events the summaries a failed write left in memory") { + let lock = UnfairLock() + var failing = false + store = EventStore.temporary(writeLog: { file, bytes in + lock.withLock { failing } ? ENOSPC : file.writeAll(bytes) + }) + _ = store.stage(EventStoreSpec.payload("written")) + _ = store.stage(EventStoreSpec.payload("summary-written"), bypassingCapacity: true) + store.commit() + lock.withLock { failing = true } + _ = store.stage(EventStoreSpec.payload("put-back")) + _ = store.stage(EventStoreSpec.payload("summary-put-back"), bypassingCapacity: true) + store.commit() + expect(store.isPersisting) == false + expect(store.pendingEventCount) == 2 + + let fromLog = store.closeBatch()! + let fromMemory = store.pendingBatches().last! + expect(EventStoreSpec.keys(ofBody: try store.body(of: fromLog))) == ["written", "summary-written"] + expect(EventStoreSpec.keys(ofBody: try store.body(of: fromMemory))) == ["put-back", "summary-put-back"] + expect(store.pendingEventCount) == 2 + _ = store.pendingBatches() + expect(store.pendingEventCount) == 2 + } + + for persistEvents in [true, false] { + it("stops counting a batch once when two deliveries both remove it (persisting: \(persistEvents))") { + store = EventStore.temporary(persistEvents: persistEvents) + _ = store.stage(EventStoreSpec.payload("first")) + let first = store.closeBatch()! + _ = store.stage(EventStoreSpec.payload("second")) + _ = store.closeBatch() + _ = store.pendingBatches() + + store.remove(first) + store.remove(first) + + expect(store.pendingEventCount) == 1 + } + } + + it("does not list a delivered batch again when its file will not delete, and deletes it when it can") { + store = EventStore.temporary() + _ = store.stage(EventStoreSpec.payload("first")) + let first = store.closeBatch()! + _ = store.stage(EventStoreSpec.payload("second")) + let second = store.closeBatch()! + + let directory = store.directory! + // A directory with no write permission is one its entries cannot be unlinked from, which is what the + // filesystem refusing a delete looks like from here. + let readOnly: [FileAttributeKey: Any] = [.posixPermissions: 0o555] + let writable: [FileAttributeKey: Any] = [.posixPermissions: 0o755] + try? FileManager.default.setAttributes(readOnly, ofItemAtPath: directory.path) + defer { try? FileManager.default.setAttributes(writable, ofItemAtPath: directory.path) } + + store.remove(first) + + // Delivered, so it neither counts against capacity nor comes back for another delivery, even though + // its file is still sitting there. + expect(store.pendingBatches().map(\.payloadId)) == [second.payloadId] + expect(store.pendingEventCount) == 1 + + try? FileManager.default.setAttributes(writable, ofItemAtPath: directory.path) + + // The listing that can delete it does, rather than leaving it for a run that would send it again. + _ = store.pendingBatches() + let left = try FileManager.default.contentsOfDirectory(at: directory, includingPropertiesForKeys: nil) + expect(left.filter { $0.lastPathComponent.hasPrefix("ready-v1-\(first.payloadId)") }).to(beEmpty()) + } + + it("stops counting a batch once when another process delivered it and a listing came between") { + store = EventStore.temporary() + _ = store.stage(EventStoreSpec.payload("first")) + let first = store.closeBatch()! + _ = store.stage(EventStoreSpec.payload("second")) + _ = store.closeBatch() + + EventStoreSpec.reader(sharing: store).remove(first) + _ = store.pendingBatches() + store.remove(first) + + expect(store.pendingEventCount) == 1 + } + + it("finds a batch deleted unsent lost where no other process shares the directory") { + store = EventStore.temporary() + store.sharesDirectoryWithOtherProcesses = false + _ = store.stage(EventStoreSpec.payload("deleted")) + let batch = store.closeBatch()! + + try FileManager.default.removeItem(at: EventStoreSpec.onlyBatchFile(in: store)) + + expect { try store.body(of: batch) }.to(throwError(errorType: VanishedBatch.self)) + } + + it("finds a batch another store in this process removed delivered, where no other process shares the directory") { + store = EventStore.temporary() + store.sharesDirectoryWithOtherProcesses = false + _ = store.stage(EventStoreSpec.payload("delivered")) + let batch = store.closeBatch()! + + EventStoreSpec.reader(sharing: store).remove(batch) + + expect(try store.body(of: batch)).to(beNil()) + } + + it("lists the events of a batch deleted unsent as vanished, once, where no other process shares the directory") { + store = EventStore.temporary() + store.sharesDirectoryWithOtherProcesses = false + _ = store.stage(EventStoreSpec.payload("deleted")) + _ = store.stage(EventStoreSpec.payload("with it")) + let deleted = store.closeBatch()! + _ = store.stage(EventStoreSpec.payload("delivered")) + let delivered = store.closeBatch()! + + try FileManager.default.removeItem( + at: store.directory!.appendingPathComponent("ready-v1-\(deleted.payloadId).\(deleted.eventCount)")) + EventStoreSpec.reader(sharing: store).remove(delivered) + + expect(store.listPendingBatches().vanishedEvents) == 2 + expect(store.listPendingBatches().vanishedEvents) == 0 + expect(store.pendingEventCount) == 0 + } + + it("lists the events of a directory deleted whole as vanished, where no other process shares it") { + store = EventStore.temporary() + store.sharesDirectoryWithOtherProcesses = false + _ = store.stage(EventStoreSpec.payload("deleted")) + _ = store.closeBatch() + + try FileManager.default.removeItem(at: store.directory!) + + expect(store.listPendingBatches().vanishedEvents) == 1 + expect(store.pendingEventCount) == 0 + } + + it("lists nothing as vanished from a directory another process shares") { + store = EventStore.temporary() + store.sharesDirectoryWithOtherProcesses = true + _ = store.stage(EventStoreSpec.payload("delivered")) + _ = store.closeBatch() + + try FileManager.default.removeItem(at: EventStoreSpec.onlyBatchFile(in: store)) + + expect(store.listPendingBatches().vanishedEvents) == 0 + expect(store.pendingEventCount) == 0 + } + + it("finds a batch gone from a directory another process shares delivered") { + store = EventStore.temporary() + store.sharesDirectoryWithOtherProcesses = true + _ = store.stage(EventStoreSpec.payload("delivered")) + let batch = store.closeBatch()! + + try FileManager.default.removeItem(at: EventStoreSpec.onlyBatchFile(in: store)) + + expect(try store.body(of: batch)).to(beNil()) + } + + it("hands over the events of an open log deleted unsent as vanished, once, where no other process shares the directory") { + store = EventStore.temporary() + store.sharesDirectoryWithOtherProcesses = false + _ = store.stage(EventStoreSpec.payload("deleted")) + _ = store.stage(EventStoreSpec.payload("with it")) + store.commit() + + try FileManager.default.removeItem(at: store.openLogUrl!) + + expect(store.closeBatch()).to(beNil()) + expect(store.holdsUnclosedEvents) == false + expect(store.pendingEventCount) == 0 + expect(store.takeVanishedLogEvents()) == 2 + expect(store.takeVanishedLogEvents()) == 0 + } + + #if os(macOS) || targetEnvironment(macCatalyst) + it("hands over the events of a locked open log deleted unsent as vanished, though another process shares the directory") { + store = EventStore.temporary() + store.sharesDirectoryWithOtherProcesses = true + store.locksOpenLog = true + _ = store.stage(EventStoreSpec.payload("deleted")) + store.commit() + + try FileManager.default.removeItem(at: store.openLogUrl!) + + expect(store.closeBatch()).to(beNil()) + expect(store.takeVanishedLogEvents()) == 1 + } + + it("hands over nothing as vanished from an unlocked open log gone from a directory another process shares") { + store = EventStore.temporary() + store.sharesDirectoryWithOtherProcesses = true + _ = store.stage(EventStoreSpec.payload("recovered elsewhere")) + store.commit() + + try FileManager.default.removeItem(at: store.openLogUrl!) + + expect(store.closeBatch()).to(beNil()) + expect(store.pendingEventCount) == 0 + expect(store.takeVanishedLogEvents()) == 0 + } + #endif + + it("hands over nothing as vanished from a log persistence gave up on, which recovery may have taken") { + var writes = 0 + store = EventStore.temporary(writeLog: { file, bytes in + writes += 1 + // The file header and the first event land; the device is full from then on. + return writes > 2 ? ENOSPC : file.writeAll(bytes) + }) + store.sharesDirectoryWithOtherProcesses = false + _ = store.stage(EventStoreSpec.payload("written")) + store.commit() + _ = store.stage(EventStoreSpec.payload("held")) + store.commit() + expect(store.isPersisting) == false + + // No longer held, the log is one recovery takes, as it would one a process that died left behind. + EventStoreSpec.reader(sharing: store).preparePayloadsFromInterruptedLogs() + + _ = store.closeBatch() + expect(store.takeVanishedLogEvents()) == 0 + expect(store.pendingBatches().flatMap { EventStoreSpec.keys(ofBody: try? store.body(of: $0)) } + .sorted()) == ["held", "written"] + } + + for persistEvents in [true, false] { + it("neither loses nor doubles a batch closed while a listing runs (persisting: \(persistEvents))") { + store = EventStore.temporary(capacity: 100_000, persistEvents: persistEvents) + let listing = store! + let doneLock = UnfairLock() + var done = false + let listerFinished = DispatchSemaphore(value: 0) + DispatchQueue.global().async { + while !doneLock.withLock({ done }) { + _ = listing.pendingBatches() + // The locks are not fair, and without a pause the listings can hold them so continuously + // that the closes starve. A yield is not enough: it returns at once while the closing thread + // is blocked on a lock rather than runnable. + usleep(200) + } + listerFinished.signal() + } + + // Checked after every close, because the next listing would correct a miscount before the end of + // the loop could see it. + var miscounts: [String] = [] + for closed in 1...150 { + _ = store.stage(EventStoreSpec.payload("event-\(closed)")) + _ = store.closeBatch() + let counted = store.pendingEventCount + if counted != closed { + miscounts.append("\(counted) after \(closed) closes") + } + } + doneLock.withLock { done = true } + listerFinished.wait() + + expect(miscounts).to(beEmpty()) + } + } + } + } + + /// Events framed in a quarter of a batch each: four of them and the file header are just past the batch size, so + /// three fit in each batch. + private func batchSizeSpec() { + describe("batch size") { + let quarter = EventLogFormat.maxBatchSize / 4 + var store: EventStore! + afterEach { + store.deleteEverything() + } + + func stage(_ indices: ClosedRange) { + for index in indices { + expect(store.stage(EventStoreSpec.payload(framedIn: quarter, key: "event-\(index)"))) == true + } + } + + func keysByBatch() -> [[String]] { + store.pendingBatches().map { batch -> [String] in + let body = try? store.body(of: batch) + expect(body?.count ?? 0) <= EventLogFormat.maxBatchSize + let keys = EventStoreSpec.keys(ofBody: body) + expect(keys.count) == batch.eventCount + return keys + } + // Batches closed within the same instant list in either order. + .sorted { $0.first ?? "" < $1.first ?? "" } + } + + func batchFileSizes() throws -> [Int] { + try FileManager.default + .contentsOfDirectory(at: store.directory!, includingPropertiesForKeys: [.fileSizeKey]) + .filter { $0.lastPathComponent.hasPrefix("ready-") } + .map { try $0.resourceValues(forKeys: [.fileSizeKey]).fileSize ?? 0 } + } + + it("closes a log a commit would take past the batch size, and carries on in a new one") { + store = EventStore.temporary() + stage(1...5) + store.commit() + _ = store.closeBatch() + + expect(keysByBatch()) == [["event-1", "event-2", "event-3"], ["event-4", "event-5"]] + expect(try batchFileSizes().allSatisfy { $0 <= EventLogFormat.maxBatchSize }) == true + expect(store.pendingEventCount) == 5 + } + + it("closes a log a later commit would take past the batch size") { + store = EventStore.temporary() + stage(1...2) + store.commit() + stage(3...5) + store.commit() + _ = store.closeBatch() + + expect(keysByBatch()) == [["event-1", "event-2", "event-3"], ["event-4", "event-5"]] + expect(try batchFileSizes().allSatisfy { $0 <= EventLogFormat.maxBatchSize }) == true + } + + it("splits what it holds in memory at the batch size too") { + store = EventStore.temporary(persistEvents: false) + stage(1...5) + _ = store.closeBatch() + + expect(keysByBatch()) == [["event-1", "event-2", "event-3"], ["event-4", "event-5"]] + expect(store.pendingEventCount) == 5 + } + + it("assembles a stored batch in the bytes it was read into, rather than in a copy") { + var readInto: UnsafeRawPointer? + store = EventStore.temporary(readFile: { url in + let file = try Data(contentsOf: url) + readInto = file.withUnsafeBytes { $0.baseAddress } + return file + }) + stage(1...3) + let batch = store.closeBatch()! + + let body = try? store.body(of: batch) + expect(body?.withUnsafeBytes { $0.baseAddress }) == readInto + expect(EventStoreSpec.keys(ofBody: body)) == ["event-1", "event-2", "event-3"] + } + + it("takes an event that fills a batch on its own, and refuses any larger") { + store = EventStore.temporary() + let largest = EventLogFormat.maxFrameSize + EventLogFormat.frameOverhead + expect(store.stage(EventStoreSpec.payload(framedIn: largest + 1, key: "too-large"))) == false + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-1"))) == true + expect(store.stage(EventStoreSpec.payload(framedIn: largest, key: "largest"))) == true + _ = store.closeBatch() + + expect(keysByBatch()) == [["event-1"], ["largest"]] + expect(try batchFileSizes().allSatisfy { $0 <= EventLogFormat.maxBatchSize }) == true + } + } + } + + /// A byte capacity of 1000 lets full events take the store to 800 bytes and summaries to 1000. Every event here is + /// framed in exactly 100 bytes, so eight full events fill it and two summaries fill the rest. + private func byteCapacitySpec() { + describe("byte capacity") { + var store: EventStore! + afterEach { + store.deleteEverything() + } + + it("refuses full events past what it holds back for summaries") { + store = EventStore.temporary(capacity: 100, byteCapacity: 1000) + + for index in 1...8 { + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-\(index)"))) == true + } + expect(store.hasRoomForEventBytes) == false + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-9"))) == false + expect(store.pendingEventCount) == 8 + } + + it("takes summaries into what it holds back, and refuses them past the byte capacity") { + store = EventStore.temporary(capacity: 100, byteCapacity: 1000) + for index in 1...8 { + _ = store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-\(index)")) + } + + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "summary-1"), bypassingCapacity: true)) == true + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "summary-2"), bypassingCapacity: true)) == true + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "summary-3"), bypassingCapacity: true)) == false + // Their bytes count, but they are not events. + expect(store.pendingEventCount) == 8 + } + + it("refuses a reserved event that turns out not to fit, and gives up its reservation") { + store = EventStore.temporary(capacity: 100, byteCapacity: 1000) + for index in 1...7 { + _ = store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-\(index)")) + } + store.reserve(1) + + expect(store.stageReserved(EventStoreSpec.payload(framedIn: 200, key: "too-big"))) == false + expect(store.pendingEventCount) == 7 + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "fits"))) == true + } + + it("makes room again once a batch is delivered") { + store = EventStore.temporary(capacity: 100, byteCapacity: 1000) + for index in 1...8 { + _ = store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-\(index)")) + } + let batch = store.closeBatch()! + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "while-held"))) == false + + store.startedSending(batch) + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "while-sending"))) == true + store.remove(batch) + expect(store.hasRoomForEventBytes) == true + } + + it("counts the bytes of a batch it read to count from what it read") { + store = EventStore.temporary(capacity: 100, byteCapacity: 1000) + for index in 1...8 { + _ = store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-\(index)")) + } + _ = store.closeBatch() + try EventStoreSpec.asUncountedName(EventStoreSpec.onlyBatchFile(in: store)) + + // Gone once it has been read, so that only what was read, or what the listing saw, can size it. + let reader = EventStore.sharing(store.directory!, byteCapacity: 1000, readFile: { url in + let read = try Data(contentsOf: url) + try? FileManager.default.removeItem(at: url) + return read + }) + expect(reader.pendingBatches().map { $0.eventCount }) == [8] + expect(reader.hasRoomForEventBytes) == false + } + + it("bounds the batches it holds in memory once it has given up on the disk") { + store = EventStore.temporary(capacity: 100, byteCapacity: 1000, writeLog: { _, _ in ENOSPC }) + for index in 1...8 { + _ = store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-\(index)")) + } + store.commit() + expect(store.isPersisting) == false + expect(store.closeBatch()).toNot(beNil()) + + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "past-capacity"))) == false + } + + it("bounds summaries by bytes even where it keeps events off the disk") { + // Nothing else bounds them: capacity never refuses a summary, and a delivery that keeps failing keeps + // every one it was given. + store = EventStore.temporary(capacity: 100, byteCapacity: 1000, persistEvents: false) + for index in 1...8 { + _ = store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-\(index)")) + } + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "summary-1"), bypassingCapacity: true)) == true + expect(store.closeBatch()?.eventCount) == 9 + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "summary-2"), bypassingCapacity: true)) == true + + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "summary-3"), bypassingCapacity: true)) == false + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-9"))) == false + } + + it("bounds full events by bytes where it keeps events off the disk, leaving the summaries' fifth") { + // Capacity counts events, so in memory through an outage it would otherwise hold that many of any size. + store = EventStore.temporary(capacity: 100, byteCapacity: 1000, persistEvents: false) + for index in 1...8 { + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-\(index)"))) == true + } + expect(store.hasRoomForEventBytes) == false + expect(store.admitsEvent(heldElsewhere: 0, capacity: 100)) == false + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-9"))) == false + expect(store.pendingEventCount) == 8 + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "summary"), bypassingCapacity: true)) == true + } + + it("admits an event by count and by bytes in one check") { + store = EventStore.temporary(capacity: 100, byteCapacity: 1000, persistEvents: false) + _ = store.stage(EventStoreSpec.payload(framedIn: 100, key: "event")) + expect(store.admitsEvent(heldElsewhere: 98, capacity: 100)) == true + expect(store.admitsEvent(heldElsewhere: 99, capacity: 100)) == false + } + + it("counts committed bytes as well as staged ones") { + store = EventStore.temporary(capacity: 100, byteCapacity: 1000) + for index in 1...8 { + _ = store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-\(index)")) + } + store.commit() + + expect(store.stage(EventStoreSpec.payload(framedIn: 100, key: "after-commit"))) == false + } + + it("counts the bytes of a batch another process closed from the listing, without reading it") { + store = EventStore.temporary() + for index in 1...8 { + _ = store.stage(EventStoreSpec.payload(framedIn: 100, key: "event-\(index)")) + } + expect(store.closeBatch()).toNot(beNil()) + + var reads = 0 + let other = EventStore.sharing(store.directory!, byteCapacity: 1000) { url in + reads += 1 + return try Data(contentsOf: url) + } + expect(other.pendingBatches().count) == 1 + + expect(reads) == 0 + expect(other.hasRoomForEventBytes) == false + expect(other.stage(EventStoreSpec.payload(framedIn: 100, key: "other"))) == false + } + } + } + + /// Each store here is the current environment of an application `app` under a root of the test's own, since the + /// first expiry also looks at every other environment beside it. + private func expirySpec() { + describe("expiry") { + var root: URL! + var store: EventStore! + beforeEach { + root = EventStore.temporaryDirectory() + store = EventStore.sharing(root.appendingPathComponent("app.current", isDirectory: true)) + } + afterEach { + try? FileManager.default.removeItem(at: root) + } + + it("discards a batch last written longer ago than the maximum age, and stops counting it") { + _ = store.stage(EventStoreSpec.payload("old-1")) + _ = store.stage(EventStoreSpec.payload("old-2")) + _ = store.closeBatch() + try EventStoreSpec.age(try EventStoreSpec.onlyBatchFile(in: store), byDays: 8) + + expect(store.discardExpired()) == 2 + + expect(store.pendingBatches()).to(beEmpty()) + expect(store.pendingEventCount) == 0 + expect(store.stage(EventStoreSpec.payload("new"))) == true + } + + it("discards an expired batch another version named without counting it") { + _ = store.stage(EventStoreSpec.payload("first")) + _ = store.stage(EventStoreSpec.payload("second")) + _ = store.closeBatch() + let batch = try EventStoreSpec.asAnotherVersionsBatch(try EventStoreSpec.onlyBatchFile(in: store)) + let reader = EventStoreSpec.reader(sharing: store) + + expect(reader.discardExpired()) == 0 + expect(FileManager.default.fileExists(atPath: batch.path)) == true + + try EventStoreSpec.age(batch, byDays: 8) + expect(reader.discardExpired()) == 0 + expect(FileManager.default.fileExists(atPath: batch.path)) == false + } + + it("keeps a batch younger than the maximum age") { + _ = store.stage(EventStoreSpec.payload("recent")) + _ = store.closeBatch() + try EventStoreSpec.age(try EventStoreSpec.onlyBatchFile(in: store), byDays: 6) + + expect(store.discardExpired()) == 0 + expect(store.pendingBatches().count) == 1 + } + + it("keeps a batch whose request is out, however old") { + _ = store.stage(EventStoreSpec.payload("sending")) + let batch = store.closeBatch()! + try EventStoreSpec.age(try EventStoreSpec.onlyBatchFile(in: store), byDays: 30) + store.startedSending(batch) + + expect(store.discardExpired()) == 0 + expect(try store.body(of: batch)).toNot(beNil()) + } + + it("leaves a batch another store in this process is sending until it stops") { + let sender = EventStore.sharing(store.directory!) + _ = sender.stage(EventStoreSpec.payload("sending")) + let batch = sender.closeBatch()! + let file = try EventStoreSpec.onlyBatchFile(in: sender) + try EventStoreSpec.age(file, byDays: 30) + sender.startedSending(batch) + + expect(store.discardExpired()) == 0 + expect(FileManager.default.fileExists(atPath: file.path)) == true + + sender.stoppedSending(batch) + expect(store.discardExpired()) == 1 + expect(FileManager.default.fileExists(atPath: file.path)) == false + } + + #if os(macOS) || targetEnvironment(macCatalyst) + it("leaves a batch another process is sending until it stops") { + _ = store.stage(EventStoreSpec.payload("sending")) + _ = store.closeBatch() + let file = try EventStoreSpec.onlyBatchFile(in: store) + try EventStoreSpec.age(file, byDays: 30) + // A lock taken through a descriptor of its own contends as another process's would. + let otherProcess = EventLogFile.open(at: file, flags: O_RDONLY)! + expect(otherProcess.tryLock(shared: true)) == true + + expect(store.discardExpired()) == 0 + expect(FileManager.default.fileExists(atPath: file.path)) == true + + otherProcess.close() + expect(store.discardExpired()) == 1 + expect(FileManager.default.fileExists(atPath: file.path)) == false + } + #endif + + it("leaves the directory of another environment open in this process") { + let open = EventStore.sharing(root.appendingPathComponent("app.open", isDirectory: true)) + _ = open.stage(EventStoreSpec.payload("open")) + _ = open.closeBatch() + try EventStoreSpec.age(contentsOf: open.directory!, byDays: 30) + + expect(store.discardExpired()) == 0 + expect(open.pendingBatches().count) == 1 + } + + it("discards a log nobody holds once nothing has written to it for the maximum age, and leaves a younger one") { + func logLeftBy(_ key: String) -> URL { + // Going away closes its log, as its process ending would. + let ended = EventStore.sharing(store.directory!) + _ = ended.stage(EventStoreSpec.payload(key)) + ended.commit() + return ended.openLogUrl! + } + let old = logLeftBy("gone") + let young = logLeftBy("recent") + try EventStoreSpec.age(old, byDays: 8) + try EventStoreSpec.age(young, byDays: 1) + + expect(store.discardExpired()) == 1 + + expect(FileManager.default.fileExists(atPath: old.path)) == false + expect(FileManager.default.fileExists(atPath: young.path)) == true + } + + it("leaves the log of another live store alone, however long since it was written") { + // A live process may go longer than the maximum age without an event. + let alive = EventStore.sharing(store.directory!) + _ = alive.stage(EventStoreSpec.payload("alive")) + alive.commit() + try EventStoreSpec.age(alive.openLogUrl!, byDays: 8) + + expect(store.discardExpired()) == 0 + expect(FileManager.default.fileExists(atPath: alive.openLogUrl!.path)) == true + } + + it("discards a log a process died while opening, which holds no events") { + try FileManager.default.createDirectory(at: store.directory!, withIntermediateDirectories: true) + let opening = store.directory!.appendingPathComponent(EventDirectory.openingLogPrefix + UUID().uuidString) + try EventLogFormat.fileHeader.write(to: opening) + try EventStoreSpec.age(opening, byDays: 8) + + expect(store.discardExpired()) == 0 + expect(FileManager.default.fileExists(atPath: opening.path)) == false + } + + it("leaves its own open log alone, however long since it was written") { + _ = store.stage(EventStoreSpec.payload("own")) + store.commit() + try EventStoreSpec.age(store.openLogUrl!, byDays: 8) + + expect(store.discardExpired()) == 0 + expect(EventStoreSpec.keys(of: store)) == ["own"] + } + + it("removes the directories of this application's other environments once nothing has written to them for the maximum age") { + let abandoned = try EventStoreSpec.environment("app.abandoned", in: root, events: 2) + let inUse = try EventStoreSpec.environment("app.in-use", in: root, events: 1) + let otherApplication = try EventStoreSpec.environment("other.abandoned", in: root, events: 1) + try EventStoreSpec.age(contentsOf: abandoned, byDays: 8) + try EventStoreSpec.age(contentsOf: inUse, byDays: 1) + try EventStoreSpec.age(contentsOf: otherApplication, byDays: 30) + + expect(store.discardExpired()) == 2 + // Once per process: the next call has nothing else to look at. + expect(store.discardExpired()) == 0 + + expect(FileManager.default.fileExists(atPath: abandoned.path)) == false + expect(try FileManager.default.contentsOfDirectory(atPath: inUse.path)).toNot(beEmpty()) + expect(try FileManager.default.contentsOfDirectory(atPath: otherApplication.path)).toNot(beEmpty()) + } + + it("empties an abandoned environment of batches another version named, without counting them") { + let abandoned = try EventStoreSpec.environment("app.abandoned", in: root, events: 2) + let batch = try FileManager.default.contentsOfDirectory(at: abandoned, includingPropertiesForKeys: nil) + .first { $0.lastPathComponent.hasPrefix("ready-") }! + try EventStoreSpec.asAnotherVersionsBatch(batch) + try EventStoreSpec.age(contentsOf: abandoned, byDays: 8) + + expect(store.discardExpired()) == 0 + expect(FileManager.default.fileExists(atPath: abandoned.path)) == false + } + + it("does nothing where the application did not ask for persistence") { + let abandoned = try EventStoreSpec.environment("app.abandoned", in: root, events: 1) + try EventStoreSpec.age(contentsOf: abandoned, byDays: 8) + store = EventStore.sharing(root.appendingPathComponent("app.current", isDirectory: true), persistEvents: false) + + expect(store.discardExpired()) == 0 + expect(try FileManager.default.contentsOfDirectory(atPath: abandoned.path)).toNot(beEmpty()) + } + } + } + + /// An environment directory under `root` holding one closed batch of `events` events. + private static func environment(_ name: String, in root: URL, events: Int) throws -> URL { + let directory = root.appendingPathComponent(name, isDirectory: true) + let writer = EventStore.sharing(directory) + for index in 0.. URL { + let directory = root.appendingPathComponent(directory, isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + let batch = directory.appendingPathComponent(name) + try Data([1, 2, 3]).write(to: batch) + return batch + } + + it("gives the batches of one directory one lock, however their URLs were come by, and another directory its own") { + let first = try batch(in: "one", named: "ready-a-1") + let second = URL(fileURLWithPath: first.deletingLastPathComponent().path + "/ready-b-1") + let elsewhere = try batch(in: "other", named: "ready-a-1") + + expect(BatchLocks.lockForTesting(first) === BatchLocks.lockForTesting(second)) == true + expect(BatchLocks.lockForTesting(first) === BatchLocks.lockForTesting(elsewhere)) == false + } + + it("expires in one directory while another directory's lock is held, and waits on its own") { + let held = try batch(in: "held", named: "ready-held-1") + let besideIt = try batch(in: "held", named: "ready-beside-1") + let elsewhere = try batch(in: "elsewhere", named: "ready-elsewhere-1") + let elsewhereDone = DispatchSemaphore(value: 0) + let besideItDone = DispatchSemaphore(value: 0) + + let lock = BatchLocks.lockForTesting(held) + lock.lock() + DispatchQueue.global().async { + _ = BatchLocks.deleteUnlessBeingSent(elsewhere) + elsewhereDone.signal() + } + let elsewhereWaited = elsewhereDone.wait(timeout: .now() + 5) == .timedOut + DispatchQueue.global().async { + _ = BatchLocks.deleteUnlessBeingSent(besideIt) + besideItDone.signal() + } + let besideItWaited = besideItDone.wait(timeout: .now() + 0.2) == .timedOut + lock.unlock() + + expect(elsewhereWaited) == false + expect(besideItWaited) == true + expect(besideItDone.wait(timeout: .now() + 5)) == .success + expect(FileManager.default.fileExists(atPath: elsewhere.path)) == false + expect(FileManager.default.fileExists(atPath: besideIt.path)) == false + } + + it("leaves a batch of the same name in another directory to expiry while one is sent") { + let sent = try batch(in: "sending", named: "ready-same-1") + let namesake = try batch(in: "expiring", named: "ready-same-1") + + BatchLocks.startedSending(sent) + expect(BatchLocks.isSentInThisProcess(sent)) == true + expect(BatchLocks.isSentInThisProcess(namesake)) == false + expect(BatchLocks.deleteUnlessBeingSent(sent)) == false + expect(BatchLocks.deleteUnlessBeingSent(namesake)) == true + BatchLocks.stoppedSending(sent) + + expect(BatchLocks.isSentInThisProcess(sent)) == false + expect(BatchLocks.deleteUnlessBeingSent(sent)) == true + } + } + + describe("a store the filesystem will not take") { + var store: EventStore! + beforeEach { + store = EventStoreSpec.unwritableStore() + } + afterEach { + // The blocking file, which is the parent of the directory the store never got to create. + try? FileManager.default.removeItem(at: store.directory!.deletingLastPathComponent()) + } + + it("delivers the events from memory rather than dropping them") { + _ = store.stage(EventStoreSpec.payload("first")) + _ = store.stage(EventStoreSpec.payload("second")) + store.commit() + + // The commit had nowhere to go, so the events are still the store's to deliver. + expect(store.pendingEventCount) == 2 + + let batch = store.closeBatch() + expect(batch?.eventCount) == 2 + expect(try store.body(of: batch!)) + == Data("[{\"kind\":\"custom\",\"key\":\"first\"},{\"kind\":\"custom\",\"key\":\"second\"}]".utf8) + } + + it("produces the same body a batch on disk would have") { + let written = EventStore.temporary() + defer { written.deleteEverything() } + _ = written.stage(EventStoreSpec.payload("same")) + let onDisk = written.closeBatch()! + + _ = store.stage(EventStoreSpec.payload("same")) + let held = store.closeBatch()! + + expect(try store.body(of: held)) == (try written.body(of: onDisk)) + } + + it("keeps accepting events after persistence has been given up on") { + _ = store.stage(EventStoreSpec.payload("before")) + store.commit() + + expect(store.stage(EventStoreSpec.payload("after"))) == true + } + + it("keeps the events in the order they were recorded across a failed commit") { + _ = store.stage(EventStoreSpec.payload("first")) + store.commit() + _ = store.stage(EventStoreSpec.payload("second")) + + let batch = store.closeBatch()! + expect(try store.body(of: batch)) + == Data("[{\"kind\":\"custom\",\"key\":\"first\"},{\"kind\":\"custom\",\"key\":\"second\"}]".utf8) + } + + it("delivers what memory was left holding beside what the disk already took") { + var writes = 0 + let interrupted = EventStore.temporary(writeLog: { file, bytes in + writes += 1 + // The file header and the first event land; the device is full from then on. + return writes > 2 ? ENOSPC : file.writeAll(bytes) + }) + defer { interrupted.deleteEverything() } + + _ = interrupted.stage(EventStoreSpec.payload("written")) + interrupted.commit() + _ = interrupted.stage(EventStoreSpec.payload("held")) + interrupted.commit() + expect(interrupted.isPersisting) == false + + _ = interrupted.closeBatch() + + // A failed write leaves events on both sides, and closing only the log would leave the rest to a + // delivery that reported success without them. + let bodies = interrupted.pendingBatches().map { EventStoreSpec.keys(ofBody: try? interrupted.body(of: $0)) } + expect(bodies) == [["written"], ["held"]] + expect(interrupted.pendingEventCount) == 2 + } + + it("sends the frames a write landed before it failed once, not from the log and memory both") { + var writes = 0 + let interrupted = EventStore.temporary(writeLog: { file, bytes in + writes += 1 + // The file header and the first event land. The next write gets a whole frame and part of another + // onto the disk before the device is full. + guard writes > 2 + else { return file.writeAll(bytes) } + _ = file.writeAll(bytes.dropLast(3)) + return ENOSPC + }) + defer { interrupted.deleteEverything() } + + _ = interrupted.stage(EventStoreSpec.payload("written")) + interrupted.commit() + _ = interrupted.stage(EventStoreSpec.payload("landed")) + _ = interrupted.stage(EventStoreSpec.payload("torn")) + interrupted.commit() + expect(interrupted.isPersisting) == false + + _ = interrupted.closeBatch() + + let bodies = interrupted.pendingBatches().map { EventStoreSpec.keys(ofBody: try? interrupted.body(of: $0)) } + expect(bodies) == [["written"], ["landed", "torn"]] + expect(interrupted.pendingEventCount) == 3 + } + + it("leaves the next run nothing to recover from a first write that failed partway") { + var writes = 0 + let interrupted = EventStore.temporary(writeLog: { file, bytes in + writes += 1 + // The file header lands, and then a whole frame and part of another before the device is full. + guard writes > 1 + else { return file.writeAll(bytes) } + _ = file.writeAll(bytes.dropLast(3)) + return ENOSPC + }) + defer { interrupted.deleteEverything() } + + _ = interrupted.stage(EventStoreSpec.payload("landed")) + _ = interrupted.stage(EventStoreSpec.payload("torn")) + interrupted.commit() + let fromMemory = interrupted.closeBatch() + expect(EventStoreSpec.keys(ofBody: fromMemory.flatMap { try? interrupted.body(of: $0) })) == ["landed", "torn"] + + // Delivered from memory this run, so the next must not find any of it again in the log. + let next = interrupted.nextRun() + next.preparePayloadsFromInterruptedLogs() + expect(next.pendingBatches()).to(beEmpty()) + } + + it("leaves the next run nothing to recover from a first write that failed partway and could not be cut back") { + var writes = 0 + let interrupted = EventStore.temporary(writeLog: { file, bytes in + writes += 1 + guard writes > 1 + else { return file.writeAll(bytes) } + _ = file.writeAll(bytes.dropLast(3)) + EventStoreSpec.refuseTruncating(file) + return ENOSPC + }) + defer { interrupted.deleteEverything() } + + _ = interrupted.stage(EventStoreSpec.payload("landed")) + _ = interrupted.stage(EventStoreSpec.payload("torn")) + interrupted.commit() + let fromMemory = interrupted.closeBatch() + expect(EventStoreSpec.keys(ofBody: fromMemory.flatMap { try? interrupted.body(of: $0) })) == ["landed", "torn"] + + // The whole frame that landed is in the log as well as in memory, under no count that would leave it + // out, so a later run recovering the log would send it a second time under another payload ID. + let next = interrupted.nextRun() + next.preparePayloadsFromInterruptedLogs() + expect(next.pendingBatches()).to(beEmpty()) + } + + it("holds a batch until it is removed, so a failed delivery can be retried") { + _ = store.stage(EventStoreSpec.payload("retry-me")) + let batch = store.closeBatch()! + + expect(store.pendingBatches()) == [batch] + expect(store.pendingEventCount) == 1 + + store.remove(batch) + + expect(store.pendingBatches()).to(beEmpty()) + expect(store.pendingEventCount) == 0 + } + + it("still refuses events once capacity is reached, so memory stays bounded") { + let bounded = EventStoreSpec.unwritableStore(capacity: 2) + defer { try? FileManager.default.removeItem(at: bounded.directory!.deletingLastPathComponent()) } + + expect(bounded.stage(EventStoreSpec.payload("first"))) == true + bounded.commit() + expect(bounded.stage(EventStoreSpec.payload("second"))) == true + expect(bounded.stage(EventStoreSpec.payload("third"))) == false + expect(bounded.pendingEventCount) == 2 + } + + it("has nothing to close when nothing was recorded") { + expect(store.closeBatch()).to(beNil()) + } + } + } +} diff --git a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/RetryStateSpec.swift b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/RetryStateSpec.swift index c3cec2665..288de3a1c 100644 --- a/LaunchDarkly/LaunchDarklyTests/ServiceObjects/RetryStateSpec.swift +++ b/LaunchDarkly/LaunchDarklyTests/ServiceObjects/RetryStateSpec.swift @@ -101,4 +101,117 @@ final class RetryStateSpec: XCTestCase { retry.recordFailure(unexpected: true) assertDelay(retry, inClosedRange: 600...600) } + + func testDeliveryNothingWaitsBeforeTheFirstFailure() { + XCTAssertEqual(DeliveryRetryState(flushInterval: 30).remainingWait, 0) + } + + func testDeliveryOrdinaryFailuresDoubleTheWaitUpToTheFlushInterval() { + let retry = DeliveryRetryState(flushInterval: 30) + for expected in [1, 2, 4, 8, 16, 30, 30] as [TimeInterval] { + XCTAssertEqual(retry.recordFailure(unexpected: false, retryAfter: nil), expected) + } + } + + func testDeliveryNormalCeilingIsNeverBelowASecond() { + let retry = DeliveryRetryState(flushInterval: 0.2) + retry.recordFailure(unexpected: false, retryAfter: nil) + XCTAssertEqual(retry.recordFailure(unexpected: false, retryAfter: nil), 1) + } + + func testDeliveryRefusalWaitsAboutEightAndAHalfMinutesAndDoublesToAnHour() { + let retry = DeliveryRetryState(flushInterval: 30) + for expected in [512, 1024, 2048, 3600] as [TimeInterval] { + XCTAssertEqual(retry.recordFailure(unexpected: true, retryAfter: nil), expected) + } + } + + func testDeliveryOrdinaryFailureAfterARefusalKeepsTheLongerCeiling() { + let retry = DeliveryRetryState(flushInterval: 30) + retry.recordFailure(unexpected: true, retryAfter: nil) + XCTAssertEqual(retry.recordFailure(unexpected: false, retryAfter: nil), 1024) + } + + func testDeliveryAcceptedDeliveryPutsEverythingBack() { + let retry = DeliveryRetryState(flushInterval: 30) + retry.recordFailure(unexpected: true, retryAfter: nil) + retry.recordFailure(unexpected: false, retryAfter: nil) + retry.recordSuccess() + XCTAssertEqual(retry.remainingWait, 0) + XCTAssertEqual(retry.recordFailure(unexpected: false, retryAfter: nil), 1) + } + + func testDeliveryRetryAfterSetsTheWaitJitteredToNoLessThanHalfOfIt() { + let retry = DeliveryRetryState(flushInterval: 30) + for _ in 0..<100 { + let wait = retry.recordFailure(unexpected: false, retryAfter: 10) + XCTAssertTrue((5...10).contains(wait), "\(wait) is outside 5...10") + } + } + + func testDeliveryRetryAfterIsCappedAtAnHour() { + let retry = DeliveryRetryState(initialDelay: 1, normalCeiling: 30, extendedCeiling: 3600, random: { 0 }) + XCTAssertEqual(retry.recordFailure(unexpected: false, retryAfter: 5 * 3600), 3600) + } + + func testDeliveryFailureWithARetryAfterStillCountsTowardTheBackoff() { + let retry = DeliveryRetryState(flushInterval: 30) + retry.recordFailure(unexpected: false, retryAfter: nil) + retry.recordFailure(unexpected: false, retryAfter: 0) + XCTAssertEqual(retry.recordFailure(unexpected: false, retryAfter: nil), 4) + } + + func testDeliveryFailureDuringTheWaitLeavesTheBackoffWhereItWas() { + let retry = DeliveryRetryState(flushInterval: 30) + retry.recordFailure(unexpected: true, retryAfter: nil) + let waiting = retry.remainingWait + for _ in 0..<5 { + XCTAssertLessThanOrEqual(retry.recordFailureDuringWait(unexpected: true, retryAfter: nil), waiting) + } + + XCTAssertLessThanOrEqual(retry.remainingWait, waiting) + // The next step is the one after the first refusal, not five steps on. + XCTAssertEqual(retry.recordFailure(unexpected: true, retryAfter: nil), 1024) + } + + func testDeliveryRetryAfterDuringTheWaitCanOnlyLengthenIt() { + let retry = DeliveryRetryState(initialDelay: 1, normalCeiling: 30, extendedCeiling: 3600, random: { 0 }) + retry.recordFailure(unexpected: true, retryAfter: nil) + let waiting = retry.remainingWait + + retry.recordFailureDuringWait(unexpected: false, retryAfter: 10) + XCTAssertLessThanOrEqual(retry.remainingWait, waiting) + + XCTAssertGreaterThan(retry.recordFailureDuringWait(unexpected: false, retryAfter: 20 * 60), 19 * 60) + XCTAssertGreaterThan(retry.remainingWait, 19 * 60) + } + + func testDeliveryFailureDuringAWaitThatHasRunOutIsAnOrdinaryOne() { + let retry = DeliveryRetryState(initialDelay: 0.02, normalCeiling: 1, extendedCeiling: 1) + retry.recordFailure(unexpected: false, retryAfter: nil) + Thread.sleep(forTimeInterval: 0.04) + + XCTAssertEqual(retry.recordFailureDuringWait(unexpected: false, retryAfter: nil), 0.04, accuracy: 0.0001) + } + + func testDeliveryWaitRunsOut() { + let retry = DeliveryRetryState(initialDelay: 0.02, normalCeiling: 0.02, extendedCeiling: 0.02) + retry.recordFailure(unexpected: false, retryAfter: nil) + XCTAssertGreaterThan(retry.remainingWait, 0) + Thread.sleep(forTimeInterval: 0.04) + XCTAssertEqual(retry.remainingWait, 0) + } + + func testRetryAfterHeaderIsReadAsSecondsOrAnHttpDate() { + func response(_ retryAfter: String?) -> HTTPURLResponse { + HTTPURLResponse(url: URL(string: "https://example.com")!, statusCode: 503, httpVersion: "1.1", + headerFields: retryAfter.map { ["retry-after": $0] })! + } + XCTAssertNil(response(nil).headerRetryAfter) + XCTAssertEqual(response("120").headerRetryAfter, 120) + XCTAssertNil(response("soon").headerRetryAfter) + XCTAssertEqual(response(DateFormatter.httpUrlHeaderFormatter.string(from: Date(timeIntervalSinceNow: -60))).headerRetryAfter, 0) + let date = DateFormatter.httpUrlHeaderFormatter.string(from: Date(timeIntervalSinceNow: 600)) + XCTAssertEqual(response(date).headerRetryAfter ?? 0, 600, accuracy: 2) + } } diff --git a/LaunchDarkly/LaunchDarklyTests/TestContext.swift b/LaunchDarkly/LaunchDarklyTests/TestContext.swift index 699805aac..7db1a7ef5 100644 --- a/LaunchDarkly/LaunchDarklyTests/TestContext.swift +++ b/LaunchDarkly/LaunchDarklyTests/TestContext.swift @@ -20,8 +20,8 @@ class TestContext { var flagSynchronizerMock: LDFlagSynchronizingMock! { subject.flagSynchronizer as? LDFlagSynchronizingMock } - var eventReporterMock: EventReportingMock! { - subject.eventReporter as? EventReportingMock + var eventProcessorMock: EventProcessorMock! { + subject.eventProcessor as? EventProcessorMock } var changeNotifierMock: FlagChangeNotifyingMock! { subject.flagChangeNotifier as? FlagChangeNotifyingMock @@ -42,7 +42,7 @@ class TestContext { serviceFactoryMock.onFlagSyncComplete } var recordedEvent: LaunchDarkly.Event? { - eventReporterMock.recordReceivedEvent + eventProcessorMock.recordReceivedEvent } var throttlerMock: ThrottlingMock? { subject.throttler as? ThrottlingMock diff --git a/LaunchDarkly/LaunchDarklyTests/TestUtil.swift b/LaunchDarkly/LaunchDarklyTests/TestUtil.swift index 523fa82a7..9d533c5ae 100644 --- a/LaunchDarkly/LaunchDarklyTests/TestUtil.swift +++ b/LaunchDarkly/LaunchDarklyTests/TestUtil.swift @@ -48,3 +48,106 @@ func valueIsArray(_ value: LDValue?, asserts: ([LDValue]) -> Void) { } asserts(arr) } + +extension EventStore { + /// A store in a directory of its own under the temporary directory, so that tests neither see each other's events + /// nor leave any in the directory a real client would use. + /// + /// Unlocked, like every store from `sharing`. + static func temporary( + capacity: Int = 100, + byteCapacity: Int = LDConfig.Defaults.eventCapacityBytes, + persistEvents: Bool = true, + readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }, + writeLog: @escaping (EventLogFile, Data) -> Int32? = { $0.writeAll($1) } + ) -> EventStore { + let directory = temporaryDirectory() + return unlocked(EventStore(directory: directory, + capacity: capacity, + byteCapacity: byteCapacity, + persistEvents: persistEvents, + logger: .disabled, + readFile: readFile, + writeLog: writeLog)) + } + + /// A directory of a test's own under the temporary directory. + /// + /// Handed in where the test has to look at the place a store would have written, which a store told to keep events + /// off the disk will not tell it: that store has no directory at all. + static func temporaryDirectory() -> URL { + FileManager.default.temporaryDirectory + .appendingPathComponent("com.launchdarkly.tests.events", isDirectory: true) + .appendingPathComponent(UUID().uuidString, isDirectory: true) + } + + /// Another store on `directory`, standing in for another process: it shares the batches, and leaves alone the log a + /// live store appends to. `nextRun` is the next run of the application. + /// + /// Unlocked on a Mac too, like every store from `temporary`; the locking itself has tests of its own. + static func sharing(_ directory: URL, + byteCapacity: Int = LDConfig.Defaults.eventCapacityBytes, + persistEvents: Bool = true, + readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }) -> EventStore { + unlocked(EventStore(directory: directory, + capacity: 100, + byteCapacity: byteCapacity, + persistEvents: persistEvents, + logger: .disabled, + readFile: readFile)) + } + + private static func unlocked(_ store: EventStore) -> EventStore { + #if os(macOS) || targetEnvironment(macCatalyst) + store.locksOpenLog = false + #endif + return store + } + + /// The next run of the application, had the process ended where it stands: a store on a copy of this one's + /// directory, which holds what reached the disk and nothing that was still in memory. No live process holds the + /// logs in the copy, as none would once the process was gone, so recovery takes them. + func nextRun(readFile: @escaping (URL) throws -> Data = { try Data(contentsOf: $0) }) -> EventStore { + let original = directory! + let copy = original.deletingLastPathComponent() + .appendingPathComponent("\(original.lastPathComponent)\(EventStore.nextRunSuffix)\(UUID().uuidString)", + isDirectory: true) + try? FileManager.default.copyItem(at: original, to: copy) + return EventStore.sharing(copy, readFile: readFile) + } + + private static let nextRunSuffix = "-next-run-" + + /// Every event in the store's directory, read straight from the files rather than through a store: the batches, + /// then the logs still open. What the next run of the application would find there. + var payloadsOnDisk: [Data] { + let files = ((try? FileManager.default.contentsOfDirectory(atPath: directory!.path)) ?? []).sorted() + let batches = files.filter { $0.hasPrefix("ready-") } + let logs = files.filter { $0.hasPrefix(EventDirectory.logPrefix) } + var payloads: [Data] = [] + for name in batches + logs { + guard let file = try? Data(contentsOf: directory!.appendingPathComponent(name)) + else { continue } + _ = EventLogFormat.forEachFrame(in: file) { type, payload in + if type == EventLogFormat.eventFrame { + payloads.append(payload) + } + } + } + return payloads + } + + /// Removes the store's directory, including any batch still waiting to be delivered, and any copy `nextRun` made of + /// it. A store told to keep events off the disk has no directory, and so nothing to remove. + func deleteEverything() { + guard let directory + else { return } + let parent = directory.deletingLastPathComponent() + let copies = ((try? FileManager.default.contentsOfDirectory(atPath: parent.path)) ?? []) + .filter { $0.hasPrefix(directory.lastPathComponent + EventStore.nextRunSuffix) } + for copy in copies { + try? FileManager.default.removeItem(at: parent.appendingPathComponent(copy)) + } + try? FileManager.default.removeItem(at: directory) + } +}