Remove LS rate config; add lsC telemetry (#593)

.github/workflows/test_ios_ut_4.yaml

@@ -20,12 +20,12 @@ jobs:
           xcode-version: 15.1.0
 
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
         with:
           fetch-depth: 0
 
       - name: Build iOS and Test 4
-        uses: sersoft-gmbh/xcodebuild-action@v1
+        uses: sersoft-gmbh/xcodebuild-action@v3
         with:
           action: build test
           build-settings: ONLY_ACTIVE_ARCH=NO TEST_AFTER_BUILD=YES

Split/Api/SplitClientConfig.swift

@@ -41,27 +41,6 @@ public class SplitClientConfig: NSObject {
     ///
     @objc public var segmentsRefreshRate: Int = 1800
 
-    ///
-    /// The SDK polls Split servers for changes to large segment definitions. This parameter controls this polling period in seconds.
-    /// This only applies when `largeSegmentsEnabled` is set to true and streaming is disabled.
-    /// Default: 60
-    ///
-    private static let kDefaultLargeSegmentsRefreshRate = 60
-    private var validLargeSegmentsRefreshRate = kDefaultLargeSegmentsRefreshRate
-    @objc public var largeSegmentsRefreshRate: Int {
-        get {
-            return validLargeSegmentsRefreshRate
-        }
-
-        set {
-            if newValue < Self.kDefaultLargeSegmentsRefreshRate {
-                Logger.w("Refresh rate value \(largeSegmentsRefreshRate) for large segments is invalid, using default")
-            } else {
-                validLargeSegmentsRefreshRate = newValue
-            }
-        }
-    }
-
     ///
     /// Default queue size for impressions. Default: 30K
     ///

Split/Common/Utils/Version.swift

@@ -9,7 +9,7 @@ import Foundation
 
 class Version {
     private static let kSdkPlatform: String = "ios"
-    private static let kVersion = "2.27.0-rc2"
+    private static let kVersion = "3.0.0-rc1"
 
     static var semantic: String {
         return kVersion

Split/FetcherEngine/Recorder/TelemetryConfigRecorderWorker.swift

@@ -45,7 +45,6 @@ class TelemetryConfigRecorderWorker: RecorderWorker {
 
         let rates = TelemetryRates(splits: splitConfig.featuresRefreshRate,
                                    mySegments: splitConfig.segmentsRefreshRate,
-                                   myLargeSegments: splitConfig.largeSegmentsRefreshRate,
                                    impressions: splitConfig.impressionRefreshRate,
                                    events: splitConfig.eventsPushRate,
                                    telemetry: splitConfig.internalTelemetryRefreshRate)

Split/FetcherEngine/Recorder/TelemetryStatsRecorderWorker.swift

@@ -14,16 +14,19 @@ class TelemetryStatsRecorderWorker: RecorderWorker {
     private let statsRecorder: HttpTelemetryStatsRecorder
     private let splitsStorage: SplitsStorage
     private let mySegmentsStorage: MySegmentsStorage
+    private let myLargeSegmentsStorage: MySegmentsStorage
 
     init(telemetryStatsRecorder: HttpTelemetryStatsRecorder,
          telemetryConsumer: TelemetryConsumer,
          splitsStorage: SplitsStorage,
-         mySegmentsStorage: MySegmentsStorage) {
+         mySegmentsStorage: MySegmentsStorage,
+         myLargeSegmentsStorage: MySegmentsStorage) {
 
         self.telemetryConsumer = telemetryConsumer
         self.statsRecorder = telemetryStatsRecorder
         self.splitsStorage = splitsStorage
         self.mySegmentsStorage = mySegmentsStorage
+        self.myLargeSegmentsStorage = myLargeSegmentsStorage
     }
 
     func flush() {
@@ -65,6 +68,7 @@ class TelemetryStatsRecorderWorker: RecorderWorker {
                               impressionsDropped: storage.getImpressionStats(type: .dropped),
                               splitCount: splitsStorage.getCount(),
                               segmentCount: mySegmentsStorage.getCount(),
+                              largeSegmentCount: myLargeSegmentsStorage.getCount(),
                               segmentKeyCount: nil,
                               sessionLengthMs: storage.getSessionLength(),
                               eventsQueued: storage.getEventStats(type: .queued),

Split/FetcherEngine/Refresh/RetryableSegmentsSyncWorker.swift

@@ -55,7 +55,7 @@ class RetryableMySegmentsSyncWorker: BaseRetryableSyncWorker {
                 return true
             }
         } catch {
-            Logger.e("Error while fetching splits in method: \(error.localizedDescription)")
+            Logger.e("Error while fetching segments in method: \(error.localizedDescription)")
             errorHandler?(error)
         }
         return false

Split/Network/Streaming/RetryableSplitsUpdateWorkerFactory.swift

@@ -214,7 +214,8 @@ class DefaultSyncWorkerFactory: SyncWorkerFactory {
         return TelemetryStatsRecorderWorker(telemetryStatsRecorder: telemetryStatsRecorder,
                                             telemetryConsumer: telemetryStorage,
                                             splitsStorage: storageContainer.splitsStorage,
-                                            mySegmentsStorage: storageContainer.mySegmentsStorage)
+                                            mySegmentsStorage: storageContainer.mySegmentsStorage,
+                                            myLargeSegmentsStorage: storageContainer.myLargeSegmentsStorage)
     }
 
     func createPeriodicTelemetryStatsRecorderWorker() -> PeriodicRecorderWorker? {
@@ -230,7 +231,8 @@ class DefaultSyncWorkerFactory: SyncWorkerFactory {
         let telemetryStatsWorker = TelemetryStatsRecorderWorker(telemetryStatsRecorder: telemetryStatsRecorder,
                                                                 telemetryConsumer: telemetryStorage,
                                                                 splitsStorage: storageContainer.splitsStorage,
-                                                                mySegmentsStorage: storageContainer.mySegmentsStorage)
+                                                                mySegmentsStorage: storageContainer.mySegmentsStorage,
+                                                                myLargeSegmentsStorage: storageContainer.myLargeSegmentsStorage)
 
         let timer = DefaultPeriodicTimer(deadline: splitConfig.internalTelemetryRefreshRate,
                                          interval: splitConfig.internalTelemetryRefreshRate)

Split/Storage/MySegments/MyLargeSegmentsStorage.swift

@@ -13,18 +13,24 @@ class MyLargeSegmentsStorage: MySegmentsStorage {
     private var inMemorySegments: SynchronizedDictionary<String, SegmentChange> = SynchronizedDictionary()
     private let persistentStorage: PersistentMySegmentsStorage
     private let defaultChangeNumber = ServiceConstants.defaultSegmentsChangeNumber
+    private let syncQueue: DispatchQueue
+    private let syncQueueKey = DispatchSpecificKey<Void>()
 
     var keys: Set<String> {
         return inMemorySegments.keys
     }
 
     init(persistentStorage: PersistentMySegmentsStorage) {
         self.persistentStorage = persistentStorage
+        self.syncQueue = DispatchQueue(label: "split-large-segments-storage")
+        syncQueue.setSpecific(key: syncQueueKey, value: ())
     }
 
     func loadLocal(forKey key: String) {
-        let change = persistentStorage.getSnapshot(forKey: key) ?? SegmentChange.empty()
-        inMemorySegments.setValue(change, forKey: key)
+        safeSync {
+            let change = persistentStorage.getSnapshot(forKey: key) ?? SegmentChange.empty()
+            inMemorySegments.setValue(change, forKey: key)
+        }
     }
 
     func changeNumber(forKey key: String) -> Int64? {
@@ -34,19 +40,24 @@ class MyLargeSegmentsStorage: MySegmentsStorage {
     func lowerChangeNumber() -> Int64 {
         return inMemorySegments.all.values.compactMap { $0.changeNumber }.min() ?? -1
     }
+
     func getAll(forKey key: String) -> Set<String> {
         return inMemorySegments.value(forKey: key)?.segments.compactMap { $0.name }.asSet() ?? Set<String>()
     }
 
     func set(_ change: SegmentChange, forKey key: String) {
-        inMemorySegments.setValue(change, forKey: key)
-        persistentStorage.set(change, forKey: key)
+        safeSync {
+            inMemorySegments.setValue(change, forKey: key)
+            persistentStorage.set(change, forKey: key)
+        }
     }
 
     func clear(forKey key: String) {
-        let clearChange = SegmentChange(segments: [])
-        inMemorySegments.setValue(clearChange, forKey: key)
-        persistentStorage.set(clearChange, forKey: key)
+        safeSync {
+            let clearChange = SegmentChange(segments: [])
+            inMemorySegments.setValue(clearChange, forKey: key)
+            persistentStorage.set(clearChange, forKey: key)
+        }
     }
 
     func destroy() {
@@ -58,11 +69,22 @@ class MyLargeSegmentsStorage: MySegmentsStorage {
     }
 
     func getCount() -> Int {
-        let keys = inMemorySegments.keys
-        var count = 0
-        for key in keys {
-            count+=(inMemorySegments.value(forKey: key)?.segments.count ?? 0)
+        safeSync {
+            let keys = inMemorySegments.keys
+            var count = 0
+            for key in keys {
+                count+=(inMemorySegments.value(forKey: key)?.segments.count ?? 0)
+            }
+            return count
+        }
+    }
+
+    // if already being executed in the queue, do not dispatch to it
+    private func safeSync<T>(_ block: () -> T) -> T {
+        if DispatchQueue.getSpecific(key: syncQueueKey) != nil {
+            return block()
+        } else {
+            return syncQueue.sync(execute: block)
         }
-        return count
     }
 }

Split/Storage/Splits/SplitsDecoder.swift

@@ -86,7 +86,7 @@ struct SplitsSerialDecoder: SplitsDecoder {
     func getSplit(_ json: String) throws -> Split {
 
         guard let data  = json.data(using: .utf8) else {
-            throw GenericError.unknown(message: "parsing pepe")
+            throw GenericError.unknown(message: "parsing error")
         }
 
         let jsonObj = try JSONSerialization.jsonObject(with: data, options: []) as? [String: Any]

Split/Telemetry/Storage/TelemetryConfig.swift

@@ -11,15 +11,13 @@ import Foundation
 struct TelemetryRates: Codable {
     var splits: Int
     var mySegments: Int
-    var myLargeSegments: Int
     var impressions: Int
     var events: Int
     var telemetry: Int
 
     enum CodingKeys: String, CodingKey {
         case splits = "sp"
         case mySegments = "ms"
-        case myLargeSegments = "mls"
         case impressions = "im"
         case events = "ev"
         case telemetry = "te"

Split/Telemetry/Storage/TelemetryStats.swift

@@ -198,6 +198,7 @@ struct TelemetryStats: Codable {
     var impressionsDropped: Int?
     var splitCount: Int?
     var segmentCount: Int?
+    var largeSegmentCount: Int?
     var segmentKeyCount: Int?
     var sessionLengthMs: Int64?
     var eventsQueued: Int?
@@ -219,6 +220,7 @@ struct TelemetryStats: Codable {
         case impressionsDropped = "iDr"
         case splitCount = "spC"
         case segmentCount = "seC"
+        case largeSegmentCount = "lsC"
         case segmentKeyCount = "skC"
         case sessionLengthMs = "sL"
         case eventsQueued = "eQ"

SplitTests/Integration/streaming/MySegmentUpdateTest.swift

@@ -64,7 +64,7 @@ class MySegmentUpdateTest: XCTestCase {
         }
 
         // Wait for hitting my segments two times (sdk ready and full sync after streaming connection)
-        wait(for: [sdkReadyExp, sseExp], timeout: 5)
+        wait(for: [sdkReadyExp, sseExp], timeout: 50)
 
         streamingBinding?.push(message: ":keepalive")
 
@@ -81,17 +81,22 @@ class MySegmentUpdateTest: XCTestCase {
         // Should not trigger any fetch to my segments because
         // this payload doesn't have "key1" enabled
 
+        Thread.sleep(forTimeInterval: 0.5)
         pushMessage(TestingData.escapedBoundedNotificationZlib(type: type, cn: mySegmentsCns[cnIndex()]))
 
         // Pushed key list message. Key 1 should add a segment
         sdkUpdExp = XCTestExpectation()
+
+        Thread.sleep(forTimeInterval: 0.5)
         pushMessage(TestingData.escapedKeyListNotificationGzip(type: type, cn: mySegmentsCns[cnIndex()]))
         wait(for: [sdkUpdExp], timeout: 5)
 
         sdkUpdExp = XCTestExpectation()
+        Thread.sleep(forTimeInterval: 0.5)
         pushMessage(TestingData.segmentRemovalNotification(type: type, cn: mySegmentsCns[cnIndex()]))
         wait(for: [sdkUpdExp], timeout: 5)
 
+        Thread.sleep(forTimeInterval: 2.0)
         var segmentEntity: [String]!
         if type == .mySegmentsUpdate {
             segmentEntity = db.mySegmentsDao.getBy(userKey: testFactory.userKey)?.segments.map { $0.name } ?? []

SplitTests/Service/Telemetry/TelemetryConfigRecorderWorkerTests.swift

@@ -51,7 +51,7 @@ class TelemetryConfigRecorderWorkerTests: XCTestCase {
 
     func createTelemetryConfig() -> TelemetryConfig {
 
-        let rates = TelemetryRates(splits: 1, mySegments: 2, myLargeSegments: 6, impressions: 3,
+        let rates = TelemetryRates(splits: 1, mySegments: 2, impressions: 3,
                                    events: 4, telemetry: 5)
         let urls = TelemetryUrlOverrides(sdk: true, events: true,
                                          auth: true, stream: true, telemetry: true)

SplitTests/Service/Telemetry/TelemetryStatsRecorderWorkerTests.swift

@@ -16,18 +16,21 @@ class TelemetryStatsRecorderWorkerTests: XCTestCase {
     var statsRecorder: HttpTelemetryStatsRecorderStub!
     var splitsStorage: SplitsStorageStub!
     var mySegmentsStorage: MySegmentsStorageStub!
+    var myLargeSegmentsStorage: MySegmentsStorageStub!
     var telemetryStorage: TelemetryStorageStub!
 
     override func setUp() {
         statsRecorder = HttpTelemetryStatsRecorderStub()
         telemetryStorage = TelemetryStorageStub()
         mySegmentsStorage = MySegmentsStorageStub()
+        myLargeSegmentsStorage = MySegmentsStorageStub()
         splitsStorage = SplitsStorageStub()
 
         worker = TelemetryStatsRecorderWorker(telemetryStatsRecorder: statsRecorder,
                                               telemetryConsumer: telemetryStorage,
                                               splitsStorage: splitsStorage,
-                                              mySegmentsStorage: mySegmentsStorage)
+                                              mySegmentsStorage: mySegmentsStorage,
+                                              myLargeSegmentsStorage: myLargeSegmentsStorage)
     }
 
     func testSendSuccess() {
@@ -38,6 +41,7 @@ class TelemetryStatsRecorderWorkerTests: XCTestCase {
         XCTAssertNotNil(statsRecorder.statsSent)
         XCTAssertEqual(1, splitsStorage.getCountCalledCount)
         XCTAssertEqual(1, mySegmentsStorage.getCountCalledCount)
+        XCTAssertEqual(1, myLargeSegmentsStorage.getCountCalledCount)
         XCTAssertEqual(1, telemetryStorage.popTagsCallCount)
 
     }
@@ -50,6 +54,7 @@ class TelemetryStatsRecorderWorkerTests: XCTestCase {
         XCTAssertEqual(3, statsRecorder.executeCallCount)
         XCTAssertEqual(1, splitsStorage.getCountCalledCount)
         XCTAssertEqual(1, mySegmentsStorage.getCountCalledCount)
+        XCTAssertEqual(1, myLargeSegmentsStorage.getCountCalledCount)
         XCTAssertEqual(1, telemetryStorage.popTagsCallCount)
 
     }
@@ -62,6 +67,7 @@ class TelemetryStatsRecorderWorkerTests: XCTestCase {
         XCTAssertEqual(3, statsRecorder.executeCallCount)
         XCTAssertEqual(1, splitsStorage.getCountCalledCount)
         XCTAssertEqual(1, mySegmentsStorage.getCountCalledCount)
+        XCTAssertEqual(1, myLargeSegmentsStorage.getCountCalledCount)
         XCTAssertEqual(1, telemetryStorage.popTagsCallCount)
 
     }
@@ -75,6 +81,7 @@ class TelemetryStatsRecorderWorkerTests: XCTestCase {
         XCTAssertEqual(0, statsRecorder.executeCallCount)
         XCTAssertEqual(0, splitsStorage.getCountCalledCount)
         XCTAssertEqual(0, mySegmentsStorage.getCountCalledCount)
+        XCTAssertEqual(0, myLargeSegmentsStorage.getCountCalledCount)
         XCTAssertEqual(0, telemetryStorage.popTagsCallCount)
     }
 
@@ -101,6 +108,7 @@ class TelemetryStatsRecorderWorkerTests: XCTestCase {
             XCTAssertNotNil(self.statsRecorder.statsSent)
             XCTAssertEqual(6, self.splitsStorage.getCountCalledCount)
             XCTAssertEqual(6, self.mySegmentsStorage.getCountCalledCount)
+            XCTAssertEqual(6, self.myLargeSegmentsStorage.getCountCalledCount)
             XCTAssertEqual(6, self.telemetryStorage.popTagsCallCount)
         }
     }