Handle child to parent broker migration

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift

@@ -25,6 +25,15 @@ struct DataBrokerScheduleConfig: Codable {
     let confirmOptOutScan: Int
     let maintenanceScan: Int
     let maxAttempts: Int
+
+    var optOutReattempt: Int {
+        let interval = maintenanceScan
+        guard interval > confirmOptOutScan else {
+            assertionFailure("We don't want another opt-out attempt before the next scan")
+            return Int.max
+        }
+        return interval
+    }
 }
 
 extension Int {

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift

@@ -115,7 +115,7 @@ class DataBrokerOperation: Operation, @unchecked Sendable {
         }
     }
 
-    private func filterAndSortOperationsData(brokerProfileQueriesData: [BrokerProfileQueryData], operationType: OperationType, priorityDate: Date?) -> [BrokerJobData] {
+    static func filterAndSortOperationsData(brokerProfileQueriesData: [BrokerProfileQueryData], operationType: OperationType, priorityDate: Date?) -> [BrokerJobData] {
         let operationsData: [BrokerJobData]
 
         switch operationType {
@@ -131,8 +131,8 @@ class DataBrokerOperation: Operation, @unchecked Sendable {
 
         if let priorityDate = priorityDate {
             filteredAndSortedOperationsData = operationsData
-                .filter { $0.preferredRunDate != nil && $0.preferredRunDate! <= priorityDate }
-                .sorted { $0.preferredRunDate! < $1.preferredRunDate! }
+                .filtered(using: priorityDate)
+                .sortedByPreferredRunDate()
         } else {
             filteredAndSortedOperationsData = operationsData
         }
@@ -152,9 +152,9 @@ class DataBrokerOperation: Operation, @unchecked Sendable {
 
         let brokerProfileQueriesData = allBrokerProfileQueryData.filter { $0.dataBroker.id == dataBrokerID }
 
-        let filteredAndSortedOperationsData = filterAndSortOperationsData(brokerProfileQueriesData: brokerProfileQueriesData,
-                                                                          operationType: operationType,
-                                                                          priorityDate: priorityDate)
+        let filteredAndSortedOperationsData = Self.filterAndSortOperationsData(brokerProfileQueriesData: brokerProfileQueriesData,
+                                                                               operationType: operationType,
+                                                                               priorityDate: priorityDate)
 
         Logger.dataBrokerProtection.debug("filteredAndSortedOperationsData count: \(filteredAndSortedOperationsData.count, privacy: .public) for brokerID \(self.dataBrokerID, privacy: .public)")
 
@@ -215,3 +215,40 @@ class DataBrokerOperation: Operation, @unchecked Sendable {
     }
 }
 // swiftlint:enable explicit_non_final_class
+
+extension Array where Element == BrokerJobData {
+    /// Filters jobs based on their preferred run date:
+    /// - Opt-out jobs with no preferred run date are included.
+    /// - Jobs with a preferred run date on or before the priority date are included.
+    ///
+    /// Note: Opt-out jobs without a preferred run date may be:
+    /// 1. From child brokers (will be skipped during runOptOutOperation).
+    /// 2. From former child brokers now acting as parent brokers (will be processed if extractedProfile hasn't been removed).
+    func filtered(using priorityDate: Date) -> [BrokerJobData] {
+        filter { jobData in
+            guard let preferredRunDate = jobData.preferredRunDate else {
+                return jobData is OptOutJobData
+            }
+
+            return preferredRunDate <= priorityDate
+        }
+    }
+
+    /// Sorts BrokerJobData array based on their preferred run dates.
+    /// - Jobs with non-nil preferred run dates are sorted in ascending order (earliest date first).
+    /// - Opt-out jobs with nil preferred run dates come last, maintaining their original relative order.
+    func sortedByPreferredRunDate() -> [BrokerJobData] {
+        sorted { lhs, rhs in
+            switch (lhs.preferredRunDate, rhs.preferredRunDate) {
+            case (nil, nil):
+                return false
+            case (_, nil):
+                return true
+            case (nil, _):
+                return false
+            case (let lhsRunDate?, let rhsRunDate?):
+                return lhsRunDate < rhsRunDate
+            }
+        }
+    }
+}

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift

@@ -291,11 +291,11 @@ struct DataBrokerProfileQueryOperationManager: OperationsManager {
         }
 
         guard extractedProfile.removedDate == nil else {
-            Logger.dataBrokerProtection.debug("Profile already extracted, skipping...")
+            Logger.dataBrokerProtection.debug("Profile already removed, skipping...")
             return
         }
 
-        guard let optOutStep = brokerProfileQueryData.dataBroker.optOutStep(), optOutStep.optOutType != .parentSiteOptOut else {
+        guard !brokerProfileQueryData.dataBroker.performsOptOutWithinParent() else {
             Logger.dataBrokerProtection.debug("Broker opts out in parent, skipping...")
             return
         }

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift

@@ -81,8 +81,10 @@ struct OperationPreferredDateCalculator {
             return date.now.addingTimeInterval(calculateNextRunDateOnError(schedulingConfig: schedulingConfig, historyEvents: historyEvents))
         case .optOutStarted, .scanStarted, .noMatchFound:
             return currentPreferredRunDate
-        case .optOutConfirmed, .optOutRequested:
+        case .optOutConfirmed:
             return nil
+        case .optOutRequested:
+            return date.now.addingTimeInterval(schedulingConfig.optOutReattempt.hoursToSeconds)
         }
     }
 

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionStatsPixels.swift

@@ -172,6 +172,10 @@ extension Date {
     static func nowMinus(hours: Int) -> Date {
         Calendar.current.date(byAdding: .hour, value: -hours, to: Date()) ?? Date()
     }
+
+    static func nowPlus(hours: Int) -> Date {
+        nowMinus(hours: -hours)
+    }
 }
 
 final class DataBrokerProtectionStatsPixels: StatsPixels {

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/backgroundcheck.run.json

@@ -1,8 +1,7 @@
 {
   "name": "backgroundcheck.run",
   "url": "backgroundcheck.run",
-  "version": "0.4.0",
-  "parent": "verecor.com",
+  "version": "0.5.0",
   "addedDatetime": 1677736800000,
   "optOutUrl": "https://backgroundcheck.run/ng/control/privacy",
   "steps": [
@@ -51,8 +50,70 @@
     },
     {
       "stepType": "optOut",
-      "optOutType": "parentSiteOptOut",
-      "actions": []
+      "optOutType": "formOptOut",
+      "actions": [
+        {
+          "actionType": "navigate",
+          "url": "https://backgroundcheck.run/ng/control/privacy",
+          "id": "fa29793c-3f85-4f01-a5fe-4ffcc26c197c"
+        },
+        {
+          "actionType": "fillForm",
+          "selector": ".ahm",
+          "elements": [
+            {
+              "type": "fullName",
+              "selector": "#name"
+            },
+            {
+              "type": "email",
+              "selector": "#email"
+            },
+            {
+              "type": "profileUrl",
+              "selector": "#url"
+            }
+          ],
+          "id": "71be571c-ce0c-43cb-afad-ae6547d44726"
+        },
+        {
+          "actionType": "click",
+          "elements": [
+            {
+              "type": "button",
+              "selector": ".//button[@type='submit']"
+            }
+          ],
+          "id": "e8794659-162d-4de7-9845-bbd140c54a00"
+        },
+        {
+          "actionType": "expectation",
+          "expectations": [
+            {
+              "type": "text",
+              "selector": "body",
+              "expect": "Your removal request has been received"
+            }
+          ],
+          "id": "6ab596e2-3642-4dba-97f0-1270d8feefd1"
+        },
+        {
+          "actionType": "emailConfirmation",
+          "pollingTime": 30,
+          "id": "9431f4e4-140a-4ade-8e74-3b7917b6ab2b"
+        },
+        {
+          "actionType": "expectation",
+          "expectations": [
+            {
+              "type": "text",
+              "selector": "body",
+              "expect": "Your information control request has been confirmed."
+            }
+          ],
+          "id": "92e1c4ff-4466-42e7-b2dd-70a319af48da"
+        }
+      ]
     }
   ],
   "schedulingConfig": {

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/clubset.com.json

@@ -1,8 +1,7 @@
 {
   "name": "Clubset",
   "url": "clubset.com",
-  "version": "0.4.0",
-  "parent": "verecor.com",
+  "version": "0.5.0",
   "addedDatetime": 1702965600000,
   "optOutUrl": "https://clubset.com/private/control/privacy",
   "steps": [
@@ -63,8 +62,80 @@
     },
     {
       "stepType": "optOut",
-      "optOutType": "parentSiteOptOut",
-      "actions": []
+      "optOutType": "formOptOut",
+      "actions": [
+        {
+          "actionType": "navigate",
+          "url": "https://clubset.com/private/control/privacy",
+          "id": "87fb7e0b-87ea-413d-847a-e88b3d023776"
+        },
+        {
+          "actionType": "fillForm",
+          "selector": ".ahm",
+          "elements": [
+            {
+              "type": "fullName",
+              "selector": "#user_name"
+            },
+            {
+              "type": "email",
+              "selector": "#user_email"
+            },
+            {
+              "type": "profileUrl",
+              "selector": "#url"
+            }
+          ],
+          "id": "48c3acd8-61fc-4680-9811-77dc5ba9c9a6"
+        },
+        {
+          "actionType": "getCaptchaInfo",
+          "selector": ".g-recaptcha",
+          "id": "47d8cf16-354f-4359-ac37-05c617d5f03e"
+        },
+        {
+          "actionType": "solveCaptcha",
+          "selector": ".g-recaptcha",
+          "id": "eb5e43df-d4dd-45e8-8192-bfcb3e814ed5"
+        },
+        {
+          "actionType": "click",
+          "elements": [
+            {
+              "type": "button",
+              "selector": ".//button[@type='submit']"
+            }
+          ],
+          "id": "2521002e-f825-4a93-aa6e-966f499096d1"
+        },
+        {
+          "actionType": "expectation",
+          "expectations": [
+            {
+              "type": "text",
+              "selector": "body",
+              "expect": "Your removal request has been received"
+            }
+          ],
+          "id": "f72634ca-60c3-450f-9656-52d924d8361f"
+        },
+        {
+          "actionType": "emailConfirmation",
+          "pollingTime": 30,
+          "id": "075640a6-f010-4135-9b46-25655e5eadd1"
+        },
+        {
+          "actionType": "expectation",
+          "expectations": [
+            {
+              "type": "text",
+              "selector": "body",
+              "expect": "Your information control request has been confirmed."
+            }
+          ],
+          "id": "96a083a1-f84b-42df-89bd-b5d0601468b2"
+        }
+      ]
     }
   ],
   "schedulingConfig": {

LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/councilon.com.json

@@ -1,8 +1,7 @@
 {
   "name": "Councilon",
   "url": "councilon.com",
-  "version": "0.4.0",
-  "parent": "verecor.com",
+  "version": "0.5.0",
   "addedDatetime": 1702965600000,
   "optOutUrl": "https://councilon.com/ex/control/privacy",
   "steps": [
@@ -63,8 +62,80 @@
     },
     {
       "stepType": "optOut",
-      "optOutType": "parentSiteOptOut",
-      "actions": []
+      "optOutType": "formOptOut",
+      "actions": [
+        {
+          "actionType": "navigate",
+          "url": "https://councilon.com/ex/control/privacy",
+          "id": "9f4ef020-811e-4d02-8622-ebe7a714a0d0"
+        },
+        {
+          "actionType": "fillForm",
+          "selector": ".ahm",
+          "elements": [
+            {
+              "type": "fullName",
+              "selector": "#user_name"
+            },
+            {
+              "type": "email",
+              "selector": "#user_email"
+            },
+            {
+              "type": "profileUrl",
+              "selector": "#url"
+            }
+          ],
+          "id": "8e7adf0a-58fb-4e57-b3b3-79cd99eeb395"
+        },
+        {
+          "actionType": "getCaptchaInfo",
+          "selector": ".g-recaptcha",
+          "id": "c80c3e5c-fa66-45c8-816b-bb320a247777"
+        },
+        {
+          "actionType": "solveCaptcha",
+          "selector": ".g-recaptcha",
+          "id": "30cfa92e-627c-4b00-9fd1-032952338468"
+        },
+        {
+          "actionType": "click",
+          "elements": [
+            {
+              "type": "button",
+              "selector": ".//button[@type='submit']"
+            }
+          ],
+          "id": "1d3fe64a-d23d-448c-bf34-aeccf32beb87"
+        },
+        {
+          "actionType": "expectation",
+          "expectations": [
+            {
+              "type": "text",
+              "selector": "body",
+              "expect": "Your removal request has been received"
+            }
+          ],
+          "id": "e6840eae-b334-431d-bbb8-94189e09023d"
+        },
+        {
+          "actionType": "emailConfirmation",
+          "pollingTime": 30,
+          "id": "8983034c-211a-4d98-8f31-bdf28a4e5011"
+        },
+        {
+          "actionType": "expectation",
+          "expectations": [
+            {
+              "type": "text",
+              "selector": "body",
+              "expect": "Your information control request has been confirmed."
+            }
+          ],
+          "id": "ffb143e1-a408-456b-9e27-d5b199cd0b52"
+        }
+      ]
     }
   ],
   "schedulingConfig": {