diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift index 4d87ae1416..501934a6ac 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Model/DataBroker.swift @@ -25,6 +25,15 @@ struct DataBrokerScheduleConfig: Codable { let confirmOptOutScan: Int let maintenanceScan: Int let maxAttempts: Int + + var optOutReattempt: Int { + let interval = maintenanceScan + guard interval > confirmOptOutScan else { + assertionFailure("We don't want another opt-out attempt before the next scan") + return Int.max + } + return interval + } } extension Int { diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift index a04ab7d0b4..b08867d313 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerOperation.swift @@ -115,7 +115,7 @@ class DataBrokerOperation: Operation, @unchecked Sendable { } } - private func filterAndSortOperationsData(brokerProfileQueriesData: [BrokerProfileQueryData], operationType: OperationType, priorityDate: Date?) -> [BrokerJobData] { + static func filterAndSortOperationsData(brokerProfileQueriesData: [BrokerProfileQueryData], operationType: OperationType, priorityDate: Date?) -> [BrokerJobData] { let operationsData: [BrokerJobData] switch operationType { @@ -131,8 +131,8 @@ class DataBrokerOperation: Operation, @unchecked Sendable { if let priorityDate = priorityDate { filteredAndSortedOperationsData = operationsData - .filter { $0.preferredRunDate != nil && $0.preferredRunDate! <= priorityDate } - .sorted { $0.preferredRunDate! < $1.preferredRunDate! } + .filtered(using: priorityDate) + .sortedByPreferredRunDate() } else { filteredAndSortedOperationsData = operationsData } @@ -152,9 +152,9 @@ class DataBrokerOperation: Operation, @unchecked Sendable { let brokerProfileQueriesData = allBrokerProfileQueryData.filter { $0.dataBroker.id == dataBrokerID } - let filteredAndSortedOperationsData = filterAndSortOperationsData(brokerProfileQueriesData: brokerProfileQueriesData, - operationType: operationType, - priorityDate: priorityDate) + let filteredAndSortedOperationsData = Self.filterAndSortOperationsData(brokerProfileQueriesData: brokerProfileQueriesData, + operationType: operationType, + priorityDate: priorityDate) Logger.dataBrokerProtection.debug("filteredAndSortedOperationsData count: \(filteredAndSortedOperationsData.count, privacy: .public) for brokerID \(self.dataBrokerID, privacy: .public)") @@ -215,3 +215,40 @@ class DataBrokerOperation: Operation, @unchecked Sendable { } } // swiftlint:enable explicit_non_final_class + +extension Array where Element == BrokerJobData { + /// Filters jobs based on their preferred run date: + /// - Opt-out jobs with no preferred run date are included. + /// - Jobs with a preferred run date on or before the priority date are included. + /// + /// Note: Opt-out jobs without a preferred run date may be: + /// 1. From child brokers (will be skipped during runOptOutOperation). + /// 2. From former child brokers now acting as parent brokers (will be processed if extractedProfile hasn't been removed). + func filtered(using priorityDate: Date) -> [BrokerJobData] { + filter { jobData in + guard let preferredRunDate = jobData.preferredRunDate else { + return jobData is OptOutJobData + } + + return preferredRunDate <= priorityDate + } + } + + /// Sorts BrokerJobData array based on their preferred run dates. + /// - Jobs with non-nil preferred run dates are sorted in ascending order (earliest date first). + /// - Opt-out jobs with nil preferred run dates come last, maintaining their original relative order. + func sortedByPreferredRunDate() -> [BrokerJobData] { + sorted { lhs, rhs in + switch (lhs.preferredRunDate, rhs.preferredRunDate) { + case (nil, nil): + return false + case (_, nil): + return true + case (nil, _): + return false + case (let lhsRunDate?, let rhsRunDate?): + return lhsRunDate < rhsRunDate + } + } + } +} diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift index 6bb37ceb99..eef9280e4c 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/DataBrokerProfileQueryOperationManager.swift @@ -291,11 +291,11 @@ struct DataBrokerProfileQueryOperationManager: OperationsManager { } guard extractedProfile.removedDate == nil else { - Logger.dataBrokerProtection.debug("Profile already extracted, skipping...") + Logger.dataBrokerProtection.debug("Profile already removed, skipping...") return } - guard let optOutStep = brokerProfileQueryData.dataBroker.optOutStep(), optOutStep.optOutType != .parentSiteOptOut else { + guard !brokerProfileQueryData.dataBroker.performsOptOutWithinParent() else { Logger.dataBrokerProtection.debug("Broker opts out in parent, skipping...") return } diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift index dae2ad1b09..10e0b7c9f6 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Operations/OperationPreferredDateCalculator.swift @@ -81,8 +81,10 @@ struct OperationPreferredDateCalculator { return date.now.addingTimeInterval(calculateNextRunDateOnError(schedulingConfig: schedulingConfig, historyEvents: historyEvents)) case .optOutStarted, .scanStarted, .noMatchFound: return currentPreferredRunDate - case .optOutConfirmed, .optOutRequested: + case .optOutConfirmed: return nil + case .optOutRequested: + return date.now.addingTimeInterval(schedulingConfig.optOutReattempt.hoursToSeconds) } } diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionStatsPixels.swift b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionStatsPixels.swift index 7534b98ee9..22798d7eb3 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionStatsPixels.swift +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Pixels/DataBrokerProtectionStatsPixels.swift @@ -172,6 +172,10 @@ extension Date { static func nowMinus(hours: Int) -> Date { Calendar.current.date(byAdding: .hour, value: -hours, to: Date()) ?? Date() } + + static func nowPlus(hours: Int) -> Date { + nowMinus(hours: -hours) + } } final class DataBrokerProtectionStatsPixels: StatsPixels { diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/backgroundcheck.run.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/backgroundcheck.run.json index d8ba2b8237..973b139e4c 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/backgroundcheck.run.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/backgroundcheck.run.json @@ -1,8 +1,7 @@ { "name": "backgroundcheck.run", "url": "backgroundcheck.run", - "version": "0.4.0", - "parent": "verecor.com", + "version": "0.5.0", "addedDatetime": 1677736800000, "optOutUrl": "https://backgroundcheck.run/ng/control/privacy", "steps": [ @@ -51,8 +50,70 @@ }, { "stepType": "optOut", - "optOutType": "parentSiteOptOut", - "actions": [] + "optOutType": "formOptOut", + "actions": [ + { + "actionType": "navigate", + "url": "https://backgroundcheck.run/ng/control/privacy", + "id": "fa29793c-3f85-4f01-a5fe-4ffcc26c197c" + }, + { + "actionType": "fillForm", + "selector": ".ahm", + "elements": [ + { + "type": "fullName", + "selector": "#name" + }, + { + "type": "email", + "selector": "#email" + }, + { + "type": "profileUrl", + "selector": "#url" + } + ], + "id": "71be571c-ce0c-43cb-afad-ae6547d44726" + }, + { + "actionType": "click", + "elements": [ + { + "type": "button", + "selector": ".//button[@type='submit']" + } + ], + "id": "e8794659-162d-4de7-9845-bbd140c54a00" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your removal request has been received" + } + ], + "id": "6ab596e2-3642-4dba-97f0-1270d8feefd1" + }, + { + "actionType": "emailConfirmation", + "pollingTime": 30, + "id": "9431f4e4-140a-4ade-8e74-3b7917b6ab2b" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your information control request has been confirmed." + } + ], + "id": "92e1c4ff-4466-42e7-b2dd-70a319af48da" + } + ] } ], "schedulingConfig": { diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/clubset.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/clubset.com.json index 3561e4963f..03f73fa09c 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/clubset.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/clubset.com.json @@ -1,8 +1,7 @@ { "name": "Clubset", "url": "clubset.com", - "version": "0.4.0", - "parent": "verecor.com", + "version": "0.5.0", "addedDatetime": 1702965600000, "optOutUrl": "https://clubset.com/private/control/privacy", "steps": [ @@ -63,8 +62,80 @@ }, { "stepType": "optOut", - "optOutType": "parentSiteOptOut", - "actions": [] + "optOutType": "formOptOut", + "actions": [ + { + "actionType": "navigate", + "url": "https://clubset.com/private/control/privacy", + "id": "87fb7e0b-87ea-413d-847a-e88b3d023776" + }, + { + "actionType": "fillForm", + "selector": ".ahm", + "elements": [ + { + "type": "fullName", + "selector": "#user_name" + }, + { + "type": "email", + "selector": "#user_email" + }, + { + "type": "profileUrl", + "selector": "#url" + } + ], + "id": "48c3acd8-61fc-4680-9811-77dc5ba9c9a6" + }, + { + "actionType": "getCaptchaInfo", + "selector": ".g-recaptcha", + "id": "47d8cf16-354f-4359-ac37-05c617d5f03e" + }, + { + "actionType": "solveCaptcha", + "selector": ".g-recaptcha", + "id": "eb5e43df-d4dd-45e8-8192-bfcb3e814ed5" + }, + { + "actionType": "click", + "elements": [ + { + "type": "button", + "selector": ".//button[@type='submit']" + } + ], + "id": "2521002e-f825-4a93-aa6e-966f499096d1" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your removal request has been received" + } + ], + "id": "f72634ca-60c3-450f-9656-52d924d8361f" + }, + { + "actionType": "emailConfirmation", + "pollingTime": 30, + "id": "075640a6-f010-4135-9b46-25655e5eadd1" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your information control request has been confirmed." + } + ], + "id": "96a083a1-f84b-42df-89bd-b5d0601468b2" + } + ] } ], "schedulingConfig": { diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/councilon.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/councilon.com.json index f15cee8c2e..e5eb48aae1 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/councilon.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/councilon.com.json @@ -1,8 +1,7 @@ { "name": "Councilon", "url": "councilon.com", - "version": "0.4.0", - "parent": "verecor.com", + "version": "0.5.0", "addedDatetime": 1702965600000, "optOutUrl": "https://councilon.com/ex/control/privacy", "steps": [ @@ -63,8 +62,80 @@ }, { "stepType": "optOut", - "optOutType": "parentSiteOptOut", - "actions": [] + "optOutType": "formOptOut", + "actions": [ + { + "actionType": "navigate", + "url": "https://councilon.com/ex/control/privacy", + "id": "9f4ef020-811e-4d02-8622-ebe7a714a0d0" + }, + { + "actionType": "fillForm", + "selector": ".ahm", + "elements": [ + { + "type": "fullName", + "selector": "#user_name" + }, + { + "type": "email", + "selector": "#user_email" + }, + { + "type": "profileUrl", + "selector": "#url" + } + ], + "id": "8e7adf0a-58fb-4e57-b3b3-79cd99eeb395" + }, + { + "actionType": "getCaptchaInfo", + "selector": ".g-recaptcha", + "id": "c80c3e5c-fa66-45c8-816b-bb320a247777" + }, + { + "actionType": "solveCaptcha", + "selector": ".g-recaptcha", + "id": "30cfa92e-627c-4b00-9fd1-032952338468" + }, + { + "actionType": "click", + "elements": [ + { + "type": "button", + "selector": ".//button[@type='submit']" + } + ], + "id": "1d3fe64a-d23d-448c-bf34-aeccf32beb87" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your removal request has been received" + } + ], + "id": "e6840eae-b334-431d-bbb8-94189e09023d" + }, + { + "actionType": "emailConfirmation", + "pollingTime": 30, + "id": "8983034c-211a-4d98-8f31-bdf28a4e5011" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your information control request has been confirmed." + } + ], + "id": "ffb143e1-a408-456b-9e27-d5b199cd0b52" + } + ] } ], "schedulingConfig": { diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/curadvisor.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/curadvisor.com.json index f415eab9f9..8176c3940e 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/curadvisor.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/curadvisor.com.json @@ -1,8 +1,7 @@ { "name": "CurAdvisor", "url": "curadvisor.com", - "version": "0.4.0", - "parent": "verecor.com", + "version": "0.5.0", "addedDatetime": 1703052000000, "optOutUrl": "https://curadvisor.com/nada/control/privacy", "steps": [ @@ -63,8 +62,80 @@ }, { "stepType": "optOut", - "optOutType": "parentSiteOptOut", - "actions": [] + "optOutType": "formOptOut", + "actions": [ + { + "actionType": "navigate", + "url": "https://curadvisor.com/nada/control/privacy", + "id": "0a0fd3f4-4505-4ebc-bbbe-819b86ac18a7" + }, + { + "actionType": "fillForm", + "selector": ".ahm", + "elements": [ + { + "type": "fullName", + "selector": "#user_name" + }, + { + "type": "email", + "selector": "#user_email" + }, + { + "type": "profileUrl", + "selector": "#url" + } + ], + "id": "0cb160f0-3475-4476-a87f-2a084bcbd4d8" + }, + { + "actionType": "getCaptchaInfo", + "selector": ".g-recaptcha", + "id": "5ebf6fc6-a3ee-41f2-875f-82e9d11d24e9" + }, + { + "actionType": "solveCaptcha", + "selector": ".g-recaptcha", + "id": "22ec8469-c3f8-41e5-8537-6406e817da06" + }, + { + "actionType": "click", + "elements": [ + { + "type": "button", + "selector": ".//button[@type='submit']" + } + ], + "id": "d02dd323-1508-4be7-b2d2-72f4bd67fc95" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your removal request has been received" + } + ], + "id": "be51eb91-bba3-4f2d-8e33-d973372e281d" + }, + { + "actionType": "emailConfirmation", + "pollingTime": 30, + "id": "e417378b-bb62-4d63-ad8b-f2e1c3b30e1a" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your information control request has been confirmed." + } + ], + "id": "e47c20c6-c45f-4d75-b514-06ffa97e35d5" + } + ] } ], "schedulingConfig": { diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json index cbe6abc734..b90a1fae58 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/kwold.com.json @@ -1,76 +1,147 @@ { - "name": "Kwold", - "url": "kwold.com", - "version": "0.4.0", - "parent": "verecor.com", - "addedDatetime": 1702965600000, - "optOutUrl": "https://kwold.com/ns/control/privacy", - "steps": [ - { - "stepType": "scan", - "scanType": "templatedUrl", - "actions": [ + "name": "Kwold", + "url": "kwold.com", + "version": "0.5.0", + "addedDatetime": 1702965600000, + "optOutUrl": "https://kwold.com/ns/control/privacy", + "steps": [ { - "actionType": "navigate", - "id": "878e00ab-dbad-4ca9-a303-645702a36ee2", - "url": "https://kwold.com/profile/search?fname=${firstName}&lname=${lastName}&state=${state}&city=${city}&fage=${age|ageRange}", - "ageRange": [ - "18-30", - "31-40", - "41-50", - "51-60", - "61-70", - "71-80", - "81+" - ] + "stepType": "scan", + "scanType": "templatedUrl", + "actions": [ + { + "actionType": "navigate", + "id": "878e00ab-dbad-4ca9-a303-645702a36ee2", + "url": "https://kwold.com/profile/search?fname=${firstName}&lname=${lastName}&state=${state}&city=${city}&fage=${age|ageRange}", + "ageRange": [ + "18-30", + "31-40", + "41-50", + "51-60", + "61-70", + "71-80", + "81+" + ] + }, + { + "actionType": "extract", + "id": "ec9f8ae6-199e-441b-9722-ffc6737b4595", + "selector": ".card", + "noResultsSelector": "//div[@class='page-404' and h1[starts-with(text(), 'Sorry')]]", + "profile": { + "name": { + "selector": ".card-title", + "beforeText": " ~" + }, + "alternativeNamesList": { + "selector": ".//div[@class='card-body']/dl[dt[text()='Known as:']]/dd/ul[@class='list-inline m-0']/li", + "findElements": true + }, + "age": { + "beforeText": "years old", + "selector": ".card-title", + "afterText": " ~" + }, + "addressCityStateList": { + "selector": ".//div[@class='card-body']/dl[dt[text()='Has lived in:']]/dd/ul[@class='list-inline m-0']/li", + "findElements": true + }, + "relativesList": { + "selector": ".//div[@class='card-body']/dl[dt[text()='Related to:']]/dd/ul[@class='list-inline m-0']/li", + "beforeText": ",", + "findElements": true + }, + "profileUrl": { + "selector": "a", + "identifierType": "path", + "identifier": "https://kwold.com/pp/${id}" + } + } + } + ] }, { - "actionType": "extract", - "id": "ec9f8ae6-199e-441b-9722-ffc6737b4595", - "selector": ".card", - "noResultsSelector": "//div[@class='page-404' and h1[starts-with(text(), 'Sorry')]]", - "profile": { - "name": { - "selector": ".card-title", - "beforeText": " ~" - }, - "alternativeNamesList": { - "selector": ".//div[@class='card-body']/dl[dt[text()='Known as:']]/dd/ul[@class='list-inline m-0']/li", - "findElements": true - }, - "age": { - "beforeText": "years old", - "selector": ".card-title", - "afterText": " ~" - }, - "addressCityStateList": { - "selector": ".//div[@class='card-body']/dl[dt[text()='Has lived in:']]/dd/ul[@class='list-inline m-0']/li", - "findElements": true - }, - "relativesList": { - "selector": ".//div[@class='card-body']/dl[dt[text()='Related to:']]/dd/ul[@class='list-inline m-0']/li", - "beforeText": ",", - "findElements": true - }, - "profileUrl": { - "selector": "a", - "identifierType": "path", - "identifier": "https://kwold.com/pp/${id}" - } - } + "stepType": "optOut", + "optOutType": "formOptOut", + "actions": [ + { + "actionType": "navigate", + "url": "https://kwold.com/ns/control/privacy", + "id": "037f7920-b9e7-4214-a937-171ec641d641" + }, + { + "actionType": "fillForm", + "selector": ".ahm", + "elements": [ + { + "type": "fullName", + "selector": "#user_name" + }, + { + "type": "email", + "selector": "#user_email" + }, + { + "type": "profileUrl", + "selector": "#url" + } + ], + "id": "5b9de12f-a52e-4bd0-b6ac-6884377d309b" + }, + { + "actionType": "getCaptchaInfo", + "selector": ".g-recaptcha", + "id": "48e5e7a8-af33-4629-a849-2cf926a518a3" + }, + { + "actionType": "solveCaptcha", + "selector": ".g-recaptcha", + "id": "bc2d26dc-3eef-478a-a04b-5671a1dbdf8b" + }, + { + "actionType": "click", + "elements": [ + { + "type": "button", + "selector": ".//button[@type='submit']" + } + ], + "id": "7f2a685e-ddad-4c5a-8e80-a6d3a690851f" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your removal request has been received" + } + ], + "id": "3a8a6e9d-c9a0-4e59-a8a4-fe4a05f3ce68" + }, + { + "actionType": "emailConfirmation", + "pollingTime": 30, + "id": "93ccf84a-a5ce-4dcf-8a78-143610723488" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your information control request has been confirmed." + } + ], + "id": "fcddc35b-6298-4f2b-a04c-08a2d6f7ceaa" + } + ] } - ] - }, - { - "stepType": "optOut", - "optOutType": "parentSiteOptOut", - "actions": [] + ], + "schedulingConfig": { + "retryError": 48, + "confirmOptOutScan": 72, + "maintenanceScan": 120, + "maxAttempts": -1 } - ], - "schedulingConfig": { - "retryError": 48, - "confirmOptOutScan": 72, - "maintenanceScan": 120, - "maxAttempts": -1 - } } diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/newenglandfacts.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/newenglandfacts.com.json index 9be7b4f7d9..ce93b0c82f 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/newenglandfacts.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/newenglandfacts.com.json @@ -1,8 +1,7 @@ { "name": "New England Facts", "url": "newenglandfacts.com", - "version": "0.3.0", - "parent": "verecor.com", + "version": "0.4.0", "addedDatetime": 1703052000000, "optOutUrl": "https://newenglandfacts.com/ng/control/privacy", "steps": [ @@ -61,8 +60,70 @@ }, { "stepType": "optOut", - "optOutType": "parentSiteOptOut", - "actions": [] + "optOutType": "formOptOut", + "actions": [ + { + "actionType": "navigate", + "url": "https://newenglandfacts.com/ng/control/privacy", + "id": "08332622-1749-4bdc-97d0-2366fef87522" + }, + { + "actionType": "fillForm", + "selector": ".ahm", + "elements": [ + { + "type": "fullName", + "selector": "#name" + }, + { + "type": "email", + "selector": "#email" + }, + { + "type": "profileUrl", + "selector": "#url" + } + ], + "id": "e3fdaaf0-70a5-4b7b-afa8-18306276f13a" + }, + { + "actionType": "click", + "elements": [ + { + "type": "button", + "selector": ".//button[@type='submit']" + } + ], + "id": "05bcd52d-21c7-4c00-bb18-b1de4b6803aa" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your removal request has been received" + } + ], + "id": "37511ba6-e622-4614-8657-d4523f650a18" + }, + { + "actionType": "emailConfirmation", + "pollingTime": 30, + "id": "80f13572-98f8-4aeb-a77c-25dcc863681a" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your information control request has been confirmed." + } + ], + "id": "fd232579-b39e-40c8-ab33-596b7e01bf33" + } + ] } ], "schedulingConfig": { diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/people-background-check.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/people-background-check.com.json index cad2226e09..ba41e564c6 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/people-background-check.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/people-background-check.com.json @@ -1,8 +1,7 @@ { "name": "People Background Check", "url": "people-background-check.com", - "version": "0.5.0", - "parent": "verecor.com", + "version": "0.6.0", "addedDatetime": 1702965600000, "optOutUrl": "https://people-background-check.com/ng/control/privacy", "steps": [ @@ -51,8 +50,70 @@ }, { "stepType": "optOut", - "optOutType": "parentSiteOptOut", - "actions": [] + "optOutType": "formOptOut", + "actions": [ + { + "actionType": "navigate", + "url": "https://people-background-check.com/ng/control/privacy", + "id": "bc160586-6a98-4eac-9fbf-fa7d24ef8bd7" + }, + { + "actionType": "fillForm", + "selector": ".ahm", + "elements": [ + { + "type": "fullName", + "selector": "#name" + }, + { + "type": "email", + "selector": "#email" + }, + { + "type": "profileUrl", + "selector": "#url" + } + ], + "id": "02e4ce59-a0e1-42cb-9adc-d07e4f28a7ff" + }, + { + "actionType": "click", + "elements": [ + { + "type": "button", + "selector": ".//button[@type='submit']" + } + ], + "id": "3ab8814b-39c1-4d3b-ab1d-5cbc69e43919" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your removal request has been received" + } + ], + "id": "b3260029-4cec-43cf-9da2-dc15467e7264" + }, + { + "actionType": "emailConfirmation", + "pollingTime": 30, + "id": "c1b7e968-a183-4d91-88da-b8c70313e9b9" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your information control request has been confirmed." + } + ], + "id": "14e8f388-a24f-4ce8-a3d5-e3e6eda1e8ef" + } + ] } ], "schedulingConfig": { diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/vericora.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/vericora.com.json index 110c78ffbb..76a017a3b1 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/vericora.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/vericora.com.json @@ -1,8 +1,7 @@ { "name": "Vericora", "url": "vericora.com", - "version": "0.4.0", - "parent": "verecor.com", + "version": "0.5.0", "addedDatetime": 1677736800000, "optOutUrl": "https://vericora.com/ng/control/privacy", "steps": [ @@ -60,8 +59,80 @@ }, { "stepType": "optOut", - "optOutType": "parentSiteOptOut", - "actions": [] + "optOutType": "formOptOut", + "actions": [ + { + "actionType": "navigate", + "url": "https://vericora.com/ng/control/privacy", + "id": "db351d7c-3ca0-4af4-9aed-b2f8fba15622" + }, + { + "actionType": "fillForm", + "selector": ".ahm", + "elements": [ + { + "type": "fullName", + "selector": "#user_name" + }, + { + "type": "email", + "selector": "#user_email" + }, + { + "type": "profileUrl", + "selector": "#url" + } + ], + "id": "c26a1a7c-46c5-4a10-a9d3-ff0660e15e32" + }, + { + "actionType": "getCaptchaInfo", + "selector": ".g-recaptcha", + "id": "09599d30-8953-477a-bb70-8042baa20cb5" + }, + { + "actionType": "solveCaptcha", + "selector": ".g-recaptcha", + "id": "a2af0d54-7170-4181-908b-669bf4dfbcea" + }, + { + "actionType": "click", + "elements": [ + { + "type": "button", + "selector": ".//button[@type='submit']" + } + ], + "id": "caf8c297-2e02-4d0b-a9ef-fb9fdb39315d" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your removal request has been received" + } + ], + "id": "b22b9195-acae-4c23-aeaf-a45e39b7a776" + }, + { + "actionType": "emailConfirmation", + "pollingTime": 30, + "id": "e8a95e02-89a0-4c6b-980d-36db2780b54b" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your information control request has been confirmed." + } + ], + "id": "9adf03c9-444f-450d-992f-adbbdf80a211" + } + ] } ], "schedulingConfig": { diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/veriforia.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/veriforia.com.json index 96a6e53137..2dd08c02e8 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/veriforia.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/veriforia.com.json @@ -1,8 +1,7 @@ { "name": "Veriforia", "url": "veriforia.com", - "version": "0.4.0", - "parent": "verecor.com", + "version": "0.5.0", "addedDatetime": 1677736800000, "optOutUrl": "https://veriforia.com/ng/control/privacy", "steps": [ @@ -60,8 +59,80 @@ }, { "stepType": "optOut", - "optOutType": "parentSiteOptOut", - "actions": [] + "optOutType": "formOptOut", + "actions": [ + { + "actionType": "navigate", + "url": "https://veriforia.com/ng/control/privacy", + "id": "943c387c-9cb2-42d6-afd5-a48cf4c2cf3d" + }, + { + "actionType": "fillForm", + "selector": ".ahm", + "elements": [ + { + "type": "fullName", + "selector": "#user_name" + }, + { + "type": "email", + "selector": "#user_email" + }, + { + "type": "profileUrl", + "selector": "#url" + } + ], + "id": "cf78bb16-b608-4a50-975b-844dad3db1db" + }, + { + "actionType": "getCaptchaInfo", + "selector": ".g-recaptcha", + "id": "4c0b8064-1e2b-4598-92a9-e832e73726db" + }, + { + "actionType": "solveCaptcha", + "selector": ".g-recaptcha", + "id": "c53bc12e-0cf6-4628-b99f-3219cb0f0c1a" + }, + { + "actionType": "click", + "elements": [ + { + "type": "button", + "selector": ".//button[@type='submit']" + } + ], + "id": "0aaae970-81d9-4d63-86d0-da4d0a3f5cf4" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your removal request has been received" + } + ], + "id": "9b6fad42-babc-4777-92af-d415ea9aef5b" + }, + { + "actionType": "emailConfirmation", + "pollingTime": 30, + "id": "d0d5f4d0-7183-40f5-b476-4315c751bcc5" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your information control request has been confirmed." + } + ], + "id": "434d001d-f6cb-4df9-acf3-e62dc821a25a" + } + ] } ], "schedulingConfig": { diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/virtory.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/virtory.com.json index 02dda9a3df..adc8d3a901 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/virtory.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/virtory.com.json @@ -1,8 +1,7 @@ { "name": "Virtory", "url": "virtory.com", - "version": "0.4.0", - "parent": "verecor.com", + "version": "0.5.0", "addedDatetime": 1703052000000, "optOutUrl": "https://virtory.com/prvt/control/privacy", "steps": [ @@ -63,8 +62,80 @@ }, { "stepType": "optOut", - "optOutType": "parentSiteOptOut", - "actions": [] + "optOutType": "formOptOut", + "actions": [ + { + "actionType": "navigate", + "url": "https://virtory.com/prvt/control/privacy", + "id": "ef272fd4-f284-47ca-aa0c-b4589a85bd81" + }, + { + "actionType": "fillForm", + "selector": ".ahm", + "elements": [ + { + "type": "fullName", + "selector": "#user_name" + }, + { + "type": "email", + "selector": "#user_email" + }, + { + "type": "profileUrl", + "selector": "#url" + } + ], + "id": "3eb2431f-0c00-440c-8a80-b6d95e0b5ee0" + }, + { + "actionType": "getCaptchaInfo", + "selector": ".g-recaptcha", + "id": "2a7550ed-6b80-4d32-adc1-20c96107afcb" + }, + { + "actionType": "solveCaptcha", + "selector": ".g-recaptcha", + "id": "d7c8d9eb-7239-42d1-9323-566ef1cf4665" + }, + { + "actionType": "click", + "elements": [ + { + "type": "button", + "selector": ".//button[@type='submit']" + } + ], + "id": "d4072304-d8c9-47ac-8c47-591a417c15b3" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your removal request has been received" + } + ], + "id": "c17b21dd-25cf-43ad-9836-01de7f2138cd" + }, + { + "actionType": "emailConfirmation", + "pollingTime": 30, + "id": "a3ea0213-bd64-4221-add5-58b517f78223" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your information control request has been confirmed." + } + ], + "id": "d4d7d0c6-3bba-470a-9fe7-6a154aa07751" + } + ] } ], "schedulingConfig": { diff --git a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/wellnut.com.json b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/wellnut.com.json index 5bcf4aa418..2c6e15a62c 100644 --- a/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/wellnut.com.json +++ b/LocalPackages/DataBrokerProtection/Sources/DataBrokerProtection/Resources/JSON/wellnut.com.json @@ -1,8 +1,7 @@ { "name": "Wellnut", "url": "wellnut.com", - "version": "0.4.0", - "parent": "verecor.com", + "version": "0.5.0", "addedDatetime": 1703052000000, "optOutUrl": "https://wellnut.com/noi/control/privacy", "steps": [ @@ -63,8 +62,80 @@ }, { "stepType": "optOut", - "optOutType": "parentSiteOptOut", - "actions": [] + "optOutType": "formOptOut", + "actions": [ + { + "actionType": "navigate", + "url": "https://wellnut.com/noi/control/privacy", + "id": "af96e7b3-fc16-4eed-9307-32da204941a1" + }, + { + "actionType": "fillForm", + "selector": ".ahm", + "elements": [ + { + "type": "fullName", + "selector": "#user_name" + }, + { + "type": "email", + "selector": "#user_email" + }, + { + "type": "profileUrl", + "selector": "#url" + } + ], + "id": "ce8f2fbd-04f7-48fd-8d67-2cf083b48df3" + }, + { + "actionType": "getCaptchaInfo", + "selector": ".g-recaptcha", + "id": "7b01c715-eb45-4b8c-94f0-cb5280a4f1d8" + }, + { + "actionType": "solveCaptcha", + "selector": ".g-recaptcha", + "id": "d3a79100-785b-4cf8-8bbe-5df701851925" + }, + { + "actionType": "click", + "elements": [ + { + "type": "button", + "selector": ".//button[@type='submit']" + } + ], + "id": "d6b36f44-291b-4951-8444-ea4a08a3c1fc" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your removal request has been received" + } + ], + "id": "a3a7cf9b-4653-4aac-9004-c2f738121ed7" + }, + { + "actionType": "emailConfirmation", + "pollingTime": 30, + "id": "cbafc394-018e-47ff-b95e-fbc521508e3f" + }, + { + "actionType": "expectation", + "expectations": [ + { + "type": "text", + "selector": "body", + "expect": "Your information control request has been confirmed." + } + ], + "id": "918f6692-c5c4-4072-95c8-795296e1a956" + } + ] } ], "schedulingConfig": { diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerOperationTests.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerOperationTests.swift new file mode 100644 index 0000000000..ad5a8aedb1 --- /dev/null +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerOperationTests.swift @@ -0,0 +1,99 @@ +// +// DataBrokerOperationTests.swift +// +// Copyright © 2024 DuckDuckGo. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +@testable import DataBrokerProtection +import XCTest + +final class DataBrokerOperationTests: XCTestCase { + lazy var mockOptOutQueryData: [BrokerProfileQueryData] = { + let brokerId: Int64 = 1 + + let mockNilPreferredRunDateQueryData = Array(1...10).map { + BrokerProfileQueryData.mock(preferredRunDate: nil, optOutJobData: [BrokerProfileQueryData.createOptOutJobData(extractedProfileId: Int64($0), brokerId: brokerId, profileQueryId: Int64($0), preferredRunDate: nil)]) + } + let mockPastQueryData = Array(1...10).map { + BrokerProfileQueryData.mock(preferredRunDate: .nowMinus(hours: $0), optOutJobData: [BrokerProfileQueryData.createOptOutJobData(extractedProfileId: Int64($0), brokerId: brokerId, profileQueryId: Int64($0), preferredRunDate: .nowMinus(hours: $0))]) + } + let mockFutureQueryData = Array(1...10).map { + BrokerProfileQueryData.mock(preferredRunDate: .nowPlus(hours: $0), optOutJobData: [BrokerProfileQueryData.createOptOutJobData(extractedProfileId: Int64($0), brokerId: brokerId, profileQueryId: Int64($0), preferredRunDate: .nowPlus(hours: $0))]) + } + + return mockNilPreferredRunDateQueryData + mockPastQueryData + mockFutureQueryData + }() + + lazy var mockScanQueryData: [BrokerProfileQueryData] = { + let mockNilPreferredRunDateQueryData = Array(1...10).map { _ in + BrokerProfileQueryData.mock(preferredRunDate: nil) + } + let mockPastQueryData = Array(1...10).map { + BrokerProfileQueryData.mock(preferredRunDate: .nowMinus(hours: $0)) + } + let mockFutureQueryData = Array(1...10).map { + BrokerProfileQueryData.mock(preferredRunDate: .nowPlus(hours: $0)) + } + + return mockNilPreferredRunDateQueryData + mockPastQueryData + mockFutureQueryData + }() + + func testWhenFilteringOptOutOperationData_thenAllButFuturePreferredRunDateIsReturned() { + let operationData1 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .optOut, priorityDate: nil) + let operationData2 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .optOut, priorityDate: .now) + let operationData3 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .optOut, priorityDate: .distantPast) + let operationData4 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .optOut, priorityDate: .distantFuture) + + XCTAssertEqual(operationData1.count, 30) // all jobs + XCTAssertEqual(operationData2.count, 20) // nil preferred run date + past jobs + XCTAssertEqual(operationData3.count, 10) // nil preferred run date jobs + XCTAssertEqual(operationData4.count, 30) // all jobs + } + + func testWhenFilteringScanOperationData_thenPreferredRunDatePriorToPriorityDateIsReturned() { + let operationData1 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockScanQueryData, operationType: .scheduledScan, priorityDate: nil) + let operationData2 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockScanQueryData, operationType: .manualScan, priorityDate: .now) + let operationData3 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockScanQueryData, operationType: .scheduledScan, priorityDate: .distantPast) + let operationData4 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockScanQueryData, operationType: .manualScan, priorityDate: .distantFuture) + + XCTAssertEqual(operationData1.count, 30) // all jobs + XCTAssertEqual(operationData2.count, 10) // past jobs + XCTAssertEqual(operationData3.count, 0) // no jobs + XCTAssertEqual(operationData4.count, 20) // past + future jobs + } + + func testFilteringAllOperationData() { + let operationData1 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .all, priorityDate: nil) + let operationData2 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .all, priorityDate: .now) + let operationData3 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .all, priorityDate: .distantPast) + let operationData4 = MockDataBrokerOperation.filterAndSortOperationsData(brokerProfileQueriesData: mockOptOutQueryData, operationType: .all, priorityDate: .distantFuture) + + XCTAssertEqual(operationData1.filter { $0 is ScanJobData }.count, 30) // all jobs + XCTAssertEqual(operationData1.filter { $0 is OptOutJobData }.count, 30) // all jobs + XCTAssertEqual(operationData1.count, 30+30) + + XCTAssertEqual(operationData2.filter { $0 is ScanJobData }.count, 10) // past jobs + XCTAssertEqual(operationData2.filter { $0 is OptOutJobData }.count, 20) // nil preferred run date + past jobs + XCTAssertEqual(operationData2.count, 10+20) + + XCTAssertEqual(operationData3.filter { $0 is ScanJobData }.count, 0) // no jobs + XCTAssertEqual(operationData3.filter { $0 is OptOutJobData }.count, 10) // nil preferred run date jobs + XCTAssertEqual(operationData3.count, 0+10) + + XCTAssertEqual(operationData4.filter { $0 is ScanJobData }.count, 20) // past + future jobs + XCTAssertEqual(operationData4.filter { $0 is OptOutJobData }.count, 30) // all jobs + XCTAssertEqual(operationData4.count, 20+30) + } +} diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift index 5e772ac0ee..8a3fa74edd 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/DataBrokerProfileQueryOperationManagerTests.swift @@ -369,24 +369,17 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { } } - func testWhenRemovedProfileIsFound_thenOptOutConfirmedIsAddedRemoveDateIsUpdatedAndPreferredRunDateIsSetToNil() async { + func testWhenRemovedProfileIsFound_thenOptOutConfirmedIsAddedRemoveDateIsUpdated() async { do { - let extractedProfileId: Int64 = 1 - let brokerId: Int64 = 1 - let profileQueryId: Int64 = 1 - let mockHistoryEvent = HistoryEvent(extractedProfileId: extractedProfileId, brokerId: brokerId, profileQueryId: profileQueryId, type: .optOutRequested) - let mockBrokerProfileQuery = BrokerProfileQueryData( - dataBroker: .mock, - profileQuery: .mock, - scanJobData: .mock, - optOutJobData: [.mock(with: .mockWithoutRemovedDate, preferredRunDate: Date(), historyEvents: [mockHistoryEvent])] - ) - mockWebOperationRunner.scanResults = [.mockWithoutId] - mockDatabase.brokerProfileQueryDataToReturn = [mockBrokerProfileQuery] _ = try await sut.runScanOperation( on: mockWebOperationRunner, - brokerProfileQueryData: mockBrokerProfileQuery, + brokerProfileQueryData: .init( + dataBroker: .mock, + profileQuery: .mock, + scanJobData: .mock, + optOutJobData: [OptOutJobData.mock(with: .mockWithoutRemovedDate)] + ), database: mockDatabase, notificationCenter: .default, pixelHandler: MockDataBrokerProtectionPixelsHandler(), @@ -396,8 +389,6 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { XCTAssertTrue(mockDatabase.optOutEvents.contains(where: { $0.type == .optOutConfirmed })) XCTAssertTrue(mockDatabase.wasUpdateRemoveDateCalled) XCTAssertNotNil(mockDatabase.extractedProfileRemovedDate) - XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForOptOutCalled) - XCTAssertNil(mockDatabase.lastPreferredRunDateOnOptOut) } catch { XCTFail("Should not throw") } @@ -841,7 +832,7 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnScan, date2: Date().addingTimeInterval(schedulingConfig.confirmOptOutScan.hoursToSeconds))) } - func testWhenUpdatingDatesAndLastEventIsOptOutRequested_thenWeSetOptOutPreferredRunDateToNil() throws { + func testWhenUpdatingDatesAndLastEventIsOptOutRequested_thenWeSetOptOutPreferredRunDateToOptOutReattempt() throws { let brokerId: Int64 = 1 let profileQueryId: Int64 = 1 let extractedProfileId: Int64 = 1 @@ -851,7 +842,7 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { try sut.updateOperationDataDates(origin: .scan, brokerId: brokerId, profileQueryId: profileQueryId, extractedProfileId: extractedProfileId, schedulingConfig: schedulingConfig, database: mockDatabase) XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForScanCalled) - XCTAssertNil(mockDatabase.lastPreferredRunDateOnOptOut) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnOptOut, date2: Date().addingTimeInterval(schedulingConfig.optOutReattempt.hoursToSeconds))) } func testWhenUpdatingDatesAndLastEventIsMatchesFound_thenWeSetScanPreferredDateToMaintanence() throws { @@ -918,7 +909,9 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { // If the date is not going to be set, we don't call the database function XCTAssertFalse(mockDatabase.wasUpdatedPreferredRunDateForScanCalled) - XCTAssertFalse(mockDatabase.wasUpdatedPreferredRunDateForOptOutCalled) + + XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForOptOutCalled) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnOptOut, date2: Date().addingTimeInterval(config.optOutReattempt.hoursToSeconds))) } func testUpdatingScanDateFromScan_thenScanDoesNotRespectMostRecentDate() throws { @@ -942,8 +935,10 @@ final class DataBrokerProfileQueryOperationManagerTests: XCTestCase { try sut.updateOperationDataDates(origin: .scan, brokerId: brokerId, profileQueryId: profileQueryId, extractedProfileId: extractedProfileId, schedulingConfig: config, database: mockDatabase) XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForScanCalled) - XCTAssertFalse(mockDatabase.wasUpdatedPreferredRunDateForOptOutCalled) XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnScan, date2: expectedPreferredRunDate), "\(String(describing: mockDatabase.lastPreferredRunDateOnScan)) is not equal to \(expectedPreferredRunDate)") + + XCTAssertTrue(mockDatabase.wasUpdatedPreferredRunDateForOptOutCalled) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: mockDatabase.lastPreferredRunDateOnOptOut, date2: Date().addingTimeInterval(config.optOutReattempt.hoursToSeconds))) } } diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/Mocks.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/Mocks.swift index 941f469659..3c32ec425a 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/Mocks.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/Mocks.swift @@ -117,6 +117,13 @@ extension BrokerProfileQueryData { return [broker1Data, broker2Data, broker3Data] } + static func createOptOutJobData(extractedProfileId: Int64, brokerId: Int64, profileQueryId: Int64, preferredRunDate: Date?) -> OptOutJobData { + + let extractedProfile = ExtractedProfile(id: extractedProfileId) + + return OptOutJobData(brokerId: brokerId, profileQueryId: profileQueryId, createdDate: .now, preferredRunDate: preferredRunDate, historyEvents: [], attemptCount: 0, extractedProfile: extractedProfile) + } + static func createOptOutJobData(extractedProfileId: Int64, brokerId: Int64, profileQueryId: Int64, startEventHoursAgo: Int, requestEventHoursAgo: Int, jobCreatedHoursAgo: Int) -> OptOutJobData { let extractedProfile = ExtractedProfile(id: extractedProfileId) diff --git a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift index 555c19a9e7..b602b1f3d8 100644 --- a/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift +++ b/LocalPackages/DataBrokerProtection/Tests/DataBrokerProtectionTests/OperationPreferredDateCalculatorTests.swift @@ -24,8 +24,8 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { private let schedulingConfig = DataBrokerScheduleConfig( retryError: 48, - confirmOptOutScan: 2000, - maintenanceScan: 3000, + confirmOptOutScan: 72, + maintenanceScan: 120, maxAttempts: 3 ) @@ -498,9 +498,7 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) } - func testOptOutConfirmedWithCurrentPreferredDate_thenOptOutIsNil() throws { - let expectedOptOutDate: Date? = nil - + func testOptOutConfirmedWithCurrentPreferredDate_thenOptOutIsNotScheduled() throws { let historyEvents = [ HistoryEvent(extractedProfileId: 1, brokerId: 1, @@ -515,12 +513,10 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { schedulingConfig: schedulingConfig, attemptCount: 0) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) + XCTAssertNil(actualOptOutDate) } - func testOptOutConfirmedWithoutCurrentPreferredDate_thenOptOutIsNil() throws { - let expectedOptOutDate: Date? = nil - + func testOptOutConfirmedWithoutCurrentPreferredDate_thenOptOutIsNotScheduled() throws { let historyEvents = [ HistoryEvent(extractedProfileId: 1, brokerId: 1, @@ -529,17 +525,17 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { let calculator = OperationPreferredDateCalculator() - let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, + let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: Date(), historyEvents: historyEvents, extractedProfileID: nil, schedulingConfig: schedulingConfig, attemptCount: 0) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) + XCTAssertNil(actualOptOutDate) } - func testOptOutRequestedWithCurrentPreferredDate_thenOptOutIsNil() throws { - let expectedOptOutDate: Date? = nil + func testOptOutRequestedWithCurrentPreferredDate_thenOptOutIsNotScheduled() throws { + let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.optOutReattempt.hoursToSeconds) let historyEvents = [ HistoryEvent(extractedProfileId: 1, @@ -553,13 +549,14 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { historyEvents: historyEvents, extractedProfileID: nil, schedulingConfig: schedulingConfig, - attemptCount: 0) + attemptCount: 0, + date: MockDate()) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: actualOptOutDate, date2: expectedOptOutDate)) } - func testOptOutRequestedWithoutCurrentPreferredDate_thenOptOutIsNil() throws { - let expectedOptOutDate: Date? = nil + func testOptOutRequestedWithoutCurrentPreferredDate_thenOptOutIsNotScheduled() throws { + let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.optOutReattempt.hoursToSeconds) let historyEvents = [ HistoryEvent(extractedProfileId: 1, @@ -573,9 +570,10 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { historyEvents: historyEvents, extractedProfileID: nil, schedulingConfig: schedulingConfig, - attemptCount: 0) + attemptCount: 0, + date: MockDate()) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: actualOptOutDate, date2: expectedOptOutDate)) } func testScanStarted_thenOptOutDoesNotChange() throws { @@ -610,10 +608,10 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { let calculator = OperationPreferredDateCalculator() let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, - historyEvents: historyEvents, - extractedProfileID: nil, - schedulingConfig: schedulingConfig, - attemptCount: 0) + historyEvents: historyEvents, + extractedProfileID: nil, + schedulingConfig: schedulingConfig, + attemptCount: 0) XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) } @@ -758,6 +756,69 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { } } + func testChildBrokerTurnsParentBroker_whenFirstOptOutSucceeds_thenOptOutDateIsNotScheduled() throws { + let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.optOutReattempt.hoursToSeconds) + + let historyEvents = [ + HistoryEvent(extractedProfileId: 1, + brokerId: 1, + profileQueryId: 1, + type: .optOutRequested), + ] + let calculator = OperationPreferredDateCalculator() + let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, + historyEvents: historyEvents, + extractedProfileID: 1, + schedulingConfig: schedulingConfig, + attemptCount: 1, + date: MockDate()) + + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: actualOptOutDate, date2: expectedOptOutDate)) + } + + func testChildBrokerTurnsParentBroker_whenFirstOptOutFails_thenOptOutIsScheduled() throws { + let expectedOptOutDate = Calendar.current.date(byAdding: .hour, value: 2, to: Date())! + + let historyEvents = [ + HistoryEvent(extractedProfileId: 1, + brokerId: 1, + profileQueryId: 1, + type: .error(error: .malformedURL)), + ] + let calculator = OperationPreferredDateCalculator() + let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, + historyEvents: historyEvents, + extractedProfileID: 1, + schedulingConfig: schedulingConfig, + attemptCount: 1) + + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) + } + + func testRequestedOptOut_whenProfileReappears_thenOptOutIsScheduled() throws { + let expectedOptOutDate = Date() + + let historyEvents = [ + HistoryEvent(extractedProfileId: 1, + brokerId: 1, + profileQueryId: 1, + type: .optOutRequested, + date: .nowMinus(hours: 24*10)), + HistoryEvent(extractedProfileId: 1, + brokerId: 1, + profileQueryId: 1, + type: .reAppearence), + ] + let calculator = OperationPreferredDateCalculator() + let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: .distantFuture, + historyEvents: historyEvents, + extractedProfileID: 1, + schedulingConfig: schedulingConfig, + attemptCount: 1) + + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) + } + func testOptOutStartedWithRecentDate_thenOptOutDateDoesNotChange() throws { let expectedOptOutDate = Date() @@ -779,8 +840,6 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { } func testOptOutConfirmedWithRecentDate_thenOptOutDateDoesNotChange() throws { - let expectedOptOutDate: Date? = nil - let historyEvents = [ HistoryEvent(extractedProfileId: 1, brokerId: 1, @@ -790,16 +849,16 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { let calculator = OperationPreferredDateCalculator() let actualOptOutDate = try calculator.dateForOptOutOperation(currentPreferredRunDate: nil, - historyEvents: historyEvents, - extractedProfileID: nil, - schedulingConfig: schedulingConfig, - attemptCount: 0) + historyEvents: historyEvents, + extractedProfileID: nil, + schedulingConfig: schedulingConfig, + attemptCount: 0) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) + XCTAssertNil(actualOptOutDate) } - func testOptOutRequestedWithRecentDate_thenOptOutDateDoesNotChange() throws { - let expectedOptOutDate: Date? = nil + func testOptOutRequestedWithRecentDate_thenOutOutIsNotScheduled() throws { + let expectedOptOutDate = MockDate().now.addingTimeInterval(schedulingConfig.optOutReattempt.hoursToSeconds) let historyEvents = [ HistoryEvent(extractedProfileId: 1, @@ -813,9 +872,10 @@ final class OperationPreferredDateCalculatorTests: XCTestCase { historyEvents: historyEvents, extractedProfileID: nil, schedulingConfig: schedulingConfig, - attemptCount: 0) + attemptCount: 0, + date: MockDate()) - XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: expectedOptOutDate, date2: actualOptOutDate)) + XCTAssertTrue(areDatesEqualIgnoringSeconds(date1: actualOptOutDate, date2: expectedOptOutDate)) } func testScanStartedWithRecentDate_thenOptOutDateDoesNotChange() throws {