Merge pull request #4873 from mozilla/mntor-3423-db

QA tool for custom data - DB changes only

src/db/migrations/20240710214906_qa_custom_brokers.js

@@ -0,0 +1,40 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+ /**
+  * @param { import("knex").Knex } knex
+  * @returns { Promise<void> }
+  */
+export function up(knex) {
+  return knex.schema
+    .createTable("qa_custom_brokers", table => {
+      table.increments("onerep_scan_result_id").primary();
+      table.integer("onerep_profile_id").notNullable();
+      table.string("link").notNullable();
+      table.integer("age").nullable();
+      table.string("data_broker").notNullable();
+      table.jsonb("emails").notNullable();
+      table.jsonb("phones").notNullable();
+      table.jsonb("addresses").notNullable();
+      table.jsonb("relatives").notNullable();
+      table.string("first_name").notNullable();
+      table.string("middle_name").nullable();
+      table.string("last_name").notNullable();
+      table.string("status").notNullable();
+      table.boolean("manually_resolved").defaultTo(false);
+      table.timestamp("created_at").defaultTo(knex.fn.now());
+      table.timestamp("updated_at").defaultTo(knex.fn.now());
+      table.integer("optout_attempts");
+    });
+}
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+export function down(knex) {
+  return knex.schema.dropTableIfExists("qa_custom_brokers");
+}
+
+

src/db/migrations/20240715110621_qa_custom_toggles.js

@@ -0,0 +1,26 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+export function up(knex) {
+  return knex.schema.createTable("qa_custom_toggles", function(table) {
+    table.string("email_hash").primary();
+    table.integer("onerep_profile_id").notNullable();
+    table.boolean("show_real_breaches").notNullable();
+    table.boolean("show_custom_breaches").notNullable();
+    table.boolean("show_real_brokers").notNullable();
+    table.boolean("show_custom_brokers").notNullable();
+  });
+}
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+export function down(knex) {
+  return knex.schema.dropTable("qa_custom_toggles");
+}

src/db/migrations/20240715115031_qa_custom_breaches.js

@@ -0,0 +1,36 @@
+/* This Source Code Form is subject to the terms of the Mozilla Public
+ * License, v. 2.0. If a copy of the MPL was not distributed with this
+ * file, You can obtain one at http://mozilla.org/MPL/2.0/. */
+
+export function up(knex) {
+  // camel case for easier insertion into table
+  return knex.schema.createTable("qa_custom_breaches", table => {
+    table.string("emailHashPrefix");
+    table.integer("Id").notNullable().primary();
+    table.string("Name").notNullable();
+    table.string("Title").notNullable();
+    table.string("Domain").notNullable();
+    table.timestamp("BreachDate").defaultTo(knex.fn.now());
+    table.timestamp("AddedDate").defaultTo(knex.fn.now());
+    table.timestamp("ModifiedDate").defaultTo(knex.fn.now());
+    table.integer("PwnCount").notNullable();
+    table.text("Description").notNullable();
+    table.string("LogoPath").notNullable();
+    table.specificType('DataClasses', 'character varying(255)[]');
+    table.boolean("IsVerified").notNullable();
+    table.boolean("IsFabricated").notNullable();
+    table.boolean("IsSensitive").notNullable();
+    table.boolean("IsRetired").notNullable();
+    table.boolean("IsSpamList").notNullable();
+    table.boolean("IsMalware").notNullable();
+    table.string("FaviconUrl").defaultTo(null);
+  });
+}
+
+/**
+ * @param { import("knex").Knex } knex
+ * @returns { Promise<void> }
+ */
+export function down(knex) {
+  return knex.schema.dropTableIfExists("qa_custom_breaches");
+}

src/db/tables/onerep_scans.ts

@@ -16,6 +16,7 @@ import {
   SubscriberRow,
 } from "knex/types/tables";
 import { RemovalStatus } from "../../app/functions/universal/scanResult.js";
+import { getQaCustomBrokers, getQaToggleRow } from "./qa_customs.ts";
 
 const knex = createDbConnection();
 
@@ -139,25 +140,46 @@ async function getLatestOnerepScan(
   return scan ?? null;
 }
 
+/*
+Note: please, don't write the results of this function back to the database!
+*/
 async function getLatestOnerepScanResults(
   onerepProfileId: number | null,
 ): Promise<LatestOnerepScanData> {
   const scan = await getLatestOnerepScan(onerepProfileId);
 
-  const results =
-    typeof scan === "undefined"
+  let results: OnerepScanResultRow[] = [];
+
+  if (typeof scan !== "undefined") {
+    const qaToggles = await getQaToggleRow(onerepProfileId);
+    let showCustomBrokers = true;
+    let showRealBrokers = true;
+
+    if (qaToggles) {
+      showCustomBrokers = qaToggles.show_custom_brokers;
+      showRealBrokers = qaToggles.show_real_brokers;
+    }
+
+    const qaBrokers = !showCustomBrokers
       ? []
-      : ((await knex("onerep_scan_results")
-          .select("onerep_scan_results.*")
-          .distinctOn("link")
-          .where("onerep_profile_id", onerepProfileId)
-          .innerJoin(
-            "onerep_scans",
-            "onerep_scan_results.onerep_scan_id",
-            "onerep_scans.onerep_scan_id",
-          )
-          .orderBy("link")
-          .orderBy("onerep_scan_result_id", "desc")) as OnerepScanResultRow[]);
+      : await getQaCustomBrokers(onerepProfileId, scan?.onerep_scan_id);
+    if (!showRealBrokers) results = qaBrokers;
+    else {
+      // Fetch initial results from onerep_scan_results
+      const scanResults = (await knex("onerep_scan_results")
+        .select("*")
+        .distinctOn("link")
+        .where("onerep_profile_id", onerepProfileId)
+        .innerJoin(
+          "onerep_scans",
+          "onerep_scan_results.onerep_scan_id",
+          "onerep_scans.onerep_scan_id",
+        )
+        .orderBy("link")
+        .orderBy("onerep_scan_result_id", "desc")) as OnerepScanResultRow[];
+      results = [...scanResults, ...qaBrokers];
+    }
+  }
 
   return {
     scan: scan ?? null,
@@ -277,7 +299,6 @@ async function markOnerepScanResultAsResolved(
   logger.info("scan_resolved", {
     onerepScanResultId,
   });
-
   await knex("onerep_scan_results")
     .update({
       manually_resolved: true,