feat(EMS-4035-4042): declarations - modern slavery - data migration

database/exip.sql

@@ -646,9 +646,9 @@ DROP TABLE IF EXISTS `DeclarationModernSlaveryVersion`;
 CREATE TABLE `DeclarationModernSlaveryVersion` (
   `id` varchar(191) COLLATE utf8mb4_unicode_ci NOT NULL,
   `declarationModernSlavery` varchar(191) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
-  `hasNoOffensesOrInvestigations` varchar(3) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '',
-  `isNotAwareOfExistingSlavery` varchar(3) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '',
-  `willAdhereToAllRequirements` varchar(3) COLLATE utf8mb4_unicode_ci NOT NULL DEFAULT '',
+  `hasNoOffensesOrInvestigations` tinyint(1) DEFAULT NULL,
+  `isNotAwareOfExistingSlavery` tinyint(1) DEFAULT NULL,
+  `willAdhereToAllRequirements` tinyint(1) DEFAULT NULL,
   PRIMARY KEY (`id`),
   KEY `DeclarationModernSlaveryVersion_declarationModernSlavery_idx` (`declarationModernSlavery`),
   CONSTRAINT `DeclarationModernSlaveryVersion_declarationModernSlavery_fkey` FOREIGN KEY (`declarationModernSlavery`) REFERENCES `DeclarationModernSlavery` (`id`) ON DELETE

src/api/data-migration/helpers/get-all-declaration-modern-slaveries.ts

@@ -0,0 +1,27 @@
+import { Connection } from 'mysql2/promise';
+import executeSqlQuery from '../execute-sql-query';
+import { ApplicationDeclarationModernSlavery } from '../../types';
+
+/**
+ * getAllDeclarationModernSlaveries
+ * Get all entries in the "DeclarationModernSlavery" table
+ * @param {Connection} connection: SQL database connection
+ * @returns {Promise<Array<ApplicationDeclarationModernSlavery>>}
+ */
+const getAllDeclarationModernSlaveries = async (connection: Connection) => {
+  const loggingMessage = 'Getting all declaration modern slaveries';
+
+  try {
+    const query = 'SELECT * FROM DeclarationModernSlavery';
+
+    const [declarations] = await executeSqlQuery({ connection, query, loggingMessage });
+
+    return declarations as Array<ApplicationDeclarationModernSlavery>;
+  } catch (error) {
+    console.error('🚨 Error %s %o', loggingMessage, error);
+
+    throw new Error(`🚨 error ${loggingMessage} ${error}`);
+  }
+};
+
+export default getAllDeclarationModernSlaveries;

src/api/data-migration/helpers/get-all-declarations.ts

@@ -0,0 +1,27 @@
+import { Connection } from 'mysql2/promise';
+import executeSqlQuery from '../execute-sql-query';
+import { ApplicationDeclaration } from '../../types';
+
+/**
+ * getAllDeclarations
+ * Get all entries in the "Declaration" table
+ * @param {Connection} connection: SQL database connection
+ * @returns {Promise<Array<ApplicationDeclaration>>}
+ */
+const getAllDeclarations = async (connection: Connection) => {
+  const loggingMessage = 'Getting all declarations';
+
+  try {
+    const query = 'SELECT * FROM Application';
+
+    const [declarations] = await executeSqlQuery({ connection, query, loggingMessage });
+
+    return declarations as Array<ApplicationDeclaration>;
+  } catch (error) {
+    console.error('🚨 Error %s %o', loggingMessage, error);
+
+    throw new Error(`🚨 error ${loggingMessage} ${error}`);
+  }
+};
+
+export default getAllDeclarations;

src/api/data-migration/version-1-to-version-2/get-all-non-submitted-applications.ts → src/api/data-migration/helpers/get-all-non-submitted-applications.ts

@@ -9,7 +9,7 @@ const { STATUS } = APPLICATION;
  * getAllNonSubmittedApplications
  * Get all entries in the "Application" table that do NOT have a SUBMITTED status
  * @param {Connection} connection: SQL database connection
- * @returns {Promise<Application>} Non-submitted applications
+ * @returns {Promise<Array<Application>>} Non-submitted applications
  */
 const getAllNonSubmittedApplications = async (connection: Connection) => {
   const loggingMessage = 'Getting all non-submitted applications';

src/api/data-migration/version-1-to-version-2/get-all-submitted-applications.ts → src/api/data-migration/helpers/get-all-submitted-applications.ts

@@ -9,7 +9,7 @@ const { STATUS } = APPLICATION;
  * getAllSubmittedApplications
  * Get all entries in the "Application" table with a SUBMITTED status
  * @param {Connection} connection: SQL database connection
- * @returns {Promise<Application>} Submitted applications
+ * @returns {Promise<Array<Application>>} Submitted applications
  */
 const getAllSubmittedApplications = async (connection: Connection) => {
   const loggingMessage = 'Getting all submitted applications';

src/api/data-migration/version-1-to-version-2/create-new-application-relationships/create-company-different-trading-address.ts

@@ -1,5 +1,5 @@
 import { Connection } from 'mysql2/promise';
-import createCuid from '../create-cuid';
+import createCuid from '../../helpers/create-cuid';
 import executeSqlQuery from '../../execute-sql-query';
 import { Application } from '../../../types';
 

src/api/data-migration/version-1-to-version-2/create-new-application-relationships/declaration-version-relationship/non-submitted-applications.ts

@@ -1,7 +1,7 @@
 import { Connection } from 'mysql2/promise';
 import { DECLARATIONS } from '../../../../constants';
-import getAllNonSubmittedApplications from '../../get-all-non-submitted-applications';
-import createCuid from '../../create-cuid';
+import getAllNonSubmittedApplications from '../../../helpers/get-all-non-submitted-applications';
+import createCuid from '../../../helpers/create-cuid';
 import executeSqlQuery from '../../../execute-sql-query';
 import { Application } from '../../../../types';
 

src/api/data-migration/version-1-to-version-2/create-new-application-relationships/declaration-version-relationship/submitted-applications.ts

@@ -1,7 +1,7 @@
 import { Connection } from 'mysql2/promise';
 import { DECLARATIONS } from '../../../../constants';
-import getAllSubmittedApplications from '../../get-all-submitted-applications';
-import createCuid from '../../create-cuid';
+import getAllSubmittedApplications from '../../../helpers/get-all-submitted-applications';
+import createCuid from '../../../helpers/create-cuid';
 import executeSqlQuery from '../../../execute-sql-query';
 import { Application } from '../../../../types';
 

src/api/data-migration/version-1-to-version-2/create-new-application-relationships/export-contract-agent/create-agent-service-charges.ts

@@ -1,5 +1,5 @@
 import { Connection } from 'mysql2/promise';
-import createCuid from '../../create-cuid';
+import createCuid from '../../../helpers/create-cuid';
 import executeSqlQuery from '../../../execute-sql-query';
 import { Application } from '../../../../types';
 

src/api/data-migration/version-1-to-version-2/create-new-application-relationships/export-contract-agent/create-agent-services.ts

@@ -1,5 +1,5 @@
 import { Connection } from 'mysql2/promise';
-import createCuid from '../../create-cuid';
+import createCuid from '../../../helpers/create-cuid';
 import executeSqlQuery from '../../../execute-sql-query';
 import { Application } from '../../../../types';
 

src/api/data-migration/version-1-to-version-2/create-new-application-relationships/export-contract-agent/create-initial-agents.ts

@@ -1,5 +1,5 @@
 import { Connection } from 'mysql2/promise';
-import createCuid from '../../create-cuid';
+import createCuid from '../../../helpers/create-cuid';
 import executeSqlQuery from '../../../execute-sql-query';
 import { Application } from '../../../../types';
 

src/api/data-migration/version-1-to-version-2/create-new-application-relationships/jointly-insured-party.ts

@@ -1,5 +1,5 @@
 import { Connection } from 'mysql2/promise';
-import createCuid from '../create-cuid';
+import createCuid from '../../helpers/create-cuid';
 import executeSqlQuery from '../../execute-sql-query';
 import { Application } from '../../../types';
 

src/api/data-migration/version-1-to-version-2/create-new-application-relationships/loss-payee/create-initial-loss-payees.ts

@@ -1,5 +1,5 @@
 import { Connection } from 'mysql2/promise';
-import createCuid from '../../create-cuid';
+import createCuid from '../../../helpers/create-cuid';
 import executeSqlQuery from '../../../execute-sql-query';
 import { Application } from '../../../../types';
 

src/api/data-migration/version-1-to-version-2/create-new-application-relationships/loss-payee/create-loss-payee-financial-international-vector.ts

@@ -1,5 +1,5 @@
 import { Connection } from 'mysql2/promise';
-import createCuid from '../../create-cuid';
+import createCuid from '../../../helpers/create-cuid';
 import executeSqlQuery from '../../../execute-sql-query';
 import { Application } from '../../../../types';
 

src/api/data-migration/version-1-to-version-2/create-new-application-relationships/loss-payee/create-loss-payee-financial-international.ts

@@ -1,6 +1,6 @@
 import { Connection } from 'mysql2/promise';
 import getAllLossPayees from '../../get-all-loss-payees';
-import createCuid from '../../create-cuid';
+import createCuid from '../../../helpers/create-cuid';
 import executeSqlQuery from '../../../execute-sql-query';
 
 /**

src/api/data-migration/version-1-to-version-2/create-new-application-relationships/loss-payee/create-loss-payee-financial-uk-vector.ts

@@ -1,5 +1,5 @@
 import { Connection } from 'mysql2/promise';
-import createCuid from '../../create-cuid';
+import createCuid from '../../../helpers/create-cuid';
 import executeSqlQuery from '../../../execute-sql-query';
 import { Application } from '../../../../types';
 

src/api/data-migration/version-1-to-version-2/create-new-application-relationships/loss-payee/create-loss-payee-financial-uk.ts

@@ -1,6 +1,6 @@
 import { Connection } from 'mysql2/promise';
 import getAllLossPayees from '../../get-all-loss-payees';
-import createCuid from '../../create-cuid';
+import createCuid from '../../../helpers/create-cuid';
 import executeSqlQuery from '../../../execute-sql-query';
 
 /**

src/api/data-migration/version-1-to-version-2/create-new-application-relationships/private-market.ts

@@ -1,5 +1,5 @@
 import { Connection } from 'mysql2/promise';
-import createCuid from '../create-cuid';
+import createCuid from '../../helpers/create-cuid';
 import executeSqlQuery from '../../execute-sql-query';
 import { Application } from '../../../types';
 

src/api/data-migration/version-1-to-version-2/update-applications/move-buyer-contact-fields.ts

@@ -1,5 +1,5 @@
 import { Connection } from 'mysql2/promise';
-import createCuid from '../create-cuid';
+import createCuid from '../../helpers/create-cuid';
 import executeSqlQuery from '../../execute-sql-query';
 import { ApplicationBuyerMvp } from '../../../types';
 

src/api/data-migration/version-1-to-version-2/update-applications/move-buyer-relationship-fields.ts

@@ -1,5 +1,5 @@
 import { Connection } from 'mysql2/promise';
-import createCuid from '../create-cuid';
+import createCuid from '../../helpers/create-cuid';
 import executeSqlQuery from '../../execute-sql-query';
 import { ApplicationBuyerMvp } from '../../../types';
 

src/api/data-migration/version-1-to-version-2/update-applications/move-buyer-trading-history-fields.ts

@@ -1,5 +1,5 @@
 import { Connection } from 'mysql2/promise';
-import createCuid from '../create-cuid';
+import createCuid from '../../helpers/create-cuid';
 import executeSqlQuery from '../../execute-sql-query';
 import { ApplicationBuyerMvp } from '../../../types';
 

src/api/data-migration/version-1-to-version-2/update-applications/update-declaration-fields.ts

@@ -1,5 +1,5 @@
 import { Connection } from 'mysql2/promise';
-import getAllNonSubmittedApplications from '../get-all-non-submitted-applications';
+import getAllNonSubmittedApplications from '../../helpers/get-all-non-submitted-applications';
 import executeSqlQuery from '../../execute-sql-query';
 import { Application } from '../../../types';
 

src/api/data-migration/version-2-to-version-3/index.ts

@@ -3,8 +3,8 @@ import updateApplications from './update-applications';
 
 /**
  * dataMigration
- * Update all accounts and applications from the MVP data model/structure,
- * to the new "No PDF" data model/structure.
+ * Update all applications from the V2 data model/structure,
+ * to the V3 data model/structure.
  * @returns {Function} process.exit()
  */
 const dataMigration = async () => {

src/api/data-migration/version-2-to-version-3/update-applications/index.ts

@@ -7,7 +7,7 @@ import updateApplicationMigrated from './update-application-migrated';
 
 /**
  * updateApplications
- * Update applications from the "No PDF" data model/structure, to the new "No PDF iterations" data model/structure.
+ * Update applications from the V2 data model/structure, to the V3 data model/structure.
  * @param {Connection} connection: SQL database connection
  * @returns {Promise<Array<object>>} executeSqlQuery responses
  */

src/api/data-migration/version-3-to-version-4/README.md

@@ -0,0 +1,58 @@
+# EXIP API - data migration - version 2 to version 3 :file_folder:
+
+This directory contains source code for migrating version 2 of EXIP data into the version 3 data structure.
+
+- Version 1 is MVP. This is the first release with account and application functionalities.
+- Version 2 is "No PDF". This is a large iteration of MVP, where we allow more applications through and collect more information, depending on answers a user provides.
+- Version 3 is a small iteration of "No PDF". This is mostly design and content improvements, however 1x field has been added to the database.
+
+## In version 3, the following data has been added
+
+- `Policy` table - 1x new field - `requestedCreditLimit`.
+
+## Prerequisites :gear:
+
+To set up and run the API locally to test this data migration, you'll need the following prerequisites:
+
+- Node.js version 16.17.0 or higher along with the corresponding `npm` package manager.
+- A MySQL database with the version 2 (No PDF) data structure.
+- An operational API (parent directory - see the API's README).
+- The `DATABASE_URL` environment variable should be configured to point to your local MySQL database, for example: `mysql://root:@localhost:1234/db-name`.
+- The local `DATABASE_USER` environment variable
+- The local `DATABASE_PASSWORD` environment variable
+- The local `CUID_FINGERPRINT` environment variable
+- `mysql2` NPM package installed as an API dependency.
+- `ts-node` NPM package installed locally.
+
+## Running Locally :computer:
+
+1. Ensure that your database has the version 2 (No PDF) data structure.
+2. In the API directory, execute `npm run data-migration`.
+
+The migration should successfully do the following:
+
+1. Connect to the database.
+2. Remove the application `migratedV1toV2` field.
+3. Create a new application `migratedV2toV3` field.
+4. Create a new requestedCreditLimit field in the policy table.
+5. Update the application version number.
+6. Update the new application `migratedV2toV3` field.
+
+## How to ensure that data migration was successful
+
+1. All applications should be aligned with the version 3 data model (listed above)
+2. In the UI, all existing applications with a status of "in progress" can be progressed and successfully submitted.
+
+## What happens to applications that are in progress :microscope:
+
+Due to the nature of GraphQL and KeystoneJS - the version 2 and version 3 data models are essentially "out of sync".
+
+If we try to run the version 3 API, with version 2 data, things will not work.
+
+Similarly, if we migrate the database and the API is running on version 3 - any applications created with the version 2 data model need to be migrated to the new version 3 model - hence why we need migration logic.
+
+This means that if a user has completed e.g all questions in an application, but has not yet submitted - and then we migrate to version 3, the user's application will be migrated from version 2 ("No PDF") to version 3 ("No PDF iterations"). Because this migration is minimal, only one section will be marked as incomplete - the "Policy" section. Because the `requestedCreditLimit` field is now required as part of version 3 ("No PDF interations).
+
+This is an intentional behaviour, so that a user can continue to complete and submit an application. The alternative to this is to ask a user to start again, which is not recommended.
+
+If you have any specific questions or need further guidance related to this data migration or the API, please feel free to ask.

src/api/data-migration/version-3-to-version-4/index.ts

@@ -0,0 +1,30 @@
+import connectToDatabase from '../connect-to-database';
+import updateApplications from './update-applications';
+
+/**
+ * dataMigration
+ * Update all applications from the V3 data model/structure,
+ * to the V4 data model/structure.
+ * @returns {Function} process.exit()
+ */
+const dataMigration = async () => {
+  try {
+    console.info('🚀 Beginning data migration (v3 to v4)');
+
+    const connection = await connectToDatabase();
+
+    await updateApplications(connection);
+
+    console.info('✅ Applications successfully updated');
+
+    console.info('🎉 Migration complete. Exiting script');
+
+    process.exit();
+  } catch (error) {
+    console.error('🚨 Error with data migration %o', error);
+
+    throw new Error(`🚨 Error with data migration ${error}`);
+  }
+};
+
+dataMigration();

src/api/data-migration/version-3-to-version-4/update-applications/add-application-migrated-v3-to-v4-field.ts

@@ -0,0 +1,18 @@
+import { Connection } from 'mysql2/promise';
+import executeSqlQuery from '../../execute-sql-query';
+
+/**
+ * addMigratedV3toV4Field
+ * Add a migratedV3toV4 field to the application table.
+ * @param {Connection} connection: SQL database connection
+ * @returns {Promise<Array<object>>} executeSqlQuery response
+ */
+const addMigratedV3toV4Field = (connection: Connection) => {
+  const loggingMessage = 'Adding FIELD migratedV3toV4 to application table';
+
+  const query = `ALTER TABLE Application ADD migratedV3toV4 tinyint(1) DEFAULT NULL`;
+
+  return executeSqlQuery({ connection, query, loggingMessage });
+};
+
+export default addMigratedV3toV4Field;

src/api/data-migration/version-3-to-version-4/update-applications/create-declaration-modern-slavery-table.ts

@@ -0,0 +1,28 @@
+import { Connection } from 'mysql2/promise';
+import executeSqlQuery from '../../execute-sql-query';
+
+/**
+ * createDeclarationModernSlaveryTable
+ * Create new "declaration modern slavery" database table.
+ * @param {Connection} connection: SQL database connection
+ * @returns {Promise<Array<object>>} executeSqlQuery response
+ */
+const createDeclarationModernSlaveryTable = (connection: Connection) => {
+  const loggingMessage = 'Creating TABLE - declaration modern slavery';
+
+  const query = `
+    CREATE TABLE DeclarationModernSlavery (
+      id varchar(191) COLLATE utf8mb4_unicode_ci NOT NULL,
+      declaration varchar(191) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+      version varchar(191) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+      willAdhereToAllRequirements tinyint(1) DEFAULT NULL,
+      hasNoOffensesOrInvestigations tinyint(1) DEFAULT NULL,
+      isNotAwareOfExistingSlavery tinyint(1) DEFAULT NULL,
+      PRIMARY KEY (id)
+    ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci;
+  `;
+
+  return executeSqlQuery({ connection, query, loggingMessage });
+};
+
+export default createDeclarationModernSlaveryTable;

src/api/data-migration/version-3-to-version-4/update-applications/create-declaration-modern-slavery-version-table.ts

@@ -0,0 +1,27 @@
+import { Connection } from 'mysql2/promise';
+import executeSqlQuery from '../../execute-sql-query';
+
+/**
+ * createDeclarationModernSlaveryVersionTable
+ * Create new "declaration modern slavery version" database table.
+ * @param {Connection} connection: SQL database connection
+ * @returns {Promise<Array<object>>} executeSqlQuery response
+ */
+const createDeclarationModernSlaveryVersionTable = (connection: Connection) => {
+  const loggingMessage = 'Creating TABLE - declaration modern slavery version';
+
+  const query = `
+    CREATE TABLE DeclarationModernSlaveryVersion (
+      id varchar(191) COLLATE utf8mb4_unicode_ci NOT NULL,
+      declarationModernSlavery varchar(191) COLLATE utf8mb4_unicode_ci DEFAULT NULL,
+      hasNoOffensesOrInvestigations tinyint(1) DEFAULT NULL,
+      isNotAwareOfExistingSlavery tinyint(1) DEFAULT NULL,
+      willAdhereToAllRequirements tinyint(1) DEFAULT NULL,
+      PRIMARY KEY (id)
+    ) ENGINE = InnoDB DEFAULT CHARSET = utf8mb4 COLLATE = utf8mb4_unicode_ci;
+  `;
+
+  return executeSqlQuery({ connection, query, loggingMessage });
+};
+
+export default createDeclarationModernSlaveryVersionTable;