Calculating and Storing Requirement Statistics

package.json

@@ -23,7 +23,8 @@
     "test": "jest",
     "cypress:open": "cypress open",
     "cypress:run": "cypress run",
-    "cypress:e2e": "npm run build:dev && concurrently \"npm run serve:build\" \"wait-on http-get://localhost:8080 && npm run cypress:run\" --kill-others --success first"
+    "cypress:e2e": "npm run build:dev && concurrently \"npm run serve:build\" \"wait-on http-get://localhost:8080 && npm run cypress:run\" --kill-others --success first",
+    "all-tests" : "npm run lint && npm run type-check && npm run test && npm run format:check"
   },
   "dependencies": {
     "@types/intro.js": "^3.0.0",

scripts/firebase-config.ts

@@ -43,6 +43,14 @@ const userCollections = {
   onboarding: 'user-onboarding-data',
 };
 
+const helperCollections = {
+  track: 'track-users',
+  courses: 'courses',
+  availableRostersForCourse: 'available-rosters-for-course',
+  crseIdToCatalogNbr: 'crseid-to-catalognbr',
+  courseFulfillmentStats: 'course-fulfillment-stats',
+};
+
 export const userCollectionNames = Object.values(userCollections);
 
 export const usernameCollection = db.collection(userCollections.name);
@@ -52,7 +60,10 @@ export const overriddenFulfillmentChoicesCollection = db.collection(userCollecti
 export const subjectColorsCollection = db.collection(userCollections.colors);
 export const uniqueIncrementerCollection = db.collection(userCollections.unique);
 export const onboardingDataCollection = db.collection(userCollections.onboarding);
-export const trackUsersCollection = db.collection('track-users');
-export const coursesCollection = db.collection('courses');
-export const availableRostersForCourseCollection = db.collection('available-rosters-for-course');
-export const crseIdToCatalogNbrCollection = db.collection('crseid-to-catalognbr');
+export const trackUsersCollection = db.collection(helperCollections.track);
+export const coursesCollection = db.collection(helperCollections.courses);
+export const availableRostersForCourseCollection = db.collection(
+  helperCollections.availableRostersForCourse
+);
+export const crseIdToCatalogNbrCollection = db.collection(helperCollections.crseIdToCatalogNbr);
+export const courseFulfillmentStats = db.collection(helperCollections.courseFulfillmentStats);

scripts/gen-req-full-stats.ts

@@ -0,0 +1,109 @@
+import {
+  onboardingDataCollection,
+  semestersCollection,
+  toggleableRequirementChoicesCollection,
+  overriddenFulfillmentChoicesCollection,
+  courseFulfillmentStats,
+} from './firebase-config';
+
+import computeGroupedRequirementFulfillmentReports from '../src/requirements/requirement-frontend-computation';
+import computeFulfillmentStats from '../src/requirements/fulfillment-stats';
+import { createAppOnboardingData } from '../src/user-data-converter';
+
+import '../src/requirements/decorated-requirements.json';
+
+let idRequirementFrequency = new Map<string, Map<number, number>[]>();
+let counter = 0;
+/**
+ * Prints to console.out the json string of frequencies each course take to
+ * fulfill a given requirement
+ */
+async function computeRequirementFullfillmentStatistics(_callback) {
+  const semQuerySnapshot = await semestersCollection.get();
+  semQuerySnapshot.forEach(async doc => {
+    // obtain the user's semesters, onboarding data, etc...
+    const semestersAndPlans = await doc.data();
+    const onboardingData = (await onboardingDataCollection.doc(doc.id).get()).data();
+    const toggleableRequirementChoices =
+      (await toggleableRequirementChoicesCollection.doc(doc.id).get()).data() ?? {};
+    const overriddenFulfillmentChoices =
+      (await overriddenFulfillmentChoicesCollection.doc(doc.id).get()).data() ?? {};
+
+    if (
+      onboardingData !== undefined &&
+      semestersAndPlans.semesters !== undefined &&
+      toggleableRequirementChoices !== undefined &&
+      overriddenFulfillmentChoices !== undefined
+    ) {
+      // if the user has onboarding data
+      try {
+        const newOnboardingData = await createAppOnboardingData(onboardingData); // convert to new format
+        const res = await computeGroupedRequirementFulfillmentReports(
+          // compute fulfillment stats
+          semestersAndPlans.semesters,
+          newOnboardingData,
+          toggleableRequirementChoices,
+          overriddenFulfillmentChoices
+        );
+        idRequirementFrequency = await computeFulfillmentStats(
+          res.groupedRequirementFulfillmentReport,
+          idRequirementFrequency
+        ); // compute fulfillment stats
+      } catch {
+        console.log(counter, ' : Error in computing fulfillment stats for user: ', doc.id);
+        counter += 1;
+      }
+    }
+  });
+
+  setTimeout(_callback, 3600 * 1000); // wait for all the data to be processed
+}
+
+/**
+ * Stores the computed requirement fulfillment statistics in firestore
+ */
+async function storeComputedRequirementFullfillmentStatistics() {
+  console.log('Keeping only the top fifty courses for each slot');
+  // iterate through all the keys in the idRequirementFrequency hashmap
+  for (const [reqID, slots] of idRequirementFrequency) {
+    const newSlots: Map<number, number>[] = [];
+    for (const slot of slots) {
+      const newSlot = new Map<number, number>();
+      const sorted = [...slot.entries()].sort((a, b) => b[1] - a[1]);
+      let count = 0;
+      for (const [course, freq] of sorted) {
+        if (count === 50) {
+          break;
+        }
+        newSlot.set(course, freq);
+        count += 1;
+      }
+      newSlots.push(newSlot);
+    }
+    idRequirementFrequency.set(reqID, newSlots);
+  }
+
+  console.log('Storing fulfillment stats in firestore');
+  // iterate through all the keys in the idRequirementFrequency hashmap
+  for (const [reqID, slots] of idRequirementFrequency) {
+    const json = {};
+
+    let i = 0;
+    for (const slot of slots) {
+      const tempJson = {};
+      for (const [course, freq] of slot) {
+        tempJson[course] = freq;
+      }
+      json[i] = tempJson;
+      i += 1;
+    }
+
+    const data = {
+      reqID,
+      slots: json,
+    };
+    courseFulfillmentStats.doc().set(data);
+  }
+}
+
+computeRequirementFullfillmentStatistics(storeComputedRequirementFullfillmentStatistics);

src/requirements/fulfillment-stats.ts

@@ -0,0 +1,37 @@
+export default function computeFulfillmentStats(
+  groups: readonly GroupedRequirementFulfillmentReport[],
+  idRequirementFrequency: Map<string, Map<number, number>[]>
+) {
+  const res = idRequirementFrequency;
+  groups.forEach(currentGroup => {
+    const { reqs } = currentGroup;
+    reqs.forEach(reqFulfillment => {
+      const key: string = reqFulfillment.requirement.id;
+      const { safeCourses } = reqFulfillment.fulfillment;
+
+      // Obtain the frequency list for this particular group's requirements
+      const freqList = res.get(key) ?? [];
+
+      // Iterate over all slots in the requirement group
+      // console.log(safeCourses.length);
+      for (let slotNumber = 0; slotNumber < safeCourses.length; slotNumber += 1) {
+        if (freqList.length === slotNumber) {
+          freqList.push(new Map<number, number>());
+        }
+        const currentCourseSlot = safeCourses[slotNumber];
+        const currentRequirementSlotFreq = freqList[slotNumber];
+
+        // Iterate over all courses taken to fulfill the req-slot
+        for (let j = 0; j < currentCourseSlot.length; j += 1) {
+          const currentCourseId = currentCourseSlot[j].courseId;
+          const pastFreq = currentRequirementSlotFreq.get(currentCourseId) ?? 0;
+          currentRequirementSlotFreq.set(currentCourseId, pastFreq + 1);
+        }
+        freqList[slotNumber] = currentRequirementSlotFreq; // Update the frequency list
+      }
+      res.set(key, freqList); // Update the hashmap with the new frequency list
+    });
+  });
+
+  return res; // return the hashmap
+}