mining_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b71875e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "516465c8",
   "metadata": {},
   "source": [
    "### Additional Requirements\n",
    "\n",
    "If you wish to run this entire notebook, you need to install OCTIS\n",
    "\n",
    "~~~\n",
    "pip install octis\n",
    "~~~"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "6bddc160",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "os.environ['CUDA_DEVICE_ORDER'] = 'PCI_BUS_ID'\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = str(0)\n",
    "\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from mlxtend.frequent_patterns import apriori"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0f0522f3",
   "metadata": {},
   "source": [
    "#### Train the model and get topic-word, topic-document distribution\n",
    "\n",
    "#### For this notebook, we use OCTIS to train, you may substitute it with any NTMof your choosing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c59994d5",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jiapeng/conda/envs/p3/lib/python3.10/site-packages/tqdm/auto.py:22: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [1/100]\tSamples: [11415/1141500]\tTrain Loss: 382.95891251231933\tTime: 0:00:01.207505\n",
      "Epoch: [1/100]\tSamples: [2447/244700]\tValidation Loss: 338.8774008990601\tTime: 0:00:00.063604\n",
      "Epoch: [2/100]\tSamples: [22830/1141500]\tTrain Loss: 360.0117293583005\tTime: 0:00:00.672641\n",
      "Epoch: [2/100]\tSamples: [2447/244700]\tValidation Loss: 336.9096943084006\tTime: 0:00:00.062449\n",
      "Epoch: [3/100]\tSamples: [34245/1141500]\tTrain Loss: 355.4035380820877\tTime: 0:00:00.656063\n",
      "Epoch: [3/100]\tSamples: [2447/244700]\tValidation Loss: 329.9719924708508\tTime: 0:00:00.062872\n",
      "Epoch: [4/100]\tSamples: [45660/1141500]\tTrain Loss: 350.0400070664969\tTime: 0:00:00.655187\n",
      "Epoch: [4/100]\tSamples: [2447/244700]\tValidation Loss: 328.23213453335717\tTime: 0:00:00.062800\n",
      "Epoch: [5/100]\tSamples: [57075/1141500]\tTrain Loss: 347.6271623822821\tTime: 0:00:00.656386\n",
      "Epoch: [5/100]\tSamples: [2447/244700]\tValidation Loss: 423.54268140835717\tTime: 0:00:00.062621\n",
      "Epoch: [6/100]\tSamples: [68490/1141500]\tTrain Loss: 343.9628962285781\tTime: 0:00:00.655047\n",
      "Epoch: [6/100]\tSamples: [2447/244700]\tValidation Loss: 325.76223297181497\tTime: 0:00:00.062862\n",
      "Epoch: [7/100]\tSamples: [79905/1141500]\tTrain Loss: 341.9796277855344\tTime: 0:00:00.655547\n",
      "Epoch: [7/100]\tSamples: [2447/244700]\tValidation Loss: 323.2648081715174\tTime: 0:00:00.062916\n",
      "Epoch: [8/100]\tSamples: [91320/1141500]\tTrain Loss: 340.92044972349976\tTime: 0:00:00.766995\n",
      "Epoch: [8/100]\tSamples: [2447/244700]\tValidation Loss: 321.6266158976553\tTime: 0:00:00.063241\n",
      "Epoch: [9/100]\tSamples: [102735/1141500]\tTrain Loss: 339.41852679793584\tTime: 0:00:00.657936\n",
      "Epoch: [9/100]\tSamples: [2447/244700]\tValidation Loss: 320.3644754139316\tTime: 0:00:00.062898\n",
      "Epoch: [10/100]\tSamples: [114150/1141500]\tTrain Loss: 338.96339272270586\tTime: 0:00:00.658814\n",
      "Epoch: [10/100]\tSamples: [2447/244700]\tValidation Loss: 320.52723519871273\tTime: 0:00:00.063173\n",
      "Epoch: [11/100]\tSamples: [125565/1141500]\tTrain Loss: 337.7835306990528\tTime: 0:00:00.658077\n",
      "Epoch: [11/100]\tSamples: [2447/244700]\tValidation Loss: 320.05454552529244\tTime: 0:00:00.062999\n",
      "Epoch: [12/100]\tSamples: [136980/1141500]\tTrain Loss: 337.2359833124589\tTime: 0:00:00.657126\n",
      "Epoch: [12/100]\tSamples: [2447/244700]\tValidation Loss: 319.5382172366673\tTime: 0:00:00.063218\n",
      "Epoch: [13/100]\tSamples: [148395/1141500]\tTrain Loss: 337.0113720981165\tTime: 0:00:00.658926\n",
      "Epoch: [13/100]\tSamples: [2447/244700]\tValidation Loss: 318.6994625913108\tTime: 0:00:00.065356\n",
      "Epoch: [14/100]\tSamples: [159810/1141500]\tTrain Loss: 336.56382312883267\tTime: 0:00:00.658459\n",
      "Epoch: [14/100]\tSamples: [2447/244700]\tValidation Loss: 317.7231966119228\tTime: 0:00:00.063056\n",
      "Epoch: [15/100]\tSamples: [171225/1141500]\tTrain Loss: 336.0257349242362\tTime: 0:00:00.659865\n",
      "Epoch: [15/100]\tSamples: [2447/244700]\tValidation Loss: 320.11341853545156\tTime: 0:00:00.063381\n",
      "Epoch: [16/100]\tSamples: [182640/1141500]\tTrain Loss: 335.52748961413164\tTime: 0:00:00.768037\n",
      "Epoch: [16/100]\tSamples: [2447/244700]\tValidation Loss: 316.1924945564722\tTime: 0:00:00.063353\n",
      "Epoch: [17/100]\tSamples: [194055/1141500]\tTrain Loss: 335.05097095892194\tTime: 0:00:00.660240\n",
      "Epoch: [17/100]\tSamples: [2447/244700]\tValidation Loss: 316.8716166519271\tTime: 0:00:00.063119\n",
      "Epoch: [18/100]\tSamples: [205470/1141500]\tTrain Loss: 335.51928529142026\tTime: 0:00:00.660707\n",
      "Epoch: [18/100]\tSamples: [2447/244700]\tValidation Loss: 316.75428258773496\tTime: 0:00:00.063191\n",
      "Epoch: [19/100]\tSamples: [216885/1141500]\tTrain Loss: 335.13783578693057\tTime: 0:00:00.659411\n",
      "Epoch: [19/100]\tSamples: [2447/244700]\tValidation Loss: 316.8296111948304\tTime: 0:00:00.063344\n",
      "Epoch: [20/100]\tSamples: [228300/1141500]\tTrain Loss: 335.04339761621225\tTime: 0:00:00.661197\n",
      "Epoch: [20/100]\tSamples: [2447/244700]\tValidation Loss: 316.71857599867184\tTime: 0:00:00.063450\n",
      "Epoch: [21/100]\tSamples: [239715/1141500]\tTrain Loss: 334.77508401089574\tTime: 0:00:00.660827\n",
      "Epoch: [21/100]\tSamples: [2447/244700]\tValidation Loss: 315.7910071444307\tTime: 0:00:00.063406\n",
      "Epoch: [22/100]\tSamples: [251130/1141500]\tTrain Loss: 334.72673000848664\tTime: 0:00:00.660978\n",
      "Epoch: [22/100]\tSamples: [2447/244700]\tValidation Loss: 316.1770579248825\tTime: 0:00:00.063112\n",
      "Epoch: [23/100]\tSamples: [262545/1141500]\tTrain Loss: 334.7748200866459\tTime: 0:00:00.660942\n",
      "Epoch: [23/100]\tSamples: [2447/244700]\tValidation Loss: 316.5100441867593\tTime: 0:00:00.063197\n",
      "Epoch: [24/100]\tSamples: [273960/1141500]\tTrain Loss: 334.0049805970762\tTime: 0:00:00.768116\n",
      "Epoch: [24/100]\tSamples: [2447/244700]\tValidation Loss: 314.9559028354235\tTime: 0:00:00.063617\n",
      "Epoch: [25/100]\tSamples: [285375/1141500]\tTrain Loss: 334.3512922022147\tTime: 0:00:00.662459\n",
      "Epoch: [25/100]\tSamples: [2447/244700]\tValidation Loss: 316.17023356086537\tTime: 0:00:00.063295\n",
      "Epoch: [26/100]\tSamples: [296790/1141500]\tTrain Loss: 334.27621666084923\tTime: 0:00:00.660963\n",
      "Epoch: [26/100]\tSamples: [2447/244700]\tValidation Loss: 315.5164654743053\tTime: 0:00:00.063663\n",
      "Epoch: [27/100]\tSamples: [308205/1141500]\tTrain Loss: 335.2325036615747\tTime: 0:00:00.661791\n",
      "Epoch: [27/100]\tSamples: [2447/244700]\tValidation Loss: 315.31087412520435\tTime: 0:00:00.063630\n",
      "Epoch: [28/100]\tSamples: [319620/1141500]\tTrain Loss: 334.0736072328077\tTime: 0:00:00.663122\n",
      "Epoch: [28/100]\tSamples: [2447/244700]\tValidation Loss: 314.8978093392419\tTime: 0:00:00.063911\n",
      "Epoch: [29/100]\tSamples: [331035/1141500]\tTrain Loss: 333.6839822362297\tTime: 0:00:00.662100\n",
      "Epoch: [29/100]\tSamples: [2447/244700]\tValidation Loss: 315.02570430629345\tTime: 0:00:00.063021\n",
      "Epoch: [30/100]\tSamples: [342450/1141500]\tTrain Loss: 333.3527845933257\tTime: 0:00:00.662227\n",
      "Epoch: [30/100]\tSamples: [2447/244700]\tValidation Loss: 314.84514480422456\tTime: 0:00:00.063595\n",
      "Epoch: [31/100]\tSamples: [353865/1141500]\tTrain Loss: 333.73743147380094\tTime: 0:00:00.660804\n",
      "Epoch: [31/100]\tSamples: [2447/244700]\tValidation Loss: 314.95053792750053\tTime: 0:00:00.063618\n",
      "Epoch: [32/100]\tSamples: [365280/1141500]\tTrain Loss: 333.18458742266205\tTime: 0:00:00.769214\n",
      "Epoch: [32/100]\tSamples: [2447/244700]\tValidation Loss: 314.5020908095372\tTime: 0:00:00.063782\n",
      "Epoch: [33/100]\tSamples: [376695/1141500]\tTrain Loss: 333.697413114597\tTime: 0:00:00.664327\n",
      "Epoch: [33/100]\tSamples: [2447/244700]\tValidation Loss: 314.8724578246322\tTime: 0:00:00.063795\n",
      "Epoch: [34/100]\tSamples: [388110/1141500]\tTrain Loss: 333.4878234676139\tTime: 0:00:00.662819\n",
      "Epoch: [34/100]\tSamples: [2447/244700]\tValidation Loss: 315.31488333929303\tTime: 0:00:00.063728\n",
      "Epoch: [35/100]\tSamples: [399525/1141500]\tTrain Loss: 333.19055005749016\tTime: 0:00:00.664392\n",
      "Epoch: [35/100]\tSamples: [2447/244700]\tValidation Loss: 314.57319579379725\tTime: 0:00:00.063668\n",
      "Epoch: [36/100]\tSamples: [410940/1141500]\tTrain Loss: 333.54313215204775\tTime: 0:00:00.662190\n",
      "Epoch: [36/100]\tSamples: [2447/244700]\tValidation Loss: 314.0584451903798\tTime: 0:00:00.063867\n",
      "Epoch: [37/100]\tSamples: [422355/1141500]\tTrain Loss: 333.3082277636334\tTime: 0:00:00.663989\n",
      "Epoch: [37/100]\tSamples: [2447/244700]\tValidation Loss: 314.9331423841694\tTime: 0:00:00.063564\n",
      "Epoch: [38/100]\tSamples: [433770/1141500]\tTrain Loss: 332.8443135659631\tTime: 0:00:00.662664\n",
      "Epoch: [38/100]\tSamples: [2447/244700]\tValidation Loss: 314.3881098626826\tTime: 0:00:00.064034\n",
      "Epoch: [39/100]\tSamples: [445185/1141500]\tTrain Loss: 333.17928391405223\tTime: 0:00:00.661800\n",
      "Epoch: [39/100]\tSamples: [2447/244700]\tValidation Loss: 314.2179829574479\tTime: 0:00:00.063741\n",
      "Epoch: [40/100]\tSamples: [456600/1141500]\tTrain Loss: 333.10897710318113\tTime: 0:00:00.769869\n",
      "Epoch: [40/100]\tSamples: [2447/244700]\tValidation Loss: 314.399264305623\tTime: 0:00:00.063654\n",
      "Epoch: [41/100]\tSamples: [468015/1141500]\tTrain Loss: 333.43437722432105\tTime: 0:00:00.663507\n",
      "Epoch: [41/100]\tSamples: [2447/244700]\tValidation Loss: 313.86764794502835\tTime: 0:00:00.064026\n",
      "Epoch: [42/100]\tSamples: [479430/1141500]\tTrain Loss: 332.59420872344504\tTime: 0:00:00.666904\n",
      "Epoch: [42/100]\tSamples: [2447/244700]\tValidation Loss: 314.2658170599522\tTime: 0:00:00.064129\n",
      "Epoch: [43/100]\tSamples: [490845/1141500]\tTrain Loss: 333.2971806730453\tTime: 0:00:00.663010\n",
      "Epoch: [43/100]\tSamples: [2447/244700]\tValidation Loss: 314.44047288851783\tTime: 0:00:00.063994\n",
      "Epoch: [44/100]\tSamples: [502260/1141500]\tTrain Loss: 333.25931361729357\tTime: 0:00:00.664365\n",
      "Epoch: [44/100]\tSamples: [2447/244700]\tValidation Loss: 313.88356248164206\tTime: 0:00:00.063743\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: [45/100]\tSamples: [513675/1141500]\tTrain Loss: 332.6589108440785\tTime: 0:00:00.665572\n",
      "Epoch: [45/100]\tSamples: [2447/244700]\tValidation Loss: 314.9151380357325\tTime: 0:00:00.063798\n",
      "Epoch: [46/100]\tSamples: [525090/1141500]\tTrain Loss: 332.70700815128123\tTime: 0:00:00.664554\n",
      "Epoch: [46/100]\tSamples: [2447/244700]\tValidation Loss: 314.4691105738532\tTime: 0:00:00.063970\n",
      "Early stopping\n"
     ]
    }
   ],
   "source": [
    "from octis.dataset.dataset import Dataset\n",
    "from octis.models.ProdLDA import ProdLDA\n",
    "\n",
    "dataset = Dataset()\n",
    "dataset.fetch_dataset(\"20NewsGroup\")\n",
    "# data_corpus = [' '.join(i) for i in dataset.get_corpus()]\n",
    "K = 20\n",
    "model = ProdLDA(num_topics=K)  \n",
    "model_output = model.train_model(dataset) "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b90b222",
   "metadata": {},
   "source": [
    "#### Notice that the beta (topic-word) has negative activations and has a shape of K x V"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "2d9bea75",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(20, 1612)\n",
      "[[-0.13062593 -0.08251245  0.00296721 ... -0.0295416  -0.34053457\n",
      "  -0.03513475]\n",
      " [ 0.02646185 -0.17787448 -0.16627383 ... -0.21845856 -0.5283106\n",
      "  -0.2165347 ]\n",
      " [ 0.03994723  0.18910125  0.04728971 ... -0.2283454  -0.3732265\n",
      "  -0.30879128]\n",
      " ...\n",
      " [ 0.18177032  0.00865875  0.05434528 ...  0.06564841  0.10349061\n",
      "   0.22130793]\n",
      " [-0.0826124  -0.18028025  0.05177277 ...  0.20207533 -0.06025597\n",
      "  -0.0767033 ]\n",
      " [-0.22579312 -0.27452523 -0.24259743 ... -0.386449   -0.24701835\n",
      "  -0.527656  ]]\n"
     ]
    }
   ],
   "source": [
    "print(model_output['topic-word-matrix'].shape)\n",
    "print(model_output['topic-word-matrix'])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74a213de",
   "metadata": {},
   "source": [
    "### Mining Step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "62d59ed8",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "threshold 0.05591654907795865\n",
      "Processing 8416 combinations | Sampling itemset size 4"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jiapeng/conda/envs/p3/lib/python3.10/site-packages/mlxtend/frequent_patterns/fpcommon.py:111: DeprecationWarning: DataFrames with non-bool types result in worse computationalperformance and their support might be discontinued in the future.Please use a DataFrame with bool type\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\r",
      "Processing 2105 combinations | Sampling itemset size 5\n",
      "topics generated @ s = 0.01 : 822\n"
     ]
    }
   ],
   "source": [
    "beta = model_output['topic-word-matrix'].T\n",
    "data = model_output['topic-document-matrix'].T\n",
    "mat = np.sort(data, axis=1).mean(axis=0)\n",
    "assert len(mat) == K\n",
    "\n",
    "# Find a suitable threshold kappa hyper-parameter (currently using 5th largest mean activation)\n",
    "threshold = mat[-5]\n",
    "print('threshold', threshold)\n",
    "    \n",
    "reduced_data = np.zeros((data.shape[0],data.shape[1]))\n",
    "for i,j in np.argwhere(data > threshold):\n",
    "    reduced_data[i,j] = 1\n",
    "\n",
    "reduced_data = pd.DataFrame(reduced_data)\n",
    "\n",
    "min_s = 0.01\n",
    "frequent_itemsets = apriori(reduced_data, \n",
    "                            min_support = min_s, \n",
    "                            max_len = 5, \n",
    "                            use_colnames = True,\n",
    "                           verbose = 1)\n",
    "\n",
    "topic_combinations = [list(a) for a in frequent_itemsets['itemsets']]\n",
    "print(\"topics generated @ s =\", min_s, \":\", len(topic_combinations))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f178fa55",
   "metadata": {},
   "source": [
    "### Get possible compositions ($\\beta^{V x K}$)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "cf41a3b8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "from algo.normal import calculate_compositions\n",
    "\n",
    "output, topic_combinations = calculate_compositions(beta, topic_combinations, add_pairs=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ec95e3b4",
   "metadata": {},
   "source": [
    "### Get top-$l$ vocab ($l = 10$)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fc9c404f",
   "metadata": {},
   "outputs": [],
   "source": [
    "topics = np.argpartition(np.array(output),-10)[:,-10:]\n",
    "vocab = dataset.get_vocabulary()\n",
    "topics = [[vocab[idx] for idx in topic] for topic in topics]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d28490c",
   "metadata": {},
   "source": [
    "### Calculate scores/fitness (estimation done using gensim)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "10fe8ec8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jiapeng/conda/envs/p3/lib/python3.10/site-packages/gensim/matutils.py:22: DeprecationWarning: Please use `triu` from the `scipy.linalg` namespace, the `scipy.linalg.special_matrices` namespace is deprecated.\n",
      "  from scipy.linalg.special_matrices import triu\n"
     ]
    }
   ],
   "source": [
    "from octis.evaluation_metrics.coherence_metrics import Coherence\n",
    "from gensim.models import CoherenceModel\n",
    "\n",
    "coherence = Coherence(texts=dataset.get_corpus(), measure = 'c_npmi')\n",
    "c = CoherenceModel(topics=topics, texts=coherence._texts, dictionary=coherence._dictionary,\n",
    "                                      coherence='c_npmi', processes=4, topn=coherence.topk)\n",
    "total_scores = c.get_coherence_per_topic()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3129a382",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NPMI: 0.07956555862542489\n",
      "TU: 0.92\n"
     ]
    }
   ],
   "source": [
    "from algo.normal import count_unique_words\n",
    "\n",
    "# for original\n",
    "print('NPMI:', np.mean(total_scores[:K]))\n",
    "print('TU:', count_unique_words(topics[:K]) / (K*10))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ddfd12e3",
   "metadata": {},
   "source": [
    "### Optimization Step"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "471f23e0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "===============================================================================\n",
      "                                     CVXPY                                     \n",
      "                                     v1.3.0                                    \n",
      "===============================================================================\n",
      "(CVXPY) Oct 28 09:06:12 PM: Your problem has 1647 variables, 827 constraints, and 0 parameters.\n",
      "(CVXPY) Oct 28 09:06:12 PM: It is compliant with the following grammars: DCP, DQCP\n",
      "(CVXPY) Oct 28 09:06:12 PM: (If you need to solve this problem multiple times, but with different data, consider using parameters.)\n",
      "(CVXPY) Oct 28 09:06:12 PM: CVXPY will first compile your problem; then, it will invoke a numerical solver to obtain a solution.\n",
      "-------------------------------------------------------------------------------\n",
      "                                  Compilation                                  \n",
      "-------------------------------------------------------------------------------\n",
      "(CVXPY) Oct 28 09:06:12 PM: Compiling problem (target solver=SCIP).\n",
      "(CVXPY) Oct 28 09:06:12 PM: Reduction chain: FlipObjective -> Dcp2Cone -> CvxAttr2Constr -> ConeMatrixStuffing -> SCIP\n",
      "(CVXPY) Oct 28 09:06:12 PM: Applying reduction FlipObjective\n",
      "(CVXPY) Oct 28 09:06:12 PM: Applying reduction Dcp2Cone\n",
      "(CVXPY) Oct 28 09:06:12 PM: Applying reduction CvxAttr2Constr\n",
      "(CVXPY) Oct 28 09:06:12 PM: Applying reduction ConeMatrixStuffing\n",
      "(CVXPY) Oct 28 09:06:13 PM: Applying reduction SCIP\n",
      "(CVXPY) Oct 28 09:06:13 PM: Finished problem compilation (took 1.141e+00 seconds).\n",
      "-------------------------------------------------------------------------------\n",
      "                                Numerical solver                               \n",
      "-------------------------------------------------------------------------------\n",
      "(CVXPY) Oct 28 09:06:13 PM: Invoking solver SCIP  to obtain a solution.\n",
      "presolving:\n",
      "-------------------------------------------------------------------------------\n",
      "                                    Summary                                    \n",
      "-------------------------------------------------------------------------------\n",
      "(CVXPY) Oct 28 09:06:14 PM: Problem status: optimal\n",
      "(CVXPY) Oct 28 09:06:14 PM: Optimal value: 3.226e+00\n",
      "   (0.0s) running MILP presolver\n",
      "   (0.0s) MILP presolver found nothing\n",
      "(round 1, exhaustive) 0 del vars, 0 del conss, 0 add conss, 0 chg bounds, 0 chg sides, 0 chg coeffs, 1 upgd conss, 0 impls, 0 clqs\n",
      "(round 2, exhaustive) 0 del vars, 0 del conss, 0 add conss, 0 chg bounds, 0 chg sides, 0 chg coeffs, 826 upgd conss, 0 impls, 0 clqs\n",
      "   (0.1s) probing: 51/1647 (3.1%) - 0 fixings, 0 aggregations, 0 implications, 0 bound changes\n",
      "   (0.1s) probing aborted: 50/50 successive totally useless probings\n",
      "   (0.1s) symmetry computation started: requiring (bin +, int -, cont +), (fixed: bin -, int +, cont -)\n",
      "   (0.1s) symmetry computation finished: 3 generators found (max: 1500, log10 of symmetry group size: 0.9)\n",
      "(round 3, exhaustive) 0 del vars, 0 del conss, 3 add conss, 0 chg bounds, 0 chg sides, 0 chg coeffs, 826 upgd conss, 0 impls, 0 clqs\n",
      "presolving (4 rounds: 4 fast, 4 medium, 4 exhaustive):\n",
      " 0 deleted vars, 0 deleted constraints, 3 added constraints, 0 tightened bounds, 0 added holes, 0 cha(CVXPY) Oct 28 09:06:14 PM: Compilation took 1.141e+00 seconds\n",
      "nged sides, 0 changed coefficients\n",
      " 0 implications, 0 cliques\n",
      "presolved problem has 1647 variables (1647 bin, 0 int, 0 impl, 0 cont) and 830 constraints\n",
      "      1 constraints of type <knapsack>\n",
      "      1 constraints of type <linear>\n",
      "      3 constraints of type <orbitope>\n",
      "    825 constraints of type <logicor>\n",
      "Presolving Time: 0.07\n",
      "\n",
      " time | node  | left  |LP iter|LP it/n|mem/heur|mdpt |vars |cons |rows |cuts |sepa|confs|strbr|  dualbound   | primalbound  |  gap   | compl. \n",
      "  0.6s|     1 |     0 |  1208 |     - |    29M |   0 |1647 | 830 | 827 |   0 |  0 |   0 |   0 |-3.239935e+00 |      --      |    Inf | unknown\n",
      "o 0.6s|     1 |     0 |  2286 |     - |feaspump|   0 |1647 | 830 | 827 |   0 |  0 |   0 |   0 |-3.239935e+00 |-3.064681e+00 |   5.72%| unknown\n",
      "  0.7s|     1 |     0 |  2317 |     - |    30M |   0 |1647 | 830 | 829 |   2 |  1 |   0 |   0 |-3.226367e+00 |-3.064681e+00 |   5.28%| unknown\n",
      "  0.7s|     1 |     0 |  2317 |     - |    30M |   0 |1647 | 830 | 829 |   2 |  1 |   0 |   0 |-3.226367e+00 |-3.064681e+00 |   5.28%| unknown\n",
      "  0.7s|     1 |     0 |  2318 |     - |    30M |   0 |1647 | 767 | 830 |   3 |  2 |   0 |   0 |-3.225839e+00 |-3.064681e+00 |   5.26%| unknown\n",
      "d 0.7s|     1 |     0 |  2318 |     - |farkasdi|   0 |1647 | 767 | 830 |   0 |  4 |   0 |   0 |-3.225839e+00 |-3.225839e+00 |   0.00%| unknown\n",
      "  0.7s|     1 |     0 |  2318 |     - |    30M |   0 |1647 | 767 | 830 |   3 |  4 |   0 |   0 |-3.225839e+00 |-3.225839e+00 |   0.00%| unknown\n",
      "\n",
      "SCIP Status        : problem is solved [optimal solution found]\n",
      "Solving Time (sec) : 0.73\n",
      "Solving Nodes      : 1\n",
      "Primal Bound       : -3.22583911926677e+00 (2 solutions)\n",
      "Dual Bound         : -3.22583911926677e+00\n",
      "Gap                : 0.00 %\n",
      "(CVXPY) Oct 28 09:06:14 PM: Solver (including time spent in interface) took 1.158e+00 seconds\n"
     ]
    }
   ],
   "source": [
    "from algo.cvxpy_based import mdkp\n",
    "import cvxpy as cp\n",
    "\n",
    "choices = mdkp(topics, total_scores, K, 0.935*K*10, solver = cp.SCIP)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "847a9442",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "NPMI: 0.16129195596333842\n",
      "TU: 0.97\n"
     ]
    }
   ],
   "source": [
    "print('NPMI:', np.mean(np.array(total_scores)[choices]))\n",
    "print('TU:', count_unique_words(np.array(topics)[choices]) / (K*10))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bac83164",
   "metadata": {},
   "source": [
    "#### Displaying composite topics $\\hat{T}$\n",
    "\n",
    "Composite topics shown with its components indented after"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "3c6ad5bc",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 0.229 government enforcement phone secure clipper key encryption encrypt secret chip\n",
      "\t> original component topic\n",
      " 0.250 greek village massacre population muslim russian genocide turkish people armenian\n",
      "\t> original component topic\n",
      " 0.234 bus card hard controller driver ide drive disk scsi problem\n",
      "\t> original component topic\n",
      " 0.282 win league goal playoff season game play player score team\n",
      "\t> original component topic\n",
      " 0.160 heat fuel temperature ground water power engine battery air mile\n",
      "\t> 0.195 power front oil ride brake car engine bike battery tire\n",
      "\t> 0.315 shuttle moon earth satellite solar planet mission orbit surface launch\n",
      " 0.088 claim atheism absolute truth evidence word belief make point thing\n",
      "\t> 0.114 atheist argument point moral belief science prove atheism evidence theory\n",
      "\t> 0.146 eternal relationship church verse scripture man sin homosexual word teach\n",
      " 0.167 tie red wing ranger ice star year team player series\n",
      "\t> 0.282 win league goal playoff season game play player score team\n",
      "\t> 0.315 shuttle moon earth satellite solar planet mission orbit surface launch\n",
      " 0.197 illegal rate firearm control violent weapon assault crime drug gun\n",
      "\t> 0.170 illegal gas police crime gun fire assault compound weapon batf\n",
      "\t> 0.073 people food good insurance year drug disease patient health medical\n",
      " 0.081 life serve law commit death follower die compound evil fire\n",
      "\t> 0.170 illegal gas police crime gun fire assault compound weapon batf\n",
      "\t> 0.146 eternal relationship church verse scripture man sin homosexual word teach\n",
      " 0.214 motherboard ram meg memory monitor board apple video port card\n",
      "\t> 0.134 condition offer ship apple port shipping sale monitor price video\n",
      "\t> 0.234 bus card hard controller driver ide drive disk scsi problem\n",
      " 0.155 start police burn building batf kill tear tank fire gas\n",
      "\t> 0.170 illegal gas police crime gun fire assault compound weapon batf\n",
      "\t> 0.043 floor sit start neighbor didn good time thing make people\n",
      " 0.143 motif character window network server manager library font application terminal\n",
      "\t> 0.178 list network mail newsgroup address email internet mailing domain host\n",
      "\t> 0.166 button application screen manager font run problem window widget motif\n",
      " 0.222 usenet site address mail list newsgroup email internet mailing post\n",
      "\t> 0.047 entry graphic good software format system mail file make image\n",
      "\t> 0.178 list network mail newsgroup address email internet mailing domain host\n",
      " 0.113 sit shift seat noise car clean room ride front bike\n",
      "\t> 0.195 power front oil ride brake car engine bike battery tire\n",
      "\t>-0.285 previously length huge requirement suit white powerful examine limited installation\n",
      "\t> 0.043 floor sit start neighbor didn good time thing make people\n",
      " 0.086 modem device work computer hardware software bit disk file system\n",
      "\t> 0.229 government enforcement phone secure clipper key encryption encrypt secret chip\n",
      "\t> 0.047 entry graphic good software format system mail file make image\n",
      "\t> 0.234 bus card hard controller driver ide drive disk scsi problem\n",
      " 0.086 assume conclude concept scientific observe objective scientist science explain theory\n",
      "\t> 0.114 atheist argument point moral belief science prove atheism evidence theory\n",
      "\t> 0.315 shuttle moon earth satellite solar planet mission orbit surface launch\n",
      "\t>-0.249 rule portion chapter explain freedom relation remark virtually language bias\n",
      " 0.165 company interested shipping pay cost offer condition sell price sale\n",
      "\t> 0.134 condition offer ship apple port shipping sale monitor price video\n",
      "\t> 0.315 shuttle moon earth satellite solar planet mission orbit surface launch\n",
      "\t> 0.073 people food good insurance year drug disease patient health medical\n",
      " 0.188 occupy territory party nation jewish peace conflict arab political israeli\n",
      "\t> 0.052 vote time press israeli meeting president decision yesterday people make\n",
      "\t> 0.250 greek village massacre population muslim russian genocide turkish people armenian\n",
      "\t>-0.249 rule portion chapter explain freedom relation remark virtually language bias\n",
      " 0.078 nature church understanding refer reject relationship discuss christian speak gay\n",
      "\t> 0.052 vote time press israeli meeting president decision yesterday people make\n",
      "\t> 0.146 eternal relationship church verse scripture man sin homosexual word teach\n",
      "\t>-0.249 rule portion chapter explain freedom relation remark virtually language bias\n",
      " 0.088 shot wasn defense hit bad fire guy shoot penalty watch\n",
      "\t> 0.282 win league goal playoff season game play player score team\n",
      "\t> 0.170 illegal gas police crime gun fire assault compound weapon batf\n",
      "\t> 0.043 floor sit start neighbor didn good time thing make people\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(choices)):\n",
    "    if choices[i]:\n",
    "        print(\"{0: .3f}\".format(total_scores[i]), \" \".join(topics[i]))\n",
    "        if i < K:\n",
    "            print('\\t> original component topic')\n",
    "            continue\n",
    "        for c in topic_combinations[i]:\n",
    "            print(\"\\t>{0: .3f}\".format(total_scores[c]), \" \".join(topics[c]))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1e8341c9",
   "metadata": {},
   "source": [
    "#### Original topics that are excluded\n",
    "\n",
    "Notice that some \"bad\" topics are components of \"new\" composite topics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "67c8a595",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 0.195 power front oil ride brake car engine bike battery tire\n",
      " 0.114 atheist argument point moral belief science prove atheism evidence theory\n",
      " 0.315 shuttle moon earth satellite solar planet mission orbit surface launch\n",
      " 0.073 people food good insurance year drug disease patient health medical\n",
      " 0.146 eternal relationship church verse scripture man sin homosexual word teach\n",
      " 0.134 condition offer ship apple port shipping sale monitor price video\n",
      "-0.285 previously length huge requirement suit white powerful examine limited installation\n",
      " 0.170 illegal gas police crime gun fire assault compound weapon batf\n",
      " 0.043 floor sit start neighbor didn good time thing make people\n",
      " 0.178 list network mail newsgroup address email internet mailing domain host\n",
      "-0.268 delay installation status examine task fill possibly requirement critical presence\n",
      " 0.166 button application screen manager font run problem window widget motif\n",
      "-0.249 rule portion chapter explain freedom relation remark virtually language bias\n",
      "-0.235 gather picture inform closely firm advance installation sheet requirement critical\n",
      " 0.052 vote time press israeli meeting president decision yesterday people make\n",
      " 0.047 entry graphic good software format system mail file make image\n"
     ]
    }
   ],
   "source": [
    "for i in range(K):\n",
    "    if choices[i]:\n",
    "        continue\n",
    "    print(\"{0: .3f}\".format(total_scores[i]), \" \".join(topics[i]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9c0f1b36",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}