KL_Divergence.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Filter Method of Feature Selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Kulback Leibler Divergence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from scipy.special import rel_entr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# example of calculating the kl divergence (relative entropy)\n",
    "\n",
    "# define distributions\n",
    "p = np.array([0.10, 0.40, 0.50])\n",
    "q = np.array([0.80, 0.15, 0.05])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.3356800809389957\n",
      "1.401299579859536\n"
     ]
    }
   ],
   "source": [
    "def KL_Divergence(p, q, epsilon=10e-10):\n",
    "    p = (p + epsilon)/np.sum(p + epsilon) # Epsilon to avoid division by zero\n",
    "    \n",
    "    q = (q + epsilon)/np.sum(q + epsilon)\n",
    "    return np.sum(p*np.log(p/q))\n",
    "\n",
    "print(KL_Divergence(p, q))\n",
    "print(KL_Divergence(q, p)) # Not symmetric"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "KL(P || Q): 1.336 nats\n",
      "KL(Q || P): 1.401 nats\n"
     ]
    }
   ],
   "source": [
    "# confirm with scipy\n",
    "# Difference cos input has zero\n",
    "kl_pq = rel_entr(p, q)\n",
    "print('KL(P || Q): %.3f nats' % sum(kl_pq))\n",
    "\n",
    "kl_qp = rel_entr(q, p)\n",
    "print('KL(Q || P): %.3f nats' % sum(kl_qp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_csv('diabetes.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pregnancies</th>\n",
       "      <th>Glucose</th>\n",
       "      <th>BloodPressure</th>\n",
       "      <th>SkinThickness</th>\n",
       "      <th>Insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>DiabetesPedigreeFunction</th>\n",
       "      <th>Age</th>\n",
       "      <th>Outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>148</td>\n",
       "      <td>72</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>33.6</td>\n",
       "      <td>0.627</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>85</td>\n",
       "      <td>66</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>26.6</td>\n",
       "      <td>0.351</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8</td>\n",
       "      <td>183</td>\n",
       "      <td>64</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>23.3</td>\n",
       "      <td>0.672</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>89</td>\n",
       "      <td>66</td>\n",
       "      <td>23</td>\n",
       "      <td>94</td>\n",
       "      <td>28.1</td>\n",
       "      <td>0.167</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>137</td>\n",
       "      <td>40</td>\n",
       "      <td>35</td>\n",
       "      <td>168</td>\n",
       "      <td>43.1</td>\n",
       "      <td>2.288</td>\n",
       "      <td>33</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pregnancies  Glucose  BloodPressure  SkinThickness  Insulin   BMI  \\\n",
       "0            6      148             72             35        0  33.6   \n",
       "1            1       85             66             29        0  26.6   \n",
       "2            8      183             64              0        0  23.3   \n",
       "3            1       89             66             23       94  28.1   \n",
       "4            0      137             40             35      168  43.1   \n",
       "\n",
       "   DiabetesPedigreeFunction  Age  Outcome  \n",
       "0                     0.627   50        1  \n",
       "1                     0.351   31        0  \n",
       "2                     0.672   32        1  \n",
       "3                     0.167   21        0  \n",
       "4                     2.288   33        1  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Pregnancies', 'Glucose', 'BloodPressure', 'SkinThickness', 'Insulin',\n",
       "       'BMI', 'DiabetesPedigreeFunction', 'Age', 'Outcome'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# List predictor colomns to compare with target\n",
    "predictors = df.columns[:-1].copy(); target = df.columns[-1]\n",
    "predictors = list(predictors);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def MutualInformation(X, y, predictor_cols, target_col, epsilon=10e-10):\n",
    "    \"\"\"\n",
    "    Mutual Information Calculated using Kulback Leibler Divergence\n",
    "    \n",
    "    inputs\n",
    "    ..................\n",
    "    X: Pandas dataframe of columns\n",
    "    y: Pandas dataframe of target\n",
    "    epsilon: small number to prevent division by 0; default:10e-10\n",
    "    predictor_cols: predictor column names\n",
    "    target_col: target name\n",
    "    output\n",
    "    ...................\n",
    "    Pandas dataframe of column combinations and KL divergence score\n",
    "    \"\"\"\n",
    "    def KL_Divergence(p, q, epsilon=epsilon):\n",
    "        p = (p + epsilon)/np.sum(p + epsilon)\n",
    "\n",
    "        q = (q + epsilon)/np.sum(q + epsilon)\n",
    "        return np.sum(p*np.log(p/q))\n",
    "\n",
    "    X = X.values; y = y.values\n",
    "    KL_list = {}\n",
    "    for i, col in enumerate(predictor_cols):\n",
    "        array_index = col + '||' + target_col\n",
    "        p = X[:, i]\n",
    "        q = y\n",
    "        KL_list[array_index] = KL_Divergence(p, q, epsilon)\n",
    "\n",
    "    sorted_KL_list = sorted(KL_list.items(), key=lambda x: x[1], reverse=True)\n",
    "\n",
    "    return pd.DataFrame(sorted_KL_list, columns=['Features', 'MI-Score (nats)'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "BloodPressure||Outcome\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Features</th>\n",
       "      <th>MI-Score (nats)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>BloodPressure||Outcome</td>\n",
       "      <td>12.320437</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>SkinThickness||Outcome</td>\n",
       "      <td>12.280656</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>BMI||Outcome</td>\n",
       "      <td>11.763647</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Age||Outcome</td>\n",
       "      <td>11.664003</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Insulin||Outcome</td>\n",
       "      <td>11.480395</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>DiabetesPedigreeFunction||Outcome</td>\n",
       "      <td>11.441139</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Glucose||Outcome</td>\n",
       "      <td>11.258261</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Pregnancies||Outcome</td>\n",
       "      <td>10.923133</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            Features  MI-Score (nats)\n",
       "0             BloodPressure||Outcome        12.320437\n",
       "1             SkinThickness||Outcome        12.280656\n",
       "2                       BMI||Outcome        11.763647\n",
       "3                       Age||Outcome        11.664003\n",
       "4                   Insulin||Outcome        11.480395\n",
       "5  DiabetesPedigreeFunction||Outcome        11.441139\n",
       "6                   Glucose||Outcome        11.258261\n",
       "7               Pregnancies||Outcome        10.923133"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MutualInformation(df.iloc[:, :-1], df.iloc[:,-1], predictors, target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}