NewtonsMethod.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Newton's Method of Linear Regression Solution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pregnancies</th>\n",
       "      <th>Glucose</th>\n",
       "      <th>BloodPressure</th>\n",
       "      <th>SkinThickness</th>\n",
       "      <th>Insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>DiabetesPedigreeFunction</th>\n",
       "      <th>Age</th>\n",
       "      <th>Outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>148</td>\n",
       "      <td>72</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>33.6</td>\n",
       "      <td>0.627</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>85</td>\n",
       "      <td>66</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>26.6</td>\n",
       "      <td>0.351</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8</td>\n",
       "      <td>183</td>\n",
       "      <td>64</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>23.3</td>\n",
       "      <td>0.672</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>89</td>\n",
       "      <td>66</td>\n",
       "      <td>23</td>\n",
       "      <td>94</td>\n",
       "      <td>28.1</td>\n",
       "      <td>0.167</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>137</td>\n",
       "      <td>40</td>\n",
       "      <td>35</td>\n",
       "      <td>168</td>\n",
       "      <td>43.1</td>\n",
       "      <td>2.288</td>\n",
       "      <td>33</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pregnancies  Glucose  BloodPressure  SkinThickness  Insulin   BMI  \\\n",
       "0            6      148             72             35        0  33.6   \n",
       "1            1       85             66             29        0  26.6   \n",
       "2            8      183             64              0        0  23.3   \n",
       "3            1       89             66             23       94  28.1   \n",
       "4            0      137             40             35      168  43.1   \n",
       "\n",
       "   DiabetesPedigreeFunction  Age  Outcome  \n",
       "0                     0.627   50        1  \n",
       "1                     0.351   31        0  \n",
       "2                     0.672   32        1  \n",
       "3                     0.167   21        0  \n",
       "4                     2.288   33        1  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv('diabetes.csv')\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Pregnancies', 'Glucose', 'BloodPressure', 'SkinThickness', 'Insulin',\n",
       "       'BMI', 'DiabetesPedigreeFunction', 'Age'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del df['Outcome']\n",
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's predict BMI\n",
    "features = ['Pregnancies', 'Glucose', 'BloodPressure', 'SkinThickness', \n",
    "            'Insulin', 'DiabetesPedigreeFunction']\n",
    "X = df.loc[:, features]\n",
    "y = df.loc[:, ['BMI']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "class LinReg:\n",
    "    \"\"\"\n",
    "    This implementation uses the Newton's Method for optimization.\n",
    "    \"\"\"\n",
    "    def __init__(self, num_iters=5, tolerance = 1e-10, epsilon = 10e-8):\n",
    "        self.num_iters = num_iters\n",
    "        self.tolerance = tolerance\n",
    "        self.epsilon = epsilon # subtracted to make hessian invertible\n",
    "        \n",
    "    def add_ones(self, X):\n",
    "        return np.concatenate((np.ones((len(X),1)), X), axis = 1)\n",
    "    \n",
    "    def cost(self, X, y_true):\n",
    "        return np.mean((X@self.theta - y_true)**2)\n",
    "    \n",
    "    def fit(self, X, y):\n",
    "        X = X.values.copy()\n",
    "        X = self.add_ones(X)\n",
    "        y = y.values.reshape(-1, 1)\n",
    "        \n",
    "        self.theta = np.zeros((len(X[0]), 1))\n",
    "        hessian_inv = np.linalg.inv(X.T@X + self.epsilon*np.eye(X.T.shape[0]))\n",
    "        current_iter = 1\n",
    "        norm = 1\n",
    "        while (norm >= self.tolerance and current_iter < self.num_iters):\n",
    "            old_theta = self.theta.copy()\n",
    "            grad = (X.T@X)@self.theta - X.T@y\n",
    "            #grad = (1/len(X)) * np.sum((X@self.theta - y)*X, axis=0)\n",
    "            grad= grad.reshape(-1, 1)\n",
    "            self.theta = self.theta - hessian_inv@grad\n",
    "            \n",
    "            print(f'cost for {current_iter} iteration : {self.cost(X, y)}')\n",
    "            norm = np.linalg.norm(old_theta - self.theta)\n",
    "            current_iter += 1\n",
    "            \n",
    "        return self.theta\n",
    "    \n",
    "    def predict(self, X):\n",
    "        X = self.add_ones(X)\n",
    "        \n",
    "        return X@self.theta\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "lr = LinReg()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "cost for 1 iteration : 47.872888313076686\n",
      "cost for 2 iteration : 47.87288831307668\n",
      "cost for 3 iteration : 47.87288831307668\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([[ 1.75679906e+01],\n",
       "       [-9.65695968e-03],\n",
       "       [ 4.44683347e-02],\n",
       "       [ 7.46763540e-02],\n",
       "       [ 1.74400269e-01],\n",
       "       [-2.87882105e-03],\n",
       "       [ 1.21506558e+00]])"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.fit(X, y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[36.33391646],\n",
       "       [31.75087798],\n",
       "       [31.22425196],\n",
       "       [30.38816889],\n",
       "       [35.04764576],\n",
       "       [28.4483118 ],\n",
       "       [30.37017718],\n",
       "       [22.74809878],\n",
       "       [38.01307981],\n",
       "       [30.5001029 ]])"
      ]
     },
     "execution_count": 123,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "lr.predict(X)[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}