SKU_Model_GridSearch.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 235,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras\n",
    "import datetime\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "# Setting seed for reproducability\n",
    "  \n",
    "\n",
    "from sklearn import preprocessing, svm\n",
    "from sklearn.metrics import confusion_matrix, recall_score, precision_score\n",
    "from keras import models\n",
    "from keras.wrappers.scikit_learn import KerasRegressor\n",
    "from keras.models import Sequential,model_from_json\n",
    "from keras.layers import Dense, Dropout, LSTM, Activation\n",
    "%matplotlib inline\n",
    "from decimal import Decimal\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import pickle\n",
    "from sklearn.preprocessing import OneHotEncoder,LabelEncoder,Normalizer, MinMaxScaler\n",
    "\n",
    "from sklearn.model_selection import GridSearchCV, cross_val_score, KFold\n",
    "from numpy.random import seed\n",
    "seed(1)\n",
    "from tensorflow import set_random_seed\n",
    "set_random_seed(2)\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn import ensemble\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "from sklearn.multioutput import MultiOutputRegressor\n",
    "import xgboost as xgb\n",
    "import collections\n",
    "import itertools"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 339,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv('C:\\\\Users\\\\lengada1\\\\NCSU\\\\ten_skus.csv')\n",
    "df['Date'] = pd.to_datetime(df['Date'])\n",
    "df['Year'] = df['Date'].dt.year\n",
    "df['Month'] = df['Date'].dt.month\n",
    "df['Day'] = df['Date'].dt.weekday_name\n",
    "\n",
    "day_dummy=pd.get_dummies(df.Day)\n",
    "df=pd.concat([df,day_dummy],axis=1)\n",
    "df.drop(['Day','Date'],inplace=True,axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 346,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_df(df):\n",
    "    d={}\n",
    "    split_by_list=np.unique(df.id.values)\n",
    "    for sku in split_by_list:\n",
    "        d[sku]=df[df.id==sku]\n",
    "    return d\n",
    "\n",
    "df_dict=split_df(df);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 347,
   "metadata": {},
   "outputs": [],
   "source": [
    "y={}\n",
    "for sku in list(df_dict.keys()):\n",
    "    y[sku]=df_dict[sku]['Sales']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 348,
   "metadata": {},
   "outputs": [],
   "source": [
    "X={}\n",
    "for sku in list(df_dict.keys()):\n",
    "    X[sku]=df_dict[sku]\n",
    "    X[sku]=X[sku].drop(['id','DayOfWeek','Customers','High_Var','Luxury','Sales'],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "metadata": {},
   "outputs": [],
   "source": [
    "for sku in list(df_dict.keys()):\n",
    "    for obs in range(1,8):\n",
    "        X[sku][\"Sales_T\"+str(obs)]=df_dict[sku]['Sales'].shift(obs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "metadata": {},
   "outputs": [],
   "source": [
    "for sku in list(df_dict.keys()):\n",
    "    X[sku][\"Mov_avg\"]=pd.rolling_mean(df_dict[sku]['Sales'], window=7).shift(1) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Cut lagged vars NAs off top \n",
    "cut_lag=7;\n",
    "for sku in list(df_dict.keys()):\n",
    "    y[sku]=y[sku][cut_lag:]\n",
    "    y[sku].reset_index(drop=True, inplace=True)\n",
    "    \n",
    "for sku in list(df_dict.keys()):\n",
    "    X[sku]=X[sku][cut_lag:]\n",
    "    X[sku].reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Open</th>\n",
       "      <th>Promo</th>\n",
       "      <th>SchoolHoliday</th>\n",
       "      <th>Year</th>\n",
       "      <th>Month</th>\n",
       "      <th>Friday</th>\n",
       "      <th>Monday</th>\n",
       "      <th>Saturday</th>\n",
       "      <th>Sunday</th>\n",
       "      <th>Thursday</th>\n",
       "      <th>Tuesday</th>\n",
       "      <th>Wednesday</th>\n",
       "      <th>Sales_T1</th>\n",
       "      <th>Sales_T2</th>\n",
       "      <th>Sales_T3</th>\n",
       "      <th>Sales_T4</th>\n",
       "      <th>Sales_T5</th>\n",
       "      <th>Sales_T6</th>\n",
       "      <th>Sales_T7</th>\n",
       "      <th>Mov_avg</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2013</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>7176.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4997.0</td>\n",
       "      <td>4486.0</td>\n",
       "      <td>4327.0</td>\n",
       "      <td>5530.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3788.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2013</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>5580.0</td>\n",
       "      <td>7176.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4997.0</td>\n",
       "      <td>4486.0</td>\n",
       "      <td>4327.0</td>\n",
       "      <td>5530.0</td>\n",
       "      <td>4585.142857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2013</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>5471.0</td>\n",
       "      <td>5580.0</td>\n",
       "      <td>7176.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4997.0</td>\n",
       "      <td>4486.0</td>\n",
       "      <td>4327.0</td>\n",
       "      <td>4576.714286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>2013</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4892.0</td>\n",
       "      <td>5471.0</td>\n",
       "      <td>5580.0</td>\n",
       "      <td>7176.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4997.0</td>\n",
       "      <td>4486.0</td>\n",
       "      <td>4657.428571</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2013</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>4881.0</td>\n",
       "      <td>4892.0</td>\n",
       "      <td>5471.0</td>\n",
       "      <td>5580.0</td>\n",
       "      <td>7176.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4997.0</td>\n",
       "      <td>4713.857143</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Open  Promo  SchoolHoliday  Year  Month  Friday  Monday  Saturday  Sunday  \\\n",
       "0     1      1              1  2013      1       0       0         0       0   \n",
       "1     1      1              1  2013      1       0       0         0       0   \n",
       "2     1      1              1  2013      1       0       0         0       0   \n",
       "3     1      1              1  2013      1       1       0         0       0   \n",
       "4     1      0              0  2013      1       0       0         1       0   \n",
       "\n",
       "   Thursday  Tuesday  Wednesday  Sales_T1  Sales_T2  Sales_T3  Sales_T4  \\\n",
       "0         0        1          0    7176.0       0.0    4997.0    4486.0   \n",
       "1         0        0          1    5580.0    7176.0       0.0    4997.0   \n",
       "2         1        0          0    5471.0    5580.0    7176.0       0.0   \n",
       "3         0        0          0    4892.0    5471.0    5580.0    7176.0   \n",
       "4         0        0          0    4881.0    4892.0    5471.0    5580.0   \n",
       "\n",
       "   Sales_T5  Sales_T6  Sales_T7      Mov_avg  \n",
       "0    4327.0    5530.0       0.0  3788.000000  \n",
       "1    4486.0    4327.0    5530.0  4585.142857  \n",
       "2    4997.0    4486.0    4327.0  4576.714286  \n",
       "3       0.0    4997.0    4486.0  4657.428571  \n",
       "4    7176.0       0.0    4997.0  4713.857143  "
      ]
     },
     "execution_count": 352,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X[1].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "metadata": {},
   "outputs": [],
   "source": [
    "std={}\n",
    "for sku in list(df_dict.keys()):\n",
    "    std[sku] = preprocessing.StandardScaler().fit(X[sku])\n",
    "    X[sku] = std[sku].transform(X[sku])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "metadata": {},
   "outputs": [],
   "source": [
    "trials=2;\n",
    "folds=4;\n",
    "kf=KFold(n_splits=folds, shuffle=True, random_state=i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "RF=RandomForestRegressor();\n",
    "# use a full grid over all parameters\n",
    "p_grid = {\n",
    "          \"n_estimators\": [800,1000,1200],\n",
    "             # 'min_samples_leaf': [1, 2],\n",
    "              \"criterion\": ['mae'],\n",
    "              'max_features':['auto',13, 14,15]\n",
    "         }\n",
    "rf_models=[];\n",
    "rf_params=[];\n",
    "for sku in list(df_dict.keys()):\n",
    "    #rf_models.append(cross_val_score(RF,X[sku],y[sku],cv=cv, scoring ='mean_absolute_error').mean().round(0) )\n",
    "    for i in range(trials): \n",
    "        clf = GridSearchCV(estimator=RF, param_grid=p_grid, cv=kf)\n",
    "        clf.fit(X[sku],y[sku])\n",
    "        rf_models.append(clf.best_score_)\n",
    "        rf_params.append(clf.best_params_)"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "rf_params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 356,
   "metadata": {},
   "outputs": [],
   "source": [
    "rf_keys=[]\n",
    "for i in range(len(rf_params)):\n",
    "    rf_keys.append(  list(rf_params[i].values() )   )\n",
    "counts = collections.defaultdict(int)\n",
    "for collab in rf_keys:\n",
    "    #collab.sort()\n",
    "    for pair in itertools.combinations(collab,4):\n",
    "        counts[pair] += 1\n",
    "\n",
    "for pair, freq in counts.items():\n",
    "    print(pair, freq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 258,
   "metadata": {},
   "outputs": [],
   "source": [
    "GB=ensemble.GradientBoostingRegressor()\n",
    "\n",
    "p_grid = {    'n_estimators': [ 1000],  #more = better from my results\n",
    "              'max_depth': [10], # Lower is better?  10 was better than 20, 30, 40\n",
    "              'loss':['ls'],\n",
    "              'learning_rate':[.01, .05,.1],  #small was better. 0.01 better than 0.05 or 0.10\n",
    "          'max_features':['auto',15,17]  #auto uses all features which isn't as strong, 15 > 17\n",
    "         \n",
    "         }\n",
    "gb_models=[];\n",
    "gb_params=[]\n",
    "for sku in list(df_dict.keys()):\n",
    "    #gb_models.append(cross_val_score(GB,X[sku],y[sku],cv=cv, scoring ='mean_absolute_error').mean().round(0)  )\n",
    "    for i in range(trials):         \n",
    "        kf = KFold(n_splits=4, shuffle=True, random_state=i)\n",
    "        clf = GridSearchCV(estimator=GB, param_grid=p_grid, cv=kf)\n",
    "        clf.fit(X[sku],y[sku])\n",
    "        gb_models.append(clf.best_score_)\n",
    "        gb_params.append(clf.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 261,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0.01, 'ls', 10, 15, 1000) 4\n",
      "(0.01, 'ls', 10, 15, 750) 5\n",
      "(0.05, 'ls', 10, 15, 1000) 3\n",
      "(0.05, 'ls', 30, 15, 1000) 2\n",
      "(0.01, 'ls', 10, 15, 500) 7\n",
      "(0.1, 'ls', 30, 15, 500) 1\n",
      "(0.1, 'ls', 10, 15, 1000) 2\n",
      "(0.05, 'ls', 10, 15, 500) 2\n",
      "(0.05, 'ls', 10, 15, 750) 3\n",
      "(0.1, 'ls', 10, 15, 500) 1\n"
     ]
    }
   ],
   "source": [
    "gb_keys=[]\n",
    "for i in range(len(gb_params)):\n",
    "    gb_keys.append(  list(gb_params[i].values() )   )\n",
    "\n",
    "    counts = collections.defaultdict(int)\n",
    "for collab in gb_keys:\n",
    "    #collab.sort()\n",
    "    for pair in itertools.combinations(collab,5):\n",
    "        counts[pair] += 1\n",
    "\n",
    "for pair, freq in counts.items():\n",
    "    print(pair, freq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "metadata": {},
   "outputs": [],
   "source": [
    "XG=xgb.XGBRegressor()\n",
    "\n",
    "p_grid = {'learning_rate':[.01], #.01 beat out .05 and .1 above\n",
    "          'n_estimators':[900, 1100], \n",
    "          'max_depth':[6,8,10],\n",
    "          'gamma':[0,2],  #Gamma is regularization, higher = less parameters \n",
    "    \n",
    "\n",
    "           'learning_rate':[.01, .05,.1]\n",
    "\n",
    "           }\n",
    "\n",
    "\n",
    "xg_models=[];\n",
    "xg_params=[]\n",
    "for sku in list(df_dict.keys()):\n",
    "    \n",
    "    for i in range(trials):         \n",
    "        kf = KFold(n_splits=4, shuffle=True, random_state=i)\n",
    "        clf = GridSearchCV(estimator=XG, param_grid=p_grid, cv=kf)\n",
    "        clf.fit(X[sku],y[sku])\n",
    "        xg_models.append(clf.best_score_)\n",
    "        xg_params.append(clf.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 362,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(0, 0.01, 6, 900) 6\n",
      "(2, 0.01, 6, 900) 11\n",
      "(0, 0.1, 6, 1100) 1\n",
      "(2, 0.1, 6, 900) 1\n",
      "(2, 0.1, 6, 1100) 1\n"
     ]
    }
   ],
   "source": [
    "xg_keys=[]\n",
    "for i in range(len(xg_params)):\n",
    "    xg_keys.append(  list(xg_params[i].values() )   )\n",
    "\n",
    "    counts = collections.defaultdict(int)\n",
    "for collab in xg_keys:\n",
    "    #collab.sort()\n",
    "    for pair in itertools.combinations(collab,4):\n",
    "        counts[pair] += 1\n",
    "\n",
    "for pair, freq in counts.items():\n",
    "    print(pair, freq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 332,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras import optimizers\n",
    "adam1=keras.optimizers.Adam(lr=0.001)\n",
    "adam2=keras.optimizers.Adam(lr=0.002)\n",
    "adam3=keras.optimizers.Adam(lr=0.005)\n",
    ")\n",
    "\n",
    "rms1=keras.optimizers.rmsprop(lr=0.001)\n",
    "rms2=keras.optimizers.rmsprop(lr=0.002)\n",
    "rms3=keras.optimizers.rmsprop(lr=0.1)\n",
    "rms4=keras.optimizers.rmsprop(lr=0.15)\n",
    "\n",
    "\n",
    "SGD1=keras.optimizers.SGD(lr=0.001)\n",
    "SGD2=keras.optimizers.SGD(lr=0.002)\n",
    "SGD3=keras.optimizers.SGD(lr=0.1)\n",
    "SGD4=keras.optimizers.SGD(lr=0.15)\n",
    "\n",
    "\n",
    "\n",
    "def create_MLP(optimizer=adam1):\n",
    "    mlp=models.Sequential()\n",
    "    mlp.add(Dense(20, input_dim=X[1].shape[1], activation='relu'))\n",
    "    mlp.add(Dense(20, activation='relu'))\n",
    "    #mlp.add(Dense(30, activation='relu'))\n",
    "    mlp.add(Dense(10, activation='relu'))\n",
    "    mlp.add(Dense(1))   \n",
    "    mlp.compile(loss='mae', metrics=['mae'], optimizer=optimizer)    \n",
    "    return mlp\n",
    "\n",
    "MLP = KerasRegressor(build_fn=create_MLP,\n",
    "                               verbose=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 334,
   "metadata": {},
   "outputs": [],
   "source": [
    "mlp_models=[]\n",
    "p_grid = {'epochs':[400 ],\n",
    "         'batch_size':[30],\n",
    "         'optimizer':[ adam1, adam2, adam3]\n",
    "         }\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "mlp_models=[];\n",
    "mlp_params=[]\n",
    "for sku in list(df_dict.keys()):\n",
    "    \n",
    "    for i in range(trials):         \n",
    "        kf = KFold(n_splits=4, shuffle=True, random_state=i)\n",
    "        clf = GridSearchCV(estimator=MLP, param_grid=p_grid, cv=kf)\n",
    "        clf.fit(X[sku],y[sku])\n",
    "        mlp_models.append(clf.best_score_)\n",
    "        mlp_params.append(clf.best_params_) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 335,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'LearningRateScheduler' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m-----------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-335-4075efec8387>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      6\u001b[0m            math.floor((1+epoch)/epochs_drop))\n\u001b[0;32m      7\u001b[0m    \u001b[1;32mreturn\u001b[0m \u001b[0mlrate\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 8\u001b[1;33m \u001b[0mlrate\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mLearningRateScheduler\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstep_decay\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[1;31mNameError\u001b[0m: name 'LearningRateScheduler' is not defined"
     ]
    }
   ],
   "source": [
    "def step_decay(epoch):\n",
    "   initial_lrate = 0.1\n",
    "   drop = 0.5\n",
    "   epochs_drop = 10.0\n",
    "   lrate = initial_lrate * math.pow(drop,  \n",
    "           math.floor((1+epoch)/epochs_drop))\n",
    "   return lrate\n",
    "lrate = LearningRateScheduler(step_decay)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 357,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[-361.29688162574155,\n",
       " -398.24448522883938,\n",
       " -589.76047141335232,\n",
       " -682.54352779286432,\n",
       " -352.93132389496992,\n",
       " -370.70839634859624,\n",
       " -685.96007661054477,\n",
       " -386.72510562019551,\n",
       " -490.90667953083221,\n",
       " -370.42627570004385]"
      ]
     },
     "execution_count": 357,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mlp_models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}