RandomForestModel.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Integrate Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "from matplotlib import pyplot as plt\n",
    "import seaborn as sns\n",
    "\n",
    "# get input dataset\n",
    "features_train = pd.read_csv('data/dengue_features_train.csv')\n",
    "labels_train = pd.read_csv('data/dengue_labels_train.csv')\n",
    "features_test = pd.read_csv('data/dengue_features_test.csv')\n",
    "\n",
    "# Normalize the week_start_date feature value\n",
    "features_train['week_start_date'] = pd.to_datetime(features_train['week_start_date'])\n",
    "features_test['week_start_date'] = pd.to_datetime(features_test['week_start_date'])\n",
    "\n",
    "# Divide training data into two parts w.r.t 'city'\n",
    "train_sj = features_train.loc[features_train['city'] == 'sj']\n",
    "train_iq = features_train.loc[features_train['city'] == 'iq']\n",
    "label_sj = labels_train.loc[labels_train['city'] == 'sj']\n",
    "label_iq = labels_train.loc[labels_train['city'] == 'iq']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocess the Training Data set"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Feature Selection \n",
    "\n",
    "Select features from the features in the training data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "KEYS = ['city', 'year', 'weekofyear']\n",
    "\n",
    "ALL_FEATURES= ['city', 'year', 'weekofyear', 'week_start_date', 'ndvi_ne', 'ndvi_nw', \n",
    "                      'ndvi_se', 'ndvi_sw', 'precipitation_amt_mm', 'reanalysis_air_temp_k',\n",
    "                      'reanalysis_avg_temp_k', 'reanalysis_dew_point_temp_k',\n",
    "                      'reanalysis_max_air_temp_k', 'reanalysis_min_air_temp_k',\n",
    "                      'reanalysis_precip_amt_kg_per_m2',\n",
    "                      'reanalysis_relative_humidity_percent', 'reanalysis_sat_precip_amt_mm',\n",
    "                      'reanalysis_specific_humidity_g_per_kg', 'reanalysis_tdtr_k',\n",
    "                      'station_avg_temp_c', 'station_diur_temp_rng_c', 'station_max_temp_c',\n",
    "                      'station_min_temp_c', 'station_precip_mm']\n",
    "\n",
    "RF_TRAINING_FEATURES = ['ndvi_ne', 'ndvi_nw', 'ndvi_se', 'ndvi_sw']\n",
    "\n",
    "TIMELY_TRAINING_FEATURES = ['year', 'weekofyear']\n",
    "\n",
    "COMMON_TRAINING_FEATURES = ['reanalysis_dew_point_temp_k', 'reanalysis_precip_amt_kg_per_m2', \n",
    "                   'reanalysis_specific_humidity_g_per_kg', 'station_avg_temp_c',  'station_max_temp_c', \n",
    "                   'station_min_temp_c']\n",
    "\n",
    "SJ_FEATURES = ['reanalysis_dew_point_temp_k', 'reanalysis_precip_amt_kg_per_m2', 'reanalysis_specific_humidity_g_per_kg',\n",
    "               'station_avg_temp_c',  'station_max_temp_c', 'station_min_temp_c']\n",
    "\n",
    "IQ_FEATURES = ['reanalysis_dew_point_temp_k', 'reanalysis_specific_humidity_g_per_kg',\n",
    "               'station_avg_temp_c', 'station_min_temp_c']\n",
    "\n",
    "NEW_FEATURES = ['recent_mean_dew_point', 'recent_mean_spec_humid', 'recent_sum_precip']\n",
    "\n",
    "TIME_SERIES_FEATURES = ['week_start_date']\n",
    "\n",
    "DROP_FEATURES = list(set(ALL_FEATURES)-set(COMMON_TRAINING_FEATURES)-set(KEYS)-set(TIME_SERIES_FEATURES))\n",
    "\n",
    "DROP_SJ_FEATURES = list(set(ALL_FEATURES)-set(SJ_FEATURES)-set(KEYS)-set(TIME_SERIES_FEATURES))\n",
    "DROP_IQ_FEATURES = list(set(ALL_FEATURES)-set(IQ_FEATURES)-set(KEYS)-set(TIME_SERIES_FEATURES))\n",
    "\n",
    "# Specific features for the cities\n",
    "FEATURES_SJ = COMMON_TRAINING_FEATURES\n",
    "FEATURES_IQ = COMMON_TRAINING_FEATURES "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Drop DROP_FEATURES from a given data set\n",
    "def drop_unnecessary_features(df,drop_features=DROP_FEATURES):\n",
    "    df.drop(drop_features, axis=1, inplace=True)\n",
    "    df.drop(TIME_SERIES_FEATURES, axis=1, inplace=True)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Filing Null values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/utils/deprecation.py:66: DeprecationWarning: Class Imputer is deprecated; Imputer was deprecated in version 0.20 and will be removed in 0.22. Import impute.SimpleImputer from sklearn instead.\n",
      "  warnings.warn(msg, category=DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import Imputer\n",
    "\n",
    "imputer = Imputer(strategy='mean')\n",
    "\n",
    "def fill_null_values_with_mean(df,features_list):\n",
    "    imputer.fit(df[features_list])\n",
    "    df[features_list] = imputer.transform(df[features_list])\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  import sys\n",
      "/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py:543: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self.obj[item] = s\n",
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  import sys\n",
      "/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py:543: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self.obj[item] = s\n"
     ]
    }
   ],
   "source": [
    "# Fill null values of RF_ Features of the training dataset\n",
    "train_sj = fill_null_values_with_mean(train_sj,RF_TRAINING_FEATURES)\n",
    "train_iq = fill_null_values_with_mean(train_iq,RF_TRAINING_FEATURES)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Normalization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Add Time Series features\n",
    "\n",
    "def add_time_series_features(df, window):\n",
    "    df.set_index('week_start_date', inplace=True)\n",
    "\n",
    "    roll_df = df.rolling(window=window, min_periods=1)\n",
    "    df['recent_mean_dew_point'] = roll_df.reanalysis_dew_point_temp_k.mean()\n",
    "    df['recent_mean_spec_humid'] = roll_df.reanalysis_specific_humidity_g_per_kg.mean()\n",
    "    df['recent_sum_precip'] = roll_df.reanalysis_precip_amt_kg_per_m2.sum()\n",
    "    \n",
    "    df.reset_index(inplace=True)    \n",
    "    return df\n",
    "\n",
    "# normalize data\n",
    "def normalize(feature):\n",
    "    return (feature - feature.mean()) / feature.std()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_sj = add_time_series_features(train_sj, window=10)\n",
    "# train_iq = add_time_series_features(train_iq, window=10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Drop some unnecessory Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/pandas/core/frame.py:3940: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  errors=errors)\n"
     ]
    }
   ],
   "source": [
    "rf_drop_features = list(set(ALL_FEATURES)-set(RF_TRAINING_FEATURES)-set(KEYS)-set(TIME_SERIES_FEATURES))\n",
    "\n",
    "train_sj = drop_unnecessary_features(train_sj,drop_features=rf_drop_features)\n",
    "train_iq = drop_unnecessary_features(train_iq,drop_features=rf_drop_features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Normalize the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/pandas/core/frame.py:3391: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self[k1] = value[k2]\n"
     ]
    }
   ],
   "source": [
    "FEATURES_TO_NRMLZE = RF_TRAINING_FEATURES\n",
    "\n",
    "train_sj[FEATURES_TO_NRMLZE] = train_sj[FEATURES_TO_NRMLZE].apply(normalize, axis=0)\n",
    "train_iq[FEATURES_TO_NRMLZE] = train_iq[FEATURES_TO_NRMLZE].apply(normalize, axis=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Spliting the training data set into train and test data sets\n",
    "\n",
    "Here we divide the tarining data set into two parts: for training and for testing.\n",
    "**X_cross_sj** is the testing data set extracted from the training data of city 'sj'\n",
    "**y_cross_sj** is the testing label values extracted from the training data of city 'sj'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "# train_sj.set_index('index', inplace=True)\n",
    "# train_iq.set_index('index', inplace=True)\n",
    "\n",
    "y_sj = labels_train.loc[labels_train['city'] == 'sj',:]\n",
    "y_iq = labels_train.loc[labels_train['city'] == 'iq',:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train_sj: (748, 7)\n",
      "y_train_sj: (748, 4)\n",
      "X_cross_sj: (188, 7)\n",
      "y_cross_sj: (188, 4)\n"
     ]
    }
   ],
   "source": [
    "X_train_sj, X_cross_sj, y_train_sj, y_cross_sj = train_test_split(train_sj, \n",
    "                                                                  y_sj,\n",
    "                                                                  test_size=0.2,\n",
    "                                                                  stratify=train_sj.weekofyear)\n",
    "\n",
    "print(f'X_train_sj: {X_train_sj.shape}')\n",
    "print(f'y_train_sj: {y_train_sj.shape}')\n",
    "print(f'X_cross_sj: {X_cross_sj.shape}')\n",
    "print(f'y_cross_sj: {y_cross_sj.shape}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train_iq: (416, 7)\n",
      "y_train_iq: (416, 4)\n",
      "X_cross_iq: (104, 7)\n",
      "y_cross_iq: (104, 4)\n"
     ]
    }
   ],
   "source": [
    "X_train_iq, X_cross_iq, y_train_iq, y_cross_iq = train_test_split(train_iq, \n",
    "                                                                  y_iq, \n",
    "                                                                  test_size=0.2,\n",
    "                                                                  stratify=train_iq.weekofyear)\n",
    "\n",
    "print(f'X_train_iq: {X_train_iq.shape}')\n",
    "print(f'y_train_iq: {y_train_iq.shape}')\n",
    "print(f'X_cross_iq: {X_cross_iq.shape}')\n",
    "print(f'y_cross_iq: {y_cross_iq.shape}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Prepare for Training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def drop_unnecessary_columns(df,features):\n",
    "    return df[features]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X_train_iq: (416, 6)\n",
      "y_train_iq: (416, 1)\n",
      "X_cross_iq: (104, 6)\n",
      "y_cross_iq: (104, 1)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ndvi_ne</th>\n",
       "      <th>ndvi_nw</th>\n",
       "      <th>ndvi_se</th>\n",
       "      <th>ndvi_sw</th>\n",
       "      <th>year</th>\n",
       "      <th>weekofyear</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>271</th>\n",
       "      <td>7.259491e-17</td>\n",
       "      <td>1.168379</td>\n",
       "      <td>1.145138</td>\n",
       "      <td>0.967838</td>\n",
       "      <td>1995</td>\n",
       "      <td>28</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>929</th>\n",
       "      <td>-3.968507e+00</td>\n",
       "      <td>-2.317955</td>\n",
       "      <td>-1.184328</td>\n",
       "      <td>-0.449390</td>\n",
       "      <td>2008</td>\n",
       "      <td>11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>267</th>\n",
       "      <td>3.308645e-01</td>\n",
       "      <td>0.135865</td>\n",
       "      <td>0.071738</td>\n",
       "      <td>-0.179638</td>\n",
       "      <td>1995</td>\n",
       "      <td>24</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>806</th>\n",
       "      <td>-9.196096e-01</td>\n",
       "      <td>-0.179981</td>\n",
       "      <td>-0.125447</td>\n",
       "      <td>-0.065611</td>\n",
       "      <td>2005</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>454</th>\n",
       "      <td>-1.336260e+00</td>\n",
       "      <td>-1.171025</td>\n",
       "      <td>0.055579</td>\n",
       "      <td>-0.902923</td>\n",
       "      <td>1999</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "          ndvi_ne   ndvi_nw   ndvi_se   ndvi_sw  year  weekofyear\n",
       "271  7.259491e-17  1.168379  1.145138  0.967838  1995          28\n",
       "929 -3.968507e+00 -2.317955 -1.184328 -0.449390  2008          11\n",
       "267  3.308645e-01  0.135865  0.071738 -0.179638  1995          24\n",
       "806 -9.196096e-01 -0.179981 -0.125447 -0.065611  2005          43\n",
       "454 -1.336260e+00 -1.171025  0.055579 -0.902923  1999           3"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_FEATURES = RF_TRAINING_FEATURES + TIMELY_TRAINING_FEATURES\n",
    "Y_FEATURES = ['total_cases']\n",
    "\n",
    "X_train_sj = drop_unnecessary_columns(X_train_sj,X_FEATURES)\n",
    "X_train_iq = drop_unnecessary_columns(X_train_iq,X_FEATURES)\n",
    "X_cross_sj = drop_unnecessary_columns(X_cross_sj,X_FEATURES)\n",
    "X_cross_iq = drop_unnecessary_columns(X_cross_iq,X_FEATURES)\n",
    "\n",
    "y_train_iq = drop_unnecessary_columns(y_train_iq,Y_FEATURES)\n",
    "y_cross_iq = drop_unnecessary_columns(y_cross_iq,Y_FEATURES)\n",
    "\n",
    "# X_train_sj.drop(['station_min_temp_c'], axis=1, inplace=True)\n",
    "# X_train_iq.drop(['station_min_temp_c'], axis=1, inplace=True)\n",
    "# X_cross_sj.drop(['station_min_temp_c'], axis=1, inplace=True)\n",
    "# X_cross_iq.drop(['station_min_temp_c'], axis=1, inplace=True)\n",
    "\n",
    "print(f'X_train_iq: {X_train_iq.shape}')\n",
    "print(f'y_train_iq: {y_train_iq.shape}')\n",
    "print(f'X_cross_iq: {X_cross_iq.shape}')\n",
    "print(f'y_cross_iq: {y_cross_iq.shape}')\n",
    "X_train_sj.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train the Model with Data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.ensemble import GradientBoostingRegressor\n",
    "\n",
    "# find the ccurancy of the model with the mean absolute value\n",
    "def cross_validate_out_of_sample(reg, X_train, y_train, X_cross, y_cross):\n",
    "    reg.fit(X_train, y_train)\n",
    "    y_pred = reg.predict(X_cross)\n",
    "    return mean_absolute_error(y_true=y_cross, y_pred=y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Grid search to get the best score hyper parameters\n",
    "def grid_search_cross_val(reg, X, y, param_grid, scoring='neg_mean_absolute_error'):\n",
    "    grid = GridSearchCV(reg, param_grid=param_grid, scoring=scoring)\n",
    "    grid.fit(X, y)\n",
    "    print(\"Best score: {}\".format(np.abs(grid.best_score_)))\n",
    "    print(\"Best params: {}\".format(grid.best_params_))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### Check the best hyper param values for GradientBoostingRegression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "# reg = GradientBoostingRegressor(random_state=67)\n",
    "\n",
    "# param_grid = [\n",
    "#     {'learning_rate': [0.1, 0.3, 1.0, 3.0], 'n_estimators': [10, 30, 100, 300, 500], \n",
    "#      'max_depth': [3, 5, 7, 9]}\n",
    "# ]\n",
    "\n",
    "# grid_search_cross_val(reg, X_train_sj, y_train_sj.total_cases, param_grid)\n",
    "# grid_search_cross_val(reg, X_train_iq, y_train_iq.total_cases, param_grid)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "###### Check the best hyper param values for RandomForestRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "# reg = RandomForestRegressor(random_state=67)\n",
    "\n",
    "# param_grid = [\n",
    "#     {\n",
    "#       'n_estimators': [10, 30, 100, 300, 500], \n",
    "#       'max_depth': [3, 5, 7, None]\n",
    "#     } \n",
    "# ]\n",
    "\n",
    "# grid_search_cross_val(reg, X_train_sj, y_train_sj.total_cases, param_grid)\n",
    "# grid_search_cross_val(reg, X_train_iq, y_train_iq.total_cases, param_grid)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ndvi_ne</th>\n",
       "      <th>ndvi_nw</th>\n",
       "      <th>ndvi_se</th>\n",
       "      <th>ndvi_sw</th>\n",
       "      <th>year</th>\n",
       "      <th>weekofyear</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1099</th>\n",
       "      <td>1.988958</td>\n",
       "      <td>1.616379</td>\n",
       "      <td>1.380631</td>\n",
       "      <td>1.379286</td>\n",
       "      <td>2003</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1291</th>\n",
       "      <td>-0.852350</td>\n",
       "      <td>-1.106541</td>\n",
       "      <td>0.591055</td>\n",
       "      <td>-1.088251</td>\n",
       "      <td>2007</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1422</th>\n",
       "      <td>1.538207</td>\n",
       "      <td>1.751404</td>\n",
       "      <td>-0.002934</td>\n",
       "      <td>1.233931</td>\n",
       "      <td>2009</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1217</th>\n",
       "      <td>0.230505</td>\n",
       "      <td>-0.237594</td>\n",
       "      <td>0.493816</td>\n",
       "      <td>-0.310042</td>\n",
       "      <td>2005</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1411</th>\n",
       "      <td>-0.532952</td>\n",
       "      <td>-0.580507</td>\n",
       "      <td>0.368795</td>\n",
       "      <td>-0.073427</td>\n",
       "      <td>2009</td>\n",
       "      <td>34</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       ndvi_ne   ndvi_nw   ndvi_se   ndvi_sw  year  weekofyear\n",
       "1099  1.988958  1.616379  1.380631  1.379286  2003          34\n",
       "1291 -0.852350 -1.106541  0.591055 -1.088251  2007          18\n",
       "1422  1.538207  1.751404 -0.002934  1.233931  2009          45\n",
       "1217  0.230505 -0.237594  0.493816 -0.310042  2005          47\n",
       "1411 -0.532952 -0.580507  0.368795 -0.073427  2009          34"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train_iq.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12.238670212765955"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train the model with SJ data set\n",
    "reg_sj = RandomForestRegressor(max_depth=None, n_estimators=500, random_state=67)\n",
    "cross_validate_out_of_sample(reg_sj, X_train_sj, y_train_sj.total_cases, X_cross_sj, y_cross_sj.total_cases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4.657457008761076"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train the model with IQ data set\n",
    "reg_iq = GradientBoostingRegressor(max_depth=None, n_estimators=500, random_state=67)\n",
    "cross_validate_out_of_sample(reg_iq, X_train_iq, y_train_iq.total_cases, X_cross_iq, y_cross_iq.total_cases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Submission the results\n",
    "def submission(pred):\n",
    "    submission = pd.read_csv(\"data/submission_format.csv\", index_col=[0, 1, 2])\n",
    "    submission['total_cases'] = pred['total_cases']\n",
    "    submission.to_csv(\"./submissions/sub_DengAI.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preprocess the Test Data set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  import sys\n",
      "/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py:543: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self.obj[item] = s\n",
      "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  import sys\n",
      "/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py:543: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self.obj[item] = s\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train_sj: (936, 6)\n",
      "train_iq: (520, 6)\n",
      "X_test_sj: (260, 6)\n",
      "X_test_iq: (156, 6)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/pandas/core/frame.py:3391: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self[k1] = value[k2]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ndvi_ne</th>\n",
       "      <th>ndvi_nw</th>\n",
       "      <th>ndvi_se</th>\n",
       "      <th>ndvi_sw</th>\n",
       "      <th>year</th>\n",
       "      <th>weekofyear</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>-3.577464e-01</td>\n",
       "      <td>-0.653177</td>\n",
       "      <td>-1.113613</td>\n",
       "      <td>-1.127334</td>\n",
       "      <td>2008</td>\n",
       "      <td>18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-3.503788e-01</td>\n",
       "      <td>-0.576857</td>\n",
       "      <td>-1.423542</td>\n",
       "      <td>-1.470585</td>\n",
       "      <td>2008</td>\n",
       "      <td>19</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>-2.153074e-01</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-0.389128</td>\n",
       "      <td>-1.121362</td>\n",
       "      <td>2008</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>-2.840141e-17</td>\n",
       "      <td>-0.664527</td>\n",
       "      <td>-0.789986</td>\n",
       "      <td>-0.500550</td>\n",
       "      <td>2008</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.619450e-01</td>\n",
       "      <td>0.036443</td>\n",
       "      <td>-1.719843</td>\n",
       "      <td>-1.405155</td>\n",
       "      <td>2008</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        ndvi_ne   ndvi_nw   ndvi_se   ndvi_sw  year  weekofyear\n",
       "0 -3.577464e-01 -0.653177 -1.113613 -1.127334  2008          18\n",
       "1 -3.503788e-01 -0.576857 -1.423542 -1.470585  2008          19\n",
       "2 -2.153074e-01  0.000000 -0.389128 -1.121362  2008          20\n",
       "3 -2.840141e-17 -0.664527 -0.789986 -0.500550  2008          21\n",
       "4  2.619450e-01  0.036443 -1.719843 -1.405155  2008          22"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_sj = drop_unnecessary_columns(train_sj,X_FEATURES)\n",
    "train_iq = drop_unnecessary_columns(train_iq,X_FEATURES)\n",
    "\n",
    "# Impute the Values\n",
    "X_test_sj = features_test.loc[features_test['city']=='sj',:]\n",
    "X_test_iq = features_test.loc[features_test['city']=='iq',:]\n",
    "\n",
    "predict_sj = X_test_sj[KEYS].copy()\n",
    "predict_iq = X_test_iq[KEYS].copy()\n",
    "\n",
    "# FIll values\n",
    "X_test_sj = fill_null_values_with_mean(X_test_sj,RF_TRAINING_FEATURES)\n",
    "X_test_iq = fill_null_values_with_mean(X_test_iq,RF_TRAINING_FEATURES)\n",
    "\n",
    "\n",
    "# Normalization\n",
    "# X_test_sj = add_time_series_features(X_test_sj, window=10)\n",
    "# X_test_iq = add_time_series_features(X_test_iq, window=10)\n",
    "\n",
    "X_test_sj[FEATURES_TO_NRMLZE] = X_test_sj[FEATURES_TO_NRMLZE].apply(normalize, axis=0)\n",
    "X_test_iq[FEATURES_TO_NRMLZE] = X_test_iq[FEATURES_TO_NRMLZE].apply(normalize, axis=0)\n",
    "\n",
    "\n",
    "columns = RF_TRAINING_FEATURES + TIMELY_TRAINING_FEATURES\n",
    "# Drop columns \n",
    "X_test_sj = drop_unnecessary_columns(X_test_sj,columns)\n",
    "X_test_iq = drop_unnecessary_columns(X_test_iq,columns)\n",
    "\n",
    "# train_sj.drop(['station_min_temp_c'], axis=1, inplace=True)\n",
    "# train_iq.drop(['station_min_temp_c'], axis=1, inplace=True)\n",
    "# X_test_sj.drop(['station_min_temp_c'], axis=1, inplace=True)\n",
    "# X_test_iq.drop(['station_min_temp_c'], axis=1, inplace=True)\n",
    "\n",
    "print(f'train_sj: {train_sj.shape}')\n",
    "print(f'train_iq: {train_iq.shape}')\n",
    "print(f'X_test_sj: {X_test_sj.shape}')\n",
    "print(f'X_test_iq: {X_test_iq.shape}')\n",
    "X_test_sj.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,\n",
       "                      max_features='auto', max_leaf_nodes=None,\n",
       "                      min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                      min_samples_leaf=1, min_samples_split=2,\n",
       "                      min_weight_fraction_leaf=0.0, n_estimators=500,\n",
       "                      n_jobs=None, oob_score=False, random_state=67, verbose=0,\n",
       "                      warm_start=False)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train models with full dataset\n",
    "reg_sj = RandomForestRegressor(max_depth=None, n_estimators=500, random_state=67)\n",
    "reg_sj.fit(train_sj, label_sj.total_cases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "RandomForestRegressor(bootstrap=True, criterion='mse', max_depth=None,\n",
       "                      max_features='auto', max_leaf_nodes=None,\n",
       "                      min_impurity_decrease=0.0, min_impurity_split=None,\n",
       "                      min_samples_leaf=1, min_samples_split=2,\n",
       "                      min_weight_fraction_leaf=0.0, n_estimators=500,\n",
       "                      n_jobs=None, oob_score=False, random_state=67, verbose=0,\n",
       "                      warm_start=False)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reg_iq = RandomForestRegressor(max_depth=None, n_estimators=500, random_state=67)\n",
    "reg_iq.fit(train_iq, label_iq.total_cases)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city</th>\n",
       "      <th>year</th>\n",
       "      <th>weekofyear</th>\n",
       "      <th>total_cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sj</td>\n",
       "      <td>2008</td>\n",
       "      <td>18</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sj</td>\n",
       "      <td>2008</td>\n",
       "      <td>19</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sj</td>\n",
       "      <td>2008</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sj</td>\n",
       "      <td>2008</td>\n",
       "      <td>21</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sj</td>\n",
       "      <td>2008</td>\n",
       "      <td>22</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  city  year  weekofyear  total_cases\n",
       "0   sj  2008          18            5\n",
       "1   sj  2008          19            5\n",
       "2   sj  2008          20            6\n",
       "3   sj  2008          21            7\n",
       "4   sj  2008          22           13"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Predict SJ\n",
    "y_sj_pred = reg_sj.predict(X_test_sj)\n",
    "predict_sj['total_cases'] = y_sj_pred.round().astype(int)\n",
    "predict_sj.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city</th>\n",
       "      <th>year</th>\n",
       "      <th>weekofyear</th>\n",
       "      <th>total_cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>260</th>\n",
       "      <td>iq</td>\n",
       "      <td>2010</td>\n",
       "      <td>26</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>261</th>\n",
       "      <td>iq</td>\n",
       "      <td>2010</td>\n",
       "      <td>27</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>262</th>\n",
       "      <td>iq</td>\n",
       "      <td>2010</td>\n",
       "      <td>28</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>263</th>\n",
       "      <td>iq</td>\n",
       "      <td>2010</td>\n",
       "      <td>29</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>264</th>\n",
       "      <td>iq</td>\n",
       "      <td>2010</td>\n",
       "      <td>30</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    city  year  weekofyear  total_cases\n",
       "260   iq  2010          26            3\n",
       "261   iq  2010          27            3\n",
       "262   iq  2010          28            3\n",
       "263   iq  2010          29            5\n",
       "264   iq  2010          30            3"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Predict IQ\n",
    "y_iq_pred = reg_iq.predict(X_test_iq)\n",
    "predict_iq['total_cases'] = y_iq_pred.round().astype(int)\n",
    "predict_iq.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city</th>\n",
       "      <th>year</th>\n",
       "      <th>weekofyear</th>\n",
       "      <th>total_cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [city, year, weekofyear, total_cases]\n",
       "Index: []"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict_df = pd.concat([predict_sj, predict_iq], axis=0)\n",
    "predict_df[predict_df.total_cases < 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city</th>\n",
       "      <th>year</th>\n",
       "      <th>weekofyear</th>\n",
       "      <th>total_cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: [city, year, weekofyear, total_cases]\n",
       "Index: []"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict_df.loc[predict_df.total_cases < 0, 'total_cases'] = 0\n",
    "predict_df[predict_df.total_cases < 0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>city</th>\n",
       "      <th>year</th>\n",
       "      <th>weekofyear</th>\n",
       "      <th>total_cases</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>sj</td>\n",
       "      <td>2008</td>\n",
       "      <td>18</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>sj</td>\n",
       "      <td>2008</td>\n",
       "      <td>19</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>sj</td>\n",
       "      <td>2008</td>\n",
       "      <td>20</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>sj</td>\n",
       "      <td>2008</td>\n",
       "      <td>21</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sj</td>\n",
       "      <td>2008</td>\n",
       "      <td>22</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  city  year  weekofyear  total_cases\n",
       "0   sj  2008          18            5\n",
       "1   sj  2008          19            5\n",
       "2   sj  2008          20            6\n",
       "3   sj  2008          21            7\n",
       "4   sj  2008          22           13"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "submission = pd.read_csv(\"./data/submission_format.csv\")\n",
    "\n",
    "submission['total_cases'] = predict_df['total_cases']\n",
    "submission.set_index('city', inplace=True)\n",
    "submission.to_csv(\"./submissions/sub_DengAIRF.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}