transfer-learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "tensorflow version: 1.12.0-rc0\n",
    "scikit-learn version: 0.17\n",
    "keras version: 2.2.4\n",
    "tensorboard version: 1.10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Transfer Learning\n",
    "In this assignment, we will use the weights of a network pre-trained in a particular problem as starting point to train our CNN to a different problem. As training a network from scratch is time-consuming and demands a lot of data, this is a frequent strategy, specially if both datasets (the one used for pre-training and the target) shares similar structures/elements/concepts. \n",
    "\n",
    "This is specially true when working with images. Most filters learned in initial convolutional layers will detect low-level elements, such as borders, corners and color blobs, which are common to most problems in the image domain. \n",
    "\n",
    "In this notebook, we will load the SqueezeNet architecture trained in the ImageNet dataset and fine-tune it to CIFAR-10.\n",
    "\n",
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: keras==2.2.4 in /home/azael/Trabalhos/MO444/MO444/lib/python3.5/site-packages (2.2.4)\r\n",
      "Requirement already satisfied: scipy>=0.14 in /home/azael/Trabalhos/MO444/MO444/lib/python3.5/site-packages (from keras==2.2.4) (1.1.0)\r\n",
      "Requirement already satisfied: h5py in /home/azael/Trabalhos/MO444/MO444/lib/python3.5/site-packages (from keras==2.2.4) (2.8.0)\r\n",
      "Requirement already satisfied: six>=1.9.0 in /home/azael/Trabalhos/MO444/MO444/lib/python3.5/site-packages (from keras==2.2.4) (1.11.0)\r\n",
      "Requirement already satisfied: keras-preprocessing>=1.0.5 in /home/azael/Trabalhos/MO444/MO444/lib/python3.5/site-packages (from keras==2.2.4) (1.0.5)\r\n",
      "Requirement already satisfied: pyyaml in /home/azael/Trabalhos/MO444/MO444/lib/python3.5/site-packages (from keras==2.2.4) (3.13)\r\n",
      "Requirement already satisfied: numpy>=1.9.1 in /home/azael/Trabalhos/MO444/MO444/lib/python3.5/site-packages (from keras==2.2.4) (1.15.1)\r\n",
      "Requirement already satisfied: keras-applications>=1.0.6 in /home/azael/Trabalhos/MO444/MO444/lib/python3.5/site-packages (from keras==2.2.4) (1.0.6)\r\n"
     ]
    }
   ],
   "source": [
    "!pip install keras==2.2.4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The scikit-learn version is 0.20.0.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import sklearn\n",
    "import keras\n",
    "print('The scikit-learn version is {}.'.format(sklearn.__version__))\n",
    "from random import sample, seed\n",
    "seed(42)\n",
    "np.random.seed(42)\n",
    "from time import time\n",
    "import matplotlib.pyplot as plt\n",
    "# plt.rcParams['figure.figsize'] = (15,15) # Make the figures a bit bigger\n",
    "\n",
    "# Keras imports\n",
    "from keras.layers import Input, Convolution2D, MaxPooling2D, Activation, concatenate, Dropout, GlobalAveragePooling2D\n",
    "from keras.models import Model\n",
    "from keras import regularizers\n",
    "from keras import optimizers\n",
    "from keras.optimizers import Adam\n",
    "from keras.utils import np_utils\n",
    "from keras.preprocessing.image import load_img, img_to_array\n",
    "from keras.datasets import cifar10\n",
    "from keras.callbacks import TensorBoard\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.utils.np_utils import to_categorical\n",
    "if (sklearn.__version__ == '0.20.0'):\n",
    "    from sklearn.model_selection import StratifiedShuffleSplit\n",
    "else:\n",
    "    from sklearn.cross_validation import StratifiedShuffleSplit\n",
    "\n",
    "\n",
    "#======================================\n",
    "# Global definitions\n",
    "#======================================\n",
    "n_epochs         = 100\n",
    "learning_rate    = 1e-4\n",
    "n_classes        = 10\n",
    "train_batch_size = 384\n",
    "val_batch_size   = 128\n",
    "    \n",
    "#Utility to plot\n",
    "def plotImages(imgList):\n",
    "    for i in range(len(imgList)):\n",
    "        plotImage(imgList[i])\n",
    "        \n",
    "        \n",
    "def plotImage(img):\n",
    "    fig = plt.figure(figsize=(3,3))\n",
    "    ax = fig.add_subplot(111)\n",
    "\n",
    "    ax.imshow(np.uint8(img), interpolation='nearest')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## SqueezeNet definition\n",
    "These methods define our architecture and load the weights obtained using ImageNet data."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Fire Module Definition\n",
    "sq1x1 = \"squeeze1x1\"\n",
    "exp1x1 = \"expand1x1\"\n",
    "exp3x3 = \"expand3x3\"\n",
    "relu = \"relu_\"\n",
    "\n",
    "def fire_module(x, fire_id, squeeze=16, expand=64):\n",
    "    s_id = 'fire' + str(fire_id) + '/'\n",
    "\n",
    "    channel_axis = 3\n",
    "    \n",
    "    x = Convolution2D(squeeze, (1, 1), padding='valid', name=s_id + sq1x1)(x)\n",
    "    x = Activation('relu', name=s_id + relu + sq1x1)(x)\n",
    "\n",
    "    left = Convolution2D(expand, (1, 1), padding='valid', name=s_id + exp1x1)(x)\n",
    "    left = Activation('relu', name=s_id + relu + exp1x1)(left)\n",
    "\n",
    "    right = Convolution2D(expand, (3, 3), padding='same', name=s_id + exp3x3)(x)\n",
    "    right = Activation('relu', name=s_id + relu + exp3x3)(right)\n",
    "\n",
    "    x = concatenate([left, right], axis=channel_axis, name=s_id + 'concat')\n",
    "    return x\n",
    "\n",
    "#SqueezeNet model definition\n",
    "def SqueezeNet(input_shape):\n",
    "    img_input = Input(shape=input_shape) #placeholder\n",
    "    \n",
    "    x = Convolution2D(64, (3, 3), strides=(2, 2), padding='valid', name='conv1')(img_input)\n",
    "    x = Activation('relu', name='relu_conv1')(x)\n",
    "    x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), name='pool1')(x)\n",
    "\n",
    "    x = fire_module(x, fire_id=2, squeeze=16, expand=64)\n",
    "    x = fire_module(x, fire_id=3, squeeze=16, expand=64)\n",
    "    x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), name='pool3')(x)\n",
    "\n",
    "    x = fire_module(x, fire_id=4, squeeze=32, expand=128)\n",
    "    x = fire_module(x, fire_id=5, squeeze=32, expand=128)\n",
    "    x = MaxPooling2D(pool_size=(3, 3), strides=(2, 2), name='pool5')(x)\n",
    "\n",
    "    x = fire_module(x, fire_id=6, squeeze=48, expand=192)\n",
    "    x = fire_module(x, fire_id=7, squeeze=48, expand=192)\n",
    "    x = fire_module(x, fire_id=8, squeeze=64, expand=256)\n",
    "    x = fire_module(x, fire_id=9, squeeze=64, expand=256)\n",
    "    \n",
    "    x = Dropout(0.5, name='drop9')(x)\n",
    "\n",
    "    x = Convolution2D(1000, (1, 1), padding='valid', name='conv10')(x)\n",
    "    x = Activation('relu', name='relu_conv10')(x)\n",
    "    x = GlobalAveragePooling2D()(x)\n",
    "    x = Activation('softmax', name='loss')(x)\n",
    "\n",
    "    model = Model(img_input, x, name='squeezenet')\n",
    "\n",
    "    # Download and load ImageNet weights\n",
    "    model.load_weights('./squeezenet_weights_tf_dim_ordering_tf_kernels.h5')\n",
    "    \n",
    "    return model    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CIFAR-10\n",
    "\n",
    "The CIFAR-10 dataset consists of 60000 32x32 colour images in 10 classes, with 6000 images per class. There are 50000 training images and 10000 test images. The class are **airplane, automobile, bird, cat, deer, dog, frog, horse, ship, truck**."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train/Val data. X:  (50000, 32, 32, 3) , Y:  (50000, 1)\n",
      "Test data. X:  (10000, 32, 32, 3) , Y:  (10000, 1)\n",
      "--- Splitting data into train and val\n",
      "Train data. X: (5, 40000, 32, 32, 3) Y: (5, 40000, 1)\n",
      "Val data. X: (5, 10000, 32, 32, 3) Y: (5, 10000, 1)\n"
     ]
    }
   ],
   "source": [
    "#Load data\n",
    "(trainVal_data, trainVal_label), (X_test, y_test) = cifar10.load_data()\n",
    "print(\"Train/Val data. X: \", trainVal_data.shape, \", Y: \", trainVal_label.shape)\n",
    "print(\"Test data. X: \", X_test.shape, \", Y: \", y_test.shape)\n",
    "\n",
    "# Normalizing the data\n",
    "trainVal_data = (trainVal_data / 127.5) - 1.\n",
    "X_test = (X_test  / 127.5) - 1.\n",
    "\n",
    "#=====================================\n",
    "# Prepare the data\n",
    "#=====================================\n",
    "\n",
    "#--- Dividing the data into training and validation\n",
    "folds = 5\n",
    "if (sklearn.__version__ == '0.20.0'):\n",
    "    sss = StratifiedShuffleSplit(folds, test_size=0.2, random_state=42)\n",
    "    sss = sss.split(trainVal_data,trainVal_label)\n",
    "else:\n",
    "    sss = StratifiedShuffleSplit(trainVal_label, folds, test_size=0.2, random_state=42)\n",
    "X_train = []\n",
    "y_train = []\n",
    "X_val = []\n",
    "y_val = []\n",
    "for train_index, val_index in sss:\n",
    "    X_train.append(trainVal_data[train_index])\n",
    "    X_val.append(trainVal_data[val_index])\n",
    "    y_train.append(trainVal_label[train_index])\n",
    "    y_val.append(trainVal_label[val_index])\n",
    "\n",
    "X_train = np.array(X_train)\n",
    "y_train = np.array(y_train)\n",
    "X_val = np.array(X_val)\n",
    "y_val = np.array(y_val)\n",
    "    \n",
    "print(\"--- Splitting data into train and val\")\n",
    "print(\"Train data. X:\",X_train.shape,\"Y:\",y_train.shape)\n",
    "print(\"Val data. X:\",X_val.shape,\"Y:\",y_val.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4000 4000 4000 4000 4000 4000 4000 4000 4000 4000]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEJCAYAAACdePCvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAFkNJREFUeJzt3X/wXXV95/Hny/BD118E+ZbGJJi0TceN3Tay3wY62tZKhYC7BqfUxW1rdJhJXXHXzrrbgu4MrZQZ6a6yOkW2tGQNTtvI0rpkMJVGxHGdWX4EDYFAKV8Fl6SBRIMoy0o3+N4/7id6N/1+87033/v9Aef5mLnzPed9Pufc982P+/rec849J1WFJKl7XjDfDUiS5ocBIEkdZQBIUkcZAJLUUQaAJHWUASBJHWUASFJHGQCS1FEGgCR11HHz3cDRnHLKKbVixYr5bkOSnlPuvvvub1bV2HTjFnQArFixgh07dsx3G5L0nJLkG4OMcxeQJHWUASBJHWUASFJHGQCS1FEGgCR11MABkGRRkq8mubnNr0xyR5KJJJ9OckKrn9jmJ9ryFX3buLTVH0xyzqhfjCRpcMN8Angf8EDf/JXAVVX1E8ATwEWtfhHwRKtf1caRZDVwIfAaYB3wiSSLZta+JOlYDRQASZYBbwb+pM0HeCNwYxuyGTi/Ta9v87TlZ7Xx64EtVfVMVT0MTABrR/EiJEnDG/QTwH8Gfhv4fpt/BfDtqjrU5vcAS9v0UuBRgLb8yTb+B/VJ1pEkzbFpvwmc5J8B+6vq7iRvmO2GkmwENgKcdtppM9rWiks+O4qWJGnOPfLhN8/6cwzyCeB1wFuSPAJsobfr52PASUkOB8gyYG+b3gssB2jLXw58q78+yTo/UFXXVtV4VY2PjU17KQtJ0jGaNgCq6tKqWlZVK+gdxP1CVf0acBtwQRu2AbipTW9t87TlX6iqavUL21lCK4FVwJ0jeyWSpKHM5GJwvwNsSfL7wFeB61r9OuBTSSaAg/RCg6raneQG4H7gEHBxVT07g+eXJM3AUAFQVV8Evtimv84kZ/FU1feAX51i/SuAK4ZtUpI0en4TWJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOmraAEjywiR3Jrknye4kv9fqn0zycJKd7bGm1ZPk40kmkuxKcnrftjYkeag9Nkz1nJKk2TfILSGfAd5YVU8lOR74cpK/asv+fVXdeMT4c+nd8H0VcAZwDXBGkpOBy4BxoIC7k2ytqidG8UIkScOZ9hNA9TzVZo9vjzrKKuuB69t6twMnJVkCnANsr6qD7U1/O7BuZu1Lko7VQMcAkixKshPYT+9N/I626Iq2m+eqJCe22lLg0b7V97TaVHVJ0jwYKACq6tmqWgMsA9Ym+SngUuDVwM8CJwO/M4qGkmxMsiPJjgMHDoxik5KkSQx1FlBVfRu4DVhXVfvabp5ngP8KrG3D9gLL+1Zb1mpT1Y98jmuraryqxsfGxoZpT5I0hEHOAhpLclKbfhHwJuBv2n59kgQ4H7ivrbIVeEc7G+hM4Mmq2gfcApydZHGSxcDZrSZJmgeDnAW0BNicZBG9wLihqm5O8oUkY0CAncC72/htwHnABPA08C6AqjqY5HLgrjbuQ1V1cHQvRZI0jGkDoKp2Aa+dpP7GKcYXcPEUyzYBm4bsUZI0C/wmsCR1lAEgSR1lAEhSRxkAktRRBoAkdZQBIEkdZQBIUkcZAJLUUQaAJHWUASBJHWUASFJHGQCS1FEGgCR1lAEgSR1lAEhSRxkAktRRBoAkddQg9wR+YZI7k9yTZHeS32v1lUnuSDKR5NNJTmj1E9v8RFu+om9bl7b6g0nOma0XJUma3iCfAJ4B3lhVPwOsAda1m71fCVxVVT8BPAFc1MZfBDzR6le1cSRZDVwIvAZYB3yi3WdYkjQPpg2A6nmqzR7fHgW8Ebix1TcD57fp9W2etvysJGn1LVX1TFU9TO+m8WtH8iokSUMb6BhAkkVJdgL7ge3A14BvV9WhNmQPsLRNLwUeBWjLnwRe0V+fZJ3+59qYZEeSHQcOHBj+FUmSBjJQAFTVs1W1BlhG77f2V89WQ1V1bVWNV9X42NjYbD2NJHXeUGcBVdW3gduAnwNOSnJcW7QM2Num9wLLAdrylwPf6q9Pso4kaY4NchbQWJKT2vSLgDcBD9ALggvasA3ATW16a5unLf9CVVWrX9jOEloJrALuHNULkSQN57jph7AE2NzO2HkBcENV3ZzkfmBLkt8Hvgpc18ZfB3wqyQRwkN6ZP1TV7iQ3APcDh4CLq+rZ0b4cSdKgpg2AqtoFvHaS+teZ5Cyeqvoe8KtTbOsK4Irh25QkjZrfBJakjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4a5J7Ay5PcluT+JLuTvK/VfzfJ3iQ72+O8vnUuTTKR5MEk5/TV17XaRJJLZuclSZIGMcg9gQ8B76+qryR5KXB3ku1t2VVV9Z/6BydZTe8+wK8BXgl8PslPtsVX07up/B7griRbq+r+UbwQSdJwBrkn8D5gX5v+bpIHgKVHWWU9sKWqngEebjeHP3zv4Il2L2GSbGljDQBJmgdDHQNIsoLeDeLvaKX3JtmVZFOSxa22FHi0b7U9rTZV/cjn2JhkR5IdBw4cGKY9SdIQBg6AJC8B/gL4rar6DnAN8OPAGnqfED4yioaq6tqqGq+q8bGxsVFsUpI0iUGOAZDkeHpv/n9aVX8JUFWP9y3/Y+DmNrsXWN63+rJW4yh1SdIcG+QsoADXAQ9U1Uf76kv6hr0VuK9NbwUuTHJikpXAKuBO4C5gVZKVSU6gd6B462hehiRpWIN8Angd8BvAvUl2ttoHgLcnWQMU8AjwmwBVtTvJDfQO7h4CLq6qZwGSvBe4BVgEbKqq3SN8LZKkIQxyFtCXgUyyaNtR1rkCuGKS+rajrSdJmjt+E1iSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjpqkHsCL09yW5L7k+xO8r5WPznJ9iQPtZ+LWz1JPp5kIsmuJKf3bWtDG/9Qkg2z97IkSdMZ5BPAIeD9VbUaOBO4OMlq4BLg1qpaBdza5gHOpXcj+FXARuAa6AUGcBlwBrAWuOxwaEiS5t60AVBV+6rqK236u8ADwFJgPbC5DdsMnN+m1wPXV8/twElJlgDnANur6mBVPQFsB9aN9NVIkgY21DGAJCuA1wJ3AKdW1b626DHg1Da9FHi0b7U9rTZV/cjn2JhkR5IdBw4cGKY9SdIQBg6AJC8B/gL4rar6Tv+yqiqgRtFQVV1bVeNVNT42NjaKTUqSJjFQACQ5nt6b/59W1V+28uNt1w7t5/5W3wss71t9WatNVZckzYNBzgIKcB3wQFV9tG/RVuDwmTwbgJv66u9oZwOdCTzZdhXdApydZHE7+Ht2q0mS5sFxA4x5HfAbwL1JdrbaB4APAzckuQj4BvC2tmwbcB4wATwNvAugqg4muRy4q437UFUdHMmrkCQNbdoAqKovA5li8VmTjC/g4im2tQnYNEyDkqTZ4TeBJamjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4a5JaQm5LsT3JfX+13k+xNsrM9zutbdmmSiSQPJjmnr76u1SaSXDL6lyJJGsYgnwA+CaybpH5VVa1pj20ASVYDFwKvaet8IsmiJIuAq4FzgdXA29tYSdI8GeSWkF9KsmLA7a0HtlTVM8DDSSaAtW3ZRFV9HSDJljb2/qE7liSNxEyOAbw3ya62i2hxqy0FHu0bs6fVpqpLkubJsQbANcCPA2uAfcBHRtVQko1JdiTZceDAgVFtVpJ0hGMKgKp6vKqerarvA3/MD3fz7AWW9w1d1mpT1Sfb9rVVNV5V42NjY8fSniRpAMcUAEmW9M2+FTh8htBW4MIkJyZZCawC7gTuAlYlWZnkBHoHircee9uSpJma9iBwkj8H3gCckmQPcBnwhiRrgAIeAX4ToKp2J7mB3sHdQ8DFVfVs2857gVuARcCmqto98lcjSRrYIGcBvX2S8nVHGX8FcMUk9W3AtqG6kyTNGr8JLEkdZQBIUkcZAJLUUQaAJHWUASBJHWUASFJHGQCS1FEGgCR1lAEgSR1lAEhSRxkAktRRBoAkdZQBIEkdZQBIUkcZAJLUUQaAJHWUASBJHTVtACTZlGR/kvv6aicn2Z7kofZzcasnyceTTCTZleT0vnU2tPEPJdkwOy9HkjSoQT4BfBJYd0TtEuDWqloF3NrmAc6ldyP4VcBG4BroBQa9ewmfAawFLjscGpKk+TFtAFTVl4CDR5TXA5vb9Gbg/L769dVzO3BSkiXAOcD2qjpYVU8A2/mHoSJJmkPHegzg1Kra16YfA05t00uBR/vG7Wm1qeqSpHky44PAVVVAjaAXAJJsTLIjyY4DBw6MarOSpCMcawA83nbt0H7ub/W9wPK+cctabar6P1BV11bVeFWNj42NHWN7kqTpHGsAbAUOn8mzAbipr/6OdjbQmcCTbVfRLcDZSRa3g79nt5okaZ4cN92AJH8OvAE4JckeemfzfBi4IclFwDeAt7Xh24DzgAngaeBdAFV1MMnlwF1t3Ieq6sgDy5KkOTRtAFTV26dYdNYkYwu4eIrtbAI2DdWdJGnW+E1gSeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqqBkFQJJHktybZGeSHa12cpLtSR5qPxe3epJ8PMlEkl1JTh/FC5AkHZtRfAL4papaU1Xjbf4S4NaqWgXc2uYBzgVWtcdG4JoRPLck6RjNxi6g9cDmNr0ZOL+vfn313A6clGTJLDy/JGkAMw2AAv46yd1JNrbaqVW1r00/BpzappcCj/atu6fVJEnz4LgZrv/6qtqb5EeA7Un+pn9hVVWSGmaDLUg2Apx22mkzbE+SNJUZfQKoqr3t537gM8Ba4PHDu3baz/1t+F5ged/qy1rtyG1eW1XjVTU+NjY2k/YkSUdxzAGQ5MVJXnp4GjgbuA/YCmxowzYAN7XprcA72tlAZwJP9u0qkiTNsZnsAjoV+EySw9v5s6r6XJK7gBuSXAR8A3hbG78NOA+YAJ4G3jWD55YkzdAxB0BVfR34mUnq3wLOmqRewMXH+nySpNHym8CS1FEGgCR1lAEgSR1lAEhSRxkAktRRBoAkdZQBIEkdZQBIUkcZAJLUUQaAJHWUASBJHWUASFJHGQCS1FEGgCR1lAEgSR1lAEhSRxkAktRRcx4ASdYleTDJRJJL5vr5JUk9cxoASRYBVwPnAquBtydZPZc9SJJ65voTwFpgoqq+XlV/D2wB1s9xD5Ik5j4AlgKP9s3vaTVJ0hw7br4bOFKSjcDGNvtUkgdnsLlTgG/OvKuRs6/h2Ndw7Gs4C7KvXDmjvl41yKC5DoC9wPK++WWt9gNVdS1w7SieLMmOqhofxbZGyb6GY1/Dsa/hdLmvud4FdBewKsnKJCcAFwJb57gHSRJz/Amgqg4leS9wC7AI2FRVu+eyB0lSz5wfA6iqbcC2OXq6kexKmgX2NRz7Go59DaezfaWqZvs5JEkLkJeCkKSOel4FQJKTk2xP8lD7ufgoY1+WZE+SP1wIfSV5VZKvJNmZZHeSdy+QvtYk+Z+tp11J/sVC6KuN+1ySbye5eZb7OerlS5KcmOTTbfkdSVbMZj9D9PUL7d/UoSQXzEVPA/b1b5Pc3/493ZpkoFMW56Cvdye5t/0f/PJcXaVg0MvjJPmVJJVkdGcGVdXz5gH8AXBJm74EuPIoYz8G/BnwhwuhL+AE4MQ2/RLgEeCVC6CvnwRWtelXAvuAk+a7r7bsLOCfAzfPYi+LgK8BP9b+ju4BVh8x5j3Af2nTFwKfnoN/U4P0tQL4aeB64ILZ7mmIvn4J+Edt+l8toD+vl/VNvwX43ELoq417KfAl4HZgfFTP/7z6BEDvshKb2/Rm4PzJBiX5p8CpwF8vlL6q6u+r6pk2eyJz8+lskL7+tqoeatN/B+wHxua7r9bPrcB3Z7mXQS5f0t/vjcBZSTLffVXVI1W1C/j+LPcybF+3VdXTbfZ2et8HWgh9fadv9sXAXBwgHfTyOJcDVwLfG+WTP98C4NSq2temH6P3Jv//SfIC4CPAv1tIfQEkWZ5kF73LZVzZ3nDnva++/tbS+y3lawupr1k2yOVLfjCmqg4BTwKvWAB9zYdh+7oI+KtZ7ahnoL6SXJzka/Q+hf6bhdBXktOB5VX12VE/+YK7FMR0knwe+NFJFn2wf6aqKslkCf4eYFtV7RnlL2kj6IuqehT46SSvBP57khur6vH57qttZwnwKWBDVc34N8pR9aXnriS/DowDvzjfvRxWVVcDVyf5l8B/ADbMZz/tF9aPAu+cje0/5wKgqn55qmVJHk+ypKr2tTes/ZMM+zng55O8h96+9hOSPFVVM7o3wQj66t/W3yW5D/h5ersU5rWvJC8DPgt8sKpun0k/o+xrjkx7+ZK+MXuSHAe8HPjWAuhrPgzUV5Jfphf2v9i363Pe++qzBbhmVjvqma6vlwI/BXyx/cL6o8DWJG+pqh0zffLn2y6grfwwsTcANx05oKp+rapOq6oV9HYDXT/TN/9R9JVkWZIXtenFwOuBmVwIb1R9nQB8ht6f04zCaJR9zaFBLl/S3+8FwBeqHbmb577mw7R9JXkt8EfAW6pqrsJ9kL5W9c2+GXhovvuqqier6pSqWtHes26n9+c24zf/w0/wvHnQ2+96K72/uM8DJ7f6OPAnk4x/J3NzFtC0fQFvAnbROwtgF7BxgfT168D/BXb2PdbMd19t/n8AB4D/Q2/f6Tmz1M95wN/SO/bxwVb7EL3/iAAvBP4bMAHcCfzYbP/dDdjXz7Y/l/9N7xPJ7gXS1+eBx/v+PW1dIH19DNjderoNeM1C6OuIsV9khGcB+U1gSeqo59suIEnSgAwASeooA0CSOsoAkKSOMgAkaQFJ8v520bdTphm3tl24bmeSe5K8ddjnMgAkaY4leUOST05SXw6cDfyvATZzH71TQtcA64A/al9EHJgBIEkLx1XAb9N3IbokL06yKcmdSb6aZD1AVT1dvWtPQe+7KEOf028ASNIC0N7Y91bVPUcs+iC9b5evpXcp7f+Y5MVtnTOS7AbuBd7dFwiDPadfBJOkuZHkDnqXe38JcDI/3NVzGfAB4OyqejLJI/R273wzyQ56v+EffnM/md633h/o2+4/pndJ8l+oqoEvGf2cuxicJD1XVdUZ0DsGALyzqt7Z5v8JsBK4p130bRnwlXYJ9gC/UlVTXhusqh5I8hS9C8cNfJ0gdwFJ0jyrqnur6kfqhxd92wOcXlWPAbcA//rwTYbaxfRoF5A7rk2/Cng1vTsJDswAkKSF7XLgeGBX299/eau/nt4nhp30rtj7nqr65jAb9hiAJHWUnwAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI76f5AaBwteePPdAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4000 4000 4000 4000 4000 4000 4000 4000 4000 4000]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEJCAYAAACdePCvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAFkNJREFUeJzt3X/wXXV95/Hny/BD118E+ZbGJJi0TceN3Tay3wY62tZKhYC7BqfUxW1rdJhJXXHXzrrbgu4MrZQZ6a6yOkW2tGQNTtvI0rpkMJVGxHGdWX4EDYFAKV8Fl6SBRIMoy0o3+N4/7id6N/1+87033/v9Aef5mLnzPed9Pufc982P+/rec849J1WFJKl7XjDfDUiS5ocBIEkdZQBIUkcZAJLUUQaAJHWUASBJHWUASFJHGQCS1FEGgCR11HHz3cDRnHLKKbVixYr5bkOSnlPuvvvub1bV2HTjFnQArFixgh07dsx3G5L0nJLkG4OMcxeQJHWUASBJHWUASFJHGQCS1FEGgCR11MABkGRRkq8mubnNr0xyR5KJJJ9OckKrn9jmJ9ryFX3buLTVH0xyzqhfjCRpcMN8Angf8EDf/JXAVVX1E8ATwEWtfhHwRKtf1caRZDVwIfAaYB3wiSSLZta+JOlYDRQASZYBbwb+pM0HeCNwYxuyGTi/Ta9v87TlZ7Xx64EtVfVMVT0MTABrR/EiJEnDG/QTwH8Gfhv4fpt/BfDtqjrU5vcAS9v0UuBRgLb8yTb+B/VJ1pEkzbFpvwmc5J8B+6vq7iRvmO2GkmwENgKcdtppM9rWiks+O4qWJGnOPfLhN8/6cwzyCeB1wFuSPAJsobfr52PASUkOB8gyYG+b3gssB2jLXw58q78+yTo/UFXXVtV4VY2PjU17KQtJ0jGaNgCq6tKqWlZVK+gdxP1CVf0acBtwQRu2AbipTW9t87TlX6iqavUL21lCK4FVwJ0jeyWSpKHM5GJwvwNsSfL7wFeB61r9OuBTSSaAg/RCg6raneQG4H7gEHBxVT07g+eXJM3AUAFQVV8Evtimv84kZ/FU1feAX51i/SuAK4ZtUpI0en4TWJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOmraAEjywiR3Jrknye4kv9fqn0zycJKd7bGm1ZPk40kmkuxKcnrftjYkeag9Nkz1nJKk2TfILSGfAd5YVU8lOR74cpK/asv+fVXdeMT4c+nd8H0VcAZwDXBGkpOBy4BxoIC7k2ytqidG8UIkScOZ9hNA9TzVZo9vjzrKKuuB69t6twMnJVkCnANsr6qD7U1/O7BuZu1Lko7VQMcAkixKshPYT+9N/I626Iq2m+eqJCe22lLg0b7V97TaVHVJ0jwYKACq6tmqWgMsA9Ym+SngUuDVwM8CJwO/M4qGkmxMsiPJjgMHDoxik5KkSQx1FlBVfRu4DVhXVfvabp5ngP8KrG3D9gLL+1Zb1mpT1Y98jmuraryqxsfGxoZpT5I0hEHOAhpLclKbfhHwJuBv2n59kgQ4H7ivrbIVeEc7G+hM4Mmq2gfcApydZHGSxcDZrSZJmgeDnAW0BNicZBG9wLihqm5O8oUkY0CAncC72/htwHnABPA08C6AqjqY5HLgrjbuQ1V1cHQvRZI0jGkDoKp2Aa+dpP7GKcYXcPEUyzYBm4bsUZI0C/wmsCR1lAEgSR1lAEhSRxkAktRRBoAkdZQBIEkdZQBIUkcZAJLUUQaAJHWUASBJHWUASFJHGQCS1FEGgCR1lAEgSR1lAEhSRxkAktRRBoAkddQg9wR+YZI7k9yTZHeS32v1lUnuSDKR5NNJTmj1E9v8RFu+om9bl7b6g0nOma0XJUma3iCfAJ4B3lhVPwOsAda1m71fCVxVVT8BPAFc1MZfBDzR6le1cSRZDVwIvAZYB3yi3WdYkjQPpg2A6nmqzR7fHgW8Ebix1TcD57fp9W2etvysJGn1LVX1TFU9TO+m8WtH8iokSUMb6BhAkkVJdgL7ge3A14BvV9WhNmQPsLRNLwUeBWjLnwRe0V+fZJ3+59qYZEeSHQcOHBj+FUmSBjJQAFTVs1W1BlhG77f2V89WQ1V1bVWNV9X42NjYbD2NJHXeUGcBVdW3gduAnwNOSnJcW7QM2Num9wLLAdrylwPf6q9Pso4kaY4NchbQWJKT2vSLgDcBD9ALggvasA3ATW16a5unLf9CVVWrX9jOEloJrALuHNULkSQN57jph7AE2NzO2HkBcENV3ZzkfmBLkt8Hvgpc18ZfB3wqyQRwkN6ZP1TV7iQ3APcDh4CLq+rZ0b4cSdKgpg2AqtoFvHaS+teZ5Cyeqvoe8KtTbOsK4Irh25QkjZrfBJakjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4a5J7Ay5PcluT+JLuTvK/VfzfJ3iQ72+O8vnUuTTKR5MEk5/TV17XaRJJLZuclSZIGMcg9gQ8B76+qryR5KXB3ku1t2VVV9Z/6BydZTe8+wK8BXgl8PslPtsVX07up/B7griRbq+r+UbwQSdJwBrkn8D5gX5v+bpIHgKVHWWU9sKWqngEebjeHP3zv4Il2L2GSbGljDQBJmgdDHQNIsoLeDeLvaKX3JtmVZFOSxa22FHi0b7U9rTZV/cjn2JhkR5IdBw4cGKY9SdIQBg6AJC8B/gL4rar6DnAN8OPAGnqfED4yioaq6tqqGq+q8bGxsVFsUpI0iUGOAZDkeHpv/n9aVX8JUFWP9y3/Y+DmNrsXWN63+rJW4yh1SdIcG+QsoADXAQ9U1Uf76kv6hr0VuK9NbwUuTHJikpXAKuBO4C5gVZKVSU6gd6B462hehiRpWIN8Angd8BvAvUl2ttoHgLcnWQMU8AjwmwBVtTvJDfQO7h4CLq6qZwGSvBe4BVgEbKqq3SN8LZKkIQxyFtCXgUyyaNtR1rkCuGKS+rajrSdJmjt+E1iSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjpqkHsCL09yW5L7k+xO8r5WPznJ9iQPtZ+LWz1JPp5kIsmuJKf3bWtDG/9Qkg2z97IkSdMZ5BPAIeD9VbUaOBO4OMlq4BLg1qpaBdza5gHOpXcj+FXARuAa6AUGcBlwBrAWuOxwaEiS5t60AVBV+6rqK236u8ADwFJgPbC5DdsMnN+m1wPXV8/twElJlgDnANur6mBVPQFsB9aN9NVIkgY21DGAJCuA1wJ3AKdW1b626DHg1Da9FHi0b7U9rTZV/cjn2JhkR5IdBw4cGKY9SdIQBg6AJC8B/gL4rar6Tv+yqiqgRtFQVV1bVeNVNT42NjaKTUqSJjFQACQ5nt6b/59W1V+28uNt1w7t5/5W3wss71t9WatNVZckzYNBzgIKcB3wQFV9tG/RVuDwmTwbgJv66u9oZwOdCTzZdhXdApydZHE7+Ht2q0mS5sFxA4x5HfAbwL1JdrbaB4APAzckuQj4BvC2tmwbcB4wATwNvAugqg4muRy4q437UFUdHMmrkCQNbdoAqKovA5li8VmTjC/g4im2tQnYNEyDkqTZ4TeBJamjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4a5JaQm5LsT3JfX+13k+xNsrM9zutbdmmSiSQPJjmnr76u1SaSXDL6lyJJGsYgnwA+CaybpH5VVa1pj20ASVYDFwKvaet8IsmiJIuAq4FzgdXA29tYSdI8GeSWkF9KsmLA7a0HtlTVM8DDSSaAtW3ZRFV9HSDJljb2/qE7liSNxEyOAbw3ya62i2hxqy0FHu0bs6fVpqpLkubJsQbANcCPA2uAfcBHRtVQko1JdiTZceDAgVFtVpJ0hGMKgKp6vKqerarvA3/MD3fz7AWW9w1d1mpT1Sfb9rVVNV5V42NjY8fSniRpAMcUAEmW9M2+FTh8htBW4MIkJyZZCawC7gTuAlYlWZnkBHoHircee9uSpJma9iBwkj8H3gCckmQPcBnwhiRrgAIeAX4ToKp2J7mB3sHdQ8DFVfVs2857gVuARcCmqto98lcjSRrYIGcBvX2S8nVHGX8FcMUk9W3AtqG6kyTNGr8JLEkdZQBIUkcZAJLUUQaAJHWUASBJHWUASFJHGQCS1FEGgCR1lAEgSR1lAEhSRxkAktRRBoAkdZQBIEkdZQBIUkcZAJLUUQaAJHWUASBJHTVtACTZlGR/kvv6aicn2Z7kofZzcasnyceTTCTZleT0vnU2tPEPJdkwOy9HkjSoQT4BfBJYd0TtEuDWqloF3NrmAc6ldyP4VcBG4BroBQa9ewmfAawFLjscGpKk+TFtAFTVl4CDR5TXA5vb9Gbg/L769dVzO3BSkiXAOcD2qjpYVU8A2/mHoSJJmkPHegzg1Kra16YfA05t00uBR/vG7Wm1qeqSpHky44PAVVVAjaAXAJJsTLIjyY4DBw6MarOSpCMcawA83nbt0H7ub/W9wPK+cctabar6P1BV11bVeFWNj42NHWN7kqTpHGsAbAUOn8mzAbipr/6OdjbQmcCTbVfRLcDZSRa3g79nt5okaZ4cN92AJH8OvAE4JckeemfzfBi4IclFwDeAt7Xh24DzgAngaeBdAFV1MMnlwF1t3Ieq6sgDy5KkOTRtAFTV26dYdNYkYwu4eIrtbAI2DdWdJGnW+E1gSeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqqBkFQJJHktybZGeSHa12cpLtSR5qPxe3epJ8PMlEkl1JTh/FC5AkHZtRfAL4papaU1Xjbf4S4NaqWgXc2uYBzgVWtcdG4JoRPLck6RjNxi6g9cDmNr0ZOL+vfn313A6clGTJLDy/JGkAMw2AAv46yd1JNrbaqVW1r00/BpzappcCj/atu6fVJEnz4LgZrv/6qtqb5EeA7Un+pn9hVVWSGmaDLUg2Apx22mkzbE+SNJUZfQKoqr3t537gM8Ba4PHDu3baz/1t+F5ged/qy1rtyG1eW1XjVTU+NjY2k/YkSUdxzAGQ5MVJXnp4GjgbuA/YCmxowzYAN7XprcA72tlAZwJP9u0qkiTNsZnsAjoV+EySw9v5s6r6XJK7gBuSXAR8A3hbG78NOA+YAJ4G3jWD55YkzdAxB0BVfR34mUnq3wLOmqRewMXH+nySpNHym8CS1FEGgCR1lAEgSR1lAEhSRxkAktRRBoAkdZQBIEkdZQBIUkcZAJLUUQaAJHWUASBJHWUASFJHGQCS1FEGgCR1lAEgSR1lAEhSRxkAktRRcx4ASdYleTDJRJJL5vr5JUk9cxoASRYBVwPnAquBtydZPZc9SJJ65voTwFpgoqq+XlV/D2wB1s9xD5Ik5j4AlgKP9s3vaTVJ0hw7br4bOFKSjcDGNvtUkgdnsLlTgG/OvKuRs6/h2Ndw7Gs4C7KvXDmjvl41yKC5DoC9wPK++WWt9gNVdS1w7SieLMmOqhofxbZGyb6GY1/Dsa/hdLmvud4FdBewKsnKJCcAFwJb57gHSRJz/Amgqg4leS9wC7AI2FRVu+eyB0lSz5wfA6iqbcC2OXq6kexKmgX2NRz7Go59DaezfaWqZvs5JEkLkJeCkKSOel4FQJKTk2xP8lD7ufgoY1+WZE+SP1wIfSV5VZKvJNmZZHeSdy+QvtYk+Z+tp11J/sVC6KuN+1ySbye5eZb7OerlS5KcmOTTbfkdSVbMZj9D9PUL7d/UoSQXzEVPA/b1b5Pc3/493ZpkoFMW56Cvdye5t/0f/PJcXaVg0MvjJPmVJJVkdGcGVdXz5gH8AXBJm74EuPIoYz8G/BnwhwuhL+AE4MQ2/RLgEeCVC6CvnwRWtelXAvuAk+a7r7bsLOCfAzfPYi+LgK8BP9b+ju4BVh8x5j3Af2nTFwKfnoN/U4P0tQL4aeB64ILZ7mmIvn4J+Edt+l8toD+vl/VNvwX43ELoq417KfAl4HZgfFTP/7z6BEDvshKb2/Rm4PzJBiX5p8CpwF8vlL6q6u+r6pk2eyJz8+lskL7+tqoeatN/B+wHxua7r9bPrcB3Z7mXQS5f0t/vjcBZSTLffVXVI1W1C/j+LPcybF+3VdXTbfZ2et8HWgh9fadv9sXAXBwgHfTyOJcDVwLfG+WTP98C4NSq2temH6P3Jv//SfIC4CPAv1tIfQEkWZ5kF73LZVzZ3nDnva++/tbS+y3lawupr1k2yOVLfjCmqg4BTwKvWAB9zYdh+7oI+KtZ7ahnoL6SXJzka/Q+hf6bhdBXktOB5VX12VE/+YK7FMR0knwe+NFJFn2wf6aqKslkCf4eYFtV7RnlL2kj6IuqehT46SSvBP57khur6vH57qttZwnwKWBDVc34N8pR9aXnriS/DowDvzjfvRxWVVcDVyf5l8B/ADbMZz/tF9aPAu+cje0/5wKgqn55qmVJHk+ypKr2tTes/ZMM+zng55O8h96+9hOSPFVVM7o3wQj66t/W3yW5D/h5ersU5rWvJC8DPgt8sKpun0k/o+xrjkx7+ZK+MXuSHAe8HPjWAuhrPgzUV5Jfphf2v9i363Pe++qzBbhmVjvqma6vlwI/BXyx/cL6o8DWJG+pqh0zffLn2y6grfwwsTcANx05oKp+rapOq6oV9HYDXT/TN/9R9JVkWZIXtenFwOuBmVwIb1R9nQB8ht6f04zCaJR9zaFBLl/S3+8FwBeqHbmb577mw7R9JXkt8EfAW6pqrsJ9kL5W9c2+GXhovvuqqier6pSqWtHes26n9+c24zf/w0/wvHnQ2+96K72/uM8DJ7f6OPAnk4x/J3NzFtC0fQFvAnbROwtgF7BxgfT168D/BXb2PdbMd19t/n8AB4D/Q2/f6Tmz1M95wN/SO/bxwVb7EL3/iAAvBP4bMAHcCfzYbP/dDdjXz7Y/l/9N7xPJ7gXS1+eBx/v+PW1dIH19DNjderoNeM1C6OuIsV9khGcB+U1gSeqo59suIEnSgAwASeooA0CSOsoAkKSOMgAkaQFJ8v520bdTphm3tl24bmeSe5K8ddjnMgAkaY4leUOST05SXw6cDfyvATZzH71TQtcA64A/al9EHJgBIEkLx1XAb9N3IbokL06yKcmdSb6aZD1AVT1dvWtPQe+7KEOf028ASNIC0N7Y91bVPUcs+iC9b5evpXcp7f+Y5MVtnTOS7AbuBd7dFwiDPadfBJOkuZHkDnqXe38JcDI/3NVzGfAB4OyqejLJI/R273wzyQ56v+EffnM/md633h/o2+4/pndJ8l+oqoEvGf2cuxicJD1XVdUZ0DsGALyzqt7Z5v8JsBK4p130bRnwlXYJ9gC/UlVTXhusqh5I8hS9C8cNfJ0gdwFJ0jyrqnur6kfqhxd92wOcXlWPAbcA//rwTYbaxfRoF5A7rk2/Cng1vTsJDswAkKSF7XLgeGBX299/eau/nt4nhp30rtj7nqr65jAb9hiAJHWUnwAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI76f5AaBwteePPdAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4000 4000 4000 4000 4000 4000 4000 4000 4000 4000]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEJCAYAAACdePCvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAFkNJREFUeJzt3X/wXXV95/Hny/BD118E+ZbGJJi0TceN3Tay3wY62tZKhYC7BqfUxW1rdJhJXXHXzrrbgu4MrZQZ6a6yOkW2tGQNTtvI0rpkMJVGxHGdWX4EDYFAKV8Fl6SBRIMoy0o3+N4/7id6N/1+87033/v9Aef5mLnzPed9Pufc982P+/rec849J1WFJKl7XjDfDUiS5ocBIEkdZQBIUkcZAJLUUQaAJHWUASBJHWUASFJHGQCS1FEGgCR11HHz3cDRnHLKKbVixYr5bkOSnlPuvvvub1bV2HTjFnQArFixgh07dsx3G5L0nJLkG4OMcxeQJHWUASBJHWUASFJHGQCS1FEGgCR11MABkGRRkq8mubnNr0xyR5KJJJ9OckKrn9jmJ9ryFX3buLTVH0xyzqhfjCRpcMN8Angf8EDf/JXAVVX1E8ATwEWtfhHwRKtf1caRZDVwIfAaYB3wiSSLZta+JOlYDRQASZYBbwb+pM0HeCNwYxuyGTi/Ta9v87TlZ7Xx64EtVfVMVT0MTABrR/EiJEnDG/QTwH8Gfhv4fpt/BfDtqjrU5vcAS9v0UuBRgLb8yTb+B/VJ1pEkzbFpvwmc5J8B+6vq7iRvmO2GkmwENgKcdtppM9rWiks+O4qWJGnOPfLhN8/6cwzyCeB1wFuSPAJsobfr52PASUkOB8gyYG+b3gssB2jLXw58q78+yTo/UFXXVtV4VY2PjU17KQtJ0jGaNgCq6tKqWlZVK+gdxP1CVf0acBtwQRu2AbipTW9t87TlX6iqavUL21lCK4FVwJ0jeyWSpKHM5GJwvwNsSfL7wFeB61r9OuBTSSaAg/RCg6raneQG4H7gEHBxVT07g+eXJM3AUAFQVV8Evtimv84kZ/FU1feAX51i/SuAK4ZtUpI0en4TWJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOmraAEjywiR3Jrknye4kv9fqn0zycJKd7bGm1ZPk40kmkuxKcnrftjYkeag9Nkz1nJKk2TfILSGfAd5YVU8lOR74cpK/asv+fVXdeMT4c+nd8H0VcAZwDXBGkpOBy4BxoIC7k2ytqidG8UIkScOZ9hNA9TzVZo9vjzrKKuuB69t6twMnJVkCnANsr6qD7U1/O7BuZu1Lko7VQMcAkixKshPYT+9N/I626Iq2m+eqJCe22lLg0b7V97TaVHVJ0jwYKACq6tmqWgMsA9Ym+SngUuDVwM8CJwO/M4qGkmxMsiPJjgMHDoxik5KkSQx1FlBVfRu4DVhXVfvabp5ngP8KrG3D9gLL+1Zb1mpT1Y98jmuraryqxsfGxoZpT5I0hEHOAhpLclKbfhHwJuBv2n59kgQ4H7ivrbIVeEc7G+hM4Mmq2gfcApydZHGSxcDZrSZJmgeDnAW0BNicZBG9wLihqm5O8oUkY0CAncC72/htwHnABPA08C6AqjqY5HLgrjbuQ1V1cHQvRZI0jGkDoKp2Aa+dpP7GKcYXcPEUyzYBm4bsUZI0C/wmsCR1lAEgSR1lAEhSRxkAktRRBoAkdZQBIEkdZQBIUkcZAJLUUQaAJHWUASBJHWUASFJHGQCS1FEGgCR1lAEgSR1lAEhSRxkAktRRBoAkddQg9wR+YZI7k9yTZHeS32v1lUnuSDKR5NNJTmj1E9v8RFu+om9bl7b6g0nOma0XJUma3iCfAJ4B3lhVPwOsAda1m71fCVxVVT8BPAFc1MZfBDzR6le1cSRZDVwIvAZYB3yi3WdYkjQPpg2A6nmqzR7fHgW8Ebix1TcD57fp9W2etvysJGn1LVX1TFU9TO+m8WtH8iokSUMb6BhAkkVJdgL7ge3A14BvV9WhNmQPsLRNLwUeBWjLnwRe0V+fZJ3+59qYZEeSHQcOHBj+FUmSBjJQAFTVs1W1BlhG77f2V89WQ1V1bVWNV9X42NjYbD2NJHXeUGcBVdW3gduAnwNOSnJcW7QM2Num9wLLAdrylwPf6q9Pso4kaY4NchbQWJKT2vSLgDcBD9ALggvasA3ATW16a5unLf9CVVWrX9jOEloJrALuHNULkSQN57jph7AE2NzO2HkBcENV3ZzkfmBLkt8Hvgpc18ZfB3wqyQRwkN6ZP1TV7iQ3APcDh4CLq+rZ0b4cSdKgpg2AqtoFvHaS+teZ5Cyeqvoe8KtTbOsK4Irh25QkjZrfBJakjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4a5J7Ay5PcluT+JLuTvK/VfzfJ3iQ72+O8vnUuTTKR5MEk5/TV17XaRJJLZuclSZIGMcg9gQ8B76+qryR5KXB3ku1t2VVV9Z/6BydZTe8+wK8BXgl8PslPtsVX07up/B7griRbq+r+UbwQSdJwBrkn8D5gX5v+bpIHgKVHWWU9sKWqngEebjeHP3zv4Il2L2GSbGljDQBJmgdDHQNIsoLeDeLvaKX3JtmVZFOSxa22FHi0b7U9rTZV/cjn2JhkR5IdBw4cGKY9SdIQBg6AJC8B/gL4rar6DnAN8OPAGnqfED4yioaq6tqqGq+q8bGxsVFsUpI0iUGOAZDkeHpv/n9aVX8JUFWP9y3/Y+DmNrsXWN63+rJW4yh1SdIcG+QsoADXAQ9U1Uf76kv6hr0VuK9NbwUuTHJikpXAKuBO4C5gVZKVSU6gd6B462hehiRpWIN8Angd8BvAvUl2ttoHgLcnWQMU8AjwmwBVtTvJDfQO7h4CLq6qZwGSvBe4BVgEbKqq3SN8LZKkIQxyFtCXgUyyaNtR1rkCuGKS+rajrSdJmjt+E1iSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjpqkHsCL09yW5L7k+xO8r5WPznJ9iQPtZ+LWz1JPp5kIsmuJKf3bWtDG/9Qkg2z97IkSdMZ5BPAIeD9VbUaOBO4OMlq4BLg1qpaBdza5gHOpXcj+FXARuAa6AUGcBlwBrAWuOxwaEiS5t60AVBV+6rqK236u8ADwFJgPbC5DdsMnN+m1wPXV8/twElJlgDnANur6mBVPQFsB9aN9NVIkgY21DGAJCuA1wJ3AKdW1b626DHg1Da9FHi0b7U9rTZV/cjn2JhkR5IdBw4cGKY9SdIQBg6AJC8B/gL4rar6Tv+yqiqgRtFQVV1bVeNVNT42NjaKTUqSJjFQACQ5nt6b/59W1V+28uNt1w7t5/5W3wss71t9WatNVZckzYNBzgIKcB3wQFV9tG/RVuDwmTwbgJv66u9oZwOdCTzZdhXdApydZHE7+Ht2q0mS5sFxA4x5HfAbwL1JdrbaB4APAzckuQj4BvC2tmwbcB4wATwNvAugqg4muRy4q437UFUdHMmrkCQNbdoAqKovA5li8VmTjC/g4im2tQnYNEyDkqTZ4TeBJamjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4a5JaQm5LsT3JfX+13k+xNsrM9zutbdmmSiSQPJjmnr76u1SaSXDL6lyJJGsYgnwA+CaybpH5VVa1pj20ASVYDFwKvaet8IsmiJIuAq4FzgdXA29tYSdI8GeSWkF9KsmLA7a0HtlTVM8DDSSaAtW3ZRFV9HSDJljb2/qE7liSNxEyOAbw3ya62i2hxqy0FHu0bs6fVpqpLkubJsQbANcCPA2uAfcBHRtVQko1JdiTZceDAgVFtVpJ0hGMKgKp6vKqerarvA3/MD3fz7AWW9w1d1mpT1Sfb9rVVNV5V42NjY8fSniRpAMcUAEmW9M2+FTh8htBW4MIkJyZZCawC7gTuAlYlWZnkBHoHircee9uSpJma9iBwkj8H3gCckmQPcBnwhiRrgAIeAX4ToKp2J7mB3sHdQ8DFVfVs2857gVuARcCmqto98lcjSRrYIGcBvX2S8nVHGX8FcMUk9W3AtqG6kyTNGr8JLEkdZQBIUkcZAJLUUQaAJHWUASBJHWUASFJHGQCS1FEGgCR1lAEgSR1lAEhSRxkAktRRBoAkdZQBIEkdZQBIUkcZAJLUUQaAJHWUASBJHTVtACTZlGR/kvv6aicn2Z7kofZzcasnyceTTCTZleT0vnU2tPEPJdkwOy9HkjSoQT4BfBJYd0TtEuDWqloF3NrmAc6ldyP4VcBG4BroBQa9ewmfAawFLjscGpKk+TFtAFTVl4CDR5TXA5vb9Gbg/L769dVzO3BSkiXAOcD2qjpYVU8A2/mHoSJJmkPHegzg1Kra16YfA05t00uBR/vG7Wm1qeqSpHky44PAVVVAjaAXAJJsTLIjyY4DBw6MarOSpCMcawA83nbt0H7ub/W9wPK+cctabar6P1BV11bVeFWNj42NHWN7kqTpHGsAbAUOn8mzAbipr/6OdjbQmcCTbVfRLcDZSRa3g79nt5okaZ4cN92AJH8OvAE4JckeemfzfBi4IclFwDeAt7Xh24DzgAngaeBdAFV1MMnlwF1t3Ieq6sgDy5KkOTRtAFTV26dYdNYkYwu4eIrtbAI2DdWdJGnW+E1gSeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqqBkFQJJHktybZGeSHa12cpLtSR5qPxe3epJ8PMlEkl1JTh/FC5AkHZtRfAL4papaU1Xjbf4S4NaqWgXc2uYBzgVWtcdG4JoRPLck6RjNxi6g9cDmNr0ZOL+vfn313A6clGTJLDy/JGkAMw2AAv46yd1JNrbaqVW1r00/BpzappcCj/atu6fVJEnz4LgZrv/6qtqb5EeA7Un+pn9hVVWSGmaDLUg2Apx22mkzbE+SNJUZfQKoqr3t537gM8Ba4PHDu3baz/1t+F5ged/qy1rtyG1eW1XjVTU+NjY2k/YkSUdxzAGQ5MVJXnp4GjgbuA/YCmxowzYAN7XprcA72tlAZwJP9u0qkiTNsZnsAjoV+EySw9v5s6r6XJK7gBuSXAR8A3hbG78NOA+YAJ4G3jWD55YkzdAxB0BVfR34mUnq3wLOmqRewMXH+nySpNHym8CS1FEGgCR1lAEgSR1lAEhSRxkAktRRBoAkdZQBIEkdZQBIUkcZAJLUUQaAJHWUASBJHWUASFJHGQCS1FEGgCR1lAEgSR1lAEhSRxkAktRRcx4ASdYleTDJRJJL5vr5JUk9cxoASRYBVwPnAquBtydZPZc9SJJ65voTwFpgoqq+XlV/D2wB1s9xD5Ik5j4AlgKP9s3vaTVJ0hw7br4bOFKSjcDGNvtUkgdnsLlTgG/OvKuRs6/h2Ndw7Gs4C7KvXDmjvl41yKC5DoC9wPK++WWt9gNVdS1w7SieLMmOqhofxbZGyb6GY1/Dsa/hdLmvud4FdBewKsnKJCcAFwJb57gHSRJz/Amgqg4leS9wC7AI2FRVu+eyB0lSz5wfA6iqbcC2OXq6kexKmgX2NRz7Go59DaezfaWqZvs5JEkLkJeCkKSOel4FQJKTk2xP8lD7ufgoY1+WZE+SP1wIfSV5VZKvJNmZZHeSdy+QvtYk+Z+tp11J/sVC6KuN+1ySbye5eZb7OerlS5KcmOTTbfkdSVbMZj9D9PUL7d/UoSQXzEVPA/b1b5Pc3/493ZpkoFMW56Cvdye5t/0f/PJcXaVg0MvjJPmVJJVkdGcGVdXz5gH8AXBJm74EuPIoYz8G/BnwhwuhL+AE4MQ2/RLgEeCVC6CvnwRWtelXAvuAk+a7r7bsLOCfAzfPYi+LgK8BP9b+ju4BVh8x5j3Af2nTFwKfnoN/U4P0tQL4aeB64ILZ7mmIvn4J+Edt+l8toD+vl/VNvwX43ELoq417KfAl4HZgfFTP/7z6BEDvshKb2/Rm4PzJBiX5p8CpwF8vlL6q6u+r6pk2eyJz8+lskL7+tqoeatN/B+wHxua7r9bPrcB3Z7mXQS5f0t/vjcBZSTLffVXVI1W1C/j+LPcybF+3VdXTbfZ2et8HWgh9fadv9sXAXBwgHfTyOJcDVwLfG+WTP98C4NSq2temH6P3Jv//SfIC4CPAv1tIfQEkWZ5kF73LZVzZ3nDnva++/tbS+y3lawupr1k2yOVLfjCmqg4BTwKvWAB9zYdh+7oI+KtZ7ahnoL6SXJzka/Q+hf6bhdBXktOB5VX12VE/+YK7FMR0knwe+NFJFn2wf6aqKslkCf4eYFtV7RnlL2kj6IuqehT46SSvBP57khur6vH57qttZwnwKWBDVc34N8pR9aXnriS/DowDvzjfvRxWVVcDVyf5l8B/ADbMZz/tF9aPAu+cje0/5wKgqn55qmVJHk+ypKr2tTes/ZMM+zng55O8h96+9hOSPFVVM7o3wQj66t/W3yW5D/h5ersU5rWvJC8DPgt8sKpun0k/o+xrjkx7+ZK+MXuSHAe8HPjWAuhrPgzUV5Jfphf2v9i363Pe++qzBbhmVjvqma6vlwI/BXyx/cL6o8DWJG+pqh0zffLn2y6grfwwsTcANx05oKp+rapOq6oV9HYDXT/TN/9R9JVkWZIXtenFwOuBmVwIb1R9nQB8ht6f04zCaJR9zaFBLl/S3+8FwBeqHbmb577mw7R9JXkt8EfAW6pqrsJ9kL5W9c2+GXhovvuqqier6pSqWtHes26n9+c24zf/w0/wvHnQ2+96K72/uM8DJ7f6OPAnk4x/J3NzFtC0fQFvAnbROwtgF7BxgfT168D/BXb2PdbMd19t/n8AB4D/Q2/f6Tmz1M95wN/SO/bxwVb7EL3/iAAvBP4bMAHcCfzYbP/dDdjXz7Y/l/9N7xPJ7gXS1+eBx/v+PW1dIH19DNjderoNeM1C6OuIsV9khGcB+U1gSeqo59suIEnSgAwASeooA0CSOsoAkKSOMgAkaQFJ8v520bdTphm3tl24bmeSe5K8ddjnMgAkaY4leUOST05SXw6cDfyvATZzH71TQtcA64A/al9EHJgBIEkLx1XAb9N3IbokL06yKcmdSb6aZD1AVT1dvWtPQe+7KEOf028ASNIC0N7Y91bVPUcs+iC9b5evpXcp7f+Y5MVtnTOS7AbuBd7dFwiDPadfBJOkuZHkDnqXe38JcDI/3NVzGfAB4OyqejLJI/R273wzyQ56v+EffnM/md633h/o2+4/pndJ8l+oqoEvGf2cuxicJD1XVdUZ0DsGALyzqt7Z5v8JsBK4p130bRnwlXYJ9gC/UlVTXhusqh5I8hS9C8cNfJ0gdwFJ0jyrqnur6kfqhxd92wOcXlWPAbcA//rwTYbaxfRoF5A7rk2/Cng1vTsJDswAkKSF7XLgeGBX299/eau/nt4nhp30rtj7nqr65jAb9hiAJHWUnwAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI76f5AaBwteePPdAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4000 4000 4000 4000 4000 4000 4000 4000 4000 4000]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEJCAYAAACdePCvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAFkNJREFUeJzt3X/wXXV95/Hny/BD118E+ZbGJJi0TceN3Tay3wY62tZKhYC7BqfUxW1rdJhJXXHXzrrbgu4MrZQZ6a6yOkW2tGQNTtvI0rpkMJVGxHGdWX4EDYFAKV8Fl6SBRIMoy0o3+N4/7id6N/1+87033/v9Aef5mLnzPed9Pufc982P+/rec849J1WFJKl7XjDfDUiS5ocBIEkdZQBIUkcZAJLUUQaAJHWUASBJHWUASFJHGQCS1FEGgCR11HHz3cDRnHLKKbVixYr5bkOSnlPuvvvub1bV2HTjFnQArFixgh07dsx3G5L0nJLkG4OMcxeQJHWUASBJHWUASFJHGQCS1FEGgCR11MABkGRRkq8mubnNr0xyR5KJJJ9OckKrn9jmJ9ryFX3buLTVH0xyzqhfjCRpcMN8Angf8EDf/JXAVVX1E8ATwEWtfhHwRKtf1caRZDVwIfAaYB3wiSSLZta+JOlYDRQASZYBbwb+pM0HeCNwYxuyGTi/Ta9v87TlZ7Xx64EtVfVMVT0MTABrR/EiJEnDG/QTwH8Gfhv4fpt/BfDtqjrU5vcAS9v0UuBRgLb8yTb+B/VJ1pEkzbFpvwmc5J8B+6vq7iRvmO2GkmwENgKcdtppM9rWiks+O4qWJGnOPfLhN8/6cwzyCeB1wFuSPAJsobfr52PASUkOB8gyYG+b3gssB2jLXw58q78+yTo/UFXXVtV4VY2PjU17KQtJ0jGaNgCq6tKqWlZVK+gdxP1CVf0acBtwQRu2AbipTW9t87TlX6iqavUL21lCK4FVwJ0jeyWSpKHM5GJwvwNsSfL7wFeB61r9OuBTSSaAg/RCg6raneQG4H7gEHBxVT07g+eXJM3AUAFQVV8Evtimv84kZ/FU1feAX51i/SuAK4ZtUpI0en4TWJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOmraAEjywiR3Jrknye4kv9fqn0zycJKd7bGm1ZPk40kmkuxKcnrftjYkeag9Nkz1nJKk2TfILSGfAd5YVU8lOR74cpK/asv+fVXdeMT4c+nd8H0VcAZwDXBGkpOBy4BxoIC7k2ytqidG8UIkScOZ9hNA9TzVZo9vjzrKKuuB69t6twMnJVkCnANsr6qD7U1/O7BuZu1Lko7VQMcAkixKshPYT+9N/I626Iq2m+eqJCe22lLg0b7V97TaVHVJ0jwYKACq6tmqWgMsA9Ym+SngUuDVwM8CJwO/M4qGkmxMsiPJjgMHDoxik5KkSQx1FlBVfRu4DVhXVfvabp5ngP8KrG3D9gLL+1Zb1mpT1Y98jmuraryqxsfGxoZpT5I0hEHOAhpLclKbfhHwJuBv2n59kgQ4H7ivrbIVeEc7G+hM4Mmq2gfcApydZHGSxcDZrSZJmgeDnAW0BNicZBG9wLihqm5O8oUkY0CAncC72/htwHnABPA08C6AqjqY5HLgrjbuQ1V1cHQvRZI0jGkDoKp2Aa+dpP7GKcYXcPEUyzYBm4bsUZI0C/wmsCR1lAEgSR1lAEhSRxkAktRRBoAkdZQBIEkdZQBIUkcZAJLUUQaAJHWUASBJHWUASFJHGQCS1FEGgCR1lAEgSR1lAEhSRxkAktRRBoAkddQg9wR+YZI7k9yTZHeS32v1lUnuSDKR5NNJTmj1E9v8RFu+om9bl7b6g0nOma0XJUma3iCfAJ4B3lhVPwOsAda1m71fCVxVVT8BPAFc1MZfBDzR6le1cSRZDVwIvAZYB3yi3WdYkjQPpg2A6nmqzR7fHgW8Ebix1TcD57fp9W2etvysJGn1LVX1TFU9TO+m8WtH8iokSUMb6BhAkkVJdgL7ge3A14BvV9WhNmQPsLRNLwUeBWjLnwRe0V+fZJ3+59qYZEeSHQcOHBj+FUmSBjJQAFTVs1W1BlhG77f2V89WQ1V1bVWNV9X42NjYbD2NJHXeUGcBVdW3gduAnwNOSnJcW7QM2Num9wLLAdrylwPf6q9Pso4kaY4NchbQWJKT2vSLgDcBD9ALggvasA3ATW16a5unLf9CVVWrX9jOEloJrALuHNULkSQN57jph7AE2NzO2HkBcENV3ZzkfmBLkt8Hvgpc18ZfB3wqyQRwkN6ZP1TV7iQ3APcDh4CLq+rZ0b4cSdKgpg2AqtoFvHaS+teZ5Cyeqvoe8KtTbOsK4Irh25QkjZrfBJakjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4a5J7Ay5PcluT+JLuTvK/VfzfJ3iQ72+O8vnUuTTKR5MEk5/TV17XaRJJLZuclSZIGMcg9gQ8B76+qryR5KXB3ku1t2VVV9Z/6BydZTe8+wK8BXgl8PslPtsVX07up/B7griRbq+r+UbwQSdJwBrkn8D5gX5v+bpIHgKVHWWU9sKWqngEebjeHP3zv4Il2L2GSbGljDQBJmgdDHQNIsoLeDeLvaKX3JtmVZFOSxa22FHi0b7U9rTZV/cjn2JhkR5IdBw4cGKY9SdIQBg6AJC8B/gL4rar6DnAN8OPAGnqfED4yioaq6tqqGq+q8bGxsVFsUpI0iUGOAZDkeHpv/n9aVX8JUFWP9y3/Y+DmNrsXWN63+rJW4yh1SdIcG+QsoADXAQ9U1Uf76kv6hr0VuK9NbwUuTHJikpXAKuBO4C5gVZKVSU6gd6B462hehiRpWIN8Angd8BvAvUl2ttoHgLcnWQMU8AjwmwBVtTvJDfQO7h4CLq6qZwGSvBe4BVgEbKqq3SN8LZKkIQxyFtCXgUyyaNtR1rkCuGKS+rajrSdJmjt+E1iSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjpqkHsCL09yW5L7k+xO8r5WPznJ9iQPtZ+LWz1JPp5kIsmuJKf3bWtDG/9Qkg2z97IkSdMZ5BPAIeD9VbUaOBO4OMlq4BLg1qpaBdza5gHOpXcj+FXARuAa6AUGcBlwBrAWuOxwaEiS5t60AVBV+6rqK236u8ADwFJgPbC5DdsMnN+m1wPXV8/twElJlgDnANur6mBVPQFsB9aN9NVIkgY21DGAJCuA1wJ3AKdW1b626DHg1Da9FHi0b7U9rTZV/cjn2JhkR5IdBw4cGKY9SdIQBg6AJC8B/gL4rar6Tv+yqiqgRtFQVV1bVeNVNT42NjaKTUqSJjFQACQ5nt6b/59W1V+28uNt1w7t5/5W3wss71t9WatNVZckzYNBzgIKcB3wQFV9tG/RVuDwmTwbgJv66u9oZwOdCTzZdhXdApydZHE7+Ht2q0mS5sFxA4x5HfAbwL1JdrbaB4APAzckuQj4BvC2tmwbcB4wATwNvAugqg4muRy4q437UFUdHMmrkCQNbdoAqKovA5li8VmTjC/g4im2tQnYNEyDkqTZ4TeBJamjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4a5JaQm5LsT3JfX+13k+xNsrM9zutbdmmSiSQPJjmnr76u1SaSXDL6lyJJGsYgnwA+CaybpH5VVa1pj20ASVYDFwKvaet8IsmiJIuAq4FzgdXA29tYSdI8GeSWkF9KsmLA7a0HtlTVM8DDSSaAtW3ZRFV9HSDJljb2/qE7liSNxEyOAbw3ya62i2hxqy0FHu0bs6fVpqpLkubJsQbANcCPA2uAfcBHRtVQko1JdiTZceDAgVFtVpJ0hGMKgKp6vKqerarvA3/MD3fz7AWW9w1d1mpT1Sfb9rVVNV5V42NjY8fSniRpAMcUAEmW9M2+FTh8htBW4MIkJyZZCawC7gTuAlYlWZnkBHoHircee9uSpJma9iBwkj8H3gCckmQPcBnwhiRrgAIeAX4ToKp2J7mB3sHdQ8DFVfVs2857gVuARcCmqto98lcjSRrYIGcBvX2S8nVHGX8FcMUk9W3AtqG6kyTNGr8JLEkdZQBIUkcZAJLUUQaAJHWUASBJHWUASFJHGQCS1FEGgCR1lAEgSR1lAEhSRxkAktRRBoAkdZQBIEkdZQBIUkcZAJLUUQaAJHWUASBJHTVtACTZlGR/kvv6aicn2Z7kofZzcasnyceTTCTZleT0vnU2tPEPJdkwOy9HkjSoQT4BfBJYd0TtEuDWqloF3NrmAc6ldyP4VcBG4BroBQa9ewmfAawFLjscGpKk+TFtAFTVl4CDR5TXA5vb9Gbg/L769dVzO3BSkiXAOcD2qjpYVU8A2/mHoSJJmkPHegzg1Kra16YfA05t00uBR/vG7Wm1qeqSpHky44PAVVVAjaAXAJJsTLIjyY4DBw6MarOSpCMcawA83nbt0H7ub/W9wPK+cctabar6P1BV11bVeFWNj42NHWN7kqTpHGsAbAUOn8mzAbipr/6OdjbQmcCTbVfRLcDZSRa3g79nt5okaZ4cN92AJH8OvAE4JckeemfzfBi4IclFwDeAt7Xh24DzgAngaeBdAFV1MMnlwF1t3Ieq6sgDy5KkOTRtAFTV26dYdNYkYwu4eIrtbAI2DdWdJGnW+E1gSeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqqBkFQJJHktybZGeSHa12cpLtSR5qPxe3epJ8PMlEkl1JTh/FC5AkHZtRfAL4papaU1Xjbf4S4NaqWgXc2uYBzgVWtcdG4JoRPLck6RjNxi6g9cDmNr0ZOL+vfn313A6clGTJLDy/JGkAMw2AAv46yd1JNrbaqVW1r00/BpzappcCj/atu6fVJEnz4LgZrv/6qtqb5EeA7Un+pn9hVVWSGmaDLUg2Apx22mkzbE+SNJUZfQKoqr3t537gM8Ba4PHDu3baz/1t+F5ged/qy1rtyG1eW1XjVTU+NjY2k/YkSUdxzAGQ5MVJXnp4GjgbuA/YCmxowzYAN7XprcA72tlAZwJP9u0qkiTNsZnsAjoV+EySw9v5s6r6XJK7gBuSXAR8A3hbG78NOA+YAJ4G3jWD55YkzdAxB0BVfR34mUnq3wLOmqRewMXH+nySpNHym8CS1FEGgCR1lAEgSR1lAEhSRxkAktRRBoAkdZQBIEkdZQBIUkcZAJLUUQaAJHWUASBJHWUASFJHGQCS1FEGgCR1lAEgSR1lAEhSRxkAktRRcx4ASdYleTDJRJJL5vr5JUk9cxoASRYBVwPnAquBtydZPZc9SJJ65voTwFpgoqq+XlV/D2wB1s9xD5Ik5j4AlgKP9s3vaTVJ0hw7br4bOFKSjcDGNvtUkgdnsLlTgG/OvKuRs6/h2Ndw7Gs4C7KvXDmjvl41yKC5DoC9wPK++WWt9gNVdS1w7SieLMmOqhofxbZGyb6GY1/Dsa/hdLmvud4FdBewKsnKJCcAFwJb57gHSRJz/Amgqg4leS9wC7AI2FRVu+eyB0lSz5wfA6iqbcC2OXq6kexKmgX2NRz7Go59DaezfaWqZvs5JEkLkJeCkKSOel4FQJKTk2xP8lD7ufgoY1+WZE+SP1wIfSV5VZKvJNmZZHeSdy+QvtYk+Z+tp11J/sVC6KuN+1ySbye5eZb7OerlS5KcmOTTbfkdSVbMZj9D9PUL7d/UoSQXzEVPA/b1b5Pc3/493ZpkoFMW56Cvdye5t/0f/PJcXaVg0MvjJPmVJJVkdGcGVdXz5gH8AXBJm74EuPIoYz8G/BnwhwuhL+AE4MQ2/RLgEeCVC6CvnwRWtelXAvuAk+a7r7bsLOCfAzfPYi+LgK8BP9b+ju4BVh8x5j3Af2nTFwKfnoN/U4P0tQL4aeB64ILZ7mmIvn4J+Edt+l8toD+vl/VNvwX43ELoq417KfAl4HZgfFTP/7z6BEDvshKb2/Rm4PzJBiX5p8CpwF8vlL6q6u+r6pk2eyJz8+lskL7+tqoeatN/B+wHxua7r9bPrcB3Z7mXQS5f0t/vjcBZSTLffVXVI1W1C/j+LPcybF+3VdXTbfZ2et8HWgh9fadv9sXAXBwgHfTyOJcDVwLfG+WTP98C4NSq2temH6P3Jv//SfIC4CPAv1tIfQEkWZ5kF73LZVzZ3nDnva++/tbS+y3lawupr1k2yOVLfjCmqg4BTwKvWAB9zYdh+7oI+KtZ7ahnoL6SXJzka/Q+hf6bhdBXktOB5VX12VE/+YK7FMR0knwe+NFJFn2wf6aqKslkCf4eYFtV7RnlL2kj6IuqehT46SSvBP57khur6vH57qttZwnwKWBDVc34N8pR9aXnriS/DowDvzjfvRxWVVcDVyf5l8B/ADbMZz/tF9aPAu+cje0/5wKgqn55qmVJHk+ypKr2tTes/ZMM+zng55O8h96+9hOSPFVVM7o3wQj66t/W3yW5D/h5ersU5rWvJC8DPgt8sKpun0k/o+xrjkx7+ZK+MXuSHAe8HPjWAuhrPgzUV5Jfphf2v9i363Pe++qzBbhmVjvqma6vlwI/BXyx/cL6o8DWJG+pqh0zffLn2y6grfwwsTcANx05oKp+rapOq6oV9HYDXT/TN/9R9JVkWZIXtenFwOuBmVwIb1R9nQB8ht6f04zCaJR9zaFBLl/S3+8FwBeqHbmb577mw7R9JXkt8EfAW6pqrsJ9kL5W9c2+GXhovvuqqier6pSqWtHes26n9+c24zf/w0/wvHnQ2+96K72/uM8DJ7f6OPAnk4x/J3NzFtC0fQFvAnbROwtgF7BxgfT168D/BXb2PdbMd19t/n8AB4D/Q2/f6Tmz1M95wN/SO/bxwVb7EL3/iAAvBP4bMAHcCfzYbP/dDdjXz7Y/l/9N7xPJ7gXS1+eBx/v+PW1dIH19DNjderoNeM1C6OuIsV9khGcB+U1gSeqo59suIEnSgAwASeooA0CSOsoAkKSOMgAkaQFJ8v520bdTphm3tl24bmeSe5K8ddjnMgAkaY4leUOST05SXw6cDfyvATZzH71TQtcA64A/al9EHJgBIEkLx1XAb9N3IbokL06yKcmdSb6aZD1AVT1dvWtPQe+7KEOf028ASNIC0N7Y91bVPUcs+iC9b5evpXcp7f+Y5MVtnTOS7AbuBd7dFwiDPadfBJOkuZHkDnqXe38JcDI/3NVzGfAB4OyqejLJI/R273wzyQ56v+EffnM/md633h/o2+4/pndJ8l+oqoEvGf2cuxicJD1XVdUZ0DsGALyzqt7Z5v8JsBK4p130bRnwlXYJ9gC/UlVTXhusqh5I8hS9C8cNfJ0gdwFJ0jyrqnur6kfqhxd92wOcXlWPAbcA//rwTYbaxfRoF5A7rk2/Cng1vTsJDswAkKSF7XLgeGBX299/eau/nt4nhp30rtj7nqr65jAb9hiAJHWUnwAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI76f5AaBwteePPdAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[4000 4000 4000 4000 4000 4000 4000 4000 4000 4000]\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYAAAAEJCAYAAACdePCvAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4zLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvIxREBQAAFkNJREFUeJzt3X/wXXV95/Hny/BD118E+ZbGJJi0TceN3Tay3wY62tZKhYC7BqfUxW1rdJhJXXHXzrrbgu4MrZQZ6a6yOkW2tGQNTtvI0rpkMJVGxHGdWX4EDYFAKV8Fl6SBRIMoy0o3+N4/7id6N/1+87033/v9Aef5mLnzPed9Pufc982P+/rec849J1WFJKl7XjDfDUiS5ocBIEkdZQBIUkcZAJLUUQaAJHWUASBJHWUASFJHGQCS1FEGgCR11HHz3cDRnHLKKbVixYr5bkOSnlPuvvvub1bV2HTjFnQArFixgh07dsx3G5L0nJLkG4OMcxeQJHWUASBJHWUASFJHGQCS1FEGgCR11MABkGRRkq8mubnNr0xyR5KJJJ9OckKrn9jmJ9ryFX3buLTVH0xyzqhfjCRpcMN8Angf8EDf/JXAVVX1E8ATwEWtfhHwRKtf1caRZDVwIfAaYB3wiSSLZta+JOlYDRQASZYBbwb+pM0HeCNwYxuyGTi/Ta9v87TlZ7Xx64EtVfVMVT0MTABrR/EiJEnDG/QTwH8Gfhv4fpt/BfDtqjrU5vcAS9v0UuBRgLb8yTb+B/VJ1pEkzbFpvwmc5J8B+6vq7iRvmO2GkmwENgKcdtppM9rWiks+O4qWJGnOPfLhN8/6cwzyCeB1wFuSPAJsobfr52PASUkOB8gyYG+b3gssB2jLXw58q78+yTo/UFXXVtV4VY2PjU17KQtJ0jGaNgCq6tKqWlZVK+gdxP1CVf0acBtwQRu2AbipTW9t87TlX6iqavUL21lCK4FVwJ0jeyWSpKHM5GJwvwNsSfL7wFeB61r9OuBTSSaAg/RCg6raneQG4H7gEHBxVT07g+eXJM3AUAFQVV8Evtimv84kZ/FU1feAX51i/SuAK4ZtUpI0en4TWJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOmraAEjywiR3Jrknye4kv9fqn0zycJKd7bGm1ZPk40kmkuxKcnrftjYkeag9Nkz1nJKk2TfILSGfAd5YVU8lOR74cpK/asv+fVXdeMT4c+nd8H0VcAZwDXBGkpOBy4BxoIC7k2ytqidG8UIkScOZ9hNA9TzVZo9vjzrKKuuB69t6twMnJVkCnANsr6qD7U1/O7BuZu1Lko7VQMcAkixKshPYT+9N/I626Iq2m+eqJCe22lLg0b7V97TaVHVJ0jwYKACq6tmqWgMsA9Ym+SngUuDVwM8CJwO/M4qGkmxMsiPJjgMHDoxik5KkSQx1FlBVfRu4DVhXVfvabp5ngP8KrG3D9gLL+1Zb1mpT1Y98jmuraryqxsfGxoZpT5I0hEHOAhpLclKbfhHwJuBv2n59kgQ4H7ivrbIVeEc7G+hM4Mmq2gfcApydZHGSxcDZrSZJmgeDnAW0BNicZBG9wLihqm5O8oUkY0CAncC72/htwHnABPA08C6AqjqY5HLgrjbuQ1V1cHQvRZI0jGkDoKp2Aa+dpP7GKcYXcPEUyzYBm4bsUZI0C/wmsCR1lAEgSR1lAEhSRxkAktRRBoAkdZQBIEkdZQBIUkcZAJLUUQaAJHWUASBJHWUASFJHGQCS1FEGgCR1lAEgSR1lAEhSRxkAktRRBoAkddQg9wR+YZI7k9yTZHeS32v1lUnuSDKR5NNJTmj1E9v8RFu+om9bl7b6g0nOma0XJUma3iCfAJ4B3lhVPwOsAda1m71fCVxVVT8BPAFc1MZfBDzR6le1cSRZDVwIvAZYB3yi3WdYkjQPpg2A6nmqzR7fHgW8Ebix1TcD57fp9W2etvysJGn1LVX1TFU9TO+m8WtH8iokSUMb6BhAkkVJdgL7ge3A14BvV9WhNmQPsLRNLwUeBWjLnwRe0V+fZJ3+59qYZEeSHQcOHBj+FUmSBjJQAFTVs1W1BlhG77f2V89WQ1V1bVWNV9X42NjYbD2NJHXeUGcBVdW3gduAnwNOSnJcW7QM2Num9wLLAdrylwPf6q9Pso4kaY4NchbQWJKT2vSLgDcBD9ALggvasA3ATW16a5unLf9CVVWrX9jOEloJrALuHNULkSQN57jph7AE2NzO2HkBcENV3ZzkfmBLkt8Hvgpc18ZfB3wqyQRwkN6ZP1TV7iQ3APcDh4CLq+rZ0b4cSdKgpg2AqtoFvHaS+teZ5Cyeqvoe8KtTbOsK4Irh25QkjZrfBJakjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4a5J7Ay5PcluT+JLuTvK/VfzfJ3iQ72+O8vnUuTTKR5MEk5/TV17XaRJJLZuclSZIGMcg9gQ8B76+qryR5KXB3ku1t2VVV9Z/6BydZTe8+wK8BXgl8PslPtsVX07up/B7griRbq+r+UbwQSdJwBrkn8D5gX5v+bpIHgKVHWWU9sKWqngEebjeHP3zv4Il2L2GSbGljDQBJmgdDHQNIsoLeDeLvaKX3JtmVZFOSxa22FHi0b7U9rTZV/cjn2JhkR5IdBw4cGKY9SdIQBg6AJC8B/gL4rar6DnAN8OPAGnqfED4yioaq6tqqGq+q8bGxsVFsUpI0iUGOAZDkeHpv/n9aVX8JUFWP9y3/Y+DmNrsXWN63+rJW4yh1SdIcG+QsoADXAQ9U1Uf76kv6hr0VuK9NbwUuTHJikpXAKuBO4C5gVZKVSU6gd6B462hehiRpWIN8Angd8BvAvUl2ttoHgLcnWQMU8AjwmwBVtTvJDfQO7h4CLq6qZwGSvBe4BVgEbKqq3SN8LZKkIQxyFtCXgUyyaNtR1rkCuGKS+rajrSdJmjt+E1iSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjpqkHsCL09yW5L7k+xO8r5WPznJ9iQPtZ+LWz1JPp5kIsmuJKf3bWtDG/9Qkg2z97IkSdMZ5BPAIeD9VbUaOBO4OMlq4BLg1qpaBdza5gHOpXcj+FXARuAa6AUGcBlwBrAWuOxwaEiS5t60AVBV+6rqK236u8ADwFJgPbC5DdsMnN+m1wPXV8/twElJlgDnANur6mBVPQFsB9aN9NVIkgY21DGAJCuA1wJ3AKdW1b626DHg1Da9FHi0b7U9rTZV/cjn2JhkR5IdBw4cGKY9SdIQBg6AJC8B/gL4rar6Tv+yqiqgRtFQVV1bVeNVNT42NjaKTUqSJjFQACQ5nt6b/59W1V+28uNt1w7t5/5W3wss71t9WatNVZckzYNBzgIKcB3wQFV9tG/RVuDwmTwbgJv66u9oZwOdCTzZdhXdApydZHE7+Ht2q0mS5sFxA4x5HfAbwL1JdrbaB4APAzckuQj4BvC2tmwbcB4wATwNvAugqg4muRy4q437UFUdHMmrkCQNbdoAqKovA5li8VmTjC/g4im2tQnYNEyDkqTZ4TeBJamjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4a5JaQm5LsT3JfX+13k+xNsrM9zutbdmmSiSQPJjmnr76u1SaSXDL6lyJJGsYgnwA+CaybpH5VVa1pj20ASVYDFwKvaet8IsmiJIuAq4FzgdXA29tYSdI8GeSWkF9KsmLA7a0HtlTVM8DDSSaAtW3ZRFV9HSDJljb2/qE7liSNxEyOAbw3ya62i2hxqy0FHu0bs6fVpqpLkubJsQbANcCPA2uAfcBHRtVQko1JdiTZceDAgVFtVpJ0hGMKgKp6vKqerarvA3/MD3fz7AWW9w1d1mpT1Sfb9rVVNV5V42NjY8fSniRpAMcUAEmW9M2+FTh8htBW4MIkJyZZCawC7gTuAlYlWZnkBHoHircee9uSpJma9iBwkj8H3gCckmQPcBnwhiRrgAIeAX4ToKp2J7mB3sHdQ8DFVfVs2857gVuARcCmqto98lcjSRrYIGcBvX2S8nVHGX8FcMUk9W3AtqG6kyTNGr8JLEkdZQBIUkcZAJLUUQaAJHWUASBJHWUASFJHGQCS1FEGgCR1lAEgSR1lAEhSRxkAktRRBoAkdZQBIEkdZQBIUkcZAJLUUQaAJHWUASBJHTVtACTZlGR/kvv6aicn2Z7kofZzcasnyceTTCTZleT0vnU2tPEPJdkwOy9HkjSoQT4BfBJYd0TtEuDWqloF3NrmAc6ldyP4VcBG4BroBQa9ewmfAawFLjscGpKk+TFtAFTVl4CDR5TXA5vb9Gbg/L769dVzO3BSkiXAOcD2qjpYVU8A2/mHoSJJmkPHegzg1Kra16YfA05t00uBR/vG7Wm1qeqSpHky44PAVVVAjaAXAJJsTLIjyY4DBw6MarOSpCMcawA83nbt0H7ub/W9wPK+cctabar6P1BV11bVeFWNj42NHWN7kqTpHGsAbAUOn8mzAbipr/6OdjbQmcCTbVfRLcDZSRa3g79nt5okaZ4cN92AJH8OvAE4JckeemfzfBi4IclFwDeAt7Xh24DzgAngaeBdAFV1MMnlwF1t3Ieq6sgDy5KkOTRtAFTV26dYdNYkYwu4eIrtbAI2DdWdJGnW+E1gSeooA0CSOsoAkKSOMgAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI4yACSpowwASeooA0CSOsoAkKSOMgAkqaMMAEnqqBkFQJJHktybZGeSHa12cpLtSR5qPxe3epJ8PMlEkl1JTh/FC5AkHZtRfAL4papaU1Xjbf4S4NaqWgXc2uYBzgVWtcdG4JoRPLck6RjNxi6g9cDmNr0ZOL+vfn313A6clGTJLDy/JGkAMw2AAv46yd1JNrbaqVW1r00/BpzappcCj/atu6fVJEnz4LgZrv/6qtqb5EeA7Un+pn9hVVWSGmaDLUg2Apx22mkzbE+SNJUZfQKoqr3t537gM8Ba4PHDu3baz/1t+F5ged/qy1rtyG1eW1XjVTU+NjY2k/YkSUdxzAGQ5MVJXnp4GjgbuA/YCmxowzYAN7XprcA72tlAZwJP9u0qkiTNsZnsAjoV+EySw9v5s6r6XJK7gBuSXAR8A3hbG78NOA+YAJ4G3jWD55YkzdAxB0BVfR34mUnq3wLOmqRewMXH+nySpNHym8CS1FEGgCR1lAEgSR1lAEhSRxkAktRRBoAkdZQBIEkdZQBIUkcZAJLUUQaAJHWUASBJHWUASFJHGQCS1FEGgCR1lAEgSR1lAEhSRxkAktRRcx4ASdYleTDJRJJL5vr5JUk9cxoASRYBVwPnAquBtydZPZc9SJJ65voTwFpgoqq+XlV/D2wB1s9xD5Ik5j4AlgKP9s3vaTVJ0hw7br4bOFKSjcDGNvtUkgdnsLlTgG/OvKuRs6/h2Ndw7Gs4C7KvXDmjvl41yKC5DoC9wPK++WWt9gNVdS1w7SieLMmOqhofxbZGyb6GY1/Dsa/hdLmvud4FdBewKsnKJCcAFwJb57gHSRJz/Amgqg4leS9wC7AI2FRVu+eyB0lSz5wfA6iqbcC2OXq6kexKmgX2NRz7Go59DaezfaWqZvs5JEkLkJeCkKSOel4FQJKTk2xP8lD7ufgoY1+WZE+SP1wIfSV5VZKvJNmZZHeSdy+QvtYk+Z+tp11J/sVC6KuN+1ySbye5eZb7OerlS5KcmOTTbfkdSVbMZj9D9PUL7d/UoSQXzEVPA/b1b5Pc3/493ZpkoFMW56Cvdye5t/0f/PJcXaVg0MvjJPmVJJVkdGcGVdXz5gH8AXBJm74EuPIoYz8G/BnwhwuhL+AE4MQ2/RLgEeCVC6CvnwRWtelXAvuAk+a7r7bsLOCfAzfPYi+LgK8BP9b+ju4BVh8x5j3Af2nTFwKfnoN/U4P0tQL4aeB64ILZ7mmIvn4J+Edt+l8toD+vl/VNvwX43ELoq417KfAl4HZgfFTP/7z6BEDvshKb2/Rm4PzJBiX5p8CpwF8vlL6q6u+r6pk2eyJz8+lskL7+tqoeatN/B+wHxua7r9bPrcB3Z7mXQS5f0t/vjcBZSTLffVXVI1W1C/j+LPcybF+3VdXTbfZ2et8HWgh9fadv9sXAXBwgHfTyOJcDVwLfG+WTP98C4NSq2temH6P3Jv//SfIC4CPAv1tIfQEkWZ5kF73LZVzZ3nDnva++/tbS+y3lawupr1k2yOVLfjCmqg4BTwKvWAB9zYdh+7oI+KtZ7ahnoL6SXJzka/Q+hf6bhdBXktOB5VX12VE/+YK7FMR0knwe+NFJFn2wf6aqKslkCf4eYFtV7RnlL2kj6IuqehT46SSvBP57khur6vH57qttZwnwKWBDVc34N8pR9aXnriS/DowDvzjfvRxWVVcDVyf5l8B/ADbMZz/tF9aPAu+cje0/5wKgqn55qmVJHk+ypKr2tTes/ZMM+zng55O8h96+9hOSPFVVM7o3wQj66t/W3yW5D/h5ersU5rWvJC8DPgt8sKpun0k/o+xrjkx7+ZK+MXuSHAe8HPjWAuhrPgzUV5Jfphf2v9i363Pe++qzBbhmVjvqma6vlwI/BXyx/cL6o8DWJG+pqh0zffLn2y6grfwwsTcANx05oKp+rapOq6oV9HYDXT/TN/9R9JVkWZIXtenFwOuBmVwIb1R9nQB8ht6f04zCaJR9zaFBLl/S3+8FwBeqHbmb577mw7R9JXkt8EfAW6pqrsJ9kL5W9c2+GXhovvuqqier6pSqWtHes26n9+c24zf/w0/wvHnQ2+96K72/uM8DJ7f6OPAnk4x/J3NzFtC0fQFvAnbROwtgF7BxgfT168D/BXb2PdbMd19t/n8AB4D/Q2/f6Tmz1M95wN/SO/bxwVb7EL3/iAAvBP4bMAHcCfzYbP/dDdjXz7Y/l/9N7xPJ7gXS1+eBx/v+PW1dIH19DNjderoNeM1C6OuIsV9khGcB+U1gSeqo59suIEnSgAwASeooA0CSOsoAkKSOMgAkaQFJ8v520bdTphm3tl24bmeSe5K8ddjnMgAkaY4leUOST05SXw6cDfyvATZzH71TQtcA64A/al9EHJgBIEkLx1XAb9N3IbokL06yKcmdSb6aZD1AVT1dvWtPQe+7KEOf028ASNIC0N7Y91bVPUcs+iC9b5evpXcp7f+Y5MVtnTOS7AbuBd7dFwiDPadfBJOkuZHkDnqXe38JcDI/3NVzGfAB4OyqejLJI/R273wzyQ56v+EffnM/md633h/o2+4/pndJ8l+oqoEvGf2cuxicJD1XVdUZ0DsGALyzqt7Z5v8JsBK4p130bRnwlXYJ9gC/UlVTXhusqh5I8hS9C8cNfJ0gdwFJ0jyrqnur6kfqhxd92wOcXlWPAbcA//rwTYbaxfRoF5A7rk2/Cng1vTsJDswAkKSF7XLgeGBX299/eau/nt4nhp30rtj7nqr65jAb9hiAJHWUnwAkqaMMAEnqKANAkjrKAJCkjjIAJKmjDABJ6igDQJI6ygCQpI76f5AaBwteePPdAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "<Figure size 432x288 with 0 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Cheking if the splits are balanced\n",
    "for i in range(folds):\n",
    "    hist = np.histogram(np.squeeze(y_train[i]))[0]\n",
    "    print(hist)    \n",
    "    plt.bar(hist,np.amax(hist),alpha=0.5)\n",
    "    plt.show()\n",
    "    plt.clf()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-----------------\n",
    "## SqueezeNet with frozen layers\n",
    "Our initial attempt will be to remove SqueezeNet's top layers --- responsible for the classification into ImageNet classes --- and train a new set of layers to our CIFAR-10 classes. We will also freeze the layers before `drop9`. Our architecture will be like this:\n",
    "\n",
    "<img src=\"frozenSqueezeNet.png\" width=70% height=70%>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/azael/Trabalhos/MO444/MO444/lib/python3.5/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 32, 32, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv1 (Conv2D)                  (None, 15, 15, 64)   1792        input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "relu_conv1 (Activation)         (None, 15, 15, 64)   0           conv1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "pool1 (MaxPooling2D)            (None, 7, 7, 64)     0           relu_conv1[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "fire2/squeeze1x1 (Conv2D)       (None, 7, 7, 16)     1040        pool1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "fire2/relu_squeeze1x1 (Activati (None, 7, 7, 16)     0           fire2/squeeze1x1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "fire2/expand1x1 (Conv2D)        (None, 7, 7, 64)     1088        fire2/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire2/expand3x3 (Conv2D)        (None, 7, 7, 64)     9280        fire2/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire2/relu_expand1x1 (Activatio (None, 7, 7, 64)     0           fire2/expand1x1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire2/relu_expand3x3 (Activatio (None, 7, 7, 64)     0           fire2/expand3x3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire2/concat (Concatenate)      (None, 7, 7, 128)    0           fire2/relu_expand1x1[0][0]       \n",
      "                                                                 fire2/relu_expand3x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "fire3/squeeze1x1 (Conv2D)       (None, 7, 7, 16)     2064        fire2/concat[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "fire3/relu_squeeze1x1 (Activati (None, 7, 7, 16)     0           fire3/squeeze1x1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "fire3/expand1x1 (Conv2D)        (None, 7, 7, 64)     1088        fire3/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire3/expand3x3 (Conv2D)        (None, 7, 7, 64)     9280        fire3/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire3/relu_expand1x1 (Activatio (None, 7, 7, 64)     0           fire3/expand1x1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire3/relu_expand3x3 (Activatio (None, 7, 7, 64)     0           fire3/expand3x3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire3/concat (Concatenate)      (None, 7, 7, 128)    0           fire3/relu_expand1x1[0][0]       \n",
      "                                                                 fire3/relu_expand3x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "pool3 (MaxPooling2D)            (None, 3, 3, 128)    0           fire3/concat[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "fire4/squeeze1x1 (Conv2D)       (None, 3, 3, 32)     4128        pool3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "fire4/relu_squeeze1x1 (Activati (None, 3, 3, 32)     0           fire4/squeeze1x1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "fire4/expand1x1 (Conv2D)        (None, 3, 3, 128)    4224        fire4/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire4/expand3x3 (Conv2D)        (None, 3, 3, 128)    36992       fire4/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire4/relu_expand1x1 (Activatio (None, 3, 3, 128)    0           fire4/expand1x1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire4/relu_expand3x3 (Activatio (None, 3, 3, 128)    0           fire4/expand3x3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire4/concat (Concatenate)      (None, 3, 3, 256)    0           fire4/relu_expand1x1[0][0]       \n",
      "                                                                 fire4/relu_expand3x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "fire5/squeeze1x1 (Conv2D)       (None, 3, 3, 32)     8224        fire4/concat[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "fire5/relu_squeeze1x1 (Activati (None, 3, 3, 32)     0           fire5/squeeze1x1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "fire5/expand1x1 (Conv2D)        (None, 3, 3, 128)    4224        fire5/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire5/expand3x3 (Conv2D)        (None, 3, 3, 128)    36992       fire5/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire5/relu_expand1x1 (Activatio (None, 3, 3, 128)    0           fire5/expand1x1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire5/relu_expand3x3 (Activatio (None, 3, 3, 128)    0           fire5/expand3x3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire5/concat (Concatenate)      (None, 3, 3, 256)    0           fire5/relu_expand1x1[0][0]       \n",
      "                                                                 fire5/relu_expand3x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "pool5 (MaxPooling2D)            (None, 1, 1, 256)    0           fire5/concat[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "fire6/squeeze1x1 (Conv2D)       (None, 1, 1, 48)     12336       pool5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "fire6/relu_squeeze1x1 (Activati (None, 1, 1, 48)     0           fire6/squeeze1x1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "fire6/expand1x1 (Conv2D)        (None, 1, 1, 192)    9408        fire6/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire6/expand3x3 (Conv2D)        (None, 1, 1, 192)    83136       fire6/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire6/relu_expand1x1 (Activatio (None, 1, 1, 192)    0           fire6/expand1x1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire6/relu_expand3x3 (Activatio (None, 1, 1, 192)    0           fire6/expand3x3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire6/concat (Concatenate)      (None, 1, 1, 384)    0           fire6/relu_expand1x1[0][0]       \n",
      "                                                                 fire6/relu_expand3x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "fire7/squeeze1x1 (Conv2D)       (None, 1, 1, 48)     18480       fire6/concat[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "fire7/relu_squeeze1x1 (Activati (None, 1, 1, 48)     0           fire7/squeeze1x1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "fire7/expand1x1 (Conv2D)        (None, 1, 1, 192)    9408        fire7/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire7/expand3x3 (Conv2D)        (None, 1, 1, 192)    83136       fire7/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire7/relu_expand1x1 (Activatio (None, 1, 1, 192)    0           fire7/expand1x1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire7/relu_expand3x3 (Activatio (None, 1, 1, 192)    0           fire7/expand3x3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire7/concat (Concatenate)      (None, 1, 1, 384)    0           fire7/relu_expand1x1[0][0]       \n",
      "                                                                 fire7/relu_expand3x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "fire8/squeeze1x1 (Conv2D)       (None, 1, 1, 64)     24640       fire7/concat[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "fire8/relu_squeeze1x1 (Activati (None, 1, 1, 64)     0           fire8/squeeze1x1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "fire8/expand1x1 (Conv2D)        (None, 1, 1, 256)    16640       fire8/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire8/expand3x3 (Conv2D)        (None, 1, 1, 256)    147712      fire8/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire8/relu_expand1x1 (Activatio (None, 1, 1, 256)    0           fire8/expand1x1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire8/relu_expand3x3 (Activatio (None, 1, 1, 256)    0           fire8/expand3x3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire8/concat (Concatenate)      (None, 1, 1, 512)    0           fire8/relu_expand1x1[0][0]       \n",
      "                                                                 fire8/relu_expand3x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "fire9/squeeze1x1 (Conv2D)       (None, 1, 1, 64)     32832       fire8/concat[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "fire9/relu_squeeze1x1 (Activati (None, 1, 1, 64)     0           fire9/squeeze1x1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "fire9/expand1x1 (Conv2D)        (None, 1, 1, 256)    16640       fire9/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire9/expand3x3 (Conv2D)        (None, 1, 1, 256)    147712      fire9/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire9/relu_expand1x1 (Activatio (None, 1, 1, 256)    0           fire9/expand1x1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire9/relu_expand3x3 (Activatio (None, 1, 1, 256)    0           fire9/expand3x3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire9/concat (Concatenate)      (None, 1, 1, 512)    0           fire9/relu_expand1x1[0][0]       \n",
      "                                                                 fire9/relu_expand3x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "drop9 (Dropout)                 (None, 1, 1, 512)    0           fire9/concat[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv10 (Conv2D)                 (None, 1, 1, 10)     5130        drop9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "relu_conv10 (Activation)        (None, 1, 1, 10)     0           conv10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling2d_2 (Glo (None, 10)           0           relu_conv10[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "loss (Activation)               (None, 10)           0           global_average_pooling2d_2[0][0] \n",
      "==================================================================================================\n",
      "Total params: 727,626\n",
      "Trainable params: 5,130\n",
      "Non-trainable params: 722,496\n",
      "__________________________________________________________________________________________________\n",
      "input_1 False\n",
      "conv1 False\n",
      "relu_conv1 False\n",
      "pool1 False\n",
      "fire2/squeeze1x1 False\n",
      "fire2/relu_squeeze1x1 False\n",
      "fire2/expand1x1 False\n",
      "fire2/expand3x3 False\n",
      "fire2/relu_expand1x1 False\n",
      "fire2/relu_expand3x3 False\n",
      "fire2/concat False\n",
      "fire3/squeeze1x1 False\n",
      "fire3/relu_squeeze1x1 False\n",
      "fire3/expand1x1 False\n",
      "fire3/expand3x3 False\n",
      "fire3/relu_expand1x1 False\n",
      "fire3/relu_expand3x3 False\n",
      "fire3/concat False\n",
      "pool3 False\n",
      "fire4/squeeze1x1 False\n",
      "fire4/relu_squeeze1x1 False\n",
      "fire4/expand1x1 False\n",
      "fire4/expand3x3 False\n",
      "fire4/relu_expand1x1 False\n",
      "fire4/relu_expand3x3 False\n",
      "fire4/concat False\n",
      "fire5/squeeze1x1 False\n",
      "fire5/relu_squeeze1x1 False\n",
      "fire5/expand1x1 False\n",
      "fire5/expand3x3 False\n",
      "fire5/relu_expand1x1 False\n",
      "fire5/relu_expand3x3 False\n",
      "fire5/concat False\n",
      "pool5 False\n",
      "fire6/squeeze1x1 False\n",
      "fire6/relu_squeeze1x1 False\n",
      "fire6/expand1x1 False\n",
      "fire6/expand3x3 False\n",
      "fire6/relu_expand1x1 False\n",
      "fire6/relu_expand3x3 False\n",
      "fire6/concat False\n",
      "fire7/squeeze1x1 False\n",
      "fire7/relu_squeeze1x1 False\n",
      "fire7/expand1x1 False\n",
      "fire7/expand3x3 False\n",
      "fire7/relu_expand1x1 False\n",
      "fire7/relu_expand3x3 False\n",
      "fire7/concat False\n",
      "fire8/squeeze1x1 False\n",
      "fire8/relu_squeeze1x1 False\n",
      "fire8/expand1x1 False\n",
      "fire8/expand3x3 False\n",
      "fire8/relu_expand1x1 False\n",
      "fire8/relu_expand3x3 False\n",
      "fire8/concat False\n",
      "fire9/squeeze1x1 False\n",
      "fire9/relu_squeeze1x1 False\n",
      "fire9/expand1x1 False\n",
      "fire9/expand3x3 False\n",
      "fire9/relu_expand1x1 False\n",
      "fire9/relu_expand3x3 False\n",
      "fire9/concat False\n",
      "drop9 False\n",
      "conv10 True\n",
      "relu_conv10 True\n",
      "global_average_pooling2d_2 True\n",
      "loss True\n"
     ]
    }
   ],
   "source": [
    "def get_squeezenet_ft():\n",
    "    #=====================================\n",
    "    # Freezing layers\n",
    "    #=====================================\n",
    "\n",
    "    squeezeNetModel = SqueezeNet((32,32,3))\n",
    "\n",
    "    #freeze layers\n",
    "    for layer in squeezeNetModel.layers:\n",
    "        layer.trainable = False\n",
    "\n",
    "    #=====================================\n",
    "    # Add new classification layers\n",
    "    #=====================================\n",
    "\n",
    "    #--- Removing layers until drop 9\n",
    "    squeezeNetModel.layers.pop() #Convolution2D\n",
    "    squeezeNetModel.layers.pop() #Activation ReLU\n",
    "    squeezeNetModel.layers.pop() #Global Avg Pool\n",
    "    squeezeNetModel.layers.pop() #Activation Softmax\n",
    "\n",
    "    #--- Adding classification layer for 10 classes\n",
    "    out = Convolution2D(n_classes, (1, 1), padding='valid', name='conv10')(squeezeNetModel.layers[-1].output)\n",
    "    out = Activation('relu', name='relu_conv10')(out)\n",
    "\n",
    "    out = GlobalAveragePooling2D()(out)\n",
    "    out = Activation('softmax', name='loss')(out)\n",
    "\n",
    "    #=====================================\n",
    "    # New Model\n",
    "    #=====================================\n",
    "    model = Model(squeezeNetModel.inputs, out, name='squeezenet_new')\n",
    "    #=====================================\n",
    "    # Compile model\n",
    "    #=====================================\n",
    "\n",
    "\n",
    "    opt = optimizer=optimizers.Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)#SGD(lr=learning_rate)\n",
    "    model.compile(loss='categorical_crossentropy',\n",
    "                  optimizer=opt,\n",
    "                  metrics=['accuracy'])\n",
    "    return model\n",
    "\n",
    "model  = get_squeezenet_ft()\n",
    "model.summary()\n",
    "\n",
    "#--- Check the trainable status of the individual layers\n",
    "for layer in model.layers:\n",
    "    print(layer.name, layer.trainable)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we compile our model and train it:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Split: 0\n",
      "Number of Epochs: 100\n",
      "Train shape: (40000, 32, 32, 3)\n",
      "Train batch size: 384\n",
      "Val batch size: 128\n",
      "================================================\n",
      "--- Preparing tensorboard\n",
      "Log Dir:  logs/1542818922.0200126_fold[0]_pre-trained\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/100\n",
      "40000/40000 [==============================] - 2s 50us/step - loss: 2.3574 - acc: 0.1033 - val_loss: 2.3015 - val_acc: 0.1100\n",
      "Epoch 2/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 2.3157 - acc: 0.1139 - val_loss: 2.2908 - val_acc: 0.1386\n",
      "Epoch 3/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 2.2946 - acc: 0.1317 - val_loss: 2.2773 - val_acc: 0.1856\n",
      "Epoch 4/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 2.2816 - acc: 0.1451 - val_loss: 2.2580 - val_acc: 0.2170\n",
      "Epoch 5/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 2.2631 - acc: 0.1669 - val_loss: 2.2302 - val_acc: 0.2444\n",
      "Epoch 6/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 2.2456 - acc: 0.1823 - val_loss: 2.2048 - val_acc: 0.2632\n",
      "Epoch 7/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 2.2227 - acc: 0.1976 - val_loss: 2.1605 - val_acc: 0.2756\n",
      "Epoch 8/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 2.1918 - acc: 0.2153 - val_loss: 2.1158 - val_acc: 0.2960\n",
      "Epoch 9/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 2.1639 - acc: 0.2297 - val_loss: 2.0767 - val_acc: 0.3122\n",
      "Epoch 10/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 2.1287 - acc: 0.2423 - val_loss: 2.0392 - val_acc: 0.3296\n",
      "Epoch 11/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 2.1025 - acc: 0.2564 - val_loss: 2.0144 - val_acc: 0.3458\n",
      "Epoch 12/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 2.0803 - acc: 0.2647 - val_loss: 1.9947 - val_acc: 0.3553\n",
      "Epoch 13/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 2.0557 - acc: 0.2727 - val_loss: 1.9672 - val_acc: 0.3624\n",
      "Epoch 14/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 2.0270 - acc: 0.2786 - val_loss: 1.9378 - val_acc: 0.3646\n",
      "Epoch 15/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 2.0033 - acc: 0.2845 - val_loss: 1.9176 - val_acc: 0.3723\n",
      "Epoch 16/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.9828 - acc: 0.2910 - val_loss: 1.9012 - val_acc: 0.3803\n",
      "Epoch 17/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.9652 - acc: 0.2979 - val_loss: 1.8869 - val_acc: 0.3794\n",
      "Epoch 18/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.9534 - acc: 0.3039 - val_loss: 1.8745 - val_acc: 0.3847\n",
      "Epoch 19/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.9387 - acc: 0.3083 - val_loss: 1.8631 - val_acc: 0.3852\n",
      "Epoch 20/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.9269 - acc: 0.3166 - val_loss: 1.8532 - val_acc: 0.3895\n",
      "Epoch 21/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.9204 - acc: 0.3151 - val_loss: 1.8441 - val_acc: 0.3909\n",
      "Epoch 22/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.9079 - acc: 0.3223 - val_loss: 1.8357 - val_acc: 0.3908\n",
      "Epoch 23/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.9038 - acc: 0.3214 - val_loss: 1.8283 - val_acc: 0.3927\n",
      "Epoch 24/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8973 - acc: 0.3247 - val_loss: 1.8214 - val_acc: 0.3948\n",
      "Epoch 25/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8946 - acc: 0.3248 - val_loss: 1.8149 - val_acc: 0.3951\n",
      "Epoch 26/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8848 - acc: 0.3301 - val_loss: 1.8089 - val_acc: 0.3960\n",
      "Epoch 27/100\n",
      "40000/40000 [==============================] - 1s 31us/step - loss: 1.8780 - acc: 0.3318 - val_loss: 1.8032 - val_acc: 0.3971\n",
      "Epoch 28/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8810 - acc: 0.3309 - val_loss: 1.7979 - val_acc: 0.3976\n",
      "Epoch 29/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8714 - acc: 0.3316 - val_loss: 1.7930 - val_acc: 0.3994\n",
      "Epoch 30/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8651 - acc: 0.3356 - val_loss: 1.7882 - val_acc: 0.3985\n",
      "Epoch 31/100\n",
      "40000/40000 [==============================] - 1s 30us/step - loss: 1.8621 - acc: 0.3363 - val_loss: 1.7838 - val_acc: 0.3998\n",
      "Epoch 32/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8593 - acc: 0.3402 - val_loss: 1.7798 - val_acc: 0.4010\n",
      "Epoch 33/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8550 - acc: 0.3390 - val_loss: 1.7758 - val_acc: 0.4019\n",
      "Epoch 34/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8520 - acc: 0.3420 - val_loss: 1.7721 - val_acc: 0.4024\n",
      "Epoch 35/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8484 - acc: 0.3435 - val_loss: 1.7686 - val_acc: 0.4022\n",
      "Epoch 36/100\n",
      "40000/40000 [==============================] - 1s 31us/step - loss: 1.8452 - acc: 0.3420 - val_loss: 1.7652 - val_acc: 0.4031\n",
      "Epoch 37/100\n",
      "40000/40000 [==============================] - 1s 30us/step - loss: 1.8463 - acc: 0.3444 - val_loss: 1.7621 - val_acc: 0.4048\n",
      "Epoch 38/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8431 - acc: 0.3450 - val_loss: 1.7589 - val_acc: 0.4044\n",
      "Epoch 39/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8366 - acc: 0.3465 - val_loss: 1.7559 - val_acc: 0.4041\n",
      "Epoch 40/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8382 - acc: 0.3450 - val_loss: 1.7532 - val_acc: 0.4060\n",
      "Epoch 41/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8331 - acc: 0.3462 - val_loss: 1.7505 - val_acc: 0.4039\n",
      "Epoch 42/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8322 - acc: 0.3472 - val_loss: 1.7480 - val_acc: 0.4076\n",
      "Epoch 43/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8293 - acc: 0.3467 - val_loss: 1.7455 - val_acc: 0.4067\n",
      "Epoch 44/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8319 - acc: 0.3457 - val_loss: 1.7430 - val_acc: 0.4073\n",
      "Epoch 45/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8264 - acc: 0.3502 - val_loss: 1.7409 - val_acc: 0.4084\n",
      "Epoch 46/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8266 - acc: 0.3472 - val_loss: 1.7388 - val_acc: 0.4082\n",
      "Epoch 47/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8245 - acc: 0.3489 - val_loss: 1.7367 - val_acc: 0.4072\n",
      "Epoch 48/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8206 - acc: 0.3487 - val_loss: 1.7348 - val_acc: 0.4080\n",
      "Epoch 49/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8198 - acc: 0.3514 - val_loss: 1.7328 - val_acc: 0.4080\n",
      "Epoch 50/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8206 - acc: 0.3497 - val_loss: 1.7309 - val_acc: 0.4085\n",
      "Epoch 51/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8188 - acc: 0.3513 - val_loss: 1.7289 - val_acc: 0.4106\n",
      "Epoch 52/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8122 - acc: 0.3535 - val_loss: 1.7270 - val_acc: 0.4096\n",
      "Epoch 53/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8126 - acc: 0.3540 - val_loss: 1.7254 - val_acc: 0.4128\n",
      "Epoch 54/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8111 - acc: 0.3545 - val_loss: 1.7236 - val_acc: 0.4125\n",
      "Epoch 55/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8133 - acc: 0.3519 - val_loss: 1.7223 - val_acc: 0.4110\n",
      "Epoch 56/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8120 - acc: 0.3548 - val_loss: 1.7208 - val_acc: 0.4135\n",
      "Epoch 57/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8105 - acc: 0.3530 - val_loss: 1.7193 - val_acc: 0.4140\n",
      "Epoch 58/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8053 - acc: 0.3575 - val_loss: 1.7178 - val_acc: 0.4137\n",
      "Epoch 59/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8078 - acc: 0.3561 - val_loss: 1.7164 - val_acc: 0.4122\n",
      "Epoch 60/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8104 - acc: 0.3530 - val_loss: 1.7151 - val_acc: 0.4146\n",
      "Epoch 61/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8068 - acc: 0.3539 - val_loss: 1.7139 - val_acc: 0.4142\n",
      "Epoch 62/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8078 - acc: 0.3556 - val_loss: 1.7126 - val_acc: 0.4144\n",
      "Epoch 63/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8027 - acc: 0.3548 - val_loss: 1.7117 - val_acc: 0.4136\n",
      "Epoch 64/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8023 - acc: 0.3555 - val_loss: 1.7104 - val_acc: 0.4161\n",
      "Epoch 65/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7985 - acc: 0.3564 - val_loss: 1.7090 - val_acc: 0.4158\n",
      "Epoch 66/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8040 - acc: 0.3570 - val_loss: 1.7079 - val_acc: 0.4157\n",
      "Epoch 67/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8013 - acc: 0.3581 - val_loss: 1.7069 - val_acc: 0.4161\n",
      "Epoch 68/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8010 - acc: 0.3578 - val_loss: 1.7057 - val_acc: 0.4156\n",
      "Epoch 69/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7953 - acc: 0.3575 - val_loss: 1.7046 - val_acc: 0.4163\n",
      "Epoch 70/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8006 - acc: 0.3555 - val_loss: 1.7038 - val_acc: 0.4160\n",
      "Epoch 71/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7973 - acc: 0.3584 - val_loss: 1.7028 - val_acc: 0.4171\n",
      "Epoch 72/100\n",
      "40000/40000 [==============================] - 1s 30us/step - loss: 1.7961 - acc: 0.3581 - val_loss: 1.7017 - val_acc: 0.4171\n",
      "Epoch 73/100\n",
      "40000/40000 [==============================] - 1s 30us/step - loss: 1.7960 - acc: 0.3639 - val_loss: 1.7009 - val_acc: 0.4178\n",
      "Epoch 74/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7952 - acc: 0.3579 - val_loss: 1.7001 - val_acc: 0.4166\n",
      "Epoch 75/100\n",
      "40000/40000 [==============================] - 1s 30us/step - loss: 1.7914 - acc: 0.3636 - val_loss: 1.6990 - val_acc: 0.4153\n",
      "Epoch 76/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7943 - acc: 0.3599 - val_loss: 1.6979 - val_acc: 0.4163\n",
      "Epoch 77/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7913 - acc: 0.3594 - val_loss: 1.6971 - val_acc: 0.4175\n",
      "Epoch 78/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7919 - acc: 0.3592 - val_loss: 1.6962 - val_acc: 0.4185\n",
      "Epoch 79/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7906 - acc: 0.3590 - val_loss: 1.6955 - val_acc: 0.4183\n",
      "Epoch 80/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7927 - acc: 0.3594 - val_loss: 1.6947 - val_acc: 0.4180\n",
      "Epoch 81/100\n",
      "40000/40000 [==============================] - 1s 30us/step - loss: 1.7902 - acc: 0.3637 - val_loss: 1.6939 - val_acc: 0.4171\n",
      "Epoch 82/100\n",
      "40000/40000 [==============================] - 1s 30us/step - loss: 1.7867 - acc: 0.3606 - val_loss: 1.6930 - val_acc: 0.4186\n",
      "Epoch 83/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7874 - acc: 0.3606 - val_loss: 1.6921 - val_acc: 0.4175\n",
      "Epoch 84/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7891 - acc: 0.3605 - val_loss: 1.6915 - val_acc: 0.4169\n",
      "Epoch 85/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7917 - acc: 0.3588 - val_loss: 1.6909 - val_acc: 0.4187\n",
      "Epoch 86/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7865 - acc: 0.3626 - val_loss: 1.6901 - val_acc: 0.4184\n",
      "Epoch 87/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7893 - acc: 0.3621 - val_loss: 1.6893 - val_acc: 0.4188\n",
      "Epoch 88/100\n",
      "40000/40000 [==============================] - 1s 30us/step - loss: 1.7865 - acc: 0.3621 - val_loss: 1.6888 - val_acc: 0.4198\n",
      "Epoch 89/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.7852 - acc: 0.3630 - val_loss: 1.6880 - val_acc: 0.4190\n",
      "Epoch 90/100\n",
      "40000/40000 [==============================] - 1s 31us/step - loss: 1.7865 - acc: 0.3602 - val_loss: 1.6874 - val_acc: 0.4192\n",
      "Epoch 91/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7819 - acc: 0.3634 - val_loss: 1.6870 - val_acc: 0.4181\n",
      "Epoch 92/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7859 - acc: 0.3651 - val_loss: 1.6863 - val_acc: 0.4196\n",
      "Epoch 93/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7817 - acc: 0.3647 - val_loss: 1.6857 - val_acc: 0.4193\n",
      "Epoch 94/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7836 - acc: 0.3621 - val_loss: 1.6850 - val_acc: 0.4195\n",
      "Epoch 95/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7831 - acc: 0.3663 - val_loss: 1.6844 - val_acc: 0.4184\n",
      "Epoch 96/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7839 - acc: 0.3618 - val_loss: 1.6838 - val_acc: 0.4195\n",
      "Epoch 97/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7850 - acc: 0.3654 - val_loss: 1.6833 - val_acc: 0.4187\n",
      "Epoch 98/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7803 - acc: 0.3613 - val_loss: 1.6826 - val_acc: 0.4195\n",
      "Epoch 99/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7795 - acc: 0.3659 - val_loss: 1.6820 - val_acc: 0.4187\n",
      "Epoch 100/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7798 - acc: 0.3639 - val_loss: 1.6816 - val_acc: 0.4202\n",
      "10000/10000 [==============================] - 0s 32us/step\n",
      "Validation loss: 1.6816067134857178\n",
      "Validation accuracy (NORMALIZED): 0.4202\n",
      "================================================\n",
      "Split: 1\n",
      "Number of Epochs: 100\n",
      "Train shape: (40000, 32, 32, 3)\n",
      "Train batch size: 384\n",
      "Val batch size: 128\n",
      "================================================\n",
      "--- Preparing tensorboard\n",
      "Log Dir:  logs/1542819042.807682_fold[1]_pre-trained\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/100\n",
      "40000/40000 [==============================] - 1s 32us/step - loss: 2.3834 - acc: 0.1023 - val_loss: 2.3046 - val_acc: 0.1248\n",
      "Epoch 2/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 2.3294 - acc: 0.1134 - val_loss: 2.2877 - val_acc: 0.1494\n",
      "Epoch 3/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 2.3015 - acc: 0.1305 - val_loss: 2.2775 - val_acc: 0.1853\n",
      "Epoch 4/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 2.2831 - acc: 0.1481 - val_loss: 2.2554 - val_acc: 0.2281\n",
      "Epoch 5/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 2.2618 - acc: 0.1718 - val_loss: 2.2264 - val_acc: 0.2481\n",
      "Epoch 6/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 2.2432 - acc: 0.1874 - val_loss: 2.1954 - val_acc: 0.2585\n",
      "Epoch 7/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 2.2159 - acc: 0.2038 - val_loss: 2.1523 - val_acc: 0.2795\n",
      "Epoch 8/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 2.1905 - acc: 0.2173 - val_loss: 2.1105 - val_acc: 0.2987\n",
      "Epoch 9/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 2.1627 - acc: 0.2303 - val_loss: 2.0740 - val_acc: 0.3203\n",
      "Epoch 10/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 2.1364 - acc: 0.2421 - val_loss: 2.0454 - val_acc: 0.3348\n",
      "Epoch 11/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 2.1062 - acc: 0.2554 - val_loss: 2.0236 - val_acc: 0.3431\n",
      "Epoch 12/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 2.0872 - acc: 0.2633 - val_loss: 2.0054 - val_acc: 0.3499\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 13/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 2.0620 - acc: 0.2697 - val_loss: 1.9660 - val_acc: 0.3537\n",
      "Epoch 14/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 2.0341 - acc: 0.2739 - val_loss: 1.9401 - val_acc: 0.3564\n",
      "Epoch 15/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 2.0081 - acc: 0.2826 - val_loss: 1.9206 - val_acc: 0.3647\n",
      "Epoch 16/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.9894 - acc: 0.2922 - val_loss: 1.9049 - val_acc: 0.3694\n",
      "Epoch 17/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.9704 - acc: 0.2963 - val_loss: 1.8911 - val_acc: 0.3714\n",
      "Epoch 18/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.9575 - acc: 0.3032 - val_loss: 1.8790 - val_acc: 0.3784\n",
      "Epoch 19/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.9438 - acc: 0.3072 - val_loss: 1.8682 - val_acc: 0.3791\n",
      "Epoch 20/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.9359 - acc: 0.3081 - val_loss: 1.8583 - val_acc: 0.3826\n",
      "Epoch 21/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.9235 - acc: 0.3195 - val_loss: 1.8494 - val_acc: 0.3850\n",
      "Epoch 22/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.9146 - acc: 0.3152 - val_loss: 1.8414 - val_acc: 0.3861\n",
      "Epoch 23/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.9113 - acc: 0.3218 - val_loss: 1.8338 - val_acc: 0.3894\n",
      "Epoch 24/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8976 - acc: 0.3290 - val_loss: 1.8268 - val_acc: 0.3897\n",
      "Epoch 25/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8903 - acc: 0.3273 - val_loss: 1.8202 - val_acc: 0.3925\n",
      "Epoch 26/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8877 - acc: 0.3252 - val_loss: 1.8141 - val_acc: 0.3944\n",
      "Epoch 27/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8826 - acc: 0.3272 - val_loss: 1.8086 - val_acc: 0.3955\n",
      "Epoch 28/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8787 - acc: 0.3310 - val_loss: 1.8032 - val_acc: 0.3955\n",
      "Epoch 29/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8734 - acc: 0.3339 - val_loss: 1.7984 - val_acc: 0.3978\n",
      "Epoch 30/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8715 - acc: 0.3355 - val_loss: 1.7940 - val_acc: 0.3979\n",
      "Epoch 31/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8632 - acc: 0.3359 - val_loss: 1.7894 - val_acc: 0.3999\n",
      "Epoch 32/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8612 - acc: 0.3366 - val_loss: 1.7852 - val_acc: 0.4003\n",
      "Epoch 33/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8568 - acc: 0.3381 - val_loss: 1.7811 - val_acc: 0.4019\n",
      "Epoch 34/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8532 - acc: 0.3380 - val_loss: 1.7774 - val_acc: 0.4020\n",
      "Epoch 35/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8510 - acc: 0.3394 - val_loss: 1.7738 - val_acc: 0.4035\n",
      "Epoch 36/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8478 - acc: 0.3422 - val_loss: 1.7705 - val_acc: 0.4040\n",
      "Epoch 37/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8448 - acc: 0.3441 - val_loss: 1.7673 - val_acc: 0.4045\n",
      "Epoch 38/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8441 - acc: 0.3432 - val_loss: 1.7642 - val_acc: 0.4034\n",
      "Epoch 39/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8388 - acc: 0.3425 - val_loss: 1.7613 - val_acc: 0.4043\n",
      "Epoch 40/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8378 - acc: 0.3469 - val_loss: 1.7584 - val_acc: 0.4054\n",
      "Epoch 41/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8349 - acc: 0.3474 - val_loss: 1.7558 - val_acc: 0.4042\n",
      "Epoch 42/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8357 - acc: 0.3447 - val_loss: 1.7532 - val_acc: 0.4049\n",
      "Epoch 43/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8331 - acc: 0.3477 - val_loss: 1.7508 - val_acc: 0.4056\n",
      "Epoch 44/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8260 - acc: 0.3511 - val_loss: 1.7487 - val_acc: 0.4049\n",
      "Epoch 45/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8241 - acc: 0.3479 - val_loss: 1.7463 - val_acc: 0.4060\n",
      "Epoch 46/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8252 - acc: 0.3499 - val_loss: 1.7440 - val_acc: 0.4055\n",
      "Epoch 47/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8251 - acc: 0.3480 - val_loss: 1.7419 - val_acc: 0.4065\n",
      "Epoch 48/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8249 - acc: 0.3480 - val_loss: 1.7401 - val_acc: 0.4074\n",
      "Epoch 49/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8154 - acc: 0.3510 - val_loss: 1.7380 - val_acc: 0.4071\n",
      "Epoch 50/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8176 - acc: 0.3498 - val_loss: 1.7359 - val_acc: 0.4079\n",
      "Epoch 51/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8149 - acc: 0.3529 - val_loss: 1.7343 - val_acc: 0.4089\n",
      "Epoch 52/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8197 - acc: 0.3492 - val_loss: 1.7326 - val_acc: 0.4088\n",
      "Epoch 53/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8126 - acc: 0.3560 - val_loss: 1.7307 - val_acc: 0.4098\n",
      "Epoch 54/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8125 - acc: 0.3537 - val_loss: 1.7290 - val_acc: 0.4104\n",
      "Epoch 55/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8092 - acc: 0.3555 - val_loss: 1.7274 - val_acc: 0.4103\n",
      "Epoch 56/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8081 - acc: 0.3539 - val_loss: 1.7258 - val_acc: 0.4098\n",
      "Epoch 57/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8087 - acc: 0.3566 - val_loss: 1.7242 - val_acc: 0.4107\n",
      "Epoch 58/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8088 - acc: 0.3519 - val_loss: 1.7227 - val_acc: 0.4106\n",
      "Epoch 59/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8086 - acc: 0.3522 - val_loss: 1.7214 - val_acc: 0.4108\n",
      "Epoch 60/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8057 - acc: 0.3538 - val_loss: 1.7199 - val_acc: 0.4128\n",
      "Epoch 61/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8063 - acc: 0.3564 - val_loss: 1.7188 - val_acc: 0.4109\n",
      "Epoch 62/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8057 - acc: 0.3555 - val_loss: 1.7174 - val_acc: 0.4116\n",
      "Epoch 63/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8031 - acc: 0.3585 - val_loss: 1.7163 - val_acc: 0.4120\n",
      "Epoch 64/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8007 - acc: 0.3573 - val_loss: 1.7149 - val_acc: 0.4134\n",
      "Epoch 65/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7977 - acc: 0.3598 - val_loss: 1.7136 - val_acc: 0.4127\n",
      "Epoch 66/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8051 - acc: 0.3557 - val_loss: 1.7126 - val_acc: 0.4127\n",
      "Epoch 67/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7970 - acc: 0.3594 - val_loss: 1.7115 - val_acc: 0.4125\n",
      "Epoch 68/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8010 - acc: 0.3574 - val_loss: 1.7105 - val_acc: 0.4133\n",
      "Epoch 69/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7954 - acc: 0.3596 - val_loss: 1.7094 - val_acc: 0.4132\n",
      "Epoch 70/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7974 - acc: 0.3585 - val_loss: 1.7085 - val_acc: 0.4158\n",
      "Epoch 71/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7984 - acc: 0.3571 - val_loss: 1.7073 - val_acc: 0.4136\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 72/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7955 - acc: 0.3608 - val_loss: 1.7065 - val_acc: 0.4124\n",
      "Epoch 73/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7923 - acc: 0.3613 - val_loss: 1.7055 - val_acc: 0.4158\n",
      "Epoch 74/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7914 - acc: 0.3603 - val_loss: 1.7043 - val_acc: 0.4149\n",
      "Epoch 75/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7925 - acc: 0.3606 - val_loss: 1.7034 - val_acc: 0.4148\n",
      "Epoch 76/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7932 - acc: 0.3574 - val_loss: 1.7027 - val_acc: 0.4147\n",
      "Epoch 77/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7919 - acc: 0.3584 - val_loss: 1.7020 - val_acc: 0.4160\n",
      "Epoch 78/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7892 - acc: 0.3621 - val_loss: 1.7010 - val_acc: 0.4158\n",
      "Epoch 79/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7894 - acc: 0.3622 - val_loss: 1.7000 - val_acc: 0.4164\n",
      "Epoch 80/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7879 - acc: 0.3607 - val_loss: 1.6993 - val_acc: 0.4169\n",
      "Epoch 81/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7897 - acc: 0.3618 - val_loss: 1.6985 - val_acc: 0.4167\n",
      "Epoch 82/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7890 - acc: 0.3610 - val_loss: 1.6977 - val_acc: 0.4166\n",
      "Epoch 83/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7879 - acc: 0.3629 - val_loss: 1.6970 - val_acc: 0.4174\n",
      "Epoch 84/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7853 - acc: 0.3631 - val_loss: 1.6961 - val_acc: 0.4171\n",
      "Epoch 85/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7867 - acc: 0.3624 - val_loss: 1.6955 - val_acc: 0.4168\n",
      "Epoch 86/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7826 - acc: 0.3621 - val_loss: 1.6946 - val_acc: 0.4191\n",
      "Epoch 87/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7824 - acc: 0.3627 - val_loss: 1.6943 - val_acc: 0.4161\n",
      "Epoch 88/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7840 - acc: 0.3617 - val_loss: 1.6934 - val_acc: 0.4180\n",
      "Epoch 89/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7851 - acc: 0.3599 - val_loss: 1.6929 - val_acc: 0.4180\n",
      "Epoch 90/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7812 - acc: 0.3618 - val_loss: 1.6921 - val_acc: 0.4190\n",
      "Epoch 91/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7832 - acc: 0.3618 - val_loss: 1.6915 - val_acc: 0.4180\n",
      "Epoch 92/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7830 - acc: 0.3616 - val_loss: 1.6910 - val_acc: 0.4191\n",
      "Epoch 93/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7851 - acc: 0.3626 - val_loss: 1.6905 - val_acc: 0.4182\n",
      "Epoch 94/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7806 - acc: 0.3644 - val_loss: 1.6899 - val_acc: 0.4195\n",
      "Epoch 95/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7828 - acc: 0.3645 - val_loss: 1.6894 - val_acc: 0.4180\n",
      "Epoch 96/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7806 - acc: 0.3629 - val_loss: 1.6888 - val_acc: 0.4194\n",
      "Epoch 97/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7801 - acc: 0.3631 - val_loss: 1.6880 - val_acc: 0.4183\n",
      "Epoch 98/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7818 - acc: 0.3642 - val_loss: 1.6878 - val_acc: 0.4193\n",
      "Epoch 99/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7779 - acc: 0.3636 - val_loss: 1.6869 - val_acc: 0.4187\n",
      "Epoch 100/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7755 - acc: 0.3640 - val_loss: 1.6864 - val_acc: 0.4201\n",
      "10000/10000 [==============================] - 0s 29us/step\n",
      "Validation loss: 1.6863978813171387\n",
      "Validation accuracy (NORMALIZED): 0.4201\n",
      "================================================\n",
      "Split: 2\n",
      "Number of Epochs: 100\n",
      "Train shape: (40000, 32, 32, 3)\n",
      "Train batch size: 384\n",
      "Val batch size: 128\n",
      "================================================\n",
      "--- Preparing tensorboard\n",
      "Log Dir:  logs/1542819154.058151_fold[2]_pre-trained\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 2.3967 - acc: 0.1075 - val_loss: 2.3082 - val_acc: 0.1036\n",
      "Epoch 2/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 2.3322 - acc: 0.1218 - val_loss: 2.2781 - val_acc: 0.1551\n",
      "Epoch 3/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 2.3036 - acc: 0.1339 - val_loss: 2.2636 - val_acc: 0.2119\n",
      "Epoch 4/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 2.2822 - acc: 0.1517 - val_loss: 2.2459 - val_acc: 0.2377\n",
      "Epoch 5/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 2.2645 - acc: 0.1637 - val_loss: 2.2220 - val_acc: 0.2418\n",
      "Epoch 6/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 2.2445 - acc: 0.1807 - val_loss: 2.1930 - val_acc: 0.2477\n",
      "Epoch 7/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 2.2246 - acc: 0.1957 - val_loss: 2.1611 - val_acc: 0.2615\n",
      "Epoch 8/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 2.2039 - acc: 0.2044 - val_loss: 2.1247 - val_acc: 0.2813\n",
      "Epoch 9/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 2.1728 - acc: 0.2190 - val_loss: 2.0830 - val_acc: 0.3010\n",
      "Epoch 10/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 2.1380 - acc: 0.2301 - val_loss: 2.0243 - val_acc: 0.3342\n",
      "Epoch 11/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 2.1000 - acc: 0.2449 - val_loss: 1.9873 - val_acc: 0.3529\n",
      "Epoch 12/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 2.0687 - acc: 0.2585 - val_loss: 1.9600 - val_acc: 0.3612\n",
      "Epoch 13/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 2.0385 - acc: 0.2700 - val_loss: 1.9373 - val_acc: 0.3710\n",
      "Epoch 14/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 2.0147 - acc: 0.2789 - val_loss: 1.9182 - val_acc: 0.3775\n",
      "Epoch 15/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.9940 - acc: 0.2891 - val_loss: 1.9017 - val_acc: 0.3797\n",
      "Epoch 16/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.9768 - acc: 0.2989 - val_loss: 1.8869 - val_acc: 0.3829\n",
      "Epoch 17/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.9681 - acc: 0.2992 - val_loss: 1.8742 - val_acc: 0.3840\n",
      "Epoch 18/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.9517 - acc: 0.3076 - val_loss: 1.8628 - val_acc: 0.3861\n",
      "Epoch 19/100\n",
      "40000/40000 [==============================] - 1s 30us/step - loss: 1.9425 - acc: 0.3047 - val_loss: 1.8525 - val_acc: 0.3875\n",
      "Epoch 20/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.9335 - acc: 0.3112 - val_loss: 1.8431 - val_acc: 0.3916\n",
      "Epoch 21/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.9245 - acc: 0.3162 - val_loss: 1.8343 - val_acc: 0.3916\n",
      "Epoch 22/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.9172 - acc: 0.3169 - val_loss: 1.8266 - val_acc: 0.3926\n",
      "Epoch 23/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.9112 - acc: 0.3197 - val_loss: 1.8190 - val_acc: 0.3940\n",
      "Epoch 24/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.9007 - acc: 0.3232 - val_loss: 1.8121 - val_acc: 0.3952\n",
      "Epoch 25/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8999 - acc: 0.3233 - val_loss: 1.8058 - val_acc: 0.3949\n",
      "Epoch 26/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8869 - acc: 0.3297 - val_loss: 1.7998 - val_acc: 0.3974\n",
      "Epoch 27/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8820 - acc: 0.3338 - val_loss: 1.7941 - val_acc: 0.3984\n",
      "Epoch 28/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8779 - acc: 0.3322 - val_loss: 1.7887 - val_acc: 0.3992\n",
      "Epoch 29/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8750 - acc: 0.3337 - val_loss: 1.7839 - val_acc: 0.4004\n",
      "Epoch 30/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8707 - acc: 0.3347 - val_loss: 1.7790 - val_acc: 0.3990\n",
      "Epoch 31/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8645 - acc: 0.3373 - val_loss: 1.7746 - val_acc: 0.3992\n",
      "Epoch 32/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8637 - acc: 0.3383 - val_loss: 1.7704 - val_acc: 0.4002\n",
      "Epoch 33/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8575 - acc: 0.3390 - val_loss: 1.7664 - val_acc: 0.4008\n",
      "Epoch 34/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8526 - acc: 0.3423 - val_loss: 1.7625 - val_acc: 0.4021\n",
      "Epoch 35/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8544 - acc: 0.3381 - val_loss: 1.7588 - val_acc: 0.4023\n",
      "Epoch 36/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8505 - acc: 0.3414 - val_loss: 1.7554 - val_acc: 0.4030\n",
      "Epoch 37/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8504 - acc: 0.3414 - val_loss: 1.7522 - val_acc: 0.4034\n",
      "Epoch 38/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8415 - acc: 0.3430 - val_loss: 1.7490 - val_acc: 0.4039\n",
      "Epoch 39/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8397 - acc: 0.3458 - val_loss: 1.7461 - val_acc: 0.4049\n",
      "Epoch 40/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8383 - acc: 0.3434 - val_loss: 1.7432 - val_acc: 0.4040\n",
      "Epoch 41/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8387 - acc: 0.3463 - val_loss: 1.7404 - val_acc: 0.4068\n",
      "Epoch 42/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8378 - acc: 0.3444 - val_loss: 1.7379 - val_acc: 0.4076\n",
      "Epoch 43/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8298 - acc: 0.3471 - val_loss: 1.7352 - val_acc: 0.4071\n",
      "Epoch 44/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8329 - acc: 0.3465 - val_loss: 1.7328 - val_acc: 0.4078\n",
      "Epoch 45/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8330 - acc: 0.3460 - val_loss: 1.7305 - val_acc: 0.4093\n",
      "Epoch 46/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8293 - acc: 0.3511 - val_loss: 1.7283 - val_acc: 0.4076\n",
      "Epoch 47/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8238 - acc: 0.3516 - val_loss: 1.7261 - val_acc: 0.4077\n",
      "Epoch 48/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8251 - acc: 0.3500 - val_loss: 1.7243 - val_acc: 0.4090\n",
      "Epoch 49/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8253 - acc: 0.3475 - val_loss: 1.7223 - val_acc: 0.4100\n",
      "Epoch 50/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8229 - acc: 0.3521 - val_loss: 1.7203 - val_acc: 0.4103\n",
      "Epoch 51/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8227 - acc: 0.3475 - val_loss: 1.7186 - val_acc: 0.4116\n",
      "Epoch 52/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8168 - acc: 0.3533 - val_loss: 1.7168 - val_acc: 0.4108\n",
      "Epoch 53/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8175 - acc: 0.3491 - val_loss: 1.7151 - val_acc: 0.4106\n",
      "Epoch 54/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8179 - acc: 0.3534 - val_loss: 1.7135 - val_acc: 0.4108\n",
      "Epoch 55/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8113 - acc: 0.3556 - val_loss: 1.7117 - val_acc: 0.4102\n",
      "Epoch 56/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8116 - acc: 0.3552 - val_loss: 1.7101 - val_acc: 0.4113\n",
      "Epoch 57/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8119 - acc: 0.3561 - val_loss: 1.7086 - val_acc: 0.4124\n",
      "Epoch 58/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8127 - acc: 0.3567 - val_loss: 1.7071 - val_acc: 0.4121\n",
      "Epoch 59/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8090 - acc: 0.3543 - val_loss: 1.7057 - val_acc: 0.4129\n",
      "Epoch 60/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8074 - acc: 0.3550 - val_loss: 1.7041 - val_acc: 0.4130\n",
      "Epoch 61/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8086 - acc: 0.3559 - val_loss: 1.7028 - val_acc: 0.4149\n",
      "Epoch 62/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8066 - acc: 0.3585 - val_loss: 1.7015 - val_acc: 0.4157\n",
      "Epoch 63/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8047 - acc: 0.3555 - val_loss: 1.7003 - val_acc: 0.4147\n",
      "Epoch 64/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8017 - acc: 0.3564 - val_loss: 1.6992 - val_acc: 0.4145\n",
      "Epoch 65/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8023 - acc: 0.3586 - val_loss: 1.6978 - val_acc: 0.4147\n",
      "Epoch 66/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8045 - acc: 0.3537 - val_loss: 1.6967 - val_acc: 0.4156\n",
      "Epoch 67/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8004 - acc: 0.3572 - val_loss: 1.6955 - val_acc: 0.4168\n",
      "Epoch 68/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8019 - acc: 0.3570 - val_loss: 1.6944 - val_acc: 0.4153\n",
      "Epoch 69/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7984 - acc: 0.3585 - val_loss: 1.6933 - val_acc: 0.4165\n",
      "Epoch 70/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8007 - acc: 0.3565 - val_loss: 1.6923 - val_acc: 0.4164\n",
      "Epoch 71/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7982 - acc: 0.3600 - val_loss: 1.6914 - val_acc: 0.4173\n",
      "Epoch 72/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8007 - acc: 0.3581 - val_loss: 1.6905 - val_acc: 0.4174\n",
      "Epoch 73/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7944 - acc: 0.3614 - val_loss: 1.6894 - val_acc: 0.4175\n",
      "Epoch 74/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7987 - acc: 0.3583 - val_loss: 1.6885 - val_acc: 0.4176\n",
      "Epoch 75/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7953 - acc: 0.3607 - val_loss: 1.6875 - val_acc: 0.4191\n",
      "Epoch 76/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7946 - acc: 0.3629 - val_loss: 1.6865 - val_acc: 0.4177\n",
      "Epoch 77/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7915 - acc: 0.3595 - val_loss: 1.6855 - val_acc: 0.4171\n",
      "Epoch 78/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7902 - acc: 0.3604 - val_loss: 1.6846 - val_acc: 0.4179\n",
      "Epoch 79/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7927 - acc: 0.3590 - val_loss: 1.6839 - val_acc: 0.4190\n",
      "Epoch 80/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7944 - acc: 0.3581 - val_loss: 1.6830 - val_acc: 0.4183\n",
      "Epoch 81/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7928 - acc: 0.3602 - val_loss: 1.6822 - val_acc: 0.4183\n",
      "Epoch 82/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7940 - acc: 0.3598 - val_loss: 1.6815 - val_acc: 0.4194\n",
      "Epoch 83/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7868 - acc: 0.3626 - val_loss: 1.6808 - val_acc: 0.4209\n",
      "Epoch 84/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7934 - acc: 0.3604 - val_loss: 1.6800 - val_acc: 0.4204\n",
      "Epoch 85/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7927 - acc: 0.3599 - val_loss: 1.6795 - val_acc: 0.4198\n",
      "Epoch 86/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7890 - acc: 0.3629 - val_loss: 1.6787 - val_acc: 0.4216\n",
      "Epoch 87/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7913 - acc: 0.3610 - val_loss: 1.6779 - val_acc: 0.4196\n",
      "Epoch 88/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7897 - acc: 0.3590 - val_loss: 1.6774 - val_acc: 0.4207\n",
      "Epoch 89/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7885 - acc: 0.3627 - val_loss: 1.6767 - val_acc: 0.4215\n",
      "Epoch 90/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7843 - acc: 0.3648 - val_loss: 1.6760 - val_acc: 0.4223\n",
      "Epoch 91/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7852 - acc: 0.3623 - val_loss: 1.6754 - val_acc: 0.4223\n",
      "Epoch 92/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7874 - acc: 0.3641 - val_loss: 1.6747 - val_acc: 0.4215\n",
      "Epoch 93/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7882 - acc: 0.3633 - val_loss: 1.6742 - val_acc: 0.4237\n",
      "Epoch 94/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7838 - acc: 0.3629 - val_loss: 1.6735 - val_acc: 0.4224\n",
      "Epoch 95/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7853 - acc: 0.3639 - val_loss: 1.6728 - val_acc: 0.4208\n",
      "Epoch 96/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7829 - acc: 0.3642 - val_loss: 1.6722 - val_acc: 0.4220\n",
      "Epoch 97/100\n",
      "40000/40000 [==============================] - 1s 30us/step - loss: 1.7865 - acc: 0.3627 - val_loss: 1.6716 - val_acc: 0.4218\n",
      "Epoch 98/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7824 - acc: 0.3641 - val_loss: 1.6712 - val_acc: 0.4221\n",
      "Epoch 99/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7830 - acc: 0.3619 - val_loss: 1.6706 - val_acc: 0.4208\n",
      "Epoch 100/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7808 - acc: 0.3613 - val_loss: 1.6700 - val_acc: 0.4208\n",
      "10000/10000 [==============================] - 0s 28us/step\n",
      "Validation loss: 1.6700160438537597\n",
      "Validation accuracy (NORMALIZED): 0.4208\n",
      "================================================\n",
      "Split: 3\n",
      "Number of Epochs: 100\n",
      "Train shape: (40000, 32, 32, 3)\n",
      "Train batch size: 384\n",
      "Val batch size: 128\n",
      "================================================\n",
      "--- Preparing tensorboard\n",
      "Log Dir:  logs/1542819268.8453317_fold[3]_pre-trained\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 2.3576 - acc: 0.1058 - val_loss: 2.2944 - val_acc: 0.1316\n",
      "Epoch 2/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 2.3127 - acc: 0.1241 - val_loss: 2.2830 - val_acc: 0.1742\n",
      "Epoch 3/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 2.2908 - acc: 0.1418 - val_loss: 2.2688 - val_acc: 0.2150\n",
      "Epoch 4/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 2.2733 - acc: 0.1596 - val_loss: 2.2509 - val_acc: 0.2323\n",
      "Epoch 5/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 2.2596 - acc: 0.1724 - val_loss: 2.2309 - val_acc: 0.2381\n",
      "Epoch 6/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 2.2396 - acc: 0.1868 - val_loss: 2.2010 - val_acc: 0.2505\n",
      "Epoch 7/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 2.2186 - acc: 0.2013 - val_loss: 2.1644 - val_acc: 0.2658\n",
      "Epoch 8/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 2.1974 - acc: 0.2134 - val_loss: 2.1299 - val_acc: 0.2766\n",
      "Epoch 9/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 2.1727 - acc: 0.2258 - val_loss: 2.0981 - val_acc: 0.2902\n",
      "Epoch 10/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 2.1434 - acc: 0.2354 - val_loss: 2.0536 - val_acc: 0.3073\n",
      "Epoch 11/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 2.1043 - acc: 0.2478 - val_loss: 2.0017 - val_acc: 0.3264\n",
      "Epoch 12/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 2.0760 - acc: 0.2567 - val_loss: 1.9688 - val_acc: 0.3395\n",
      "Epoch 13/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 2.0465 - acc: 0.2694 - val_loss: 1.9440 - val_acc: 0.3504\n",
      "Epoch 14/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 2.0212 - acc: 0.2768 - val_loss: 1.9235 - val_acc: 0.3602\n",
      "Epoch 15/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 2.0000 - acc: 0.2825 - val_loss: 1.9063 - val_acc: 0.3653\n",
      "Epoch 16/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.9837 - acc: 0.2876 - val_loss: 1.8915 - val_acc: 0.3699\n",
      "Epoch 17/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.9688 - acc: 0.2967 - val_loss: 1.8786 - val_acc: 0.3740\n",
      "Epoch 18/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.9520 - acc: 0.3040 - val_loss: 1.8671 - val_acc: 0.3778\n",
      "Epoch 19/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.9412 - acc: 0.3075 - val_loss: 1.8567 - val_acc: 0.3823\n",
      "Epoch 20/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.9323 - acc: 0.3122 - val_loss: 1.8470 - val_acc: 0.3819\n",
      "Epoch 21/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.9237 - acc: 0.3126 - val_loss: 1.8382 - val_acc: 0.3844\n",
      "Epoch 22/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.9096 - acc: 0.3190 - val_loss: 1.8299 - val_acc: 0.3843\n",
      "Epoch 23/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.9060 - acc: 0.3178 - val_loss: 1.8227 - val_acc: 0.3865\n",
      "Epoch 24/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.9016 - acc: 0.3209 - val_loss: 1.8159 - val_acc: 0.3880\n",
      "Epoch 25/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8908 - acc: 0.3265 - val_loss: 1.8097 - val_acc: 0.3916\n",
      "Epoch 26/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8846 - acc: 0.3306 - val_loss: 1.8039 - val_acc: 0.3939\n",
      "Epoch 27/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8795 - acc: 0.3319 - val_loss: 1.7981 - val_acc: 0.3946\n",
      "Epoch 28/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8763 - acc: 0.3339 - val_loss: 1.7929 - val_acc: 0.3947\n",
      "Epoch 29/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8714 - acc: 0.3320 - val_loss: 1.7879 - val_acc: 0.3951\n",
      "Epoch 30/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8684 - acc: 0.3330 - val_loss: 1.7836 - val_acc: 0.3981\n",
      "Epoch 31/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8677 - acc: 0.3348 - val_loss: 1.7792 - val_acc: 0.3955\n",
      "Epoch 32/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8632 - acc: 0.3376 - val_loss: 1.7751 - val_acc: 0.3969\n",
      "Epoch 33/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8574 - acc: 0.3370 - val_loss: 1.7715 - val_acc: 0.3978\n",
      "Epoch 34/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8542 - acc: 0.3408 - val_loss: 1.7678 - val_acc: 0.3993\n",
      "Epoch 35/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8539 - acc: 0.3386 - val_loss: 1.7644 - val_acc: 0.4001\n",
      "Epoch 36/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8485 - acc: 0.3440 - val_loss: 1.7612 - val_acc: 0.3985\n",
      "Epoch 37/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8446 - acc: 0.3435 - val_loss: 1.7580 - val_acc: 0.4020\n",
      "Epoch 38/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8450 - acc: 0.3422 - val_loss: 1.7550 - val_acc: 0.4026\n",
      "Epoch 39/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8386 - acc: 0.3446 - val_loss: 1.7521 - val_acc: 0.4021\n",
      "Epoch 40/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8379 - acc: 0.3453 - val_loss: 1.7492 - val_acc: 0.4035\n",
      "Epoch 41/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8361 - acc: 0.3449 - val_loss: 1.7468 - val_acc: 0.4039\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 42/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8352 - acc: 0.3454 - val_loss: 1.7442 - val_acc: 0.4054\n",
      "Epoch 43/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8306 - acc: 0.3461 - val_loss: 1.7416 - val_acc: 0.4060\n",
      "Epoch 44/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8254 - acc: 0.3488 - val_loss: 1.7394 - val_acc: 0.4066\n",
      "Epoch 45/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8278 - acc: 0.3480 - val_loss: 1.7369 - val_acc: 0.4073\n",
      "Epoch 46/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8246 - acc: 0.3501 - val_loss: 1.7347 - val_acc: 0.4058\n",
      "Epoch 47/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8257 - acc: 0.3491 - val_loss: 1.7327 - val_acc: 0.4067\n",
      "Epoch 48/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8256 - acc: 0.3466 - val_loss: 1.7307 - val_acc: 0.4091\n",
      "Epoch 49/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8216 - acc: 0.3520 - val_loss: 1.7290 - val_acc: 0.4104\n",
      "Epoch 50/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8162 - acc: 0.3525 - val_loss: 1.7269 - val_acc: 0.4089\n",
      "Epoch 51/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8187 - acc: 0.3485 - val_loss: 1.7253 - val_acc: 0.4108\n",
      "Epoch 52/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8153 - acc: 0.3550 - val_loss: 1.7234 - val_acc: 0.4096\n",
      "Epoch 53/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8155 - acc: 0.3504 - val_loss: 1.7218 - val_acc: 0.4105\n",
      "Epoch 54/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8128 - acc: 0.3539 - val_loss: 1.7201 - val_acc: 0.4104\n",
      "Epoch 55/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8118 - acc: 0.3549 - val_loss: 1.7185 - val_acc: 0.4097\n",
      "Epoch 56/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8105 - acc: 0.3542 - val_loss: 1.7169 - val_acc: 0.4133\n",
      "Epoch 57/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8099 - acc: 0.3565 - val_loss: 1.7155 - val_acc: 0.4116\n",
      "Epoch 58/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8047 - acc: 0.3551 - val_loss: 1.7142 - val_acc: 0.4126\n",
      "Epoch 59/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8089 - acc: 0.3537 - val_loss: 1.7127 - val_acc: 0.4120\n",
      "Epoch 60/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8094 - acc: 0.3544 - val_loss: 1.7114 - val_acc: 0.4146\n",
      "Epoch 61/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8062 - acc: 0.3566 - val_loss: 1.7102 - val_acc: 0.4120\n",
      "Epoch 62/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8054 - acc: 0.3549 - val_loss: 1.7089 - val_acc: 0.4124\n",
      "Epoch 63/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8057 - acc: 0.3561 - val_loss: 1.7078 - val_acc: 0.4126\n",
      "Epoch 64/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8052 - acc: 0.3576 - val_loss: 1.7065 - val_acc: 0.4131\n",
      "Epoch 65/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7987 - acc: 0.3594 - val_loss: 1.7051 - val_acc: 0.4133\n",
      "Epoch 66/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8000 - acc: 0.3543 - val_loss: 1.7041 - val_acc: 0.4154\n",
      "Epoch 67/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8004 - acc: 0.3588 - val_loss: 1.7031 - val_acc: 0.4141\n",
      "Epoch 68/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7983 - acc: 0.3622 - val_loss: 1.7019 - val_acc: 0.4131\n",
      "Epoch 69/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8013 - acc: 0.3575 - val_loss: 1.7010 - val_acc: 0.4138\n",
      "Epoch 70/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7961 - acc: 0.3572 - val_loss: 1.7003 - val_acc: 0.4148\n",
      "Epoch 71/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7967 - acc: 0.3573 - val_loss: 1.6992 - val_acc: 0.4155\n",
      "Epoch 72/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7957 - acc: 0.3577 - val_loss: 1.6981 - val_acc: 0.4152\n",
      "Epoch 73/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7972 - acc: 0.3589 - val_loss: 1.6971 - val_acc: 0.4152\n",
      "Epoch 74/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7991 - acc: 0.3598 - val_loss: 1.6963 - val_acc: 0.4161\n",
      "Epoch 75/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7935 - acc: 0.3598 - val_loss: 1.6954 - val_acc: 0.4164\n",
      "Epoch 76/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7915 - acc: 0.3599 - val_loss: 1.6947 - val_acc: 0.4150\n",
      "Epoch 77/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7974 - acc: 0.3614 - val_loss: 1.6939 - val_acc: 0.4162\n",
      "Epoch 78/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7900 - acc: 0.3594 - val_loss: 1.6928 - val_acc: 0.4155\n",
      "Epoch 79/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7937 - acc: 0.3616 - val_loss: 1.6921 - val_acc: 0.4164\n",
      "Epoch 80/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7881 - acc: 0.3649 - val_loss: 1.6912 - val_acc: 0.4160\n",
      "Epoch 81/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7883 - acc: 0.3620 - val_loss: 1.6903 - val_acc: 0.4169\n",
      "Epoch 82/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7903 - acc: 0.3603 - val_loss: 1.6895 - val_acc: 0.4166\n",
      "Epoch 83/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7875 - acc: 0.3625 - val_loss: 1.6887 - val_acc: 0.4172\n",
      "Epoch 84/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7879 - acc: 0.3619 - val_loss: 1.6881 - val_acc: 0.4159\n",
      "Epoch 85/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7875 - acc: 0.3607 - val_loss: 1.6875 - val_acc: 0.4172\n",
      "Epoch 86/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7895 - acc: 0.3597 - val_loss: 1.6868 - val_acc: 0.4156\n",
      "Epoch 87/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7855 - acc: 0.3627 - val_loss: 1.6861 - val_acc: 0.4182\n",
      "Epoch 88/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7862 - acc: 0.3627 - val_loss: 1.6853 - val_acc: 0.4174\n",
      "Epoch 89/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7886 - acc: 0.3608 - val_loss: 1.6848 - val_acc: 0.4180\n",
      "Epoch 90/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7876 - acc: 0.3600 - val_loss: 1.6842 - val_acc: 0.4184\n",
      "Epoch 91/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7846 - acc: 0.3629 - val_loss: 1.6834 - val_acc: 0.4195\n",
      "Epoch 92/100\n",
      "40000/40000 [==============================] - 1s 30us/step - loss: 1.7838 - acc: 0.3647 - val_loss: 1.6828 - val_acc: 0.4192\n",
      "Epoch 93/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7861 - acc: 0.3613 - val_loss: 1.6823 - val_acc: 0.4190\n",
      "Epoch 94/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7843 - acc: 0.3611 - val_loss: 1.6817 - val_acc: 0.4186\n",
      "Epoch 95/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7843 - acc: 0.3621 - val_loss: 1.6813 - val_acc: 0.4190\n",
      "Epoch 96/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7845 - acc: 0.3637 - val_loss: 1.6807 - val_acc: 0.4193\n",
      "Epoch 97/100\n",
      "40000/40000 [==============================] - 1s 30us/step - loss: 1.7864 - acc: 0.3656 - val_loss: 1.6802 - val_acc: 0.4197\n",
      "Epoch 98/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7829 - acc: 0.3628 - val_loss: 1.6797 - val_acc: 0.4190\n",
      "Epoch 99/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7795 - acc: 0.3675 - val_loss: 1.6791 - val_acc: 0.4203\n",
      "Epoch 100/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7822 - acc: 0.3654 - val_loss: 1.6785 - val_acc: 0.4196\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 0s 29us/step\n",
      "Validation loss: 1.678535626602173\n",
      "Validation accuracy (NORMALIZED): 0.4196\n",
      "================================================\n",
      "Split: 4\n",
      "Number of Epochs: 100\n",
      "Train shape: (40000, 32, 32, 3)\n",
      "Train batch size: 384\n",
      "Val batch size: 128\n",
      "================================================\n",
      "--- Preparing tensorboard\n",
      "Log Dir:  logs/1542819386.8009765_fold[4]_pre-trained\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 2.3281 - acc: 0.1118 - val_loss: 2.2958 - val_acc: 0.1300\n",
      "Epoch 2/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 2.2993 - acc: 0.1286 - val_loss: 2.2835 - val_acc: 0.1718\n",
      "Epoch 3/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 2.2864 - acc: 0.1389 - val_loss: 2.2691 - val_acc: 0.2082\n",
      "Epoch 4/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 2.2667 - acc: 0.1595 - val_loss: 2.2359 - val_acc: 0.2422\n",
      "Epoch 5/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 2.2428 - acc: 0.1842 - val_loss: 2.1964 - val_acc: 0.2705\n",
      "Epoch 6/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 2.2182 - acc: 0.2020 - val_loss: 2.1643 - val_acc: 0.2854\n",
      "Epoch 7/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 2.1947 - acc: 0.2148 - val_loss: 2.1363 - val_acc: 0.2991\n",
      "Epoch 8/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 2.1714 - acc: 0.2284 - val_loss: 2.1005 - val_acc: 0.3050\n",
      "Epoch 9/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 2.1479 - acc: 0.2370 - val_loss: 2.0675 - val_acc: 0.3208\n",
      "Epoch 10/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 2.1219 - acc: 0.2483 - val_loss: 2.0340 - val_acc: 0.3340\n",
      "Epoch 11/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 2.0955 - acc: 0.2582 - val_loss: 2.0037 - val_acc: 0.3499\n",
      "Epoch 12/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 2.0743 - acc: 0.2671 - val_loss: 1.9829 - val_acc: 0.3575\n",
      "Epoch 13/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 2.0558 - acc: 0.2751 - val_loss: 1.9667 - val_acc: 0.3647\n",
      "Epoch 14/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 2.0288 - acc: 0.2815 - val_loss: 1.9313 - val_acc: 0.3667\n",
      "Epoch 15/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 2.0059 - acc: 0.2881 - val_loss: 1.9071 - val_acc: 0.3783\n",
      "Epoch 16/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.9805 - acc: 0.2961 - val_loss: 1.8891 - val_acc: 0.3812\n",
      "Epoch 17/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.9628 - acc: 0.3046 - val_loss: 1.8744 - val_acc: 0.3857\n",
      "Epoch 18/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.9502 - acc: 0.3069 - val_loss: 1.8617 - val_acc: 0.3854\n",
      "Epoch 19/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.9389 - acc: 0.3114 - val_loss: 1.8509 - val_acc: 0.3916\n",
      "Epoch 20/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.9269 - acc: 0.3176 - val_loss: 1.8408 - val_acc: 0.3924\n",
      "Epoch 21/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.9188 - acc: 0.3195 - val_loss: 1.8320 - val_acc: 0.3934\n",
      "Epoch 22/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.9087 - acc: 0.3254 - val_loss: 1.8239 - val_acc: 0.3941\n",
      "Epoch 23/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.9016 - acc: 0.3234 - val_loss: 1.8166 - val_acc: 0.3961\n",
      "Epoch 24/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8960 - acc: 0.3260 - val_loss: 1.8100 - val_acc: 0.3956\n",
      "Epoch 25/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8904 - acc: 0.3270 - val_loss: 1.8034 - val_acc: 0.3966\n",
      "Epoch 26/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8840 - acc: 0.3309 - val_loss: 1.7976 - val_acc: 0.3989\n",
      "Epoch 27/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8762 - acc: 0.3335 - val_loss: 1.7920 - val_acc: 0.4009\n",
      "Epoch 28/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8731 - acc: 0.3347 - val_loss: 1.7869 - val_acc: 0.4012\n",
      "Epoch 29/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8656 - acc: 0.3398 - val_loss: 1.7818 - val_acc: 0.4016\n",
      "Epoch 30/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8654 - acc: 0.3365 - val_loss: 1.7772 - val_acc: 0.4006\n",
      "Epoch 31/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8598 - acc: 0.3413 - val_loss: 1.7730 - val_acc: 0.4011\n",
      "Epoch 32/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8568 - acc: 0.3430 - val_loss: 1.7690 - val_acc: 0.4023\n",
      "Epoch 33/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8528 - acc: 0.3394 - val_loss: 1.7653 - val_acc: 0.4046\n",
      "Epoch 34/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8504 - acc: 0.3430 - val_loss: 1.7616 - val_acc: 0.4023\n",
      "Epoch 35/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8467 - acc: 0.3430 - val_loss: 1.7581 - val_acc: 0.4034\n",
      "Epoch 36/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8425 - acc: 0.3445 - val_loss: 1.7548 - val_acc: 0.4033\n",
      "Epoch 37/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8411 - acc: 0.3456 - val_loss: 1.7517 - val_acc: 0.4052\n",
      "Epoch 38/100\n",
      "40000/40000 [==============================] - 1s 31us/step - loss: 1.8392 - acc: 0.3459 - val_loss: 1.7488 - val_acc: 0.4040\n",
      "Epoch 39/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8382 - acc: 0.3481 - val_loss: 1.7459 - val_acc: 0.4061\n",
      "Epoch 40/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8340 - acc: 0.3477 - val_loss: 1.7431 - val_acc: 0.4050\n",
      "Epoch 41/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8315 - acc: 0.3483 - val_loss: 1.7406 - val_acc: 0.4073\n",
      "Epoch 42/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8322 - acc: 0.3477 - val_loss: 1.7382 - val_acc: 0.4061\n",
      "Epoch 43/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8303 - acc: 0.3489 - val_loss: 1.7360 - val_acc: 0.4095\n",
      "Epoch 44/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8254 - acc: 0.3489 - val_loss: 1.7338 - val_acc: 0.4066\n",
      "Epoch 45/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8275 - acc: 0.3486 - val_loss: 1.7316 - val_acc: 0.4084\n",
      "Epoch 46/100\n",
      "40000/40000 [==============================] - 1s 30us/step - loss: 1.8225 - acc: 0.3514 - val_loss: 1.7294 - val_acc: 0.4085\n",
      "Epoch 47/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8232 - acc: 0.3526 - val_loss: 1.7275 - val_acc: 0.4096\n",
      "Epoch 48/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8209 - acc: 0.3525 - val_loss: 1.7255 - val_acc: 0.4099\n",
      "Epoch 49/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8231 - acc: 0.3525 - val_loss: 1.7237 - val_acc: 0.4097\n",
      "Epoch 50/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8148 - acc: 0.3552 - val_loss: 1.7220 - val_acc: 0.4089\n",
      "Epoch 51/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8169 - acc: 0.3511 - val_loss: 1.7200 - val_acc: 0.4091\n",
      "Epoch 52/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8140 - acc: 0.3544 - val_loss: 1.7185 - val_acc: 0.4113\n",
      "Epoch 53/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.8155 - acc: 0.3539 - val_loss: 1.7169 - val_acc: 0.4110\n",
      "Epoch 54/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8138 - acc: 0.3514 - val_loss: 1.7154 - val_acc: 0.4111\n",
      "Epoch 55/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8098 - acc: 0.3552 - val_loss: 1.7137 - val_acc: 0.4101\n",
      "Epoch 56/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8081 - acc: 0.3543 - val_loss: 1.7122 - val_acc: 0.4115\n",
      "Epoch 57/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8126 - acc: 0.3546 - val_loss: 1.7111 - val_acc: 0.4108\n",
      "Epoch 58/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8031 - acc: 0.3580 - val_loss: 1.7095 - val_acc: 0.4104\n",
      "Epoch 59/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8075 - acc: 0.3579 - val_loss: 1.7082 - val_acc: 0.4124\n",
      "Epoch 60/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8054 - acc: 0.3572 - val_loss: 1.7068 - val_acc: 0.4119\n",
      "Epoch 61/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.8066 - acc: 0.3564 - val_loss: 1.7056 - val_acc: 0.4121\n",
      "Epoch 62/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8015 - acc: 0.3578 - val_loss: 1.7044 - val_acc: 0.4124\n",
      "Epoch 63/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8021 - acc: 0.3567 - val_loss: 1.7033 - val_acc: 0.4116\n",
      "Epoch 64/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.8050 - acc: 0.3569 - val_loss: 1.7018 - val_acc: 0.4149\n",
      "Epoch 65/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8027 - acc: 0.3566 - val_loss: 1.7009 - val_acc: 0.4142\n",
      "Epoch 66/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8007 - acc: 0.3570 - val_loss: 1.6997 - val_acc: 0.4151\n",
      "Epoch 67/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7991 - acc: 0.3555 - val_loss: 1.6987 - val_acc: 0.4138\n",
      "Epoch 68/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.8017 - acc: 0.3546 - val_loss: 1.6977 - val_acc: 0.4138\n",
      "Epoch 69/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7975 - acc: 0.3590 - val_loss: 1.6967 - val_acc: 0.4143\n",
      "Epoch 70/100\n",
      "40000/40000 [==============================] - 1s 31us/step - loss: 1.7995 - acc: 0.3577 - val_loss: 1.6957 - val_acc: 0.4145\n",
      "Epoch 71/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7963 - acc: 0.3585 - val_loss: 1.6947 - val_acc: 0.4154\n",
      "Epoch 72/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7958 - acc: 0.3589 - val_loss: 1.6939 - val_acc: 0.4139\n",
      "Epoch 73/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7958 - acc: 0.3616 - val_loss: 1.6930 - val_acc: 0.4149\n",
      "Epoch 74/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7947 - acc: 0.3576 - val_loss: 1.6921 - val_acc: 0.4144\n",
      "Epoch 75/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7926 - acc: 0.3619 - val_loss: 1.6913 - val_acc: 0.4153\n",
      "Epoch 76/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7934 - acc: 0.3594 - val_loss: 1.6903 - val_acc: 0.4155\n",
      "Epoch 77/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7879 - acc: 0.3612 - val_loss: 1.6895 - val_acc: 0.4148\n",
      "Epoch 78/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7932 - acc: 0.3633 - val_loss: 1.6888 - val_acc: 0.4150\n",
      "Epoch 79/100\n",
      "40000/40000 [==============================] - 1s 29us/step - loss: 1.7918 - acc: 0.3588 - val_loss: 1.6881 - val_acc: 0.4157\n",
      "Epoch 80/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7938 - acc: 0.3618 - val_loss: 1.6874 - val_acc: 0.4170\n",
      "Epoch 81/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7919 - acc: 0.3582 - val_loss: 1.6866 - val_acc: 0.4156\n",
      "Epoch 82/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7887 - acc: 0.3624 - val_loss: 1.6858 - val_acc: 0.4167\n",
      "Epoch 83/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7900 - acc: 0.3599 - val_loss: 1.6851 - val_acc: 0.4180\n",
      "Epoch 84/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7883 - acc: 0.3622 - val_loss: 1.6842 - val_acc: 0.4176\n",
      "Epoch 85/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7890 - acc: 0.3617 - val_loss: 1.6837 - val_acc: 0.4161\n",
      "Epoch 86/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7893 - acc: 0.3612 - val_loss: 1.6832 - val_acc: 0.4171\n",
      "Epoch 87/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7864 - acc: 0.3612 - val_loss: 1.6825 - val_acc: 0.4182\n",
      "Epoch 88/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7863 - acc: 0.3629 - val_loss: 1.6819 - val_acc: 0.4165\n",
      "Epoch 89/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7876 - acc: 0.3597 - val_loss: 1.6813 - val_acc: 0.4158\n",
      "Epoch 90/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7853 - acc: 0.3638 - val_loss: 1.6806 - val_acc: 0.4166\n",
      "Epoch 91/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7842 - acc: 0.3636 - val_loss: 1.6801 - val_acc: 0.4172\n",
      "Epoch 92/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7847 - acc: 0.3642 - val_loss: 1.6794 - val_acc: 0.4173\n",
      "Epoch 93/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7797 - acc: 0.3635 - val_loss: 1.6787 - val_acc: 0.4177\n",
      "Epoch 94/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7845 - acc: 0.3628 - val_loss: 1.6783 - val_acc: 0.4171\n",
      "Epoch 95/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7873 - acc: 0.3641 - val_loss: 1.6778 - val_acc: 0.4194\n",
      "Epoch 96/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7814 - acc: 0.3612 - val_loss: 1.6772 - val_acc: 0.4175\n",
      "Epoch 97/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7780 - acc: 0.3680 - val_loss: 1.6766 - val_acc: 0.4173\n",
      "Epoch 98/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7791 - acc: 0.3656 - val_loss: 1.6760 - val_acc: 0.4180\n",
      "Epoch 99/100\n",
      "40000/40000 [==============================] - 1s 28us/step - loss: 1.7816 - acc: 0.3640 - val_loss: 1.6756 - val_acc: 0.4177\n",
      "Epoch 100/100\n",
      "40000/40000 [==============================] - 1s 27us/step - loss: 1.7816 - acc: 0.3637 - val_loss: 1.6750 - val_acc: 0.4183\n",
      "10000/10000 [==============================] - 0s 28us/step\n",
      "Validation loss: 1.674973847579956\n",
      "Validation accuracy (NORMALIZED): 0.4183\n",
      "[[1.6816067134857178, 0.4202], [1.6863978813171387, 0.4201], [1.6700160438537597, 0.4208], [1.678535626602173, 0.4196], [1.674973847579956, 0.4183]]\n"
     ]
    }
   ],
   "source": [
    "#=====================================\n",
    "# Training model and Evaluation\n",
    "#=====================================\n",
    "\n",
    "y_train_categorical = to_categorical(y_train, num_classes=n_classes)\n",
    "y_val_categorical = to_categorical(y_val, num_classes=n_classes)\n",
    "\n",
    "scores = []\n",
    "for i in range(folds):\n",
    "    #--- Loading model\n",
    "    model  = get_squeezenet_ft()\n",
    "\n",
    "    \n",
    "    #--- Evaluating the model for split i\n",
    "    print(\"================================================\")\n",
    "    print(\"Split: \"+str(i))\n",
    "    print(\"Number of Epochs: \"+str(n_epochs))\n",
    "    print(\"Train shape:\",X_train[i].shape)\n",
    "    print(\"Train batch size: \"+str(train_batch_size))\n",
    "    print(\"Val batch size: \"+str(val_batch_size))\n",
    "    print(\"================================================\")\n",
    "    \n",
    "    #=====================================\n",
    "    # Tensorboard callback\n",
    "    #=====================================\n",
    "    print(\"--- Preparing tensorboard\")\n",
    "    log_dir = \"logs/{}_fold[{}]_{}\".format(time(), i, 'pre-trained')\n",
    "    print(\"Log Dir: \", log_dir)\n",
    "    tbCallBack = TensorBoard(log_dir=log_dir, write_graph=True)    \n",
    "    \n",
    "    # Training without data augmentation\n",
    "    model.fit(X_train[i],y_train_categorical[i], batch_size=train_batch_size, \n",
    "              epochs=n_epochs, \n",
    "              validation_data=(X_val[i],y_val_categorical[i]),\n",
    "              verbose=1,\n",
    "              callbacks=[tbCallBack])\n",
    "    \n",
    "    #--- Evaluating the model for split i\n",
    "    score = model.evaluate(x=X_val[i], y=y_val_categorical[i], batch_size=val_batch_size, verbose=1)\n",
    "    scores.append(score)\n",
    "    print('Validation loss:', score[0])\n",
    "    print('Validation accuracy (NORMALIZED):', score[1])\n",
    "    \n",
    "#--- Showing scores\n",
    "print(scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, let's evaluate on our test set:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Validation loss: 1.678306022567749\n",
      "Mean Validation accuracy (NORMALIZED): 0.41980000000000006\n"
     ]
    }
   ],
   "source": [
    "#=====================================\n",
    "# Evaluate on validation\n",
    "#=====================================\n",
    "\n",
    "#--- The evaluation of the model \n",
    "np_aux = np.array(scores).mean(axis=0)\n",
    "\n",
    "print('Mean Validation loss:', np_aux[0])\n",
    "print('Mean Validation accuracy (NORMALIZED):', np_aux[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-----------------\n",
    "-----------------\n",
    "\n",
    "# Training last 2 Fire Modules + classification layers\n",
    "As we could see, the frozen network performed very poorly. By freezing most layers, we do not allow SqueezeNet to adapt its weights to features present in CIFAR-10.\n",
    "\n",
    "Let's try to unfreeze the last two fire modules and train once more. The architecture will be:\n",
    "<img src=\"partFrozenSqueezeNet.png\" width=70% height=70%>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_7 (InputLayer)            (None, 32, 32, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv1 (Conv2D)                  (None, 15, 15, 64)   1792        input_7[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "relu_conv1 (Activation)         (None, 15, 15, 64)   0           conv1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "pool1 (MaxPooling2D)            (None, 7, 7, 64)     0           relu_conv1[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "fire2/squeeze1x1 (Conv2D)       (None, 7, 7, 16)     1040        pool1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "fire2/relu_squeeze1x1 (Activati (None, 7, 7, 16)     0           fire2/squeeze1x1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "fire2/expand1x1 (Conv2D)        (None, 7, 7, 64)     1088        fire2/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire2/expand3x3 (Conv2D)        (None, 7, 7, 64)     9280        fire2/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire2/relu_expand1x1 (Activatio (None, 7, 7, 64)     0           fire2/expand1x1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire2/relu_expand3x3 (Activatio (None, 7, 7, 64)     0           fire2/expand3x3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire2/concat (Concatenate)      (None, 7, 7, 128)    0           fire2/relu_expand1x1[0][0]       \n",
      "                                                                 fire2/relu_expand3x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "fire3/squeeze1x1 (Conv2D)       (None, 7, 7, 16)     2064        fire2/concat[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "fire3/relu_squeeze1x1 (Activati (None, 7, 7, 16)     0           fire3/squeeze1x1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "fire3/expand1x1 (Conv2D)        (None, 7, 7, 64)     1088        fire3/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire3/expand3x3 (Conv2D)        (None, 7, 7, 64)     9280        fire3/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire3/relu_expand1x1 (Activatio (None, 7, 7, 64)     0           fire3/expand1x1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire3/relu_expand3x3 (Activatio (None, 7, 7, 64)     0           fire3/expand3x3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire3/concat (Concatenate)      (None, 7, 7, 128)    0           fire3/relu_expand1x1[0][0]       \n",
      "                                                                 fire3/relu_expand3x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "pool3 (MaxPooling2D)            (None, 3, 3, 128)    0           fire3/concat[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "fire4/squeeze1x1 (Conv2D)       (None, 3, 3, 32)     4128        pool3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "fire4/relu_squeeze1x1 (Activati (None, 3, 3, 32)     0           fire4/squeeze1x1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "fire4/expand1x1 (Conv2D)        (None, 3, 3, 128)    4224        fire4/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire4/expand3x3 (Conv2D)        (None, 3, 3, 128)    36992       fire4/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire4/relu_expand1x1 (Activatio (None, 3, 3, 128)    0           fire4/expand1x1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire4/relu_expand3x3 (Activatio (None, 3, 3, 128)    0           fire4/expand3x3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire4/concat (Concatenate)      (None, 3, 3, 256)    0           fire4/relu_expand1x1[0][0]       \n",
      "                                                                 fire4/relu_expand3x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "fire5/squeeze1x1 (Conv2D)       (None, 3, 3, 32)     8224        fire4/concat[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "fire5/relu_squeeze1x1 (Activati (None, 3, 3, 32)     0           fire5/squeeze1x1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "fire5/expand1x1 (Conv2D)        (None, 3, 3, 128)    4224        fire5/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire5/expand3x3 (Conv2D)        (None, 3, 3, 128)    36992       fire5/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire5/relu_expand1x1 (Activatio (None, 3, 3, 128)    0           fire5/expand1x1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire5/relu_expand3x3 (Activatio (None, 3, 3, 128)    0           fire5/expand3x3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire5/concat (Concatenate)      (None, 3, 3, 256)    0           fire5/relu_expand1x1[0][0]       \n",
      "                                                                 fire5/relu_expand3x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "pool5 (MaxPooling2D)            (None, 1, 1, 256)    0           fire5/concat[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "fire6/squeeze1x1 (Conv2D)       (None, 1, 1, 48)     12336       pool5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "fire6/relu_squeeze1x1 (Activati (None, 1, 1, 48)     0           fire6/squeeze1x1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "fire6/expand1x1 (Conv2D)        (None, 1, 1, 192)    9408        fire6/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire6/expand3x3 (Conv2D)        (None, 1, 1, 192)    83136       fire6/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire6/relu_expand1x1 (Activatio (None, 1, 1, 192)    0           fire6/expand1x1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire6/relu_expand3x3 (Activatio (None, 1, 1, 192)    0           fire6/expand3x3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire6/concat (Concatenate)      (None, 1, 1, 384)    0           fire6/relu_expand1x1[0][0]       \n",
      "                                                                 fire6/relu_expand3x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "fire7/squeeze1x1 (Conv2D)       (None, 1, 1, 48)     18480       fire6/concat[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "fire7/relu_squeeze1x1 (Activati (None, 1, 1, 48)     0           fire7/squeeze1x1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "fire7/expand1x1 (Conv2D)        (None, 1, 1, 192)    9408        fire7/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire7/expand3x3 (Conv2D)        (None, 1, 1, 192)    83136       fire7/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire7/relu_expand1x1 (Activatio (None, 1, 1, 192)    0           fire7/expand1x1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire7/relu_expand3x3 (Activatio (None, 1, 1, 192)    0           fire7/expand3x3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire7/concat (Concatenate)      (None, 1, 1, 384)    0           fire7/relu_expand1x1[0][0]       \n",
      "                                                                 fire7/relu_expand3x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "fire8/squeeze1x1 (Conv2D)       (None, 1, 1, 64)     24640       fire7/concat[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "fire8/relu_squeeze1x1 (Activati (None, 1, 1, 64)     0           fire8/squeeze1x1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "fire8/expand1x1 (Conv2D)        (None, 1, 1, 256)    16640       fire8/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire8/expand3x3 (Conv2D)        (None, 1, 1, 256)    147712      fire8/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire8/relu_expand1x1 (Activatio (None, 1, 1, 256)    0           fire8/expand1x1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire8/relu_expand3x3 (Activatio (None, 1, 1, 256)    0           fire8/expand3x3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire8/concat (Concatenate)      (None, 1, 1, 512)    0           fire8/relu_expand1x1[0][0]       \n",
      "                                                                 fire8/relu_expand3x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "fire9/squeeze1x1 (Conv2D)       (None, 1, 1, 64)     32832       fire8/concat[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "fire9/relu_squeeze1x1 (Activati (None, 1, 1, 64)     0           fire9/squeeze1x1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "fire9/expand1x1 (Conv2D)        (None, 1, 1, 256)    16640       fire9/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire9/expand3x3 (Conv2D)        (None, 1, 1, 256)    147712      fire9/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire9/relu_expand1x1 (Activatio (None, 1, 1, 256)    0           fire9/expand1x1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire9/relu_expand3x3 (Activatio (None, 1, 1, 256)    0           fire9/expand3x3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire9/concat (Concatenate)      (None, 1, 1, 512)    0           fire9/relu_expand1x1[0][0]       \n",
      "                                                                 fire9/relu_expand3x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "drop9 (Dropout)                 (None, 1, 1, 512)    0           fire9/concat[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv10 (Conv2D)                 (None, 1, 1, 10)     5130        drop9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "relu_conv10 (Activation)        (None, 1, 1, 10)     0           conv10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling2d_14 (Gl (None, 10)           0           relu_conv10[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "loss (Activation)               (None, 10)           0           global_average_pooling2d_14[0][0]\n",
      "==================================================================================================\n",
      "Total params: 727,626\n",
      "Trainable params: 391,306\n",
      "Non-trainable params: 336,320\n",
      "__________________________________________________________________________________________________\n",
      "input_7 False\n",
      "conv1 False\n",
      "relu_conv1 False\n",
      "pool1 False\n",
      "fire2/squeeze1x1 False\n",
      "fire2/relu_squeeze1x1 False\n",
      "fire2/expand1x1 False\n",
      "fire2/expand3x3 False\n",
      "fire2/relu_expand1x1 False\n",
      "fire2/relu_expand3x3 False\n",
      "fire2/concat False\n",
      "fire3/squeeze1x1 False\n",
      "fire3/relu_squeeze1x1 False\n",
      "fire3/expand1x1 False\n",
      "fire3/expand3x3 False\n",
      "fire3/relu_expand1x1 False\n",
      "fire3/relu_expand3x3 False\n",
      "fire3/concat False\n",
      "pool3 False\n",
      "fire4/squeeze1x1 False\n",
      "fire4/relu_squeeze1x1 False\n",
      "fire4/expand1x1 False\n",
      "fire4/expand3x3 False\n",
      "fire4/relu_expand1x1 False\n",
      "fire4/relu_expand3x3 False\n",
      "fire4/concat False\n",
      "fire5/squeeze1x1 False\n",
      "fire5/relu_squeeze1x1 False\n",
      "fire5/expand1x1 False\n",
      "fire5/expand3x3 False\n",
      "fire5/relu_expand1x1 False\n",
      "fire5/relu_expand3x3 False\n",
      "fire5/concat False\n",
      "pool5 False\n",
      "fire6/squeeze1x1 False\n",
      "fire6/relu_squeeze1x1 False\n",
      "fire6/expand1x1 False\n",
      "fire6/expand3x3 False\n",
      "fire6/relu_expand1x1 False\n",
      "fire6/relu_expand3x3 False\n",
      "fire6/concat False\n",
      "fire7/squeeze1x1 False\n",
      "fire7/relu_squeeze1x1 False\n",
      "fire7/expand1x1 False\n",
      "fire7/expand3x3 False\n",
      "fire7/relu_expand1x1 False\n",
      "fire7/relu_expand3x3 False\n",
      "fire7/concat False\n",
      "fire8/squeeze1x1 True\n",
      "fire8/relu_squeeze1x1 True\n",
      "fire8/expand1x1 True\n",
      "fire8/expand3x3 True\n",
      "fire8/relu_expand1x1 True\n",
      "fire8/relu_expand3x3 True\n",
      "fire8/concat True\n",
      "fire9/squeeze1x1 True\n",
      "fire9/relu_squeeze1x1 True\n",
      "fire9/expand1x1 True\n",
      "fire9/expand3x3 True\n",
      "fire9/relu_expand1x1 True\n",
      "fire9/relu_expand3x3 True\n",
      "fire9/concat True\n",
      "drop9 True\n",
      "conv10 True\n",
      "relu_conv10 True\n",
      "global_average_pooling2d_14 True\n",
      "loss True\n"
     ]
    }
   ],
   "source": [
    "def get_squeezenet_ft2():\n",
    "    squeezeNetModel = SqueezeNet((32,32,3))\n",
    "\n",
    "    #=====================================\n",
    "    # Freezing mentioned layers\n",
    "    #=====================================\n",
    "\n",
    "    trainable_layer_index = 19\n",
    "    for i in range(len(squeezeNetModel.layers)-trainable_layer_index):\n",
    "        squeezeNetModel.layers[i].trainable = False\n",
    "\n",
    "    #=====================================\n",
    "    # Add new classification layers\n",
    "    #=====================================\n",
    "\n",
    "    #--- Removing layers until drop 9\n",
    "    squeezeNetModel.layers.pop() #Convolution2D\n",
    "    squeezeNetModel.layers.pop() #Activation ReLU\n",
    "    squeezeNetModel.layers.pop() #Global Avg Pool\n",
    "    squeezeNetModel.layers.pop() #Activation Softmax\n",
    "\n",
    "    #--- Adding classification layer for 10 classes\n",
    "    out = Convolution2D(n_classes, (1, 1), padding='valid', name='conv10')(squeezeNetModel.layers[-1].output)\n",
    "    out = Activation('relu', name='relu_conv10')(out)\n",
    "\n",
    "    out = GlobalAveragePooling2D()(out)\n",
    "    out = Activation('softmax', name='loss')(out)\n",
    "\n",
    "    #=====================================\n",
    "    # New Model\n",
    "    #=====================================\n",
    "    model = Model(squeezeNetModel.inputs, out, name='squeezenet_new')\n",
    "    opt = optimizer=optimizers.Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)#SGD(lr=learning_rate)\n",
    "    model.compile(loss='categorical_crossentropy',\n",
    "              optimizer=opt,\n",
    "              metrics=['accuracy'])\n",
    "    return model\n",
    "\n",
    "model = get_squeezenet_ft2()\n",
    "model.summary()\n",
    "\n",
    "#--- Check the trainable status of the individual layers\n",
    "for layer in model.layers:\n",
    "    print(layer.name, layer.trainable)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we compile our model and train it:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "================================================\n",
      "Split: 0\n",
      "Number of Epochs: 100\n",
      "Train shape: (40000, 32, 32, 3)\n",
      "Train batch size: 384\n",
      "Val batch size: 128\n",
      "================================================\n",
      "--- Preparing tensorboard\n",
      "Log Dir:  logs/1542820925.149155_fold[0]_fine-tunned\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/100\n",
      "40000/40000 [==============================] - 2s 46us/step - loss: 2.2352 - acc: 0.1857 - val_loss: 2.0410 - val_acc: 0.3121\n",
      "Epoch 2/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.8980 - acc: 0.3316 - val_loss: 1.6625 - val_acc: 0.4227\n",
      "Epoch 3/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.6973 - acc: 0.3955 - val_loss: 1.5590 - val_acc: 0.4511\n",
      "Epoch 4/100\n",
      "40000/40000 [==============================] - 2s 49us/step - loss: 1.6158 - acc: 0.4245 - val_loss: 1.5063 - val_acc: 0.4661\n",
      "Epoch 5/100\n",
      "40000/40000 [==============================] - 2s 56us/step - loss: 1.5680 - acc: 0.4431 - val_loss: 1.4784 - val_acc: 0.4794\n",
      "Epoch 6/100\n",
      "40000/40000 [==============================] - 2s 39us/step - loss: 1.5309 - acc: 0.4546 - val_loss: 1.4516 - val_acc: 0.4885\n",
      "Epoch 7/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.5086 - acc: 0.4668 - val_loss: 1.4342 - val_acc: 0.4923\n",
      "Epoch 8/100\n",
      "40000/40000 [==============================] - 2s 62us/step - loss: 1.4894 - acc: 0.4747 - val_loss: 1.4207 - val_acc: 0.4962\n",
      "Epoch 9/100\n",
      "40000/40000 [==============================] - 2s 41us/step - loss: 1.4725 - acc: 0.4807 - val_loss: 1.4114 - val_acc: 0.4999\n",
      "Epoch 10/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.4575 - acc: 0.4865 - val_loss: 1.4001 - val_acc: 0.4996\n",
      "Epoch 11/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.4459 - acc: 0.4887 - val_loss: 1.3944 - val_acc: 0.5047\n",
      "Epoch 12/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.4378 - acc: 0.4945 - val_loss: 1.3861 - val_acc: 0.5043\n",
      "Epoch 13/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.4269 - acc: 0.4951 - val_loss: 1.3793 - val_acc: 0.5075\n",
      "Epoch 14/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.4151 - acc: 0.4997 - val_loss: 1.3740 - val_acc: 0.5117\n",
      "Epoch 15/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.4084 - acc: 0.5024 - val_loss: 1.3690 - val_acc: 0.5130\n",
      "Epoch 16/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3971 - acc: 0.5070 - val_loss: 1.3651 - val_acc: 0.5144\n",
      "Epoch 17/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3938 - acc: 0.5089 - val_loss: 1.3575 - val_acc: 0.5190\n",
      "Epoch 18/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3873 - acc: 0.5100 - val_loss: 1.3568 - val_acc: 0.5203\n",
      "Epoch 19/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3794 - acc: 0.5121 - val_loss: 1.3520 - val_acc: 0.5200\n",
      "Epoch 20/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.3727 - acc: 0.5162 - val_loss: 1.3498 - val_acc: 0.5218\n",
      "Epoch 21/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.3733 - acc: 0.5140 - val_loss: 1.3445 - val_acc: 0.5234\n",
      "Epoch 22/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.3629 - acc: 0.5180 - val_loss: 1.3466 - val_acc: 0.5218\n",
      "Epoch 23/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3589 - acc: 0.5206 - val_loss: 1.3408 - val_acc: 0.5259\n",
      "Epoch 24/100\n",
      "40000/40000 [==============================] - 2s 39us/step - loss: 1.3543 - acc: 0.5226 - val_loss: 1.3377 - val_acc: 0.5248\n",
      "Epoch 25/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3505 - acc: 0.5244 - val_loss: 1.3339 - val_acc: 0.5267\n",
      "Epoch 26/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3461 - acc: 0.5273 - val_loss: 1.3349 - val_acc: 0.5266\n",
      "Epoch 27/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.3395 - acc: 0.5279 - val_loss: 1.3295 - val_acc: 0.5283\n",
      "Epoch 28/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.3352 - acc: 0.5282 - val_loss: 1.3264 - val_acc: 0.5296\n",
      "Epoch 29/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.3303 - acc: 0.5299 - val_loss: 1.3235 - val_acc: 0.5293\n",
      "Epoch 30/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.3303 - acc: 0.5305 - val_loss: 1.3232 - val_acc: 0.5301\n",
      "Epoch 31/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.3243 - acc: 0.5320 - val_loss: 1.3217 - val_acc: 0.5327\n",
      "Epoch 32/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.3245 - acc: 0.5328 - val_loss: 1.3212 - val_acc: 0.5298\n",
      "Epoch 33/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3214 - acc: 0.5346 - val_loss: 1.3170 - val_acc: 0.5339\n",
      "Epoch 34/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3136 - acc: 0.5361 - val_loss: 1.3147 - val_acc: 0.5361\n",
      "Epoch 35/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3152 - acc: 0.5369 - val_loss: 1.3138 - val_acc: 0.5355\n",
      "Epoch 36/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3097 - acc: 0.5385 - val_loss: 1.3175 - val_acc: 0.5323\n",
      "Epoch 37/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3062 - acc: 0.5382 - val_loss: 1.3094 - val_acc: 0.5355\n",
      "Epoch 38/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3064 - acc: 0.5388 - val_loss: 1.3076 - val_acc: 0.5370\n",
      "Epoch 39/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3006 - acc: 0.5422 - val_loss: 1.3101 - val_acc: 0.5337\n",
      "Epoch 40/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3011 - acc: 0.5411 - val_loss: 1.3084 - val_acc: 0.5397\n",
      "Epoch 41/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2975 - acc: 0.5431 - val_loss: 1.3054 - val_acc: 0.5383\n",
      "Epoch 42/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2946 - acc: 0.5446 - val_loss: 1.3021 - val_acc: 0.5374\n",
      "Epoch 43/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2910 - acc: 0.5432 - val_loss: 1.3044 - val_acc: 0.5387\n",
      "Epoch 44/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2879 - acc: 0.5475 - val_loss: 1.3011 - val_acc: 0.5399\n",
      "Epoch 45/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2864 - acc: 0.5441 - val_loss: 1.2992 - val_acc: 0.5398\n",
      "Epoch 46/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2833 - acc: 0.5493 - val_loss: 1.2987 - val_acc: 0.5427\n",
      "Epoch 47/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2812 - acc: 0.5470 - val_loss: 1.2997 - val_acc: 0.5438\n",
      "Epoch 48/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.2818 - acc: 0.5476 - val_loss: 1.2984 - val_acc: 0.5415\n",
      "Epoch 49/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2770 - acc: 0.5484 - val_loss: 1.2952 - val_acc: 0.5439\n",
      "Epoch 50/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2721 - acc: 0.5503 - val_loss: 1.2965 - val_acc: 0.5408\n",
      "Epoch 51/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.2727 - acc: 0.5506 - val_loss: 1.2980 - val_acc: 0.5434\n",
      "Epoch 52/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2677 - acc: 0.5527 - val_loss: 1.3032 - val_acc: 0.5394\n",
      "Epoch 53/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2690 - acc: 0.5523 - val_loss: 1.2933 - val_acc: 0.5430\n",
      "Epoch 54/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2667 - acc: 0.5523 - val_loss: 1.2911 - val_acc: 0.5434\n",
      "Epoch 55/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2641 - acc: 0.5526 - val_loss: 1.2919 - val_acc: 0.5438\n",
      "Epoch 56/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2617 - acc: 0.5533 - val_loss: 1.2886 - val_acc: 0.5464\n",
      "Epoch 57/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2593 - acc: 0.5548 - val_loss: 1.2886 - val_acc: 0.5466\n",
      "Epoch 58/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2578 - acc: 0.5546 - val_loss: 1.2906 - val_acc: 0.5469\n",
      "Epoch 59/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2591 - acc: 0.5560 - val_loss: 1.2879 - val_acc: 0.5475\n",
      "Epoch 60/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2525 - acc: 0.5565 - val_loss: 1.2914 - val_acc: 0.5448\n",
      "Epoch 61/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2503 - acc: 0.5576 - val_loss: 1.2865 - val_acc: 0.5467\n",
      "Epoch 62/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2517 - acc: 0.5575 - val_loss: 1.2862 - val_acc: 0.5513\n",
      "Epoch 63/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2476 - acc: 0.5608 - val_loss: 1.2877 - val_acc: 0.5445\n",
      "Epoch 64/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2466 - acc: 0.5594 - val_loss: 1.2871 - val_acc: 0.5497\n",
      "Epoch 65/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2462 - acc: 0.5595 - val_loss: 1.2841 - val_acc: 0.5470\n",
      "Epoch 66/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2416 - acc: 0.5609 - val_loss: 1.2882 - val_acc: 0.5466\n",
      "Epoch 67/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2412 - acc: 0.5625 - val_loss: 1.2843 - val_acc: 0.5503\n",
      "Epoch 68/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2383 - acc: 0.5635 - val_loss: 1.2821 - val_acc: 0.5493\n",
      "Epoch 69/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.2362 - acc: 0.5612 - val_loss: 1.2859 - val_acc: 0.5488\n",
      "Epoch 70/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2369 - acc: 0.5633 - val_loss: 1.2799 - val_acc: 0.5499\n",
      "Epoch 71/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2325 - acc: 0.5640 - val_loss: 1.2786 - val_acc: 0.5502\n",
      "Epoch 72/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2332 - acc: 0.5653 - val_loss: 1.2819 - val_acc: 0.5482\n",
      "Epoch 73/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2295 - acc: 0.5655 - val_loss: 1.2799 - val_acc: 0.5481\n",
      "Epoch 74/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2312 - acc: 0.5659 - val_loss: 1.2813 - val_acc: 0.5477\n",
      "Epoch 75/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2295 - acc: 0.5648 - val_loss: 1.2786 - val_acc: 0.5519\n",
      "Epoch 76/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2270 - acc: 0.5683 - val_loss: 1.2786 - val_acc: 0.5488\n",
      "Epoch 77/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2264 - acc: 0.5685 - val_loss: 1.2798 - val_acc: 0.5493\n",
      "Epoch 78/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2233 - acc: 0.5665 - val_loss: 1.2816 - val_acc: 0.5488\n",
      "Epoch 79/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2190 - acc: 0.5678 - val_loss: 1.2805 - val_acc: 0.5494\n",
      "Epoch 80/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2190 - acc: 0.5683 - val_loss: 1.2751 - val_acc: 0.5504\n",
      "Epoch 81/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2188 - acc: 0.5688 - val_loss: 1.2738 - val_acc: 0.5519\n",
      "Epoch 82/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2140 - acc: 0.5689 - val_loss: 1.2748 - val_acc: 0.5527\n",
      "Epoch 83/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2152 - acc: 0.5700 - val_loss: 1.2728 - val_acc: 0.5530\n",
      "Epoch 84/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.2124 - acc: 0.5719 - val_loss: 1.2736 - val_acc: 0.5552\n",
      "Epoch 85/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.2122 - acc: 0.5711 - val_loss: 1.2746 - val_acc: 0.5536\n",
      "Epoch 86/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.2090 - acc: 0.5715 - val_loss: 1.2721 - val_acc: 0.5534\n",
      "Epoch 87/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.2083 - acc: 0.5726 - val_loss: 1.2722 - val_acc: 0.5539\n",
      "Epoch 88/100\n",
      "40000/40000 [==============================] - 2s 38us/step - loss: 1.2066 - acc: 0.5729 - val_loss: 1.2694 - val_acc: 0.5547\n",
      "Epoch 89/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2044 - acc: 0.5736 - val_loss: 1.2725 - val_acc: 0.5547\n",
      "Epoch 90/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2024 - acc: 0.5744 - val_loss: 1.2727 - val_acc: 0.5530\n",
      "Epoch 91/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2031 - acc: 0.5743 - val_loss: 1.2733 - val_acc: 0.5514\n",
      "Epoch 92/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2016 - acc: 0.5739 - val_loss: 1.2744 - val_acc: 0.5506\n",
      "Epoch 93/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2002 - acc: 0.5750 - val_loss: 1.2732 - val_acc: 0.5531\n",
      "Epoch 94/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.1980 - acc: 0.5752 - val_loss: 1.2699 - val_acc: 0.5537\n",
      "Epoch 95/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.1957 - acc: 0.5758 - val_loss: 1.2680 - val_acc: 0.5551\n",
      "Epoch 96/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.1966 - acc: 0.5760 - val_loss: 1.2706 - val_acc: 0.5532\n",
      "Epoch 97/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.1962 - acc: 0.5755 - val_loss: 1.2761 - val_acc: 0.5529\n",
      "Epoch 98/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.1937 - acc: 0.5772 - val_loss: 1.2709 - val_acc: 0.5548\n",
      "Epoch 99/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.1912 - acc: 0.5792 - val_loss: 1.2713 - val_acc: 0.5537\n",
      "Epoch 100/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.1894 - acc: 0.5782 - val_loss: 1.2713 - val_acc: 0.5518\n",
      "10000/10000 [==============================] - 0s 29us/step\n",
      "Validation loss: 1.2712621648788451\n",
      "Validation accuracy (NORMALIZED): 0.5518\n",
      "================================================\n",
      "Split: 1\n",
      "Number of Epochs: 100\n",
      "Train shape: (40000, 32, 32, 3)\n",
      "Train batch size: 384\n",
      "Val batch size: 128\n",
      "================================================\n",
      "--- Preparing tensorboard\n",
      "Log Dir:  logs/1542821070.1326225_fold[1]_fine-tunned\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/100\n",
      "40000/40000 [==============================] - 2s 45us/step - loss: 2.2216 - acc: 0.1956 - val_loss: 2.0486 - val_acc: 0.3121\n",
      "Epoch 2/100\n",
      "40000/40000 [==============================] - 2s 39us/step - loss: 1.9253 - acc: 0.3275 - val_loss: 1.6808 - val_acc: 0.4181\n",
      "Epoch 3/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.6992 - acc: 0.3920 - val_loss: 1.5629 - val_acc: 0.4475\n",
      "Epoch 4/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.6134 - acc: 0.4246 - val_loss: 1.5137 - val_acc: 0.4596\n",
      "Epoch 5/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.5678 - acc: 0.4404 - val_loss: 1.4903 - val_acc: 0.4712\n",
      "Epoch 6/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.5310 - acc: 0.4587 - val_loss: 1.4635 - val_acc: 0.4810\n",
      "Epoch 7/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.5055 - acc: 0.4668 - val_loss: 1.4444 - val_acc: 0.4868\n",
      "Epoch 8/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.4863 - acc: 0.4729 - val_loss: 1.4358 - val_acc: 0.4884\n",
      "Epoch 9/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.4689 - acc: 0.4811 - val_loss: 1.4218 - val_acc: 0.4950\n",
      "Epoch 10/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.4562 - acc: 0.4836 - val_loss: 1.4116 - val_acc: 0.4977\n",
      "Epoch 11/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.4403 - acc: 0.4917 - val_loss: 1.4024 - val_acc: 0.5012\n",
      "Epoch 12/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.4308 - acc: 0.4956 - val_loss: 1.3954 - val_acc: 0.5050\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 13/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.4217 - acc: 0.4979 - val_loss: 1.3875 - val_acc: 0.5046\n",
      "Epoch 14/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.4142 - acc: 0.4980 - val_loss: 1.3875 - val_acc: 0.5052\n",
      "Epoch 15/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.4052 - acc: 0.5047 - val_loss: 1.3764 - val_acc: 0.5111\n",
      "Epoch 16/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3972 - acc: 0.5070 - val_loss: 1.3726 - val_acc: 0.5128\n",
      "Epoch 17/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3887 - acc: 0.5086 - val_loss: 1.3663 - val_acc: 0.5114\n",
      "Epoch 18/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3830 - acc: 0.5106 - val_loss: 1.3615 - val_acc: 0.5147\n",
      "Epoch 19/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3770 - acc: 0.5152 - val_loss: 1.3600 - val_acc: 0.5171\n",
      "Epoch 20/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3707 - acc: 0.5172 - val_loss: 1.3611 - val_acc: 0.5139\n",
      "Epoch 21/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3665 - acc: 0.5169 - val_loss: 1.3535 - val_acc: 0.5196\n",
      "Epoch 22/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3640 - acc: 0.5203 - val_loss: 1.3521 - val_acc: 0.5199\n",
      "Epoch 23/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3566 - acc: 0.5206 - val_loss: 1.3473 - val_acc: 0.5203\n",
      "Epoch 24/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.3533 - acc: 0.5210 - val_loss: 1.3451 - val_acc: 0.5244\n",
      "Epoch 25/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3503 - acc: 0.5250 - val_loss: 1.3423 - val_acc: 0.5202\n",
      "Epoch 26/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3459 - acc: 0.5245 - val_loss: 1.3384 - val_acc: 0.5233\n",
      "Epoch 27/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3405 - acc: 0.5262 - val_loss: 1.3368 - val_acc: 0.5261\n",
      "Epoch 28/100\n",
      "40000/40000 [==============================] - 2s 39us/step - loss: 1.3355 - acc: 0.5280 - val_loss: 1.3351 - val_acc: 0.5285\n",
      "Epoch 29/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3326 - acc: 0.5304 - val_loss: 1.3341 - val_acc: 0.5280\n",
      "Epoch 30/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3302 - acc: 0.5298 - val_loss: 1.3304 - val_acc: 0.5286\n",
      "Epoch 31/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.3263 - acc: 0.5323 - val_loss: 1.3288 - val_acc: 0.5284\n",
      "Epoch 32/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3242 - acc: 0.5331 - val_loss: 1.3279 - val_acc: 0.5268\n",
      "Epoch 33/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3179 - acc: 0.5378 - val_loss: 1.3290 - val_acc: 0.5288\n",
      "Epoch 34/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3180 - acc: 0.5343 - val_loss: 1.3252 - val_acc: 0.5297\n",
      "Epoch 35/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3133 - acc: 0.5356 - val_loss: 1.3268 - val_acc: 0.5302\n",
      "Epoch 36/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3107 - acc: 0.5383 - val_loss: 1.3202 - val_acc: 0.5325\n",
      "Epoch 37/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.3078 - acc: 0.5411 - val_loss: 1.3187 - val_acc: 0.5327\n",
      "Epoch 38/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3044 - acc: 0.5380 - val_loss: 1.3204 - val_acc: 0.5320\n",
      "Epoch 39/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.3015 - acc: 0.5427 - val_loss: 1.3190 - val_acc: 0.5332\n",
      "Epoch 40/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.2997 - acc: 0.5425 - val_loss: 1.3168 - val_acc: 0.5373\n",
      "Epoch 41/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2973 - acc: 0.5446 - val_loss: 1.3169 - val_acc: 0.5334\n",
      "Epoch 42/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2925 - acc: 0.5442 - val_loss: 1.3167 - val_acc: 0.5347\n",
      "Epoch 43/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2945 - acc: 0.5446 - val_loss: 1.3154 - val_acc: 0.5338\n",
      "Epoch 44/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2895 - acc: 0.5457 - val_loss: 1.3139 - val_acc: 0.5360\n",
      "Epoch 45/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2881 - acc: 0.5450 - val_loss: 1.3167 - val_acc: 0.5332\n",
      "Epoch 46/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.2843 - acc: 0.5467 - val_loss: 1.3136 - val_acc: 0.5375\n",
      "Epoch 47/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2817 - acc: 0.5476 - val_loss: 1.3126 - val_acc: 0.5353\n",
      "Epoch 48/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2824 - acc: 0.5463 - val_loss: 1.3099 - val_acc: 0.5345\n",
      "Epoch 49/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.2801 - acc: 0.5494 - val_loss: 1.3130 - val_acc: 0.5348\n",
      "Epoch 50/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2761 - acc: 0.5474 - val_loss: 1.3075 - val_acc: 0.5384\n",
      "Epoch 51/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.2738 - acc: 0.5489 - val_loss: 1.3055 - val_acc: 0.5372\n",
      "Epoch 52/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2736 - acc: 0.5498 - val_loss: 1.3081 - val_acc: 0.5391\n",
      "Epoch 53/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2705 - acc: 0.5533 - val_loss: 1.3060 - val_acc: 0.5391\n",
      "Epoch 54/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2705 - acc: 0.5498 - val_loss: 1.3066 - val_acc: 0.5394\n",
      "Epoch 55/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2646 - acc: 0.5530 - val_loss: 1.3045 - val_acc: 0.5411\n",
      "Epoch 56/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2632 - acc: 0.5521 - val_loss: 1.3017 - val_acc: 0.5383\n",
      "Epoch 57/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2602 - acc: 0.5564 - val_loss: 1.3022 - val_acc: 0.5400\n",
      "Epoch 58/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2612 - acc: 0.5548 - val_loss: 1.3044 - val_acc: 0.5381\n",
      "Epoch 59/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2624 - acc: 0.5543 - val_loss: 1.3008 - val_acc: 0.5398\n",
      "Epoch 60/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.2573 - acc: 0.5554 - val_loss: 1.3009 - val_acc: 0.5412\n",
      "Epoch 61/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2539 - acc: 0.5590 - val_loss: 1.3035 - val_acc: 0.5395\n",
      "Epoch 62/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2539 - acc: 0.5592 - val_loss: 1.2993 - val_acc: 0.5423\n",
      "Epoch 63/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2506 - acc: 0.5591 - val_loss: 1.2983 - val_acc: 0.5430\n",
      "Epoch 64/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2516 - acc: 0.5587 - val_loss: 1.2999 - val_acc: 0.5428\n",
      "Epoch 65/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2487 - acc: 0.5619 - val_loss: 1.2980 - val_acc: 0.5400\n",
      "Epoch 66/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2470 - acc: 0.5576 - val_loss: 1.2991 - val_acc: 0.5411\n",
      "Epoch 67/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2444 - acc: 0.5605 - val_loss: 1.2993 - val_acc: 0.5422\n",
      "Epoch 68/100\n",
      "40000/40000 [==============================] - 2s 39us/step - loss: 1.2455 - acc: 0.5594 - val_loss: 1.2993 - val_acc: 0.5468\n",
      "Epoch 69/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2435 - acc: 0.5640 - val_loss: 1.2977 - val_acc: 0.5415\n",
      "Epoch 70/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2379 - acc: 0.5622 - val_loss: 1.3012 - val_acc: 0.5408\n",
      "Epoch 71/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2398 - acc: 0.5631 - val_loss: 1.2952 - val_acc: 0.5435\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 72/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2363 - acc: 0.5633 - val_loss: 1.2994 - val_acc: 0.5430\n",
      "Epoch 73/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2359 - acc: 0.5642 - val_loss: 1.2947 - val_acc: 0.5430\n",
      "Epoch 74/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2347 - acc: 0.5628 - val_loss: 1.2967 - val_acc: 0.5456\n",
      "Epoch 75/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2320 - acc: 0.5659 - val_loss: 1.2950 - val_acc: 0.5441\n",
      "Epoch 76/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.2296 - acc: 0.5647 - val_loss: 1.2925 - val_acc: 0.5436\n",
      "Epoch 77/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2301 - acc: 0.5645 - val_loss: 1.3025 - val_acc: 0.5406\n",
      "Epoch 78/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2281 - acc: 0.5669 - val_loss: 1.2955 - val_acc: 0.5459\n",
      "Epoch 79/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2284 - acc: 0.5662 - val_loss: 1.2914 - val_acc: 0.5453\n",
      "Epoch 80/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2204 - acc: 0.5691 - val_loss: 1.2913 - val_acc: 0.5456\n",
      "Epoch 81/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2222 - acc: 0.5655 - val_loss: 1.2937 - val_acc: 0.5431\n",
      "Epoch 82/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2211 - acc: 0.5697 - val_loss: 1.2946 - val_acc: 0.5479\n",
      "Epoch 83/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.2189 - acc: 0.5674 - val_loss: 1.2920 - val_acc: 0.5469\n",
      "Epoch 84/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2204 - acc: 0.5671 - val_loss: 1.2957 - val_acc: 0.5426\n",
      "Epoch 85/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2162 - acc: 0.5685 - val_loss: 1.2926 - val_acc: 0.5494\n",
      "Epoch 86/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2133 - acc: 0.5710 - val_loss: 1.2898 - val_acc: 0.5470\n",
      "Epoch 87/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2149 - acc: 0.5714 - val_loss: 1.2885 - val_acc: 0.5458\n",
      "Epoch 88/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2130 - acc: 0.5715 - val_loss: 1.2914 - val_acc: 0.5480\n",
      "Epoch 89/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2120 - acc: 0.5722 - val_loss: 1.2908 - val_acc: 0.5462\n",
      "Epoch 90/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2098 - acc: 0.5735 - val_loss: 1.2899 - val_acc: 0.5491\n",
      "Epoch 91/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2086 - acc: 0.5728 - val_loss: 1.2871 - val_acc: 0.5485\n",
      "Epoch 92/100\n",
      "40000/40000 [==============================] - 2s 38us/step - loss: 1.2066 - acc: 0.5704 - val_loss: 1.2903 - val_acc: 0.5480\n",
      "Epoch 93/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2048 - acc: 0.5717 - val_loss: 1.2886 - val_acc: 0.5457\n",
      "Epoch 94/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2046 - acc: 0.5729 - val_loss: 1.2918 - val_acc: 0.5412\n",
      "Epoch 95/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2046 - acc: 0.5740 - val_loss: 1.2863 - val_acc: 0.5462\n",
      "Epoch 96/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2024 - acc: 0.5730 - val_loss: 1.2911 - val_acc: 0.5465\n",
      "Epoch 97/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2024 - acc: 0.5749 - val_loss: 1.2874 - val_acc: 0.5519\n",
      "Epoch 98/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.1992 - acc: 0.5758 - val_loss: 1.2882 - val_acc: 0.5506\n",
      "Epoch 99/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.1957 - acc: 0.5765 - val_loss: 1.2888 - val_acc: 0.5491\n",
      "Epoch 100/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.1938 - acc: 0.5755 - val_loss: 1.2851 - val_acc: 0.5467\n",
      "10000/10000 [==============================] - 0s 31us/step\n",
      "Validation loss: 1.2850975257873536\n",
      "Validation accuracy (NORMALIZED): 0.5467\n",
      "================================================\n",
      "Split: 2\n",
      "Number of Epochs: 100\n",
      "Train shape: (40000, 32, 32, 3)\n",
      "Train batch size: 384\n",
      "Val batch size: 128\n",
      "================================================\n",
      "--- Preparing tensorboard\n",
      "Log Dir:  logs/1542821212.3707876_fold[2]_fine-tunned\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/100\n",
      "40000/40000 [==============================] - 2s 48us/step - loss: 2.2761 - acc: 0.1625 - val_loss: 2.0615 - val_acc: 0.3044\n",
      "Epoch 2/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.9309 - acc: 0.3055 - val_loss: 1.7018 - val_acc: 0.4054\n",
      "Epoch 3/100\n",
      "40000/40000 [==============================] - 2s 39us/step - loss: 1.7357 - acc: 0.3758 - val_loss: 1.5782 - val_acc: 0.4463\n",
      "Epoch 4/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.6430 - acc: 0.4101 - val_loss: 1.5210 - val_acc: 0.4595\n",
      "Epoch 5/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.5947 - acc: 0.4279 - val_loss: 1.4841 - val_acc: 0.4705\n",
      "Epoch 6/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.5543 - acc: 0.4456 - val_loss: 1.4553 - val_acc: 0.4771\n",
      "Epoch 7/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.5262 - acc: 0.4572 - val_loss: 1.4402 - val_acc: 0.4866\n",
      "Epoch 8/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.5054 - acc: 0.4672 - val_loss: 1.4165 - val_acc: 0.4904\n",
      "Epoch 9/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.4866 - acc: 0.4755 - val_loss: 1.4067 - val_acc: 0.4967\n",
      "Epoch 10/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.4698 - acc: 0.4791 - val_loss: 1.3985 - val_acc: 0.5011\n",
      "Epoch 11/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.4576 - acc: 0.4855 - val_loss: 1.3893 - val_acc: 0.5053\n",
      "Epoch 12/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.4489 - acc: 0.4912 - val_loss: 1.3832 - val_acc: 0.5041\n",
      "Epoch 13/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.4350 - acc: 0.4913 - val_loss: 1.3745 - val_acc: 0.5069\n",
      "Epoch 14/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.4285 - acc: 0.4976 - val_loss: 1.3704 - val_acc: 0.5112\n",
      "Epoch 15/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.4178 - acc: 0.4989 - val_loss: 1.3659 - val_acc: 0.5110\n",
      "Epoch 16/100\n",
      "40000/40000 [==============================] - 2s 38us/step - loss: 1.4114 - acc: 0.5006 - val_loss: 1.3574 - val_acc: 0.5130\n",
      "Epoch 17/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.4056 - acc: 0.5052 - val_loss: 1.3541 - val_acc: 0.5154\n",
      "Epoch 18/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.4004 - acc: 0.5054 - val_loss: 1.3469 - val_acc: 0.5212\n",
      "Epoch 19/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3946 - acc: 0.5078 - val_loss: 1.3443 - val_acc: 0.5193\n",
      "Epoch 20/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3842 - acc: 0.5114 - val_loss: 1.3413 - val_acc: 0.5223\n",
      "Epoch 21/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3806 - acc: 0.5122 - val_loss: 1.3357 - val_acc: 0.5241\n",
      "Epoch 22/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3761 - acc: 0.5141 - val_loss: 1.3348 - val_acc: 0.5278\n",
      "Epoch 23/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.3717 - acc: 0.5167 - val_loss: 1.3319 - val_acc: 0.5258\n",
      "Epoch 24/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3657 - acc: 0.5188 - val_loss: 1.3284 - val_acc: 0.5242\n",
      "Epoch 25/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3628 - acc: 0.5222 - val_loss: 1.3232 - val_acc: 0.5281\n",
      "Epoch 26/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3585 - acc: 0.5211 - val_loss: 1.3216 - val_acc: 0.5292\n",
      "Epoch 27/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3555 - acc: 0.5207 - val_loss: 1.3184 - val_acc: 0.5291\n",
      "Epoch 28/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.3502 - acc: 0.5257 - val_loss: 1.3199 - val_acc: 0.5303\n",
      "Epoch 29/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.3480 - acc: 0.5246 - val_loss: 1.3200 - val_acc: 0.5291\n",
      "Epoch 30/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3435 - acc: 0.5263 - val_loss: 1.3183 - val_acc: 0.5309\n",
      "Epoch 31/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3390 - acc: 0.5295 - val_loss: 1.3154 - val_acc: 0.5321\n",
      "Epoch 32/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3372 - acc: 0.5285 - val_loss: 1.3108 - val_acc: 0.5324\n",
      "Epoch 33/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3302 - acc: 0.5328 - val_loss: 1.3069 - val_acc: 0.5341\n",
      "Epoch 34/100\n",
      "40000/40000 [==============================] - 2s 40us/step - loss: 1.3289 - acc: 0.5325 - val_loss: 1.3048 - val_acc: 0.5360\n",
      "Epoch 35/100\n",
      "40000/40000 [==============================] - 2s 43us/step - loss: 1.3252 - acc: 0.5342 - val_loss: 1.3071 - val_acc: 0.5346\n",
      "Epoch 36/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.3252 - acc: 0.5341 - val_loss: 1.3014 - val_acc: 0.5387\n",
      "Epoch 37/100\n",
      "40000/40000 [==============================] - 2s 38us/step - loss: 1.3199 - acc: 0.5352 - val_loss: 1.3019 - val_acc: 0.5374\n",
      "Epoch 38/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3196 - acc: 0.5370 - val_loss: 1.3008 - val_acc: 0.5378\n",
      "Epoch 39/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3124 - acc: 0.5372 - val_loss: 1.3030 - val_acc: 0.5369\n",
      "Epoch 40/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3120 - acc: 0.5382 - val_loss: 1.3005 - val_acc: 0.5385\n",
      "Epoch 41/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3094 - acc: 0.5397 - val_loss: 1.3008 - val_acc: 0.5364\n",
      "Epoch 42/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3086 - acc: 0.5420 - val_loss: 1.2934 - val_acc: 0.5389\n",
      "Epoch 43/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3048 - acc: 0.5391 - val_loss: 1.2952 - val_acc: 0.5373\n",
      "Epoch 44/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3051 - acc: 0.5401 - val_loss: 1.2931 - val_acc: 0.5433\n",
      "Epoch 45/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.3023 - acc: 0.5399 - val_loss: 1.2919 - val_acc: 0.5416\n",
      "Epoch 46/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2977 - acc: 0.5434 - val_loss: 1.2944 - val_acc: 0.5380\n",
      "Epoch 47/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2969 - acc: 0.5434 - val_loss: 1.2942 - val_acc: 0.5411\n",
      "Epoch 48/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2926 - acc: 0.5440 - val_loss: 1.2899 - val_acc: 0.5422\n",
      "Epoch 49/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.2898 - acc: 0.5449 - val_loss: 1.2894 - val_acc: 0.5385\n",
      "Epoch 50/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2914 - acc: 0.5450 - val_loss: 1.2874 - val_acc: 0.5433\n",
      "Epoch 51/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2882 - acc: 0.5469 - val_loss: 1.2873 - val_acc: 0.5410\n",
      "Epoch 52/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.2839 - acc: 0.5482 - val_loss: 1.2846 - val_acc: 0.5409\n",
      "Epoch 53/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2812 - acc: 0.5502 - val_loss: 1.2868 - val_acc: 0.5394\n",
      "Epoch 54/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2818 - acc: 0.5486 - val_loss: 1.2864 - val_acc: 0.5428\n",
      "Epoch 55/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2785 - acc: 0.5486 - val_loss: 1.2831 - val_acc: 0.5446\n",
      "Epoch 56/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2776 - acc: 0.5507 - val_loss: 1.2819 - val_acc: 0.5452\n",
      "Epoch 57/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2776 - acc: 0.5478 - val_loss: 1.2800 - val_acc: 0.5449\n",
      "Epoch 58/100\n",
      "40000/40000 [==============================] - 2s 38us/step - loss: 1.2740 - acc: 0.5514 - val_loss: 1.2813 - val_acc: 0.5438\n",
      "Epoch 59/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2724 - acc: 0.5522 - val_loss: 1.2813 - val_acc: 0.5437\n",
      "Epoch 60/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2719 - acc: 0.5526 - val_loss: 1.2814 - val_acc: 0.5453\n",
      "Epoch 61/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2712 - acc: 0.5525 - val_loss: 1.2792 - val_acc: 0.5452\n",
      "Epoch 62/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2655 - acc: 0.5539 - val_loss: 1.2787 - val_acc: 0.5458\n",
      "Epoch 63/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2643 - acc: 0.5560 - val_loss: 1.2783 - val_acc: 0.5452\n",
      "Epoch 64/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2617 - acc: 0.5571 - val_loss: 1.2780 - val_acc: 0.5454\n",
      "Epoch 65/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2603 - acc: 0.5556 - val_loss: 1.2798 - val_acc: 0.5432\n",
      "Epoch 66/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2605 - acc: 0.5556 - val_loss: 1.2758 - val_acc: 0.5488\n",
      "Epoch 67/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2580 - acc: 0.5577 - val_loss: 1.2756 - val_acc: 0.5479\n",
      "Epoch 68/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2563 - acc: 0.5557 - val_loss: 1.2785 - val_acc: 0.5445\n",
      "Epoch 69/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2529 - acc: 0.5579 - val_loss: 1.2775 - val_acc: 0.5468\n",
      "Epoch 70/100\n",
      "40000/40000 [==============================] - 1s 33us/step - loss: 1.2534 - acc: 0.5587 - val_loss: 1.2773 - val_acc: 0.5438\n",
      "Epoch 71/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2497 - acc: 0.5615 - val_loss: 1.2773 - val_acc: 0.5466\n",
      "Epoch 72/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2501 - acc: 0.5609 - val_loss: 1.2782 - val_acc: 0.5454\n",
      "Epoch 73/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2474 - acc: 0.5603 - val_loss: 1.2749 - val_acc: 0.5451\n",
      "Epoch 74/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2445 - acc: 0.5624 - val_loss: 1.2727 - val_acc: 0.5463\n",
      "Epoch 75/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2429 - acc: 0.5620 - val_loss: 1.2723 - val_acc: 0.5458\n",
      "Epoch 76/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2439 - acc: 0.5623 - val_loss: 1.2722 - val_acc: 0.5436\n",
      "Epoch 77/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2384 - acc: 0.5638 - val_loss: 1.2699 - val_acc: 0.5516\n",
      "Epoch 78/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2376 - acc: 0.5638 - val_loss: 1.2739 - val_acc: 0.5448\n",
      "Epoch 79/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2351 - acc: 0.5622 - val_loss: 1.2721 - val_acc: 0.5480\n",
      "Epoch 80/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2360 - acc: 0.5642 - val_loss: 1.2721 - val_acc: 0.5521\n",
      "Epoch 81/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2353 - acc: 0.5664 - val_loss: 1.2735 - val_acc: 0.5451\n",
      "Epoch 82/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2327 - acc: 0.5660 - val_loss: 1.2721 - val_acc: 0.5482\n",
      "Epoch 83/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2312 - acc: 0.5664 - val_loss: 1.2686 - val_acc: 0.5489\n",
      "Epoch 84/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2320 - acc: 0.5687 - val_loss: 1.2732 - val_acc: 0.5474\n",
      "Epoch 85/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2314 - acc: 0.5674 - val_loss: 1.2678 - val_acc: 0.5510\n",
      "Epoch 86/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2287 - acc: 0.5680 - val_loss: 1.2672 - val_acc: 0.5519\n",
      "Epoch 87/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2260 - acc: 0.5690 - val_loss: 1.2690 - val_acc: 0.5476\n",
      "Epoch 88/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2246 - acc: 0.5670 - val_loss: 1.2692 - val_acc: 0.5491\n",
      "Epoch 89/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2264 - acc: 0.5674 - val_loss: 1.2684 - val_acc: 0.5504\n",
      "Epoch 90/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2207 - acc: 0.5713 - val_loss: 1.2669 - val_acc: 0.5507\n",
      "Epoch 91/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2221 - acc: 0.5700 - val_loss: 1.2679 - val_acc: 0.5504\n",
      "Epoch 92/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2196 - acc: 0.5716 - val_loss: 1.2682 - val_acc: 0.5497\n",
      "Epoch 93/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2201 - acc: 0.5706 - val_loss: 1.2720 - val_acc: 0.5497\n",
      "Epoch 94/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2178 - acc: 0.5701 - val_loss: 1.2699 - val_acc: 0.5494\n",
      "Epoch 95/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2149 - acc: 0.5726 - val_loss: 1.2662 - val_acc: 0.5500\n",
      "Epoch 96/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2136 - acc: 0.5737 - val_loss: 1.2694 - val_acc: 0.5472\n",
      "Epoch 97/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2126 - acc: 0.5740 - val_loss: 1.2679 - val_acc: 0.5494\n",
      "Epoch 98/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2097 - acc: 0.5748 - val_loss: 1.2671 - val_acc: 0.5539\n",
      "Epoch 99/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2077 - acc: 0.5756 - val_loss: 1.2671 - val_acc: 0.5494\n",
      "Epoch 100/100\n",
      "40000/40000 [==============================] - 1s 34us/step - loss: 1.2068 - acc: 0.5754 - val_loss: 1.2702 - val_acc: 0.5490\n",
      "10000/10000 [==============================] - 0s 31us/step\n",
      "Validation loss: 1.2702240009307861\n",
      "Validation accuracy (NORMALIZED): 0.549\n",
      "================================================\n",
      "Split: 3\n",
      "Number of Epochs: 100\n",
      "Train shape: (40000, 32, 32, 3)\n",
      "Train batch size: 384\n",
      "Val batch size: 128\n",
      "================================================\n",
      "--- Preparing tensorboard\n",
      "Log Dir:  logs/1542821355.9681177_fold[3]_fine-tunned\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/100\n",
      "40000/40000 [==============================] - 2s 47us/step - loss: 2.2201 - acc: 0.1923 - val_loss: 1.9955 - val_acc: 0.3148\n",
      "Epoch 2/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.8505 - acc: 0.3392 - val_loss: 1.6286 - val_acc: 0.4253\n",
      "Epoch 3/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.6711 - acc: 0.3987 - val_loss: 1.5439 - val_acc: 0.4502\n",
      "Epoch 4/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.5986 - acc: 0.4289 - val_loss: 1.5020 - val_acc: 0.4639\n",
      "Epoch 5/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.5606 - acc: 0.4447 - val_loss: 1.4727 - val_acc: 0.4804\n",
      "Epoch 6/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.5227 - acc: 0.4584 - val_loss: 1.4495 - val_acc: 0.4879\n",
      "Epoch 7/100\n",
      "40000/40000 [==============================] - 2s 38us/step - loss: 1.5048 - acc: 0.4680 - val_loss: 1.4316 - val_acc: 0.4940\n",
      "Epoch 8/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.4815 - acc: 0.4739 - val_loss: 1.4183 - val_acc: 0.4960\n",
      "Epoch 9/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.4670 - acc: 0.4818 - val_loss: 1.4061 - val_acc: 0.5023\n",
      "Epoch 10/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.4516 - acc: 0.4879 - val_loss: 1.3949 - val_acc: 0.5032\n",
      "Epoch 11/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.4394 - acc: 0.4928 - val_loss: 1.3859 - val_acc: 0.5056\n",
      "Epoch 12/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.4313 - acc: 0.4944 - val_loss: 1.3850 - val_acc: 0.5053\n",
      "Epoch 13/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.4228 - acc: 0.4960 - val_loss: 1.3737 - val_acc: 0.5113\n",
      "Epoch 14/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.4110 - acc: 0.5025 - val_loss: 1.3690 - val_acc: 0.5142\n",
      "Epoch 15/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.4035 - acc: 0.5055 - val_loss: 1.3634 - val_acc: 0.5141\n",
      "Epoch 16/100\n",
      "40000/40000 [==============================] - 2s 38us/step - loss: 1.3946 - acc: 0.5088 - val_loss: 1.3571 - val_acc: 0.5182\n",
      "Epoch 17/100\n",
      "40000/40000 [==============================] - 2s 39us/step - loss: 1.3912 - acc: 0.5076 - val_loss: 1.3558 - val_acc: 0.5168\n",
      "Epoch 18/100\n",
      "40000/40000 [==============================] - 2s 40us/step - loss: 1.3825 - acc: 0.5113 - val_loss: 1.3501 - val_acc: 0.5183\n",
      "Epoch 19/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3792 - acc: 0.5149 - val_loss: 1.3439 - val_acc: 0.5211\n",
      "Epoch 20/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3738 - acc: 0.5157 - val_loss: 1.3470 - val_acc: 0.5196\n",
      "Epoch 21/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3666 - acc: 0.5162 - val_loss: 1.3416 - val_acc: 0.5218\n",
      "Epoch 22/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3612 - acc: 0.5187 - val_loss: 1.3382 - val_acc: 0.5259\n",
      "Epoch 23/100\n",
      "40000/40000 [==============================] - 2s 38us/step - loss: 1.3580 - acc: 0.5217 - val_loss: 1.3326 - val_acc: 0.5217\n",
      "Epoch 24/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3522 - acc: 0.5236 - val_loss: 1.3317 - val_acc: 0.5235\n",
      "Epoch 25/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3501 - acc: 0.5220 - val_loss: 1.3335 - val_acc: 0.5219\n",
      "Epoch 26/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3460 - acc: 0.5221 - val_loss: 1.3274 - val_acc: 0.5236\n",
      "Epoch 27/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3413 - acc: 0.5264 - val_loss: 1.3231 - val_acc: 0.5263\n",
      "Epoch 28/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3391 - acc: 0.5258 - val_loss: 1.3225 - val_acc: 0.5256\n",
      "Epoch 29/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3342 - acc: 0.5313 - val_loss: 1.3255 - val_acc: 0.5249\n",
      "Epoch 30/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3321 - acc: 0.5289 - val_loss: 1.3188 - val_acc: 0.5265\n",
      "Epoch 31/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3256 - acc: 0.5311 - val_loss: 1.3258 - val_acc: 0.5282\n",
      "Epoch 32/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3250 - acc: 0.5327 - val_loss: 1.3152 - val_acc: 0.5296\n",
      "Epoch 33/100\n",
      "40000/40000 [==============================] - 2s 41us/step - loss: 1.3206 - acc: 0.5319 - val_loss: 1.3138 - val_acc: 0.5338\n",
      "Epoch 34/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3192 - acc: 0.5342 - val_loss: 1.3112 - val_acc: 0.5329\n",
      "Epoch 35/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3152 - acc: 0.5360 - val_loss: 1.3100 - val_acc: 0.5336\n",
      "Epoch 36/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3145 - acc: 0.5358 - val_loss: 1.3079 - val_acc: 0.5343\n",
      "Epoch 37/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3117 - acc: 0.5371 - val_loss: 1.3079 - val_acc: 0.5320\n",
      "Epoch 38/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.3060 - acc: 0.5407 - val_loss: 1.3073 - val_acc: 0.5322\n",
      "Epoch 39/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3024 - acc: 0.5406 - val_loss: 1.3052 - val_acc: 0.5334\n",
      "Epoch 40/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3008 - acc: 0.5402 - val_loss: 1.3022 - val_acc: 0.5365\n",
      "Epoch 41/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2982 - acc: 0.5412 - val_loss: 1.3035 - val_acc: 0.5351\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 42/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2999 - acc: 0.5403 - val_loss: 1.2993 - val_acc: 0.5356\n",
      "Epoch 43/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2963 - acc: 0.5418 - val_loss: 1.2997 - val_acc: 0.5359\n",
      "Epoch 44/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2915 - acc: 0.5443 - val_loss: 1.2992 - val_acc: 0.5365\n",
      "Epoch 45/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2915 - acc: 0.5436 - val_loss: 1.3007 - val_acc: 0.5366\n",
      "Epoch 46/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2897 - acc: 0.5441 - val_loss: 1.2990 - val_acc: 0.5371\n",
      "Epoch 47/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2857 - acc: 0.5451 - val_loss: 1.3008 - val_acc: 0.5376\n",
      "Epoch 48/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2827 - acc: 0.5460 - val_loss: 1.2970 - val_acc: 0.5372\n",
      "Epoch 49/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2806 - acc: 0.5482 - val_loss: 1.2940 - val_acc: 0.5385\n",
      "Epoch 50/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2798 - acc: 0.5480 - val_loss: 1.2929 - val_acc: 0.5408\n",
      "Epoch 51/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2777 - acc: 0.5492 - val_loss: 1.2935 - val_acc: 0.5384\n",
      "Epoch 52/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2749 - acc: 0.5515 - val_loss: 1.2943 - val_acc: 0.5411\n",
      "Epoch 53/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2729 - acc: 0.5508 - val_loss: 1.2901 - val_acc: 0.5395\n",
      "Epoch 54/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2712 - acc: 0.5496 - val_loss: 1.2904 - val_acc: 0.5414\n",
      "Epoch 55/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2690 - acc: 0.5528 - val_loss: 1.2890 - val_acc: 0.5413\n",
      "Epoch 56/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2682 - acc: 0.5539 - val_loss: 1.2915 - val_acc: 0.5412\n",
      "Epoch 57/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2672 - acc: 0.5537 - val_loss: 1.2879 - val_acc: 0.5416\n",
      "Epoch 58/100\n",
      "40000/40000 [==============================] - 2s 40us/step - loss: 1.2653 - acc: 0.5523 - val_loss: 1.2878 - val_acc: 0.5424\n",
      "Epoch 59/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.2633 - acc: 0.5563 - val_loss: 1.2886 - val_acc: 0.5431\n",
      "Epoch 60/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2622 - acc: 0.5544 - val_loss: 1.2853 - val_acc: 0.5436\n",
      "Epoch 61/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2578 - acc: 0.5553 - val_loss: 1.2869 - val_acc: 0.5409\n",
      "Epoch 62/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2559 - acc: 0.5548 - val_loss: 1.2843 - val_acc: 0.5424\n",
      "Epoch 63/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2537 - acc: 0.5571 - val_loss: 1.2858 - val_acc: 0.5425\n",
      "Epoch 64/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2524 - acc: 0.5582 - val_loss: 1.2819 - val_acc: 0.5432\n",
      "Epoch 65/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2538 - acc: 0.5565 - val_loss: 1.2859 - val_acc: 0.5442\n",
      "Epoch 66/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2499 - acc: 0.5574 - val_loss: 1.2842 - val_acc: 0.5453\n",
      "Epoch 67/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2443 - acc: 0.5592 - val_loss: 1.2847 - val_acc: 0.5449\n",
      "Epoch 68/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2456 - acc: 0.5625 - val_loss: 1.2831 - val_acc: 0.5443\n",
      "Epoch 69/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2451 - acc: 0.5592 - val_loss: 1.2809 - val_acc: 0.5437\n",
      "Epoch 70/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2425 - acc: 0.5610 - val_loss: 1.2804 - val_acc: 0.5438\n",
      "Epoch 71/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2422 - acc: 0.5587 - val_loss: 1.2795 - val_acc: 0.5479\n",
      "Epoch 72/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2394 - acc: 0.5623 - val_loss: 1.2823 - val_acc: 0.5411\n",
      "Epoch 73/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2384 - acc: 0.5623 - val_loss: 1.2819 - val_acc: 0.5471\n",
      "Epoch 74/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2351 - acc: 0.5631 - val_loss: 1.2807 - val_acc: 0.5458\n",
      "Epoch 75/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2359 - acc: 0.5638 - val_loss: 1.2816 - val_acc: 0.5461\n",
      "Epoch 76/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2332 - acc: 0.5629 - val_loss: 1.2764 - val_acc: 0.5481\n",
      "Epoch 77/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2322 - acc: 0.5648 - val_loss: 1.2774 - val_acc: 0.5483\n",
      "Epoch 78/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2266 - acc: 0.5671 - val_loss: 1.2791 - val_acc: 0.5475\n",
      "Epoch 79/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2312 - acc: 0.5649 - val_loss: 1.2782 - val_acc: 0.5482\n",
      "Epoch 80/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2269 - acc: 0.5669 - val_loss: 1.2762 - val_acc: 0.5453\n",
      "Epoch 81/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2256 - acc: 0.5662 - val_loss: 1.2788 - val_acc: 0.5476\n",
      "Epoch 82/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2237 - acc: 0.5665 - val_loss: 1.2773 - val_acc: 0.5482\n",
      "Epoch 83/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2226 - acc: 0.5662 - val_loss: 1.2748 - val_acc: 0.5469\n",
      "Epoch 84/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2181 - acc: 0.5699 - val_loss: 1.2749 - val_acc: 0.5486\n",
      "Epoch 85/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2194 - acc: 0.5706 - val_loss: 1.2748 - val_acc: 0.5495\n",
      "Epoch 86/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2173 - acc: 0.5710 - val_loss: 1.2735 - val_acc: 0.5488\n",
      "Epoch 87/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2187 - acc: 0.5698 - val_loss: 1.2731 - val_acc: 0.5491\n",
      "Epoch 88/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2141 - acc: 0.5716 - val_loss: 1.2740 - val_acc: 0.5470\n",
      "Epoch 89/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2106 - acc: 0.5708 - val_loss: 1.2743 - val_acc: 0.5484\n",
      "Epoch 90/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.2143 - acc: 0.5700 - val_loss: 1.2744 - val_acc: 0.5484\n",
      "Epoch 91/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2109 - acc: 0.5718 - val_loss: 1.2759 - val_acc: 0.5480\n",
      "Epoch 92/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2085 - acc: 0.5720 - val_loss: 1.2741 - val_acc: 0.5471\n",
      "Epoch 93/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2095 - acc: 0.5721 - val_loss: 1.2763 - val_acc: 0.5506\n",
      "Epoch 94/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2073 - acc: 0.5733 - val_loss: 1.2735 - val_acc: 0.5491\n",
      "Epoch 95/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2065 - acc: 0.5731 - val_loss: 1.2761 - val_acc: 0.5475\n",
      "Epoch 96/100\n",
      "40000/40000 [==============================] - 2s 40us/step - loss: 1.2026 - acc: 0.5737 - val_loss: 1.2750 - val_acc: 0.5476\n",
      "Epoch 97/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.2005 - acc: 0.5746 - val_loss: 1.2718 - val_acc: 0.5468\n",
      "Epoch 98/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2014 - acc: 0.5773 - val_loss: 1.2703 - val_acc: 0.5515\n",
      "Epoch 99/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.1994 - acc: 0.5779 - val_loss: 1.2755 - val_acc: 0.5512\n",
      "Epoch 100/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.1957 - acc: 0.5758 - val_loss: 1.2718 - val_acc: 0.5494\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 0s 30us/step\n",
      "Validation loss: 1.2718006425857544\n",
      "Validation accuracy (NORMALIZED): 0.5494\n",
      "================================================\n",
      "Split: 4\n",
      "Number of Epochs: 100\n",
      "Train shape: (40000, 32, 32, 3)\n",
      "Train batch size: 384\n",
      "Val batch size: 128\n",
      "================================================\n",
      "--- Preparing tensorboard\n",
      "Log Dir:  logs/1542821503.9207437_fold[4]_fine-tunned\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/100\n",
      "40000/40000 [==============================] - 2s 51us/step - loss: 2.2545 - acc: 0.1662 - val_loss: 2.1039 - val_acc: 0.2816\n",
      "Epoch 2/100\n",
      "40000/40000 [==============================] - 2s 40us/step - loss: 1.9314 - acc: 0.3152 - val_loss: 1.6611 - val_acc: 0.4211\n",
      "Epoch 3/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.6834 - acc: 0.4022 - val_loss: 1.5517 - val_acc: 0.4519\n",
      "Epoch 4/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.6035 - acc: 0.4284 - val_loss: 1.5091 - val_acc: 0.4630\n",
      "Epoch 5/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.5620 - acc: 0.4463 - val_loss: 1.4743 - val_acc: 0.4736\n",
      "Epoch 6/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.5311 - acc: 0.4559 - val_loss: 1.4585 - val_acc: 0.4756\n",
      "Epoch 7/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.5071 - acc: 0.4678 - val_loss: 1.4380 - val_acc: 0.4862\n",
      "Epoch 8/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.4835 - acc: 0.4767 - val_loss: 1.4280 - val_acc: 0.4912\n",
      "Epoch 9/100\n",
      "40000/40000 [==============================] - 2s 38us/step - loss: 1.4717 - acc: 0.4798 - val_loss: 1.4114 - val_acc: 0.4944\n",
      "Epoch 10/100\n",
      "40000/40000 [==============================] - 2s 38us/step - loss: 1.4581 - acc: 0.4853 - val_loss: 1.4040 - val_acc: 0.4998\n",
      "Epoch 11/100\n",
      "40000/40000 [==============================] - 2s 38us/step - loss: 1.4408 - acc: 0.4917 - val_loss: 1.3952 - val_acc: 0.5022\n",
      "Epoch 12/100\n",
      "40000/40000 [==============================] - 2s 38us/step - loss: 1.4333 - acc: 0.4950 - val_loss: 1.3875 - val_acc: 0.5083\n",
      "Epoch 13/100\n",
      "40000/40000 [==============================] - 2s 38us/step - loss: 1.4199 - acc: 0.4974 - val_loss: 1.3777 - val_acc: 0.5100\n",
      "Epoch 14/100\n",
      "40000/40000 [==============================] - 2s 38us/step - loss: 1.4137 - acc: 0.4999 - val_loss: 1.3735 - val_acc: 0.5124\n",
      "Epoch 15/100\n",
      "40000/40000 [==============================] - 2s 38us/step - loss: 1.4076 - acc: 0.5008 - val_loss: 1.3727 - val_acc: 0.5153\n",
      "Epoch 16/100\n",
      "40000/40000 [==============================] - 2s 38us/step - loss: 1.4010 - acc: 0.5054 - val_loss: 1.3642 - val_acc: 0.5165\n",
      "Epoch 17/100\n",
      "40000/40000 [==============================] - 2s 38us/step - loss: 1.3919 - acc: 0.5099 - val_loss: 1.3630 - val_acc: 0.5173\n",
      "Epoch 18/100\n",
      "40000/40000 [==============================] - 2s 38us/step - loss: 1.3844 - acc: 0.5114 - val_loss: 1.3555 - val_acc: 0.5176\n",
      "Epoch 19/100\n",
      "40000/40000 [==============================] - 2s 38us/step - loss: 1.3799 - acc: 0.5150 - val_loss: 1.3504 - val_acc: 0.5230\n",
      "Epoch 20/100\n",
      "40000/40000 [==============================] - 2s 39us/step - loss: 1.3737 - acc: 0.5130 - val_loss: 1.3473 - val_acc: 0.5212\n",
      "Epoch 21/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.3666 - acc: 0.5173 - val_loss: 1.3489 - val_acc: 0.5206\n",
      "Epoch 22/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3633 - acc: 0.5185 - val_loss: 1.3410 - val_acc: 0.5250\n",
      "Epoch 23/100\n",
      "40000/40000 [==============================] - 2s 38us/step - loss: 1.3600 - acc: 0.5209 - val_loss: 1.3373 - val_acc: 0.5256\n",
      "Epoch 24/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.3512 - acc: 0.5222 - val_loss: 1.3366 - val_acc: 0.5240\n",
      "Epoch 25/100\n",
      "40000/40000 [==============================] - 2s 38us/step - loss: 1.3506 - acc: 0.5244 - val_loss: 1.3346 - val_acc: 0.5269\n",
      "Epoch 26/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3442 - acc: 0.5252 - val_loss: 1.3308 - val_acc: 0.5291\n",
      "Epoch 27/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3409 - acc: 0.5258 - val_loss: 1.3308 - val_acc: 0.5298\n",
      "Epoch 28/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.3385 - acc: 0.5279 - val_loss: 1.3261 - val_acc: 0.5306\n",
      "Epoch 29/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3327 - acc: 0.5297 - val_loss: 1.3242 - val_acc: 0.5286\n",
      "Epoch 30/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.3309 - acc: 0.5313 - val_loss: 1.3221 - val_acc: 0.5308\n",
      "Epoch 31/100\n",
      "40000/40000 [==============================] - 2s 42us/step - loss: 1.3269 - acc: 0.5325 - val_loss: 1.3220 - val_acc: 0.5309\n",
      "Epoch 32/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.3234 - acc: 0.5341 - val_loss: 1.3180 - val_acc: 0.5323\n",
      "Epoch 33/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.3198 - acc: 0.5368 - val_loss: 1.3188 - val_acc: 0.5323\n",
      "Epoch 34/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3189 - acc: 0.5356 - val_loss: 1.3151 - val_acc: 0.5349\n",
      "Epoch 35/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3144 - acc: 0.5345 - val_loss: 1.3130 - val_acc: 0.5350\n",
      "Epoch 36/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3107 - acc: 0.5384 - val_loss: 1.3139 - val_acc: 0.5377\n",
      "Epoch 37/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3107 - acc: 0.5390 - val_loss: 1.3167 - val_acc: 0.5295\n",
      "Epoch 38/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3085 - acc: 0.5374 - val_loss: 1.3099 - val_acc: 0.5356\n",
      "Epoch 39/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3049 - acc: 0.5407 - val_loss: 1.3075 - val_acc: 0.5365\n",
      "Epoch 40/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.3035 - acc: 0.5420 - val_loss: 1.3092 - val_acc: 0.5354\n",
      "Epoch 41/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2985 - acc: 0.5433 - val_loss: 1.3051 - val_acc: 0.5368\n",
      "Epoch 42/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2970 - acc: 0.5428 - val_loss: 1.3079 - val_acc: 0.5347\n",
      "Epoch 43/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2914 - acc: 0.5430 - val_loss: 1.3043 - val_acc: 0.5377\n",
      "Epoch 44/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2927 - acc: 0.5441 - val_loss: 1.3026 - val_acc: 0.5371\n",
      "Epoch 45/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2898 - acc: 0.5455 - val_loss: 1.3043 - val_acc: 0.5370\n",
      "Epoch 46/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.2860 - acc: 0.5448 - val_loss: 1.3012 - val_acc: 0.5370\n",
      "Epoch 47/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2827 - acc: 0.5460 - val_loss: 1.2999 - val_acc: 0.5378\n",
      "Epoch 48/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2827 - acc: 0.5479 - val_loss: 1.3012 - val_acc: 0.5395\n",
      "Epoch 49/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2804 - acc: 0.5466 - val_loss: 1.3019 - val_acc: 0.5385\n",
      "Epoch 50/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2761 - acc: 0.5489 - val_loss: 1.2971 - val_acc: 0.5378\n",
      "Epoch 51/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2779 - acc: 0.5497 - val_loss: 1.3018 - val_acc: 0.5375\n",
      "Epoch 52/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2733 - acc: 0.5502 - val_loss: 1.2955 - val_acc: 0.5383\n",
      "Epoch 53/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2729 - acc: 0.5503 - val_loss: 1.2953 - val_acc: 0.5395\n",
      "Epoch 54/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2692 - acc: 0.5517 - val_loss: 1.2937 - val_acc: 0.5379\n",
      "Epoch 55/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2691 - acc: 0.5528 - val_loss: 1.2975 - val_acc: 0.5410\n",
      "Epoch 56/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2649 - acc: 0.5538 - val_loss: 1.2962 - val_acc: 0.5383\n",
      "Epoch 57/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2679 - acc: 0.5511 - val_loss: 1.2972 - val_acc: 0.5388\n",
      "Epoch 58/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2622 - acc: 0.5549 - val_loss: 1.2918 - val_acc: 0.5417\n",
      "Epoch 59/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2603 - acc: 0.5555 - val_loss: 1.2961 - val_acc: 0.5385\n",
      "Epoch 60/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2599 - acc: 0.5587 - val_loss: 1.2899 - val_acc: 0.5405\n",
      "Epoch 61/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2585 - acc: 0.5572 - val_loss: 1.2888 - val_acc: 0.5422\n",
      "Epoch 62/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2557 - acc: 0.5579 - val_loss: 1.2878 - val_acc: 0.5421\n",
      "Epoch 63/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2529 - acc: 0.5576 - val_loss: 1.2914 - val_acc: 0.5382\n",
      "Epoch 64/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2519 - acc: 0.5586 - val_loss: 1.2914 - val_acc: 0.5426\n",
      "Epoch 65/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2502 - acc: 0.5592 - val_loss: 1.2885 - val_acc: 0.5439\n",
      "Epoch 66/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2488 - acc: 0.5593 - val_loss: 1.2895 - val_acc: 0.5432\n",
      "Epoch 67/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2468 - acc: 0.5589 - val_loss: 1.2856 - val_acc: 0.5422\n",
      "Epoch 68/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2459 - acc: 0.5601 - val_loss: 1.2890 - val_acc: 0.5406\n",
      "Epoch 69/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2443 - acc: 0.5620 - val_loss: 1.2905 - val_acc: 0.5425\n",
      "Epoch 70/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2423 - acc: 0.5619 - val_loss: 1.2862 - val_acc: 0.5448\n",
      "Epoch 71/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2407 - acc: 0.5635 - val_loss: 1.2861 - val_acc: 0.5409\n",
      "Epoch 72/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2377 - acc: 0.5619 - val_loss: 1.2855 - val_acc: 0.5441\n",
      "Epoch 73/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2394 - acc: 0.5629 - val_loss: 1.2839 - val_acc: 0.5452\n",
      "Epoch 74/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2373 - acc: 0.5638 - val_loss: 1.2869 - val_acc: 0.5430\n",
      "Epoch 75/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2354 - acc: 0.5642 - val_loss: 1.2848 - val_acc: 0.5447\n",
      "Epoch 76/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2289 - acc: 0.5652 - val_loss: 1.2849 - val_acc: 0.5449\n",
      "Epoch 77/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2318 - acc: 0.5649 - val_loss: 1.2823 - val_acc: 0.5457\n",
      "Epoch 78/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2284 - acc: 0.5679 - val_loss: 1.2833 - val_acc: 0.5439\n",
      "Epoch 79/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.2254 - acc: 0.5679 - val_loss: 1.2850 - val_acc: 0.5433\n",
      "Epoch 80/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.2265 - acc: 0.5655 - val_loss: 1.2830 - val_acc: 0.5455\n",
      "Epoch 81/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2273 - acc: 0.5673 - val_loss: 1.2806 - val_acc: 0.5441\n",
      "Epoch 82/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2242 - acc: 0.5673 - val_loss: 1.2790 - val_acc: 0.5456\n",
      "Epoch 83/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2217 - acc: 0.5686 - val_loss: 1.2862 - val_acc: 0.5419\n",
      "Epoch 84/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2190 - acc: 0.5690 - val_loss: 1.2804 - val_acc: 0.5441\n",
      "Epoch 85/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2192 - acc: 0.5688 - val_loss: 1.2872 - val_acc: 0.5428\n",
      "Epoch 86/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.2186 - acc: 0.5691 - val_loss: 1.2779 - val_acc: 0.5462\n",
      "Epoch 87/100\n",
      "40000/40000 [==============================] - 1s 37us/step - loss: 1.2156 - acc: 0.5718 - val_loss: 1.2801 - val_acc: 0.5472\n",
      "Epoch 88/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2162 - acc: 0.5709 - val_loss: 1.2816 - val_acc: 0.5467\n",
      "Epoch 89/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2140 - acc: 0.5708 - val_loss: 1.2832 - val_acc: 0.5445\n",
      "Epoch 90/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2140 - acc: 0.5715 - val_loss: 1.2798 - val_acc: 0.5461\n",
      "Epoch 91/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2120 - acc: 0.5733 - val_loss: 1.2792 - val_acc: 0.5450\n",
      "Epoch 92/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2092 - acc: 0.5723 - val_loss: 1.2785 - val_acc: 0.5445\n",
      "Epoch 93/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2085 - acc: 0.5731 - val_loss: 1.2778 - val_acc: 0.5466\n",
      "Epoch 94/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2075 - acc: 0.5738 - val_loss: 1.2796 - val_acc: 0.5454\n",
      "Epoch 95/100\n",
      "40000/40000 [==============================] - 1s 35us/step - loss: 1.2076 - acc: 0.5754 - val_loss: 1.2797 - val_acc: 0.5438\n",
      "Epoch 96/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2056 - acc: 0.5753 - val_loss: 1.2789 - val_acc: 0.5462\n",
      "Epoch 97/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2076 - acc: 0.5734 - val_loss: 1.2783 - val_acc: 0.5462\n",
      "Epoch 98/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2019 - acc: 0.5777 - val_loss: 1.2780 - val_acc: 0.5454\n",
      "Epoch 99/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.2003 - acc: 0.5773 - val_loss: 1.2770 - val_acc: 0.5460\n",
      "Epoch 100/100\n",
      "40000/40000 [==============================] - 1s 36us/step - loss: 1.1987 - acc: 0.5777 - val_loss: 1.2767 - val_acc: 0.5470\n",
      "10000/10000 [==============================] - 0s 34us/step\n",
      "Validation loss: 1.2767395862579345\n",
      "Validation accuracy (NORMALIZED): 0.547\n",
      "[[1.2712621648788451, 0.5518], [1.2850975257873536, 0.5467], [1.2702240009307861, 0.549], [1.2718006425857544, 0.5494], [1.2767395862579345, 0.547]]\n"
     ]
    }
   ],
   "source": [
    "#=====================================\n",
    "# Training model and Evaluation\n",
    "#=====================================\n",
    "\n",
    "y_train_categorical = to_categorical(y_train, num_classes=n_classes)\n",
    "y_val_categorical = to_categorical(y_val, num_classes=n_classes)\n",
    "\n",
    "scores = []\n",
    "for i in range(folds):\n",
    "    #--- Loading model\n",
    "    model = get_squeezenet_ft2()    \n",
    "    \n",
    "    #--- Evaluating the model for split i\n",
    "    print(\"================================================\")\n",
    "    print(\"Split: \"+str(i))\n",
    "    print(\"Number of Epochs: \"+str(n_epochs))\n",
    "    print(\"Train shape:\",X_train[i].shape)\n",
    "    print(\"Train batch size: \"+str(train_batch_size))\n",
    "    print(\"Val batch size: \"+str(val_batch_size))\n",
    "    print(\"================================================\")\n",
    "    \n",
    "    #=====================================\n",
    "    # Tensorboard callback\n",
    "    #=====================================\n",
    "    print(\"--- Preparing tensorboard\")\n",
    "    log_dir = \"logs/{}_fold[{}]_{}\".format(time(), i, 'fine-tunned')\n",
    "    print(\"Log Dir: \", log_dir)\n",
    "    tbCallBack = TensorBoard(log_dir=log_dir, write_graph=True)    \n",
    "    \n",
    "    # Training without data augmentation\n",
    "    model.fit(X_train[i],y_train_categorical[i], batch_size=train_batch_size, \n",
    "              epochs=n_epochs, \n",
    "              validation_data=(X_val[i],y_val_categorical[i]),\n",
    "              verbose=1,\n",
    "              callbacks=[tbCallBack])\n",
    "    \n",
    "    #--- Evaluating the model for split i\n",
    "    score = model.evaluate(x=X_val[i], y=y_val_categorical[i], batch_size=val_batch_size, verbose=1)\n",
    "    scores.append(score)\n",
    "    print('Validation loss:', score[0])\n",
    "    print('Validation accuracy (NORMALIZED):', score[1])\n",
    "    \n",
    "#--- Showing scores\n",
    "print(scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, let's evaluate on our test set:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Validation loss: 1.2750247840881346\n",
      "Mean Validation accuracy (NORMALIZED): 0.54878\n"
     ]
    }
   ],
   "source": [
    "#=====================================\n",
    "# Evaluate on validation\n",
    "#=====================================\n",
    "\n",
    "#--- The evaluation of the model \n",
    "np_aux = np.array(scores).mean(axis=0)\n",
    "\n",
    "print('Mean Validation loss:', np_aux[0])\n",
    "print('Mean Validation accuracy (NORMALIZED):', np_aux[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-----------\n",
    "-----------\n",
    "-----------\n",
    "# Tensorboard\n",
    "\n",
    "Tensorboard is a visualization tool for Tensorflow. Among other things, it allows us to monitor the progress of our training, plot metrics per epochs, visualize the architecture's schematics. \n",
    "\n",
    "Just like for Early Stopping, we will use the [Tensorboard callback](https://keras.io/callbacks/#tensorboard) to log the information about our training. An example of usage, would be:"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "## Just an example, DON'T RUN! \n",
    "### You will need to change <<LOG_DIR>>\n",
    "import keras.callbacks as callbacks\n",
    "tbCallBack = callbacks.TensorBoard(log_dir = \"./<<LOG_DIR>>\")\n",
    "model.fit(..., callbacks=[tbCallBack])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As your training progresses, Keras will log the metrics (e.g., loss, accuracy) to `<<LOG_DIR>>` (**make sure `<<LOG_DIR>>` is a valid directory)**. On your terminal, you will need to run Tensorboard, assign a port and access it via browser (just like jupyter).\n",
    "\n",
    "#### ----> MAKE SURE YOU USE A DIFFERENT PORT FOR JUPYTER AND TENSORBOARD <----\n",
    "\n",
    "### Docker\n",
    "For those using docker, open a new terminal and create a new container (using the same image) running Tensorboard:"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "$ docker run -it -p <<port_host>>:<<port_container>>\n",
    "            --volume=<<LOG_DIR>>:<<LOG_DIR>>\n",
    "            --name=<<container_name>> <<docker_image>> \n",
    "            tensorboard --logdir=<<LOG_DIR>> --port=<<port_container>>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For example:"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "$ docker run -it -p 8887:8887\n",
    "            --volume=/your/path/ml2018/:/ml2018\n",
    "            --name=mdc_container_tensorboard mdc-keras:cpu\n",
    "            tensorboard --logdir=/ml2018/logs --port=8887"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After starting Tensorboard, access it via browser on `http://localhost:<<port_container>>`.\n",
    "\n",
    "### Anaconda\n",
    "$ tensorboard --logdir=<<LOG_DIR>> --port=<<port>>"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "After starting Tensorboard, access it via browser on `http://localhost:<<port>>`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-----------\n",
    "-----------\n",
    "-----------\n",
    "\n",
    "# Fine-tuning all layers\n",
    "\n",
    "What if we fine-tune all layers of SqueezeNet?\n",
    "<img src=\"unfrozenSqueezeNet.png\" width=70% height=70%>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- Compiling model\n",
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_13 (InputLayer)           (None, 32, 32, 3)    0                                            \n",
      "__________________________________________________________________________________________________\n",
      "conv1 (Conv2D)                  (None, 15, 15, 64)   1792        input_13[0][0]                   \n",
      "__________________________________________________________________________________________________\n",
      "relu_conv1 (Activation)         (None, 15, 15, 64)   0           conv1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "pool1 (MaxPooling2D)            (None, 7, 7, 64)     0           relu_conv1[0][0]                 \n",
      "__________________________________________________________________________________________________\n",
      "fire2/squeeze1x1 (Conv2D)       (None, 7, 7, 16)     1040        pool1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "fire2/relu_squeeze1x1 (Activati (None, 7, 7, 16)     0           fire2/squeeze1x1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "fire2/expand1x1 (Conv2D)        (None, 7, 7, 64)     1088        fire2/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire2/expand3x3 (Conv2D)        (None, 7, 7, 64)     9280        fire2/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire2/relu_expand1x1 (Activatio (None, 7, 7, 64)     0           fire2/expand1x1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire2/relu_expand3x3 (Activatio (None, 7, 7, 64)     0           fire2/expand3x3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire2/concat (Concatenate)      (None, 7, 7, 128)    0           fire2/relu_expand1x1[0][0]       \n",
      "                                                                 fire2/relu_expand3x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "fire3/squeeze1x1 (Conv2D)       (None, 7, 7, 16)     2064        fire2/concat[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "fire3/relu_squeeze1x1 (Activati (None, 7, 7, 16)     0           fire3/squeeze1x1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "fire3/expand1x1 (Conv2D)        (None, 7, 7, 64)     1088        fire3/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire3/expand3x3 (Conv2D)        (None, 7, 7, 64)     9280        fire3/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire3/relu_expand1x1 (Activatio (None, 7, 7, 64)     0           fire3/expand1x1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire3/relu_expand3x3 (Activatio (None, 7, 7, 64)     0           fire3/expand3x3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire3/concat (Concatenate)      (None, 7, 7, 128)    0           fire3/relu_expand1x1[0][0]       \n",
      "                                                                 fire3/relu_expand3x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "pool3 (MaxPooling2D)            (None, 3, 3, 128)    0           fire3/concat[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "fire4/squeeze1x1 (Conv2D)       (None, 3, 3, 32)     4128        pool3[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "fire4/relu_squeeze1x1 (Activati (None, 3, 3, 32)     0           fire4/squeeze1x1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "fire4/expand1x1 (Conv2D)        (None, 3, 3, 128)    4224        fire4/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire4/expand3x3 (Conv2D)        (None, 3, 3, 128)    36992       fire4/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire4/relu_expand1x1 (Activatio (None, 3, 3, 128)    0           fire4/expand1x1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire4/relu_expand3x3 (Activatio (None, 3, 3, 128)    0           fire4/expand3x3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire4/concat (Concatenate)      (None, 3, 3, 256)    0           fire4/relu_expand1x1[0][0]       \n",
      "                                                                 fire4/relu_expand3x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "fire5/squeeze1x1 (Conv2D)       (None, 3, 3, 32)     8224        fire4/concat[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "fire5/relu_squeeze1x1 (Activati (None, 3, 3, 32)     0           fire5/squeeze1x1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "fire5/expand1x1 (Conv2D)        (None, 3, 3, 128)    4224        fire5/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire5/expand3x3 (Conv2D)        (None, 3, 3, 128)    36992       fire5/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire5/relu_expand1x1 (Activatio (None, 3, 3, 128)    0           fire5/expand1x1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire5/relu_expand3x3 (Activatio (None, 3, 3, 128)    0           fire5/expand3x3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire5/concat (Concatenate)      (None, 3, 3, 256)    0           fire5/relu_expand1x1[0][0]       \n",
      "                                                                 fire5/relu_expand3x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "pool5 (MaxPooling2D)            (None, 1, 1, 256)    0           fire5/concat[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "fire6/squeeze1x1 (Conv2D)       (None, 1, 1, 48)     12336       pool5[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "fire6/relu_squeeze1x1 (Activati (None, 1, 1, 48)     0           fire6/squeeze1x1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "fire6/expand1x1 (Conv2D)        (None, 1, 1, 192)    9408        fire6/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire6/expand3x3 (Conv2D)        (None, 1, 1, 192)    83136       fire6/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire6/relu_expand1x1 (Activatio (None, 1, 1, 192)    0           fire6/expand1x1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire6/relu_expand3x3 (Activatio (None, 1, 1, 192)    0           fire6/expand3x3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire6/concat (Concatenate)      (None, 1, 1, 384)    0           fire6/relu_expand1x1[0][0]       \n",
      "                                                                 fire6/relu_expand3x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "fire7/squeeze1x1 (Conv2D)       (None, 1, 1, 48)     18480       fire6/concat[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "fire7/relu_squeeze1x1 (Activati (None, 1, 1, 48)     0           fire7/squeeze1x1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "fire7/expand1x1 (Conv2D)        (None, 1, 1, 192)    9408        fire7/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire7/expand3x3 (Conv2D)        (None, 1, 1, 192)    83136       fire7/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire7/relu_expand1x1 (Activatio (None, 1, 1, 192)    0           fire7/expand1x1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire7/relu_expand3x3 (Activatio (None, 1, 1, 192)    0           fire7/expand3x3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire7/concat (Concatenate)      (None, 1, 1, 384)    0           fire7/relu_expand1x1[0][0]       \n",
      "                                                                 fire7/relu_expand3x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "fire8/squeeze1x1 (Conv2D)       (None, 1, 1, 64)     24640       fire7/concat[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "fire8/relu_squeeze1x1 (Activati (None, 1, 1, 64)     0           fire8/squeeze1x1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "fire8/expand1x1 (Conv2D)        (None, 1, 1, 256)    16640       fire8/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire8/expand3x3 (Conv2D)        (None, 1, 1, 256)    147712      fire8/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire8/relu_expand1x1 (Activatio (None, 1, 1, 256)    0           fire8/expand1x1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire8/relu_expand3x3 (Activatio (None, 1, 1, 256)    0           fire8/expand3x3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire8/concat (Concatenate)      (None, 1, 1, 512)    0           fire8/relu_expand1x1[0][0]       \n",
      "                                                                 fire8/relu_expand3x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "fire9/squeeze1x1 (Conv2D)       (None, 1, 1, 64)     32832       fire8/concat[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "fire9/relu_squeeze1x1 (Activati (None, 1, 1, 64)     0           fire9/squeeze1x1[0][0]           \n",
      "__________________________________________________________________________________________________\n",
      "fire9/expand1x1 (Conv2D)        (None, 1, 1, 256)    16640       fire9/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire9/expand3x3 (Conv2D)        (None, 1, 1, 256)    147712      fire9/relu_squeeze1x1[0][0]      \n",
      "__________________________________________________________________________________________________\n",
      "fire9/relu_expand1x1 (Activatio (None, 1, 1, 256)    0           fire9/expand1x1[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire9/relu_expand3x3 (Activatio (None, 1, 1, 256)    0           fire9/expand3x3[0][0]            \n",
      "__________________________________________________________________________________________________\n",
      "fire9/concat (Concatenate)      (None, 1, 1, 512)    0           fire9/relu_expand1x1[0][0]       \n",
      "                                                                 fire9/relu_expand3x3[0][0]       \n",
      "__________________________________________________________________________________________________\n",
      "drop9 (Dropout)                 (None, 1, 1, 512)    0           fire9/concat[0][0]               \n",
      "__________________________________________________________________________________________________\n",
      "conv10 (Conv2D)                 (None, 1, 1, 10)     5130        drop9[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "relu_conv10 (Activation)        (None, 1, 1, 10)     0           conv10[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "global_average_pooling2d_26 (Gl (None, 10)           0           relu_conv10[0][0]                \n",
      "__________________________________________________________________________________________________\n",
      "loss (Activation)               (None, 10)           0           global_average_pooling2d_26[0][0]\n",
      "==================================================================================================\n",
      "Total params: 727,626\n",
      "Trainable params: 727,626\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "def get_squeezenet_ft3():\n",
    "    squeezeNetModel = SqueezeNet((32,32,3))\n",
    "\n",
    "    for layer in squeezeNetModel.layers:\n",
    "        layer.trainable = True       #by default they are all trainable, but just for clarification\n",
    "\n",
    "    #=====================================\n",
    "    # Add new classification layers\n",
    "    #=====================================\n",
    "\n",
    "    #--- Removing layers until drop 9\n",
    "    squeezeNetModel.layers.pop() #Convolution2D\n",
    "    squeezeNetModel.layers.pop() #Activation ReLU\n",
    "    squeezeNetModel.layers.pop() #Global Avg Pool\n",
    "    squeezeNetModel.layers.pop() #Activation Softmax\n",
    "\n",
    "    #--- Adding classification layer for 10 classes\n",
    "    out = Convolution2D(n_classes, (1, 1), padding='valid', name='conv10')(squeezeNetModel.layers[-1].output)\n",
    "    out = Activation('relu', name='relu_conv10')(out)\n",
    "\n",
    "    out = GlobalAveragePooling2D()(out)\n",
    "    out = Activation('softmax', name='loss')(out)\n",
    "\n",
    "    #=====================================\n",
    "    # New Model\n",
    "    #=====================================\n",
    "    model = Model(squeezeNetModel.inputs, out, name='squeezenet_new')\n",
    "    opt = optimizer=optimizers.Adam(lr=learning_rate, beta_1=0.9, beta_2=0.999, epsilon=None, decay=0.0, amsgrad=False)#SGD(lr=learning_rate)\n",
    "    print(\"--- Compiling model\")\n",
    "    model.compile(loss='categorical_crossentropy',\n",
    "                  optimizer=opt,\n",
    "                  metrics=['accuracy'])\n",
    "    return model\n",
    "\n",
    "model = get_squeezenet_ft3()\n",
    "model.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we compile our model and train it:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- Start training\n",
      "--- Compiling model\n",
      "================================================\n",
      "Split: 0\n",
      "Number of Epochs: 100\n",
      "Train shape: (40000, 32, 32, 3)\n",
      "Train batch size: 384\n",
      "Val batch size: 128\n",
      "================================================\n",
      "--- Preparing tensorboard\n",
      "Log Dir:  logs/1542822066.911866_fold[0]_fully-trained\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/100\n",
      "40000/40000 [==============================] - 4s 94us/step - loss: 2.0651 - acc: 0.2545 - val_loss: 1.5485 - val_acc: 0.4718\n",
      "Epoch 2/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.4449 - acc: 0.4942 - val_loss: 1.1769 - val_acc: 0.5924\n",
      "Epoch 3/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.1981 - acc: 0.5910 - val_loss: 1.0688 - val_acc: 0.6320\n",
      "Epoch 4/100\n",
      "40000/40000 [==============================] - 3s 74us/step - loss: 1.0876 - acc: 0.6315 - val_loss: 1.0008 - val_acc: 0.6537\n",
      "Epoch 5/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.0082 - acc: 0.6622 - val_loss: 0.9538 - val_acc: 0.6703\n",
      "Epoch 6/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.9522 - acc: 0.6803 - val_loss: 0.9148 - val_acc: 0.6871\n",
      "Epoch 7/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.9119 - acc: 0.6952 - val_loss: 0.9132 - val_acc: 0.6879\n",
      "Epoch 8/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.8679 - acc: 0.7118 - val_loss: 0.8764 - val_acc: 0.6990\n",
      "Epoch 9/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.8372 - acc: 0.7207 - val_loss: 0.8611 - val_acc: 0.7040\n",
      "Epoch 10/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.8144 - acc: 0.7291 - val_loss: 0.8497 - val_acc: 0.7067\n",
      "Epoch 11/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.7902 - acc: 0.7362 - val_loss: 0.8366 - val_acc: 0.7138\n",
      "Epoch 12/100\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.7648 - acc: 0.7471 - val_loss: 0.8330 - val_acc: 0.7126\n",
      "Epoch 13/100\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.7421 - acc: 0.7533 - val_loss: 0.8331 - val_acc: 0.7168\n",
      "Epoch 14/100\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.7222 - acc: 0.7595 - val_loss: 0.8112 - val_acc: 0.7236\n",
      "Epoch 15/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.6979 - acc: 0.7681 - val_loss: 0.8016 - val_acc: 0.7267\n",
      "Epoch 16/100\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.6778 - acc: 0.7752 - val_loss: 0.7958 - val_acc: 0.7287\n",
      "Epoch 17/100\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.6671 - acc: 0.7791 - val_loss: 0.8129 - val_acc: 0.7260\n",
      "Epoch 18/100\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.6513 - acc: 0.7838 - val_loss: 0.7910 - val_acc: 0.7327\n",
      "Epoch 19/100\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.6334 - acc: 0.7902 - val_loss: 0.7845 - val_acc: 0.7350\n",
      "Epoch 20/100\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.6166 - acc: 0.7961 - val_loss: 0.7949 - val_acc: 0.7349\n",
      "Epoch 21/100\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.6059 - acc: 0.8010 - val_loss: 0.7848 - val_acc: 0.7402\n",
      "Epoch 22/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.5887 - acc: 0.8050 - val_loss: 0.7772 - val_acc: 0.7398\n",
      "Epoch 23/100\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.5740 - acc: 0.8095 - val_loss: 0.7812 - val_acc: 0.7380\n",
      "Epoch 24/100\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.5587 - acc: 0.8147 - val_loss: 0.7780 - val_acc: 0.7437\n",
      "Epoch 25/100\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.5533 - acc: 0.8195 - val_loss: 0.7875 - val_acc: 0.7419\n",
      "Epoch 26/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.5398 - acc: 0.8229 - val_loss: 0.7880 - val_acc: 0.7442\n",
      "Epoch 27/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.5307 - acc: 0.8247 - val_loss: 0.7866 - val_acc: 0.7457\n",
      "Epoch 28/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.5149 - acc: 0.8305 - val_loss: 0.7853 - val_acc: 0.7468\n",
      "Epoch 29/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.5019 - acc: 0.8354 - val_loss: 0.7929 - val_acc: 0.7439\n",
      "Epoch 30/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.4915 - acc: 0.8381 - val_loss: 0.7875 - val_acc: 0.7482\n",
      "Epoch 31/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.4831 - acc: 0.8408 - val_loss: 0.8036 - val_acc: 0.7474\n",
      "Epoch 32/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.4695 - acc: 0.8466 - val_loss: 0.7927 - val_acc: 0.7507\n",
      "Epoch 33/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.4590 - acc: 0.8504 - val_loss: 0.7943 - val_acc: 0.7539\n",
      "Epoch 34/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.4453 - acc: 0.8554 - val_loss: 0.8209 - val_acc: 0.7490\n",
      "Epoch 35/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.4346 - acc: 0.8584 - val_loss: 0.8164 - val_acc: 0.7526\n",
      "Epoch 36/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.4292 - acc: 0.8594 - val_loss: 0.8269 - val_acc: 0.7508\n",
      "Epoch 37/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.4164 - acc: 0.8653 - val_loss: 0.8362 - val_acc: 0.7497\n",
      "Epoch 38/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.4116 - acc: 0.8647 - val_loss: 0.8257 - val_acc: 0.7533\n",
      "Epoch 39/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.3952 - acc: 0.8725 - val_loss: 0.8500 - val_acc: 0.7522\n",
      "Epoch 40/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.3896 - acc: 0.8734 - val_loss: 0.8417 - val_acc: 0.7477\n",
      "Epoch 41/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.3766 - acc: 0.8796 - val_loss: 0.8533 - val_acc: 0.7527\n",
      "Epoch 42/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.3711 - acc: 0.8796 - val_loss: 0.8784 - val_acc: 0.7507\n",
      "Epoch 43/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.3605 - acc: 0.8838 - val_loss: 0.8667 - val_acc: 0.7537\n",
      "Epoch 44/100\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.3516 - acc: 0.8873 - val_loss: 0.8874 - val_acc: 0.7521\n",
      "Epoch 45/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.3394 - acc: 0.8919 - val_loss: 0.8882 - val_acc: 0.7559\n",
      "Epoch 46/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.3369 - acc: 0.8917 - val_loss: 0.9047 - val_acc: 0.7536\n",
      "Epoch 47/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.3278 - acc: 0.8954 - val_loss: 0.9368 - val_acc: 0.7519\n",
      "Epoch 48/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.3182 - acc: 0.8988 - val_loss: 0.9367 - val_acc: 0.7456\n",
      "Epoch 49/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.3081 - acc: 0.8996 - val_loss: 0.9489 - val_acc: 0.7506\n",
      "Epoch 50/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.3025 - acc: 0.9036 - val_loss: 0.9371 - val_acc: 0.7526\n",
      "Epoch 51/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.2927 - acc: 0.9065 - val_loss: 0.9613 - val_acc: 0.7451\n",
      "Epoch 52/100\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.2855 - acc: 0.9086 - val_loss: 0.9877 - val_acc: 0.7451\n",
      "Epoch 53/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.2833 - acc: 0.9094 - val_loss: 1.0016 - val_acc: 0.7487\n",
      "Epoch 54/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.2662 - acc: 0.9165 - val_loss: 1.0052 - val_acc: 0.7526\n",
      "Epoch 55/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.2607 - acc: 0.9181 - val_loss: 1.0417 - val_acc: 0.7503\n",
      "Epoch 56/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.2516 - acc: 0.9216 - val_loss: 1.0509 - val_acc: 0.7476\n",
      "Epoch 57/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.2471 - acc: 0.9230 - val_loss: 1.0487 - val_acc: 0.7510\n",
      "Epoch 58/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.2393 - acc: 0.9263 - val_loss: 1.0581 - val_acc: 0.7498\n",
      "Epoch 59/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.2286 - acc: 0.9290 - val_loss: 1.1082 - val_acc: 0.7469\n",
      "Epoch 60/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.2252 - acc: 0.9294 - val_loss: 1.1195 - val_acc: 0.7500\n",
      "Epoch 61/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.2294 - acc: 0.9281 - val_loss: 1.1261 - val_acc: 0.7486\n",
      "Epoch 62/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.2212 - acc: 0.9307 - val_loss: 1.1502 - val_acc: 0.7444\n",
      "Epoch 63/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.2056 - acc: 0.9361 - val_loss: 1.1573 - val_acc: 0.7528\n",
      "Epoch 64/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.2002 - acc: 0.9380 - val_loss: 1.1681 - val_acc: 0.7504\n",
      "Epoch 65/100\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.1972 - acc: 0.9393 - val_loss: 1.1931 - val_acc: 0.7495\n",
      "Epoch 66/100\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.1900 - acc: 0.9425 - val_loss: 1.2179 - val_acc: 0.7506\n",
      "Epoch 67/100\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.1838 - acc: 0.9439 - val_loss: 1.2118 - val_acc: 0.7481\n",
      "Epoch 68/100\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.1789 - acc: 0.9450 - val_loss: 1.2165 - val_acc: 0.7499\n",
      "Epoch 69/100\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.1684 - acc: 0.9486 - val_loss: 1.2359 - val_acc: 0.7461\n",
      "Epoch 70/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.1639 - acc: 0.9506 - val_loss: 1.2964 - val_acc: 0.7459\n",
      "Epoch 71/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1606 - acc: 0.9521 - val_loss: 1.2743 - val_acc: 0.7463\n",
      "Epoch 72/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.1538 - acc: 0.9543 - val_loss: 1.2944 - val_acc: 0.7495\n",
      "Epoch 73/100\n",
      "40000/40000 [==============================] - 3s 72us/step - loss: 0.1500 - acc: 0.9550 - val_loss: 1.3553 - val_acc: 0.7425\n",
      "Epoch 74/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.1513 - acc: 0.9541 - val_loss: 1.3279 - val_acc: 0.7477\n",
      "Epoch 75/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.1521 - acc: 0.9539 - val_loss: 1.4459 - val_acc: 0.7393\n",
      "Epoch 76/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.1468 - acc: 0.9561 - val_loss: 1.3615 - val_acc: 0.7486\n",
      "Epoch 77/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1365 - acc: 0.9600 - val_loss: 1.4353 - val_acc: 0.7408\n",
      "Epoch 78/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.1302 - acc: 0.9624 - val_loss: 1.4331 - val_acc: 0.7420\n",
      "Epoch 79/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.1231 - acc: 0.9637 - val_loss: 1.4622 - val_acc: 0.7444\n",
      "Epoch 80/100\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.1178 - acc: 0.9661 - val_loss: 1.4587 - val_acc: 0.7409\n",
      "Epoch 81/100\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.1207 - acc: 0.9649 - val_loss: 1.5127 - val_acc: 0.7442\n",
      "Epoch 82/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.1154 - acc: 0.9669 - val_loss: 1.5849 - val_acc: 0.7381\n",
      "Epoch 83/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1201 - acc: 0.9643 - val_loss: 1.4964 - val_acc: 0.7461\n",
      "Epoch 84/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.1088 - acc: 0.9678 - val_loss: 1.5539 - val_acc: 0.7446\n",
      "Epoch 85/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.1106 - acc: 0.9673 - val_loss: 1.5781 - val_acc: 0.7387\n",
      "Epoch 86/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1089 - acc: 0.9689 - val_loss: 1.5573 - val_acc: 0.7441\n",
      "Epoch 87/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.1078 - acc: 0.9690 - val_loss: 1.5768 - val_acc: 0.7388\n",
      "Epoch 88/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.1071 - acc: 0.9686 - val_loss: 1.5737 - val_acc: 0.7403\n",
      "Epoch 89/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.0961 - acc: 0.9726 - val_loss: 1.5849 - val_acc: 0.7449\n",
      "Epoch 90/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.0901 - acc: 0.9747 - val_loss: 1.6020 - val_acc: 0.7421\n",
      "Epoch 91/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.0855 - acc: 0.9765 - val_loss: 1.6743 - val_acc: 0.7389\n",
      "Epoch 92/100\n",
      "40000/40000 [==============================] - 3s 72us/step - loss: 0.0814 - acc: 0.9775 - val_loss: 1.6927 - val_acc: 0.7415\n",
      "Epoch 93/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.0826 - acc: 0.9774 - val_loss: 1.6996 - val_acc: 0.7409\n",
      "Epoch 94/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.0883 - acc: 0.9746 - val_loss: 1.7018 - val_acc: 0.7368\n",
      "Epoch 95/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.0877 - acc: 0.9745 - val_loss: 1.7175 - val_acc: 0.7378\n",
      "Epoch 96/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.0868 - acc: 0.9745 - val_loss: 1.8325 - val_acc: 0.7320\n",
      "Epoch 97/100\n",
      "40000/40000 [==============================] - 4s 90us/step - loss: 0.0776 - acc: 0.9788 - val_loss: 1.7570 - val_acc: 0.7422\n",
      "Epoch 98/100\n",
      "40000/40000 [==============================] - 3s 87us/step - loss: 0.0746 - acc: 0.9797 - val_loss: 1.7159 - val_acc: 0.7393\n",
      "Epoch 99/100\n",
      "40000/40000 [==============================] - 3s 80us/step - loss: 0.0703 - acc: 0.9806 - val_loss: 1.7610 - val_acc: 0.7356\n",
      "Epoch 100/100\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.0736 - acc: 0.9800 - val_loss: 1.8033 - val_acc: 0.7359\n",
      "10000/10000 [==============================] - 0s 32us/step\n",
      "Validation loss: 1.8032953178405762\n",
      "Validation accuracy (NORMALIZED): 0.7359\n",
      "--- Compiling model\n",
      "================================================\n",
      "Split: 1\n",
      "Number of Epochs: 100\n",
      "Train shape: (40000, 32, 32, 3)\n",
      "Train batch size: 384\n",
      "Val batch size: 128\n",
      "================================================\n",
      "--- Preparing tensorboard\n",
      "Log Dir:  logs/1542822344.3423727_fold[1]_fully-trained\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/100\n",
      "40000/40000 [==============================] - 4s 89us/step - loss: 1.9783 - acc: 0.3028 - val_loss: 1.4767 - val_acc: 0.4987\n",
      "Epoch 2/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.4415 - acc: 0.5043 - val_loss: 1.2354 - val_acc: 0.5741\n",
      "Epoch 3/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.2417 - acc: 0.5794 - val_loss: 1.1330 - val_acc: 0.6167\n",
      "Epoch 4/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.1395 - acc: 0.6164 - val_loss: 1.0605 - val_acc: 0.6347\n",
      "Epoch 5/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 1.0614 - acc: 0.6423 - val_loss: 1.0137 - val_acc: 0.6496\n",
      "Epoch 6/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.9972 - acc: 0.6658 - val_loss: 0.9867 - val_acc: 0.6608\n",
      "Epoch 7/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.9496 - acc: 0.6827 - val_loss: 0.9479 - val_acc: 0.6735\n",
      "Epoch 8/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.9093 - acc: 0.6978 - val_loss: 0.9252 - val_acc: 0.6816\n",
      "Epoch 9/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.8796 - acc: 0.7063 - val_loss: 0.9054 - val_acc: 0.6891\n",
      "Epoch 10/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.8566 - acc: 0.7126 - val_loss: 0.8941 - val_acc: 0.6934\n",
      "Epoch 11/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.8201 - acc: 0.7276 - val_loss: 0.8829 - val_acc: 0.6973\n",
      "Epoch 12/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.7968 - acc: 0.7353 - val_loss: 0.8724 - val_acc: 0.7069\n",
      "Epoch 13/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.7779 - acc: 0.7394 - val_loss: 0.8593 - val_acc: 0.7083\n",
      "Epoch 14/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.7559 - acc: 0.7476 - val_loss: 0.8464 - val_acc: 0.7122\n",
      "Epoch 15/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.7336 - acc: 0.7553 - val_loss: 0.8378 - val_acc: 0.7182\n",
      "Epoch 16/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.7143 - acc: 0.7629 - val_loss: 0.8369 - val_acc: 0.7203\n",
      "Epoch 17/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.6987 - acc: 0.7680 - val_loss: 0.8330 - val_acc: 0.7242\n",
      "Epoch 18/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.6812 - acc: 0.7736 - val_loss: 0.8305 - val_acc: 0.7219\n",
      "Epoch 19/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.6677 - acc: 0.7818 - val_loss: 0.8286 - val_acc: 0.7218\n",
      "Epoch 20/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.6489 - acc: 0.7848 - val_loss: 0.8234 - val_acc: 0.7294\n",
      "Epoch 21/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.6322 - acc: 0.7916 - val_loss: 0.8198 - val_acc: 0.7299\n",
      "Epoch 22/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.6173 - acc: 0.7968 - val_loss: 0.8303 - val_acc: 0.7302\n",
      "Epoch 23/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.6093 - acc: 0.7992 - val_loss: 0.8248 - val_acc: 0.7314\n",
      "Epoch 24/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.5939 - acc: 0.8021 - val_loss: 0.8278 - val_acc: 0.7314\n",
      "Epoch 25/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.5771 - acc: 0.8107 - val_loss: 0.8188 - val_acc: 0.7332\n",
      "Epoch 26/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.5675 - acc: 0.8136 - val_loss: 0.8303 - val_acc: 0.7366\n",
      "Epoch 27/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.5560 - acc: 0.8158 - val_loss: 0.8249 - val_acc: 0.7386\n",
      "Epoch 28/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.5422 - acc: 0.8231 - val_loss: 0.8151 - val_acc: 0.7390\n",
      "Epoch 29/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.5305 - acc: 0.8258 - val_loss: 0.8396 - val_acc: 0.7372\n",
      "Epoch 30/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.5187 - acc: 0.8290 - val_loss: 0.8267 - val_acc: 0.7370\n",
      "Epoch 31/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.5084 - acc: 0.8334 - val_loss: 0.8361 - val_acc: 0.7398\n",
      "Epoch 32/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.4936 - acc: 0.8380 - val_loss: 0.8349 - val_acc: 0.7403\n",
      "Epoch 33/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.4899 - acc: 0.8402 - val_loss: 0.8383 - val_acc: 0.7402\n",
      "Epoch 34/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.4783 - acc: 0.8438 - val_loss: 0.8289 - val_acc: 0.7440\n",
      "Epoch 35/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.4646 - acc: 0.8478 - val_loss: 0.8376 - val_acc: 0.7445\n",
      "Epoch 36/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.4532 - acc: 0.8529 - val_loss: 0.8741 - val_acc: 0.7339\n",
      "Epoch 37/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.4425 - acc: 0.8544 - val_loss: 0.8592 - val_acc: 0.7399\n",
      "Epoch 38/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.4300 - acc: 0.8595 - val_loss: 0.8607 - val_acc: 0.7362\n",
      "Epoch 39/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.4309 - acc: 0.8591 - val_loss: 0.8753 - val_acc: 0.7374\n",
      "Epoch 40/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.4174 - acc: 0.8639 - val_loss: 0.8819 - val_acc: 0.7405\n",
      "Epoch 41/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.4001 - acc: 0.8685 - val_loss: 0.9246 - val_acc: 0.7332\n",
      "Epoch 42/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.4028 - acc: 0.8692 - val_loss: 0.8923 - val_acc: 0.7426\n",
      "Epoch 43/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.3816 - acc: 0.8769 - val_loss: 0.9041 - val_acc: 0.7418\n",
      "Epoch 44/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.3751 - acc: 0.8793 - val_loss: 0.9156 - val_acc: 0.7431\n",
      "Epoch 45/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.3614 - acc: 0.8836 - val_loss: 0.9255 - val_acc: 0.7409\n",
      "Epoch 46/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.3561 - acc: 0.8873 - val_loss: 0.9279 - val_acc: 0.7419\n",
      "Epoch 47/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.3495 - acc: 0.8879 - val_loss: 0.9473 - val_acc: 0.7401\n",
      "Epoch 48/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.3417 - acc: 0.8900 - val_loss: 0.9682 - val_acc: 0.7394\n",
      "Epoch 49/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.3306 - acc: 0.8945 - val_loss: 0.9736 - val_acc: 0.7391\n",
      "Epoch 50/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.3232 - acc: 0.8968 - val_loss: 0.9900 - val_acc: 0.7411\n",
      "Epoch 51/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.3126 - acc: 0.9024 - val_loss: 0.9969 - val_acc: 0.7397\n",
      "Epoch 52/100\n",
      "40000/40000 [==============================] - 3s 77us/step - loss: 0.3078 - acc: 0.9000 - val_loss: 1.0115 - val_acc: 0.7431\n",
      "Epoch 53/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.2998 - acc: 0.9053 - val_loss: 1.0545 - val_acc: 0.7392\n",
      "Epoch 54/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.2884 - acc: 0.9102 - val_loss: 1.0369 - val_acc: 0.7373\n",
      "Epoch 55/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.2846 - acc: 0.9102 - val_loss: 1.0646 - val_acc: 0.7378\n",
      "Epoch 56/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.2817 - acc: 0.9101 - val_loss: 1.0596 - val_acc: 0.7405\n",
      "Epoch 57/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.2680 - acc: 0.9165 - val_loss: 1.0866 - val_acc: 0.7378\n",
      "Epoch 58/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.2613 - acc: 0.9189 - val_loss: 1.0899 - val_acc: 0.7393\n",
      "Epoch 59/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.2549 - acc: 0.9211 - val_loss: 1.1293 - val_acc: 0.7317\n",
      "Epoch 60/100\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.2477 - acc: 0.9226 - val_loss: 1.1447 - val_acc: 0.7388\n",
      "Epoch 61/100\n",
      "40000/40000 [==============================] - 3s 65us/step - loss: 0.2436 - acc: 0.9258 - val_loss: 1.1343 - val_acc: 0.7376\n",
      "Epoch 62/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.2322 - acc: 0.9275 - val_loss: 1.1495 - val_acc: 0.7393\n",
      "Epoch 63/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.2320 - acc: 0.9280 - val_loss: 1.1450 - val_acc: 0.7422\n",
      "Epoch 64/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.2320 - acc: 0.9270 - val_loss: 1.1817 - val_acc: 0.7367\n",
      "Epoch 65/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.2170 - acc: 0.9343 - val_loss: 1.2563 - val_acc: 0.7399\n",
      "Epoch 66/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.2116 - acc: 0.9348 - val_loss: 1.2347 - val_acc: 0.7350\n",
      "Epoch 67/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.2028 - acc: 0.9384 - val_loss: 1.2153 - val_acc: 0.7417\n",
      "Epoch 68/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.2031 - acc: 0.9378 - val_loss: 1.2573 - val_acc: 0.7343\n",
      "Epoch 69/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.2051 - acc: 0.9355 - val_loss: 1.2648 - val_acc: 0.7365\n",
      "Epoch 70/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1907 - acc: 0.9429 - val_loss: 1.2871 - val_acc: 0.7335\n",
      "Epoch 71/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1826 - acc: 0.9449 - val_loss: 1.2968 - val_acc: 0.7358\n",
      "Epoch 72/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.1774 - acc: 0.9464 - val_loss: 1.3338 - val_acc: 0.7330\n",
      "Epoch 73/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1742 - acc: 0.9484 - val_loss: 1.3357 - val_acc: 0.7369\n",
      "Epoch 74/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1765 - acc: 0.9464 - val_loss: 1.3149 - val_acc: 0.7376\n",
      "Epoch 75/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.1583 - acc: 0.9536 - val_loss: 1.3605 - val_acc: 0.7351\n",
      "Epoch 76/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1597 - acc: 0.9531 - val_loss: 1.4283 - val_acc: 0.7337\n",
      "Epoch 77/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.1558 - acc: 0.9537 - val_loss: 1.4155 - val_acc: 0.7337\n",
      "Epoch 78/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1524 - acc: 0.9554 - val_loss: 1.4430 - val_acc: 0.7376\n",
      "Epoch 79/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1453 - acc: 0.9564 - val_loss: 1.4620 - val_acc: 0.7338\n",
      "Epoch 80/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.1394 - acc: 0.9601 - val_loss: 1.4561 - val_acc: 0.7336\n",
      "Epoch 81/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.1430 - acc: 0.9578 - val_loss: 1.4681 - val_acc: 0.7339\n",
      "Epoch 82/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1425 - acc: 0.9576 - val_loss: 1.4741 - val_acc: 0.7357\n",
      "Epoch 83/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1276 - acc: 0.9633 - val_loss: 1.5252 - val_acc: 0.7371\n",
      "Epoch 84/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1284 - acc: 0.9626 - val_loss: 1.4979 - val_acc: 0.7334\n",
      "Epoch 85/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.1225 - acc: 0.9647 - val_loss: 1.5768 - val_acc: 0.7317\n",
      "Epoch 86/100\n",
      "40000/40000 [==============================] - 3s 72us/step - loss: 0.1246 - acc: 0.9639 - val_loss: 1.5571 - val_acc: 0.7370\n",
      "Epoch 87/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.1180 - acc: 0.9656 - val_loss: 1.5572 - val_acc: 0.7365\n",
      "Epoch 88/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1134 - acc: 0.9679 - val_loss: 1.6237 - val_acc: 0.7324\n",
      "Epoch 89/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1282 - acc: 0.9618 - val_loss: 1.6579 - val_acc: 0.7319\n",
      "Epoch 90/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1086 - acc: 0.9695 - val_loss: 1.6312 - val_acc: 0.7342\n",
      "Epoch 91/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.1030 - acc: 0.9709 - val_loss: 1.7497 - val_acc: 0.7314\n",
      "Epoch 92/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1085 - acc: 0.9682 - val_loss: 1.6848 - val_acc: 0.7314\n",
      "Epoch 93/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.0993 - acc: 0.9719 - val_loss: 1.6810 - val_acc: 0.7330\n",
      "Epoch 94/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1025 - acc: 0.9706 - val_loss: 1.7131 - val_acc: 0.7337\n",
      "Epoch 95/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1005 - acc: 0.9719 - val_loss: 1.6858 - val_acc: 0.7344\n",
      "Epoch 96/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.0945 - acc: 0.9726 - val_loss: 1.7796 - val_acc: 0.7305\n",
      "Epoch 97/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.0912 - acc: 0.9740 - val_loss: 1.7360 - val_acc: 0.7344\n",
      "Epoch 98/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.0864 - acc: 0.9755 - val_loss: 1.7239 - val_acc: 0.7332\n",
      "Epoch 99/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.0834 - acc: 0.9761 - val_loss: 1.7559 - val_acc: 0.7341\n",
      "Epoch 100/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.0781 - acc: 0.9790 - val_loss: 1.7919 - val_acc: 0.7359\n",
      "10000/10000 [==============================] - 0s 34us/step\n",
      "Validation loss: 1.791891314125061\n",
      "Validation accuracy (NORMALIZED): 0.7359\n",
      "--- Compiling model\n",
      "================================================\n",
      "Split: 2\n",
      "Number of Epochs: 100\n",
      "Train shape: (40000, 32, 32, 3)\n",
      "Train batch size: 384\n",
      "Val batch size: 128\n",
      "================================================\n",
      "--- Preparing tensorboard\n",
      "Log Dir:  logs/1542822619.153883_fold[2]_fully-trained\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/100\n",
      "40000/40000 [==============================] - 4s 97us/step - loss: 2.0793 - acc: 0.2552 - val_loss: 1.5852 - val_acc: 0.4625\n",
      "Epoch 2/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 1.4930 - acc: 0.4847 - val_loss: 1.2088 - val_acc: 0.5882\n",
      "Epoch 3/100\n",
      "40000/40000 [==============================] - 3s 72us/step - loss: 1.2410 - acc: 0.5772 - val_loss: 1.0925 - val_acc: 0.6243\n",
      "Epoch 4/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 1.1350 - acc: 0.6125 - val_loss: 1.0405 - val_acc: 0.6386\n",
      "Epoch 5/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.0579 - acc: 0.6406 - val_loss: 0.9991 - val_acc: 0.6561\n",
      "Epoch 6/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.9976 - acc: 0.6625 - val_loss: 0.9582 - val_acc: 0.6737\n",
      "Epoch 7/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.9566 - acc: 0.6783 - val_loss: 0.9369 - val_acc: 0.6806\n",
      "Epoch 8/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.9135 - acc: 0.6919 - val_loss: 0.9129 - val_acc: 0.6886\n",
      "Epoch 9/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.8815 - acc: 0.7028 - val_loss: 0.8982 - val_acc: 0.6929\n",
      "Epoch 10/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.8544 - acc: 0.7157 - val_loss: 0.8756 - val_acc: 0.7011\n",
      "Epoch 11/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.8267 - acc: 0.7220 - val_loss: 0.8689 - val_acc: 0.7072\n",
      "Epoch 12/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.8047 - acc: 0.7327 - val_loss: 0.8540 - val_acc: 0.7108\n",
      "Epoch 13/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.7803 - acc: 0.7396 - val_loss: 0.8572 - val_acc: 0.7133\n",
      "Epoch 14/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.7575 - acc: 0.7468 - val_loss: 0.8325 - val_acc: 0.7196\n",
      "Epoch 15/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.7442 - acc: 0.7522 - val_loss: 0.8359 - val_acc: 0.7193\n",
      "Epoch 16/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.7262 - acc: 0.7561 - val_loss: 0.8309 - val_acc: 0.7271\n",
      "Epoch 17/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.7059 - acc: 0.7647 - val_loss: 0.8201 - val_acc: 0.7272\n",
      "Epoch 18/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.6899 - acc: 0.7701 - val_loss: 0.8343 - val_acc: 0.7229\n",
      "Epoch 19/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.6752 - acc: 0.7745 - val_loss: 0.8446 - val_acc: 0.7254\n",
      "Epoch 20/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.6635 - acc: 0.7798 - val_loss: 0.8105 - val_acc: 0.7318\n",
      "Epoch 21/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.6500 - acc: 0.7852 - val_loss: 0.8292 - val_acc: 0.7305\n",
      "Epoch 22/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.6351 - acc: 0.7887 - val_loss: 0.8132 - val_acc: 0.7298\n",
      "Epoch 23/100\n",
      "40000/40000 [==============================] - 3s 72us/step - loss: 0.6211 - acc: 0.7944 - val_loss: 0.7989 - val_acc: 0.7346\n",
      "Epoch 24/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.6086 - acc: 0.7994 - val_loss: 0.8092 - val_acc: 0.7298\n",
      "Epoch 25/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.5977 - acc: 0.8014 - val_loss: 0.8071 - val_acc: 0.7367\n",
      "Epoch 26/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.5841 - acc: 0.8073 - val_loss: 0.8005 - val_acc: 0.7397\n",
      "Epoch 27/100\n",
      "40000/40000 [==============================] - 3s 72us/step - loss: 0.5691 - acc: 0.8106 - val_loss: 0.8206 - val_acc: 0.7363\n",
      "Epoch 28/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.5635 - acc: 0.8136 - val_loss: 0.8198 - val_acc: 0.7408\n",
      "Epoch 29/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.5511 - acc: 0.8180 - val_loss: 0.8080 - val_acc: 0.7436\n",
      "Epoch 30/100\n",
      "40000/40000 [==============================] - 3s 72us/step - loss: 0.5360 - acc: 0.8223 - val_loss: 0.8052 - val_acc: 0.7410\n",
      "Epoch 31/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.5234 - acc: 0.8278 - val_loss: 0.8060 - val_acc: 0.7459\n",
      "Epoch 32/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.5198 - acc: 0.8270 - val_loss: 0.8158 - val_acc: 0.7400\n",
      "Epoch 33/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.5093 - acc: 0.8319 - val_loss: 0.8126 - val_acc: 0.7445\n",
      "Epoch 34/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.4968 - acc: 0.8379 - val_loss: 0.8191 - val_acc: 0.7462\n",
      "Epoch 35/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.4855 - acc: 0.8394 - val_loss: 0.8201 - val_acc: 0.7472\n",
      "Epoch 36/100\n",
      "40000/40000 [==============================] - 3s 72us/step - loss: 0.4762 - acc: 0.8426 - val_loss: 0.8311 - val_acc: 0.7436\n",
      "Epoch 37/100\n",
      "40000/40000 [==============================] - 3s 72us/step - loss: 0.4665 - acc: 0.8452 - val_loss: 0.8312 - val_acc: 0.7492\n",
      "Epoch 38/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.4562 - acc: 0.8491 - val_loss: 0.8271 - val_acc: 0.7492\n",
      "Epoch 39/100\n",
      "40000/40000 [==============================] - 3s 72us/step - loss: 0.4506 - acc: 0.8495 - val_loss: 0.8274 - val_acc: 0.7505\n",
      "Epoch 40/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.4406 - acc: 0.8556 - val_loss: 0.8318 - val_acc: 0.7500\n",
      "Epoch 41/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.4331 - acc: 0.8571 - val_loss: 0.8512 - val_acc: 0.7469\n",
      "Epoch 42/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.4213 - acc: 0.8623 - val_loss: 0.8292 - val_acc: 0.7470\n",
      "Epoch 43/100\n",
      "40000/40000 [==============================] - 3s 72us/step - loss: 0.4176 - acc: 0.8622 - val_loss: 0.8449 - val_acc: 0.7517\n",
      "Epoch 44/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.4038 - acc: 0.8676 - val_loss: 0.8657 - val_acc: 0.7465\n",
      "Epoch 45/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.3969 - acc: 0.8689 - val_loss: 0.8835 - val_acc: 0.7454\n",
      "Epoch 46/100\n",
      "40000/40000 [==============================] - 3s 72us/step - loss: 0.3824 - acc: 0.8752 - val_loss: 0.8880 - val_acc: 0.7519\n",
      "Epoch 47/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.3802 - acc: 0.8750 - val_loss: 0.8774 - val_acc: 0.7519\n",
      "Epoch 48/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.3660 - acc: 0.8821 - val_loss: 0.8919 - val_acc: 0.7509\n",
      "Epoch 49/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.3599 - acc: 0.8827 - val_loss: 0.9044 - val_acc: 0.7507\n",
      "Epoch 50/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.3525 - acc: 0.8845 - val_loss: 0.9202 - val_acc: 0.7456\n",
      "Epoch 51/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.3419 - acc: 0.8888 - val_loss: 0.9360 - val_acc: 0.7492\n",
      "Epoch 52/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.3410 - acc: 0.8896 - val_loss: 0.9242 - val_acc: 0.7467\n",
      "Epoch 53/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.3290 - acc: 0.8923 - val_loss: 0.9567 - val_acc: 0.7438\n",
      "Epoch 54/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.3185 - acc: 0.8966 - val_loss: 1.0017 - val_acc: 0.7465\n",
      "Epoch 55/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.3136 - acc: 0.8991 - val_loss: 0.9702 - val_acc: 0.7465\n",
      "Epoch 56/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.3051 - acc: 0.9021 - val_loss: 0.9991 - val_acc: 0.7453\n",
      "Epoch 57/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.2969 - acc: 0.9038 - val_loss: 1.0020 - val_acc: 0.7470\n",
      "Epoch 58/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.2929 - acc: 0.9044 - val_loss: 1.0016 - val_acc: 0.7494\n",
      "Epoch 59/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.2825 - acc: 0.9097 - val_loss: 1.0331 - val_acc: 0.7497\n",
      "Epoch 60/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.2778 - acc: 0.9099 - val_loss: 1.0302 - val_acc: 0.7451\n",
      "Epoch 61/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.2726 - acc: 0.9120 - val_loss: 1.0385 - val_acc: 0.7488\n",
      "Epoch 62/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.2591 - acc: 0.9178 - val_loss: 1.0385 - val_acc: 0.7467\n",
      "Epoch 63/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.2570 - acc: 0.9189 - val_loss: 1.0783 - val_acc: 0.7490\n",
      "Epoch 64/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.2506 - acc: 0.9192 - val_loss: 1.1126 - val_acc: 0.7429\n",
      "Epoch 65/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.2432 - acc: 0.9222 - val_loss: 1.1180 - val_acc: 0.7474\n",
      "Epoch 66/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.2312 - acc: 0.9271 - val_loss: 1.1666 - val_acc: 0.7450\n",
      "Epoch 67/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.2286 - acc: 0.9273 - val_loss: 1.1316 - val_acc: 0.7458\n",
      "Epoch 68/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.2261 - acc: 0.9295 - val_loss: 1.1238 - val_acc: 0.7484\n",
      "Epoch 69/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.2164 - acc: 0.9318 - val_loss: 1.2085 - val_acc: 0.7463\n",
      "Epoch 70/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.2131 - acc: 0.9323 - val_loss: 1.1990 - val_acc: 0.7439\n",
      "Epoch 71/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.2072 - acc: 0.9350 - val_loss: 1.2394 - val_acc: 0.7388\n",
      "Epoch 72/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.2151 - acc: 0.9324 - val_loss: 1.1716 - val_acc: 0.7447\n",
      "Epoch 73/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1973 - acc: 0.9386 - val_loss: 1.2643 - val_acc: 0.7445\n",
      "Epoch 74/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1955 - acc: 0.9382 - val_loss: 1.2727 - val_acc: 0.7431\n",
      "Epoch 75/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1953 - acc: 0.9384 - val_loss: 1.2895 - val_acc: 0.7474\n",
      "Epoch 76/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1803 - acc: 0.9442 - val_loss: 1.2881 - val_acc: 0.7421\n",
      "Epoch 77/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1723 - acc: 0.9468 - val_loss: 1.3213 - val_acc: 0.7440\n",
      "Epoch 78/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1794 - acc: 0.9443 - val_loss: 1.3380 - val_acc: 0.7417\n",
      "Epoch 79/100\n",
      "40000/40000 [==============================] - 3s 72us/step - loss: 0.1737 - acc: 0.9464 - val_loss: 1.3609 - val_acc: 0.7475\n",
      "Epoch 80/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.1621 - acc: 0.9505 - val_loss: 1.3247 - val_acc: 0.7456\n",
      "Epoch 81/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1614 - acc: 0.9507 - val_loss: 1.3743 - val_acc: 0.7375\n",
      "Epoch 82/100\n",
      "40000/40000 [==============================] - 3s 72us/step - loss: 0.1651 - acc: 0.9493 - val_loss: 1.3737 - val_acc: 0.7370\n",
      "Epoch 83/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1604 - acc: 0.9513 - val_loss: 1.3951 - val_acc: 0.7461\n",
      "Epoch 84/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1430 - acc: 0.9572 - val_loss: 1.4352 - val_acc: 0.7467\n",
      "Epoch 85/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1458 - acc: 0.9564 - val_loss: 1.4262 - val_acc: 0.7423\n",
      "Epoch 86/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1365 - acc: 0.9591 - val_loss: 1.4186 - val_acc: 0.7451\n",
      "Epoch 87/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.1418 - acc: 0.9570 - val_loss: 1.4557 - val_acc: 0.7425\n",
      "Epoch 88/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.1281 - acc: 0.9623 - val_loss: 1.4740 - val_acc: 0.7404\n",
      "Epoch 89/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.1322 - acc: 0.9612 - val_loss: 1.5377 - val_acc: 0.7427\n",
      "Epoch 90/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.1260 - acc: 0.9632 - val_loss: 1.5181 - val_acc: 0.7414\n",
      "Epoch 91/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.1178 - acc: 0.9661 - val_loss: 1.5448 - val_acc: 0.7396\n",
      "Epoch 92/100\n",
      "40000/40000 [==============================] - 3s 72us/step - loss: 0.1249 - acc: 0.9628 - val_loss: 1.5456 - val_acc: 0.7455\n",
      "Epoch 93/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.1164 - acc: 0.9663 - val_loss: 1.6015 - val_acc: 0.7429\n",
      "Epoch 94/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.1297 - acc: 0.9612 - val_loss: 1.5982 - val_acc: 0.7396\n",
      "Epoch 95/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.1162 - acc: 0.9657 - val_loss: 1.5852 - val_acc: 0.7350\n",
      "Epoch 96/100\n",
      "40000/40000 [==============================] - 3s 72us/step - loss: 0.1113 - acc: 0.9672 - val_loss: 1.5845 - val_acc: 0.7429\n",
      "Epoch 97/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.1011 - acc: 0.9716 - val_loss: 1.6505 - val_acc: 0.7417\n",
      "Epoch 98/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.1004 - acc: 0.9710 - val_loss: 1.6102 - val_acc: 0.7442\n",
      "Epoch 99/100\n",
      "40000/40000 [==============================] - 3s 73us/step - loss: 0.0880 - acc: 0.9752 - val_loss: 1.6965 - val_acc: 0.7424\n",
      "Epoch 100/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.0920 - acc: 0.9737 - val_loss: 1.6884 - val_acc: 0.7439\n",
      "10000/10000 [==============================] - 0s 35us/step\n",
      "Validation loss: 1.6884147048950195\n",
      "Validation accuracy (NORMALIZED): 0.7439\n",
      "--- Compiling model\n",
      "================================================\n",
      "Split: 3\n",
      "Number of Epochs: 100\n",
      "Train shape: (40000, 32, 32, 3)\n",
      "Train batch size: 384\n",
      "Val batch size: 128\n",
      "================================================\n",
      "--- Preparing tensorboard\n",
      "Log Dir:  logs/1542822904.8992963_fold[3]_fully-trained\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/100\n",
      "40000/40000 [==============================] - 4s 98us/step - loss: 2.0687 - acc: 0.2643 - val_loss: 1.6226 - val_acc: 0.4624\n",
      "Epoch 2/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 1.4801 - acc: 0.5016 - val_loss: 1.2055 - val_acc: 0.5887\n",
      "Epoch 3/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 1.2427 - acc: 0.5825 - val_loss: 1.0962 - val_acc: 0.6247\n",
      "Epoch 4/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.1211 - acc: 0.6248 - val_loss: 1.0211 - val_acc: 0.6482\n",
      "Epoch 5/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 1.0503 - acc: 0.6495 - val_loss: 0.9794 - val_acc: 0.6653\n",
      "Epoch 6/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.9953 - acc: 0.6706 - val_loss: 0.9446 - val_acc: 0.6752\n",
      "Epoch 7/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.9489 - acc: 0.6873 - val_loss: 0.9162 - val_acc: 0.6867\n",
      "Epoch 8/100\n",
      "40000/40000 [==============================] - 3s 72us/step - loss: 0.9162 - acc: 0.6972 - val_loss: 0.8968 - val_acc: 0.6932\n",
      "Epoch 9/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.8805 - acc: 0.7088 - val_loss: 0.8875 - val_acc: 0.6942\n",
      "Epoch 10/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.8541 - acc: 0.7177 - val_loss: 0.8588 - val_acc: 0.7030\n",
      "Epoch 11/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.8250 - acc: 0.7271 - val_loss: 0.8543 - val_acc: 0.7073\n",
      "Epoch 12/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.7969 - acc: 0.7393 - val_loss: 0.8515 - val_acc: 0.7086\n",
      "Epoch 13/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.7766 - acc: 0.7464 - val_loss: 0.8339 - val_acc: 0.7138\n",
      "Epoch 14/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.7584 - acc: 0.7504 - val_loss: 0.8252 - val_acc: 0.7184\n",
      "Epoch 15/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.7328 - acc: 0.7590 - val_loss: 0.8169 - val_acc: 0.7223\n",
      "Epoch 16/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.7182 - acc: 0.7654 - val_loss: 0.8149 - val_acc: 0.7202\n",
      "Epoch 17/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.7017 - acc: 0.7691 - val_loss: 0.7973 - val_acc: 0.7285\n",
      "Epoch 18/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.6811 - acc: 0.7776 - val_loss: 0.7941 - val_acc: 0.7309\n",
      "Epoch 19/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.6660 - acc: 0.7823 - val_loss: 0.7983 - val_acc: 0.7295\n",
      "Epoch 20/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.6495 - acc: 0.7890 - val_loss: 0.7911 - val_acc: 0.7321\n",
      "Epoch 21/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.6344 - acc: 0.7930 - val_loss: 0.7916 - val_acc: 0.7285\n",
      "Epoch 22/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.6202 - acc: 0.7977 - val_loss: 0.7929 - val_acc: 0.7360\n",
      "Epoch 23/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.6088 - acc: 0.8018 - val_loss: 0.7878 - val_acc: 0.7381\n",
      "Epoch 24/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.5928 - acc: 0.8064 - val_loss: 0.7996 - val_acc: 0.7368\n",
      "Epoch 25/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.5837 - acc: 0.8104 - val_loss: 0.7859 - val_acc: 0.7438\n",
      "Epoch 26/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.5726 - acc: 0.8133 - val_loss: 0.7894 - val_acc: 0.7443\n",
      "Epoch 27/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.5537 - acc: 0.8201 - val_loss: 0.7926 - val_acc: 0.7422\n",
      "Epoch 28/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.5502 - acc: 0.8218 - val_loss: 0.7816 - val_acc: 0.7419\n",
      "Epoch 29/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.5293 - acc: 0.8278 - val_loss: 0.7823 - val_acc: 0.7459\n",
      "Epoch 30/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.5187 - acc: 0.8318 - val_loss: 0.7990 - val_acc: 0.7423\n",
      "Epoch 31/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.5116 - acc: 0.8336 - val_loss: 0.8014 - val_acc: 0.7453\n",
      "Epoch 32/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.5001 - acc: 0.8393 - val_loss: 0.8101 - val_acc: 0.7429\n",
      "Epoch 33/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.4876 - acc: 0.8428 - val_loss: 0.8012 - val_acc: 0.7501\n",
      "Epoch 34/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.4797 - acc: 0.8424 - val_loss: 0.8089 - val_acc: 0.7472\n",
      "Epoch 35/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.4664 - acc: 0.8493 - val_loss: 0.8035 - val_acc: 0.7498\n",
      "Epoch 36/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.4515 - acc: 0.8558 - val_loss: 0.8246 - val_acc: 0.7457\n",
      "Epoch 37/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.4459 - acc: 0.8553 - val_loss: 0.8314 - val_acc: 0.7481\n",
      "Epoch 38/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.4354 - acc: 0.8606 - val_loss: 0.8390 - val_acc: 0.7493\n",
      "Epoch 39/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.4246 - acc: 0.8642 - val_loss: 0.8331 - val_acc: 0.7446\n",
      "Epoch 40/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.4136 - acc: 0.8675 - val_loss: 0.8498 - val_acc: 0.7494\n",
      "Epoch 41/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.4070 - acc: 0.8700 - val_loss: 0.8541 - val_acc: 0.7501\n",
      "Epoch 42/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.4005 - acc: 0.8709 - val_loss: 0.8728 - val_acc: 0.7484\n",
      "Epoch 43/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.3925 - acc: 0.8726 - val_loss: 0.8574 - val_acc: 0.7518\n",
      "Epoch 44/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.3818 - acc: 0.8786 - val_loss: 0.8626 - val_acc: 0.7513\n",
      "Epoch 45/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.3674 - acc: 0.8827 - val_loss: 0.8959 - val_acc: 0.7456\n",
      "Epoch 46/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.3583 - acc: 0.8869 - val_loss: 0.9007 - val_acc: 0.7518\n",
      "Epoch 47/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.3527 - acc: 0.8867 - val_loss: 0.9227 - val_acc: 0.7460\n",
      "Epoch 48/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.3422 - acc: 0.8904 - val_loss: 0.9173 - val_acc: 0.7523\n",
      "Epoch 49/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.3339 - acc: 0.8957 - val_loss: 0.9363 - val_acc: 0.7504\n",
      "Epoch 50/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.3275 - acc: 0.8969 - val_loss: 0.9442 - val_acc: 0.7532\n",
      "Epoch 51/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.3160 - acc: 0.9008 - val_loss: 0.9633 - val_acc: 0.7505\n",
      "Epoch 52/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.3079 - acc: 0.9026 - val_loss: 0.9609 - val_acc: 0.7541\n",
      "Epoch 53/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.2970 - acc: 0.9077 - val_loss: 0.9836 - val_acc: 0.7514\n",
      "Epoch 54/100\n",
      "40000/40000 [==============================] - 3s 72us/step - loss: 0.2941 - acc: 0.9100 - val_loss: 0.9932 - val_acc: 0.7443\n",
      "Epoch 55/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.2871 - acc: 0.9105 - val_loss: 1.0196 - val_acc: 0.7435\n",
      "Epoch 56/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.2799 - acc: 0.9134 - val_loss: 1.0071 - val_acc: 0.7480\n",
      "Epoch 57/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.2762 - acc: 0.9143 - val_loss: 1.0368 - val_acc: 0.7527\n",
      "Epoch 58/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.2596 - acc: 0.9208 - val_loss: 1.0347 - val_acc: 0.7515\n",
      "Epoch 59/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.2535 - acc: 0.9221 - val_loss: 1.0585 - val_acc: 0.7508\n",
      "Epoch 60/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.2484 - acc: 0.9236 - val_loss: 1.0762 - val_acc: 0.7486\n",
      "Epoch 61/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.2433 - acc: 0.9253 - val_loss: 1.1035 - val_acc: 0.7489\n",
      "Epoch 62/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.2399 - acc: 0.9254 - val_loss: 1.1095 - val_acc: 0.7464\n",
      "Epoch 63/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.2315 - acc: 0.9291 - val_loss: 1.1674 - val_acc: 0.7434\n",
      "Epoch 64/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.2236 - acc: 0.9327 - val_loss: 1.1253 - val_acc: 0.7472\n",
      "Epoch 65/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.2119 - acc: 0.9362 - val_loss: 1.1599 - val_acc: 0.7510\n",
      "Epoch 66/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.2070 - acc: 0.9383 - val_loss: 1.1826 - val_acc: 0.7486\n",
      "Epoch 67/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.2086 - acc: 0.9368 - val_loss: 1.1755 - val_acc: 0.7437\n",
      "Epoch 68/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.1957 - acc: 0.9424 - val_loss: 1.2195 - val_acc: 0.7476\n",
      "Epoch 69/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1989 - acc: 0.9402 - val_loss: 1.2088 - val_acc: 0.7519\n",
      "Epoch 70/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1867 - acc: 0.9452 - val_loss: 1.2729 - val_acc: 0.7434\n",
      "Epoch 71/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1836 - acc: 0.9460 - val_loss: 1.2690 - val_acc: 0.7488\n",
      "Epoch 72/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1705 - acc: 0.9489 - val_loss: 1.2685 - val_acc: 0.7490\n",
      "Epoch 73/100\n",
      "40000/40000 [==============================] - 3s 72us/step - loss: 0.1723 - acc: 0.9489 - val_loss: 1.3030 - val_acc: 0.7414\n",
      "Epoch 74/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.1695 - acc: 0.9506 - val_loss: 1.3041 - val_acc: 0.7473\n",
      "Epoch 75/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1703 - acc: 0.9497 - val_loss: 1.3189 - val_acc: 0.7474\n",
      "Epoch 76/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.1566 - acc: 0.9549 - val_loss: 1.3181 - val_acc: 0.7480\n",
      "Epoch 77/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.1537 - acc: 0.9563 - val_loss: 1.3443 - val_acc: 0.7476\n",
      "Epoch 78/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.1497 - acc: 0.9568 - val_loss: 1.3533 - val_acc: 0.7455\n",
      "Epoch 79/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1446 - acc: 0.9582 - val_loss: 1.4176 - val_acc: 0.7473\n",
      "Epoch 80/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1450 - acc: 0.9583 - val_loss: 1.4158 - val_acc: 0.7447\n",
      "Epoch 81/100\n",
      "40000/40000 [==============================] - 3s 66us/step - loss: 0.1332 - acc: 0.9630 - val_loss: 1.4413 - val_acc: 0.7457\n",
      "Epoch 82/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1278 - acc: 0.9632 - val_loss: 1.4288 - val_acc: 0.7488\n",
      "Epoch 83/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1285 - acc: 0.9638 - val_loss: 1.4430 - val_acc: 0.7465\n",
      "Epoch 84/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1311 - acc: 0.9626 - val_loss: 1.4735 - val_acc: 0.7471\n",
      "Epoch 85/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1199 - acc: 0.9661 - val_loss: 1.5159 - val_acc: 0.7452\n",
      "Epoch 86/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1187 - acc: 0.9662 - val_loss: 1.5037 - val_acc: 0.7451\n",
      "Epoch 87/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.1090 - acc: 0.9706 - val_loss: 1.5132 - val_acc: 0.7477\n",
      "Epoch 88/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.1191 - acc: 0.9668 - val_loss: 1.5332 - val_acc: 0.7482\n",
      "Epoch 89/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.1065 - acc: 0.9706 - val_loss: 1.5608 - val_acc: 0.7451\n",
      "Epoch 90/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.1164 - acc: 0.9667 - val_loss: 1.5424 - val_acc: 0.7498\n",
      "Epoch 91/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.1121 - acc: 0.9679 - val_loss: 1.5446 - val_acc: 0.7445\n",
      "Epoch 92/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.1025 - acc: 0.9715 - val_loss: 1.5856 - val_acc: 0.7438\n",
      "Epoch 93/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.0975 - acc: 0.9731 - val_loss: 1.6199 - val_acc: 0.7435\n",
      "Epoch 94/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.0933 - acc: 0.9739 - val_loss: 1.6356 - val_acc: 0.7422\n",
      "Epoch 95/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.0877 - acc: 0.9758 - val_loss: 1.6577 - val_acc: 0.7468\n",
      "Epoch 96/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.0937 - acc: 0.9737 - val_loss: 1.6989 - val_acc: 0.7416\n",
      "Epoch 97/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.0970 - acc: 0.9727 - val_loss: 1.6674 - val_acc: 0.7455\n",
      "Epoch 98/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.0841 - acc: 0.9772 - val_loss: 1.6642 - val_acc: 0.7461\n",
      "Epoch 99/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.0827 - acc: 0.9772 - val_loss: 1.6978 - val_acc: 0.7460\n",
      "Epoch 100/100\n",
      "40000/40000 [==============================] - 3s 67us/step - loss: 0.0803 - acc: 0.9781 - val_loss: 1.7592 - val_acc: 0.7427\n",
      "10000/10000 [==============================] - 0s 36us/step\n",
      "Validation loss: 1.7591750661849976\n",
      "Validation accuracy (NORMALIZED): 0.7427\n",
      "--- Compiling model\n",
      "================================================\n",
      "Split: 4\n",
      "Number of Epochs: 100\n",
      "Train shape: (40000, 32, 32, 3)\n",
      "Train batch size: 384\n",
      "Val batch size: 128\n",
      "================================================\n",
      "--- Preparing tensorboard\n",
      "Log Dir:  logs/1542823186.0667903_fold[4]_fully-trained\n",
      "Train on 40000 samples, validate on 10000 samples\n",
      "Epoch 1/100\n",
      "40000/40000 [==============================] - 4s 99us/step - loss: 2.0294 - acc: 0.2715 - val_loss: 1.5311 - val_acc: 0.4746\n",
      "Epoch 2/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 1.4653 - acc: 0.4938 - val_loss: 1.2263 - val_acc: 0.5731\n",
      "Epoch 3/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 1.2572 - acc: 0.5701 - val_loss: 1.1156 - val_acc: 0.6152\n",
      "Epoch 4/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 1.1506 - acc: 0.6076 - val_loss: 1.0454 - val_acc: 0.6407\n",
      "Epoch 5/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 1.0644 - acc: 0.6406 - val_loss: 0.9894 - val_acc: 0.6609\n",
      "Epoch 6/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 1.0043 - acc: 0.6632 - val_loss: 0.9625 - val_acc: 0.6716\n",
      "Epoch 7/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.9583 - acc: 0.6790 - val_loss: 0.9190 - val_acc: 0.6866\n",
      "Epoch 8/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.9146 - acc: 0.6940 - val_loss: 0.9097 - val_acc: 0.6878\n",
      "Epoch 9/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.8826 - acc: 0.7043 - val_loss: 0.8977 - val_acc: 0.6940\n",
      "Epoch 10/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.8481 - acc: 0.7163 - val_loss: 0.8684 - val_acc: 0.7034\n",
      "Epoch 11/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.8170 - acc: 0.7280 - val_loss: 0.8489 - val_acc: 0.7125\n",
      "Epoch 12/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.7969 - acc: 0.7334 - val_loss: 0.8443 - val_acc: 0.7145\n",
      "Epoch 13/100\n",
      "40000/40000 [==============================] - 3s 72us/step - loss: 0.7733 - acc: 0.7423 - val_loss: 0.8362 - val_acc: 0.7153\n",
      "Epoch 14/100\n",
      "40000/40000 [==============================] - 3s 73us/step - loss: 0.7475 - acc: 0.7495 - val_loss: 0.8225 - val_acc: 0.7241\n",
      "Epoch 15/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.7330 - acc: 0.7557 - val_loss: 0.8273 - val_acc: 0.7221\n",
      "Epoch 16/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.7116 - acc: 0.7634 - val_loss: 0.8193 - val_acc: 0.7237\n",
      "Epoch 17/100\n",
      "40000/40000 [==============================] - 3s 73us/step - loss: 0.6982 - acc: 0.7677 - val_loss: 0.8121 - val_acc: 0.7321\n",
      "Epoch 18/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.6778 - acc: 0.7763 - val_loss: 0.8037 - val_acc: 0.7306\n",
      "Epoch 19/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.6635 - acc: 0.7791 - val_loss: 0.8019 - val_acc: 0.7342\n",
      "Epoch 20/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.6522 - acc: 0.7829 - val_loss: 0.8107 - val_acc: 0.7333\n",
      "Epoch 21/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.6410 - acc: 0.7870 - val_loss: 0.7994 - val_acc: 0.7336\n",
      "Epoch 22/100\n",
      "40000/40000 [==============================] - 3s 72us/step - loss: 0.6182 - acc: 0.7956 - val_loss: 0.8107 - val_acc: 0.7363\n",
      "Epoch 23/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.6023 - acc: 0.8006 - val_loss: 0.8026 - val_acc: 0.7389\n",
      "Epoch 24/100\n",
      "40000/40000 [==============================] - 3s 75us/step - loss: 0.5934 - acc: 0.8030 - val_loss: 0.7961 - val_acc: 0.7411\n",
      "Epoch 25/100\n",
      "40000/40000 [==============================] - 3s 75us/step - loss: 0.5763 - acc: 0.8073 - val_loss: 0.7846 - val_acc: 0.7430\n",
      "Epoch 26/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.5678 - acc: 0.8127 - val_loss: 0.8097 - val_acc: 0.7384\n",
      "Epoch 27/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.5510 - acc: 0.8170 - val_loss: 0.7986 - val_acc: 0.7411\n",
      "Epoch 28/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.5411 - acc: 0.8207 - val_loss: 0.8051 - val_acc: 0.7421\n",
      "Epoch 29/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.5283 - acc: 0.8253 - val_loss: 0.7939 - val_acc: 0.7427\n",
      "Epoch 30/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.5183 - acc: 0.8288 - val_loss: 0.8065 - val_acc: 0.7424\n",
      "Epoch 31/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.5054 - acc: 0.8336 - val_loss: 0.8135 - val_acc: 0.7484\n",
      "Epoch 32/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.4962 - acc: 0.8347 - val_loss: 0.8278 - val_acc: 0.7426\n",
      "Epoch 33/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.4823 - acc: 0.8399 - val_loss: 0.8171 - val_acc: 0.7465\n",
      "Epoch 34/100\n",
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.4716 - acc: 0.8437 - val_loss: 0.8409 - val_acc: 0.7412\n",
      "Epoch 35/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.4622 - acc: 0.8489 - val_loss: 0.8334 - val_acc: 0.7468\n",
      "Epoch 36/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.4563 - acc: 0.8505 - val_loss: 0.8280 - val_acc: 0.7460\n",
      "Epoch 37/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.4408 - acc: 0.8540 - val_loss: 0.8443 - val_acc: 0.7465\n",
      "Epoch 38/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.4317 - acc: 0.8576 - val_loss: 0.8581 - val_acc: 0.7432\n",
      "Epoch 39/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.4203 - acc: 0.8612 - val_loss: 0.8742 - val_acc: 0.7474\n",
      "Epoch 40/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.4088 - acc: 0.8654 - val_loss: 0.8729 - val_acc: 0.7439\n",
      "Epoch 41/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.4059 - acc: 0.8663 - val_loss: 0.8966 - val_acc: 0.7447\n",
      "Epoch 42/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.3879 - acc: 0.8720 - val_loss: 0.8889 - val_acc: 0.7458\n",
      "Epoch 43/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.3822 - acc: 0.8746 - val_loss: 0.9001 - val_acc: 0.7432\n",
      "Epoch 44/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.3725 - acc: 0.8767 - val_loss: 0.9237 - val_acc: 0.7466\n",
      "Epoch 45/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.3639 - acc: 0.8824 - val_loss: 0.9143 - val_acc: 0.7468\n",
      "Epoch 46/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.3552 - acc: 0.8833 - val_loss: 0.9330 - val_acc: 0.7490\n",
      "Epoch 47/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.3463 - acc: 0.8878 - val_loss: 0.9411 - val_acc: 0.7498\n",
      "Epoch 48/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.3383 - acc: 0.8894 - val_loss: 0.9586 - val_acc: 0.7443\n",
      "Epoch 49/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.3376 - acc: 0.8908 - val_loss: 0.9628 - val_acc: 0.7431\n",
      "Epoch 50/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.3255 - acc: 0.8944 - val_loss: 0.9627 - val_acc: 0.7446\n",
      "Epoch 51/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.3151 - acc: 0.8977 - val_loss: 0.9714 - val_acc: 0.7501\n",
      "Epoch 52/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.3015 - acc: 0.9032 - val_loss: 0.9857 - val_acc: 0.7492\n",
      "Epoch 53/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.2983 - acc: 0.9039 - val_loss: 1.0076 - val_acc: 0.7457\n",
      "Epoch 54/100\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "40000/40000 [==============================] - 3s 71us/step - loss: 0.2886 - acc: 0.9070 - val_loss: 1.0288 - val_acc: 0.7503\n",
      "Epoch 55/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.2875 - acc: 0.9058 - val_loss: 1.0508 - val_acc: 0.7491\n",
      "Epoch 56/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.2795 - acc: 0.9094 - val_loss: 1.0370 - val_acc: 0.7445\n",
      "Epoch 57/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.2664 - acc: 0.9140 - val_loss: 1.0878 - val_acc: 0.7460\n",
      "Epoch 58/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.2672 - acc: 0.9132 - val_loss: 1.0798 - val_acc: 0.7439\n",
      "Epoch 59/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.2544 - acc: 0.9177 - val_loss: 1.0975 - val_acc: 0.7437\n",
      "Epoch 60/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.2532 - acc: 0.9185 - val_loss: 1.1183 - val_acc: 0.7449\n",
      "Epoch 61/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.2402 - acc: 0.9223 - val_loss: 1.1352 - val_acc: 0.7455\n",
      "Epoch 62/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.2351 - acc: 0.9244 - val_loss: 1.1540 - val_acc: 0.7455\n",
      "Epoch 63/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.2264 - acc: 0.9276 - val_loss: 1.1758 - val_acc: 0.7426\n",
      "Epoch 64/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.2227 - acc: 0.9294 - val_loss: 1.1989 - val_acc: 0.7462\n",
      "Epoch 65/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.2160 - acc: 0.9313 - val_loss: 1.2287 - val_acc: 0.7399\n",
      "Epoch 66/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.2079 - acc: 0.9349 - val_loss: 1.2429 - val_acc: 0.7451\n",
      "Epoch 67/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.2084 - acc: 0.9330 - val_loss: 1.2320 - val_acc: 0.7451\n",
      "Epoch 68/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.2081 - acc: 0.9335 - val_loss: 1.2821 - val_acc: 0.7435\n",
      "Epoch 69/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.1969 - acc: 0.9375 - val_loss: 1.2542 - val_acc: 0.7417\n",
      "Epoch 70/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.1952 - acc: 0.9378 - val_loss: 1.2598 - val_acc: 0.7444\n",
      "Epoch 71/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.1884 - acc: 0.9386 - val_loss: 1.2981 - val_acc: 0.7430\n",
      "Epoch 72/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.1774 - acc: 0.9443 - val_loss: 1.3454 - val_acc: 0.7415\n",
      "Epoch 73/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.1700 - acc: 0.9465 - val_loss: 1.3243 - val_acc: 0.7411\n",
      "Epoch 74/100\n",
      "40000/40000 [==============================] - 3s 69us/step - loss: 0.1785 - acc: 0.9434 - val_loss: 1.3429 - val_acc: 0.7379\n",
      "Epoch 75/100\n",
      "40000/40000 [==============================] - 3s 70us/step - loss: 0.1698 - acc: 0.9467 - val_loss: 1.3584 - val_acc: 0.7435\n",
      "Epoch 76/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1618 - acc: 0.9489 - val_loss: 1.3912 - val_acc: 0.7429\n",
      "Epoch 77/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1549 - acc: 0.9511 - val_loss: 1.4687 - val_acc: 0.7393\n",
      "Epoch 78/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1643 - acc: 0.9482 - val_loss: 1.4406 - val_acc: 0.7375\n",
      "Epoch 79/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1458 - acc: 0.9559 - val_loss: 1.4677 - val_acc: 0.7415\n",
      "Epoch 80/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1423 - acc: 0.9558 - val_loss: 1.4603 - val_acc: 0.7427\n",
      "Epoch 81/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1365 - acc: 0.9584 - val_loss: 1.4929 - val_acc: 0.7409\n",
      "Epoch 82/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1463 - acc: 0.9539 - val_loss: 1.4602 - val_acc: 0.7388\n",
      "Epoch 83/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1361 - acc: 0.9579 - val_loss: 1.4625 - val_acc: 0.7415\n",
      "Epoch 84/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1302 - acc: 0.9596 - val_loss: 1.5221 - val_acc: 0.7393\n",
      "Epoch 85/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1327 - acc: 0.9591 - val_loss: 1.4883 - val_acc: 0.7383\n",
      "Epoch 86/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1478 - acc: 0.9525 - val_loss: 1.5163 - val_acc: 0.7406\n",
      "Epoch 87/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1208 - acc: 0.9631 - val_loss: 1.5508 - val_acc: 0.7411\n",
      "Epoch 88/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1156 - acc: 0.9645 - val_loss: 1.5974 - val_acc: 0.7367\n",
      "Epoch 89/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1106 - acc: 0.9664 - val_loss: 1.6160 - val_acc: 0.7402\n",
      "Epoch 90/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1094 - acc: 0.9664 - val_loss: 1.6453 - val_acc: 0.7420\n",
      "Epoch 91/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1061 - acc: 0.9673 - val_loss: 1.6886 - val_acc: 0.7385\n",
      "Epoch 92/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1115 - acc: 0.9658 - val_loss: 1.7248 - val_acc: 0.7354\n",
      "Epoch 93/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1053 - acc: 0.9684 - val_loss: 1.6879 - val_acc: 0.7371\n",
      "Epoch 94/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.0960 - acc: 0.9721 - val_loss: 1.6755 - val_acc: 0.7359\n",
      "Epoch 95/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.0962 - acc: 0.9716 - val_loss: 1.6898 - val_acc: 0.7351\n",
      "Epoch 96/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.0964 - acc: 0.9705 - val_loss: 1.7273 - val_acc: 0.7389\n",
      "Epoch 97/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.0970 - acc: 0.9701 - val_loss: 1.6847 - val_acc: 0.7373\n",
      "Epoch 98/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1028 - acc: 0.9684 - val_loss: 1.7487 - val_acc: 0.7359\n",
      "Epoch 99/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.1012 - acc: 0.9694 - val_loss: 1.7204 - val_acc: 0.7365\n",
      "Epoch 100/100\n",
      "40000/40000 [==============================] - 3s 68us/step - loss: 0.0806 - acc: 0.9767 - val_loss: 1.8170 - val_acc: 0.7365\n",
      "10000/10000 [==============================] - 0s 36us/step\n",
      "Validation loss: 1.8170050434112548\n",
      "Validation accuracy (NORMALIZED): 0.7365\n",
      "[[1.8032953178405762, 0.7359], [1.791891314125061, 0.7359], [1.6884147048950195, 0.7439], [1.7591750661849976, 0.7427], [1.8170050434112548, 0.7365]]\n"
     ]
    }
   ],
   "source": [
    "from time import time\n",
    "\n",
    "\n",
    "\n",
    "#=====================================\n",
    "# Training model and Evaluation\n",
    "#=====================================\n",
    "print(\"--- Start training\")\n",
    "y_train_categorical = to_categorical(y_train, num_classes=n_classes)\n",
    "y_val_categorical = to_categorical(y_val, num_classes=n_classes)\n",
    "scores = []\n",
    "\n",
    "for i in range(folds):\n",
    "    #--- Loading model\n",
    "    model = get_squeezenet_ft3()\n",
    "\n",
    "    \n",
    "    #--- Evaluating the model for split i\n",
    "    print(\"================================================\")\n",
    "    print(\"Split: \"+str(i))\n",
    "    print(\"Number of Epochs: \"+str(n_epochs))\n",
    "    print(\"Train shape:\",X_train[i].shape)\n",
    "    print(\"Train batch size: \"+str(train_batch_size))\n",
    "    print(\"Val batch size: \"+str(val_batch_size))\n",
    "    print(\"================================================\")\n",
    "          \n",
    "    #=====================================\n",
    "    # Tensorboard callback\n",
    "    #=====================================\n",
    "    print(\"--- Preparing tensorboard\")\n",
    "    log_dir = \"logs/{}_fold[{}]_{}\".format(time(), i, 'fully-trained')\n",
    "    print(\"Log Dir: \", log_dir)\n",
    "    tbCallBack = TensorBoard(log_dir=log_dir, write_graph=True)    \n",
    "    \n",
    "    # Training without data augmentation\n",
    "    model.fit(X_train[i],y_train_categorical[i], batch_size=train_batch_size, \n",
    "              epochs=n_epochs, \n",
    "              validation_data=(X_val[i],y_val_categorical[i]),\n",
    "              verbose=1,\n",
    "              callbacks=[tbCallBack])\n",
    "          \n",
    "    #--- Evaluating the model for split i\n",
    "    score = model.evaluate(x=X_val[i], y=y_val_categorical[i], batch_size=val_batch_size, verbose=1)\n",
    "    scores.append(score)\n",
    "    print('Validation loss:', score[0])\n",
    "    print('Validation accuracy (NORMALIZED):', score[1])\n",
    "    \n",
    "#--- Showing scores\n",
    "print(scores)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Finally, let's evaluate on our validation set:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mean Validation loss: 1.7719562892913818\n",
      "Mean Validation accuracy (NORMALIZED): 0.73898\n"
     ]
    }
   ],
   "source": [
    "#=====================================\n",
    "# Evaluate on validation\n",
    "#=====================================\n",
    "\n",
    "#--- The evaluation of the model \n",
    "np_aux = np.array(scores).mean(axis=0)\n",
    "\n",
    "print('Mean Validation loss:', np_aux[0])\n",
    "print('Mean Validation accuracy (NORMALIZED):', np_aux[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Evaluate your best model on test\n",
    "# ... \n",
    "# Done with the loading model cell"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Saving the model\n",
    "Now that we are working on more complex tasks and our trainings are starting to take more time it is usually a good idea to save the trained model from time to time. [Keras has a lot of ways of saving and loading the model](https://keras.io/getting-started/faq/#how-can-i-save-a-keras-model), but in this exercise we will use the simplest of them all: `model.save()`. It saves the architecture, the weights, the choice of loss function/optimizer/metrics and even the current state of the training, so you can resume your training later."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "## The better model obtained was when retraining the whole network \n",
    "model.save('my_model.h5')  # creates a HDF5 file 'my_model.h5'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading a model\n",
    "Once we have our model trained, we can load it using:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 1s 147us/step\n",
      "Test loss: 1.791677146911621\n",
      "Test accuracy (NORMALIZED): 0.7384\n"
     ]
    }
   ],
   "source": [
    "from keras.models import load_model\n",
    "\n",
    "del model  # Will delete model, only to check if load_model is working\n",
    "\n",
    "y_test_categorical = to_categorical(y_test, num_classes=n_classes)\n",
    "\n",
    "# returns a compiled model identical to the previous one\n",
    "model = load_model('my_model.h5')\n",
    "\n",
    "score = model.evaluate(x=X_test, y=y_test_categorical, batch_size=val_batch_size, verbose=1)    \n",
    "# evaluate test set again... should give us the same result\n",
    "# ...\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy (NORMALIZED):', score[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Best model\n",
    "\n",
    "* Using the whole traning \n",
    "* Adding early stop as a callback "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- Compiling model\n",
      "================================================\n",
      "Number of Epochs: 100\n",
      "Train shape: 47500.0\n",
      "Train batch size: 384\n",
      "Val batch size: 128\n",
      "================================================\n",
      "--- Preparing tensorboard\n",
      "Log Dir:  logs/1542829537.8803167\n",
      "Train on 47500 samples, validate on 2500 samples\n",
      "Epoch 1/100\n",
      "47500/47500 [==============================] - 5s 100us/step - loss: 1.9792 - acc: 0.2807 - val_loss: 1.3883 - val_acc: 0.5136\n",
      "Epoch 2/100\n",
      "47500/47500 [==============================] - 4s 75us/step - loss: 1.3777 - acc: 0.5172 - val_loss: 1.1616 - val_acc: 0.6012\n",
      "Epoch 3/100\n",
      "47500/47500 [==============================] - 4s 75us/step - loss: 1.1792 - acc: 0.5925 - val_loss: 1.0446 - val_acc: 0.6440\n",
      "Epoch 4/100\n",
      "47500/47500 [==============================] - 4s 74us/step - loss: 1.0774 - acc: 0.6283 - val_loss: 0.9690 - val_acc: 0.6720\n",
      "Epoch 5/100\n",
      "47500/47500 [==============================] - 3s 74us/step - loss: 1.0021 - acc: 0.6614 - val_loss: 0.9458 - val_acc: 0.6800\n",
      "Epoch 6/100\n",
      "47500/47500 [==============================] - 3s 74us/step - loss: 0.9472 - acc: 0.6805 - val_loss: 0.8846 - val_acc: 0.6976\n",
      "Epoch 7/100\n",
      "47500/47500 [==============================] - 3s 74us/step - loss: 0.9045 - acc: 0.6951 - val_loss: 0.8474 - val_acc: 0.7048\n",
      "Epoch 8/100\n",
      "47500/47500 [==============================] - 4s 74us/step - loss: 0.8637 - acc: 0.7100 - val_loss: 0.8440 - val_acc: 0.7104\n",
      "Epoch 9/100\n",
      "47500/47500 [==============================] - 3s 74us/step - loss: 0.8322 - acc: 0.7220 - val_loss: 0.8117 - val_acc: 0.7180\n",
      "Epoch 10/100\n",
      "47500/47500 [==============================] - 3s 73us/step - loss: 0.8025 - acc: 0.7317 - val_loss: 0.7946 - val_acc: 0.7236\n",
      "Epoch 11/100\n",
      "47500/47500 [==============================] - 3s 73us/step - loss: 0.7782 - acc: 0.7410 - val_loss: 0.7905 - val_acc: 0.7280\n",
      "Epoch 12/100\n",
      "47500/47500 [==============================] - 3s 74us/step - loss: 0.7592 - acc: 0.7468 - val_loss: 0.7722 - val_acc: 0.7372\n",
      "Epoch 13/100\n",
      "47500/47500 [==============================] - 3s 73us/step - loss: 0.7333 - acc: 0.7548 - val_loss: 0.7842 - val_acc: 0.7300\n",
      "Epoch 14/100\n",
      "47500/47500 [==============================] - 3s 73us/step - loss: 0.7125 - acc: 0.7619 - val_loss: 0.7705 - val_acc: 0.7336\n",
      "Epoch 15/100\n",
      "47500/47500 [==============================] - 3s 74us/step - loss: 0.6971 - acc: 0.7707 - val_loss: 0.7506 - val_acc: 0.7444\n",
      "Epoch 16/100\n",
      "47500/47500 [==============================] - 3s 73us/step - loss: 0.6794 - acc: 0.7744 - val_loss: 0.7667 - val_acc: 0.7448\n",
      "Epoch 17/100\n",
      "47500/47500 [==============================] - 3s 74us/step - loss: 0.6643 - acc: 0.7805 - val_loss: 0.7463 - val_acc: 0.7388\n",
      "Epoch 18/100\n",
      "47500/47500 [==============================] - 3s 73us/step - loss: 0.6482 - acc: 0.7868 - val_loss: 0.7382 - val_acc: 0.7428\n",
      "Epoch 19/100\n",
      "47500/47500 [==============================] - 3s 73us/step - loss: 0.6331 - acc: 0.7911 - val_loss: 0.7349 - val_acc: 0.7544\n",
      "Epoch 20/100\n",
      "47500/47500 [==============================] - 4s 74us/step - loss: 0.6153 - acc: 0.7959 - val_loss: 0.7295 - val_acc: 0.7524\n",
      "Epoch 21/100\n",
      "47500/47500 [==============================] - 4s 75us/step - loss: 0.6003 - acc: 0.8020 - val_loss: 0.7277 - val_acc: 0.7580\n",
      "Epoch 22/100\n",
      "47500/47500 [==============================] - 3s 74us/step - loss: 0.5930 - acc: 0.8053 - val_loss: 0.7336 - val_acc: 0.7536\n",
      "Epoch 23/100\n",
      "47500/47500 [==============================] - 4s 74us/step - loss: 0.5735 - acc: 0.8125 - val_loss: 0.7276 - val_acc: 0.7568\n",
      "Epoch 24/100\n",
      "47500/47500 [==============================] - 4s 74us/step - loss: 0.5629 - acc: 0.8139 - val_loss: 0.7488 - val_acc: 0.7576\n",
      "Epoch 25/100\n",
      "47500/47500 [==============================] - 4s 74us/step - loss: 0.5503 - acc: 0.8183 - val_loss: 0.7298 - val_acc: 0.7588\n",
      "Epoch 26/100\n",
      "47500/47500 [==============================] - 3s 73us/step - loss: 0.5389 - acc: 0.8216 - val_loss: 0.7329 - val_acc: 0.7584\n",
      "Epoch 27/100\n",
      "47500/47500 [==============================] - 4s 75us/step - loss: 0.5273 - acc: 0.8244 - val_loss: 0.7471 - val_acc: 0.7604\n",
      "Epoch 28/100\n",
      "47500/47500 [==============================] - 4s 74us/step - loss: 0.5192 - acc: 0.8299 - val_loss: 0.7369 - val_acc: 0.7600\n",
      "Epoch 29/100\n",
      "47500/47500 [==============================] - 3s 73us/step - loss: 0.5065 - acc: 0.8341 - val_loss: 0.7275 - val_acc: 0.7624\n",
      "Epoch 30/100\n",
      "47500/47500 [==============================] - 3s 73us/step - loss: 0.4949 - acc: 0.8373 - val_loss: 0.7316 - val_acc: 0.7640\n",
      "Epoch 31/100\n",
      "47500/47500 [==============================] - 4s 75us/step - loss: 0.4855 - acc: 0.8410 - val_loss: 0.7361 - val_acc: 0.7664\n",
      "Epoch 32/100\n",
      "47500/47500 [==============================] - 4s 76us/step - loss: 0.4737 - acc: 0.8462 - val_loss: 0.7501 - val_acc: 0.7636\n",
      "Epoch 33/100\n",
      "47500/47500 [==============================] - 4s 74us/step - loss: 0.4622 - acc: 0.8484 - val_loss: 0.7388 - val_acc: 0.7632\n",
      "Epoch 34/100\n",
      "47500/47500 [==============================] - 4s 75us/step - loss: 0.4497 - acc: 0.8533 - val_loss: 0.7444 - val_acc: 0.7632\n",
      "Epoch 35/100\n",
      "47500/47500 [==============================] - 4s 74us/step - loss: 0.4431 - acc: 0.8549 - val_loss: 0.7636 - val_acc: 0.7676\n",
      "Epoch 36/100\n",
      "47500/47500 [==============================] - 3s 73us/step - loss: 0.4322 - acc: 0.8595 - val_loss: 0.7501 - val_acc: 0.7656\n",
      "Epoch 37/100\n",
      "47500/47500 [==============================] - 3s 74us/step - loss: 0.4197 - acc: 0.8637 - val_loss: 0.7666 - val_acc: 0.7664\n",
      "Epoch 38/100\n",
      "47500/47500 [==============================] - 4s 75us/step - loss: 0.4100 - acc: 0.8647 - val_loss: 0.7659 - val_acc: 0.7664\n",
      "Epoch 39/100\n",
      "47500/47500 [==============================] - 4s 76us/step - loss: 0.4055 - acc: 0.8682 - val_loss: 0.7767 - val_acc: 0.7648\n",
      "Restoring model weights from the end of the best epoch\n",
      "Epoch 00039: early stopping\n"
     ]
    }
   ],
   "source": [
    "from time import time\n",
    "\n",
    "model = get_squeezenet_ft3()\n",
    "    \n",
    "#--- Evaluating the model for split i\n",
    "print(\"================================================\")\n",
    "print(\"Number of Epochs: \"+str(n_epochs))\n",
    "print(\"Train shape:\",trainVal_label.shape[0] * 0.95)\n",
    "print(\"Train batch size: \"+str(train_batch_size))\n",
    "print(\"Val batch size: \"+str(val_batch_size))\n",
    "#print(\"Optimizer:\", opt)\n",
    "print(\"================================================\")\n",
    "\n",
    "y_train_categorical = to_categorical(trainVal_label, num_classes=n_classes)\n",
    "\n",
    "earlyStopcb = keras.callbacks.EarlyStopping(monitor='val_loss', min_delta=0, \n",
    "                                            patience=10, verbose=1, mode='auto', \n",
    "                                            baseline=None, restore_best_weights=True)\n",
    "print(\"--- Preparing tensorboard\")\n",
    "log_dir = \"logs/{}\".format(time())\n",
    "print(\"Log Dir: \", log_dir)\n",
    "tbCallBack = TensorBoard(log_dir=log_dir, write_graph=True)\n",
    "\n",
    "\n",
    "#--- Training without data augmentation\n",
    "model.fit(trainVal_data,y_train_categorical, batch_size=train_batch_size,\n",
    "          epochs=n_epochs, \n",
    "          validation_split=0.05,\n",
    "          verbose=1,\n",
    "          callbacks=[tbCallBack, earlyStopcb])\n",
    "\n",
    "\n",
    "model.save('my_model.h5')  # creates a HDF5 file 'my_model.h5'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 2s 199us/step\n",
      "Test loss: 0.7707838417053222\n",
      "Test accuracy (NORMALIZED): 0.7548\n"
     ]
    }
   ],
   "source": [
    "from keras.models import load_model\n",
    "\n",
    "\n",
    "del model  # Will delete model, only to check if load_model is working\n",
    "\n",
    "y_test_categorical = to_categorical(y_test, num_classes=n_classes)\n",
    "\n",
    "# returns a compiled model identical to the previous one\n",
    "model = load_model('my_model.h5')\n",
    "\n",
    "score = model.evaluate(x=X_test, y=y_test_categorical, batch_size=val_batch_size, verbose=1)    \n",
    "# evaluate test set again... should give us the same result\n",
    "# ...\n",
    "print('Test loss:', score[0])\n",
    "print('Test accuracy (NORMALIZED):', score[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}