server_training_new_weights.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Server-side for updating the base_model.\n",
    "In this jupyter notebook, we simulate how the server updates the base model.  \n",
    "The server retrieves encrypted weights from both users User A and User B. The weights are added together and decrypted. The new weights are averaged and used to update the Dense layer weights."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import time\n",
    "import pickle\n",
    "import tenseal as ts\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating server dataset. \n",
    "The server dataset consists of all digits from 0 to 9 from the MNIST dataset. The dataset is split into 3 main parts: Server data, User A data and User B data.  \n",
    "In this section, the server dataset contains the first 20,000 MNIST images and correspoding labels."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of original training examples: 60000\n",
      "Number of original test examples: 10000\n"
     ]
    }
   ],
   "source": [
    "(x_train, y_train), (x_test, y_test) = tf.keras.datasets.mnist.load_data()\n",
    "\n",
    "# Rescale the images from [0,255] to the [0.0,1.0] range.\n",
    "x_train, x_test = x_train[..., np.newaxis]/255.0, x_test[..., np.newaxis]/255.0\n",
    "\n",
    "print(\"Number of original training examples:\", len(x_train))\n",
    "print(\"Number of original test examples:\", len(x_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Slicing the MNIST original dataset to retain only part of the dataset.\n",
    "x_train = x_train[:20000,:,:]\n",
    "y_train = y_train[:20000]\n",
    "x_test = x_test[:4000,:,:]\n",
    "y_test  = y_test[:4000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x_train: 20000\n",
      "y_train: 20000\n",
      "x_test: 4000\n",
      "y_test: 4000\n"
     ]
    }
   ],
   "source": [
    "# Length of the new dataset\n",
    "print(\"x_train:\",len(x_train))\n",
    "print(\"y_train:\",len(y_train))\n",
    "print(\"x_test:\",len(x_test))\n",
    "print(\"y_test:\",len(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAU4AAAD8CAYAAAAYJk2jAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABgKUlEQVR4nO2dd3gV1daH3z1zWnrvCQlJSELovakUC/YK9q5YEbG379p7w3pRsYAd9doFRUWx0HsLkBDSC+mkn5w5+/sjAYEESEjMJDnzPk80Z+bMnsXK7DW7rp+QUmJgYGBg0HoUvQ0wMDAw6G4YgdPAwMCgjRiB08DAwKCNGIHTwMDAoI0YgdPAwMCgjRiB08DAwKCNtCtwCiFOFkJsF0KkCSHu7SijDAwMDLoy4mjXcQohVGAHcCKQA6wCLpJSbu048wwMDAy6Hu1pcY4E0qSU6VJKO/ApcFbHmGVgYGDQdTG149oIIHu/zznAqMNdYBFWacOjHbdspI5q7LJetLugLkBH+QSgkrJiKWVQhxSmI4ZPWsaoP83RyyftCZytQghxHXAdgA13Ronj213mCvlru8vQk3/DJwC/yC8yO6QgHTB80jJG/WlOV/BJewJnLhC13+fIpmMHIKV8C3gLwFv4H3ZAVfHyovbYJBS7E8uSTcgGezvM67q01ieKlxfSbkcNC8EeHYCppBalpBxHfkGn2dpZtOU5cSXa6hfVzw9HcjTm3NLG66uq0YpL/l0jO5mu8Ky0J3CuAvoIIXrTGDAvBC5ujzG1xybxxGtvsr4umu+umACrNrWnuG6N2rcPKXf5oNocJIUXcmXYt6ytimZNSS+KfhtL9Fvb0EpK9TZTF9SQYCrH9ibndA1RZSJggyDw8804Kyv1Nk1fhCD3yr7cfN3XLCpOxikF6zOiSXrUGy1tl97WdSrOYwaTOd3Z+EEKHPUmfFZa8UuzY/55DbQzudFRB04ppUMIMR34CVCBd6WUW46mLGG1oo3oS+gDOxlicWAW6bzf93QCsoLRCncfrYndFsVmI+cpE6uGvcyrpSP5387B/F/aOSAFN4z+nYtu/JBTxN1EPbsS6XDobW6nodhsFFw7lItv+ImzvL5CRRKoqqSdqXK1z0xCZy3V20RdcEwaxu7hVqK+2U1tiOQCr51c5d04/VATa+e9YX356YLRODdv09nSzqM2xMoXo1+mr9mMgsCBRtGkerba/XjgiWvxn7cSnNpRl9+udZxSygVSygQpZZyU8omjLmdQAie98Seze30PQB9TAyfe/hfbnonCFBrSHhO7JSI6kqf6f8VzxeNYfv0woi7LIOHa1SRMW8XCeybwXVVfbrjkB5TEOL1N7VTqxvfntumfMcNvGwuq+nHavLuYXxlHf4tgT7+eOaxzWIRAmziUwc+tY8WMl9h5WRARvzt4p7zfvq+4CwtTvDZTcJy/joZ2Pt5/pHPpq7eT0tDAz7VubG/QCFHdmOhWh9uFBZiCA9tVvu47hxQPD6oereZ2v1Q8FSuFmp1XSofjrtj5fdLLbH20F8Jq1dvMTqU+0ofcBn9+e3kMrNiEs7p63znrwtW88e4ZjHVPJWdygI5Wdi5qQhxu9+ZxgVc+U9PO4KeLxxDzxFq+vGwSl+06mTcnzKPygtF6m9l5CEH9KcMZPWsVT4auwCxU+hyTgXtKAYsuG8OETVNJ/GUad+SPxlcxETIlE1NML72t7jRkdTUNnrC1PowHnrua26+6iYmbplIvG5iVMJ+qEdHtKl/XwCnMFvKmDeLZhC9woPFHnYXjv7mDpdOG8evMY3i/fDiPj/+S4suHgqLqaWqnIcwWMk4zk2/3JWB9RfOxGCkJWVPH+rpeVEc59TFSB0pGBfNS7Oesq1cofSUa54YUZH09cvVmcl/pQ5SpgsCbMlzmOVH79mHYY2t4KGg9SlM17utdgDSbkOu34nNZJYk3bWfT/w3my6pIpkX+SdWAUJ2t7jzqjk1m1mXv8H9rziJ43jrU39ZiejmAX2oDGWhRKRrYvgVF+gZOm5X6sZUMsth5ZPcwHr/5apL+LwVWbsL0+3o++WQSJ7pnMeOOzzGFu8YfXUnozeRj1zN32THIzTta/I65rI5SrWPWOXYHFJuN3WM1/BW4dv3leC8+0C8+i1O5J/Nc+noXIMz/+gq7LkHZIH+mB/6x7/OOBjt/PzMKLT0LpEQrKsJZXY1bVgX5DX5Mdt9NwUgVRI9YvnlYhNVKxrmCDHsgvd8UOOvqALD9soE7V04FwDmosl2+0L2rrqpOvqmOYPk9I7EsWou2Z0/jCadGxJJqltaFMMKWRdWQCH0N7STqwjwZ7JlFn/frXWri53AoYSEM6beLFfUB+L/vgVZefuAXpKTSbmWc5w6UWNfojpacVUMvkzsAxVotZyy5Gd9F25tNeOQdH8hFPuuwCjNOix6Wdj71EwfyyPivmPXVmahL/5mvlg12vP52AxrjTnvQ9/UcHcGpvbeyojIW86LVB5wSVivZEz04xlZIWoMN9+wqXKFjWhdgpszhgTm7hEOFTadFxSyOfkawu+H0ciPZO53f9/TFfeEGDpVfIcZcisPXnZ7fpjqQaelTSbo7B62sDGhcy+mMi6DoITv/SfyQQMXCe3uiiFzc0O5lOF0dYTKReUrjcE3UL/X/2lpwXVuc9eGeXOa/DKc80AzV14eM+4fx5JXv4wSuWXc5IqPZ2vqehxAUjoHe1sMswVJU8sZ7MdQtA58drhEiHN42hrhnUu80IevrD/m9SqcFxe4arfT9e5m1DjPCwx1lUF8KbhtL1jvhnDbvTxYPmceZHmVUyQZe/OxsbEu362dwZyEU1KDGrrlSd+jGRW11+yacu8SAkEnRUDw8UIIDkRYzKXf48fbxb3KMrY73KhIIe9WKVl6ht5mdgtBo9iLZn4YThjDyvI28VTCB0E+34QrtTsXhpMjhhZvagOLhccAqAwDh6cEVUcu47I9pJKak0LPbVI2YNnhSNrYWH8XG7D6fsOC7fngpdZzokUagYsEsVMBCjbQzPfNM4t7LweFCGwRCTRXUhtlwP+i4VBvHg4MXWdrV+ta1xWmqbGBtXS/O9VtN6lsJ3PrzAr5a/Alpp73JcTY7lU574zjF72v1NLPzkBLfrYJyzR3pduAb0RTTi/LLxxD44C76eeaR83iffV2zno6yfgcvf3oWZ/qsJXv6IITpwPd96bgIfNUaen8AzpoanazsXCIXV7G0rjF3SW+TjZt9d3K5dy5hqltT0IQtdgeDf5hB5c3BODKy9DS30znGVo16QyGO44ehhgTDyAHUnDMKy8lFzNx5PgF/57er/CO2OIUQUcD7QAgggbeklC8LIR4GpgFFTV+9X0q5oE0335bFoytO59cJr7BlwltoUqJJqKOeV0uHMP/DScR/nnvIsb6eiG96PaHmCopeVCkuHL7v+KikdB4Kf4U8hx//987lRP66xiVaVgDOujp6f5zP/x17Dv6T8lHnh+PIyEKYTIjkeKJv3sH9G84mZnO2S7TAAZSNadz33pX0mjaLfpYDq3GFs44Ndm9u/mg6Sc+51lZU6WjAutGdimPsLEj+jPdf7c3SijjG+iwm1rIbJwoPP3wVPhkr2nWf1nTVHcAdUsq1QggvYI0Q4uemc7OklM8f7c21sjL6PujJmRfczakXLeUG/794uXgCS3LiCX7KQvjypS4VNAEs63fx4BuXM+umN5kwpGHf8Q8qQ7l5yyWILwKImLfskBMkPRUtbRfmhwbh9kQROS/54PPeSIoHmDj2zHUkuhdS+GisS23PddbU0OvFtVxZdRuTr1rK48Fr+KwqmPkFI9iyJoY+H1USvXY5Thd7TpCSqAWlXDDxMn7t/wXX+GRxjU9ja/u3Whv3vXQNwR8t+/f3qksp84H8pt8rhRApNObi7BAcmdlEzCpk4099uaL3SDzS9hBeUo5WXOoyLar90crKCH9lNY9vvor/+P7z5/HMqSNoRy5aUctrO10BsWwjcmYSDWf6EXhHKuWlgSxaPYDt3/fH9vtGl3tenHV1hL21lo1/JDM+cTQeefVYMopIKN3UbBzYlXBu3Ib7vf3oP2U6x52wkT8zY/H82ROvrAZC/1zfIS+TNk0OCSFigCHACmAcMF0IcTmwmsZWabNBt4Nz57WEbLAjN6TgtgGc0OOXHR3JJ7LBjuWn1Ry87K4nd0Nb85wgJc4NKURtFNQ8ayFcK0M6JTi1Hhs0j+QXZ10drN+K1/rGz67QQ2tVTFm3hZj1guyHzcTIlH1rojsqtrR6ckgI4Qn8D5gppdwDzAbigME0tkhfaOk6KeVbUsrhUsrhZlxrz/mhMHzSnDb5RMrG7ZYOR7sy3HQHjGelOa32iZSNjbJ/YSNJq8TahBBm4HvgJynli/sdPxl4GbACNinlYfdFCiEqgfYuJgsEPHqKHIIQogioBorbUUxg0/XRPcEvhk9axqg/zdHLJ62ZVRfAO0DKQUEzAnidRpXLc4EHhBDJR1C53C6lHH6Y80dECLFaShnTnjK6ElLKoKZ/01H7pb3XdzUMnxwSo/40RxeftGaMcxxwGbBJCLG+6dj9wC1AMPA1kEFj1/0swJAHNjAw6NG0R1d9CnCylPLaps+XAaOklNMPdY2h0tccQ9GxOYZPWsaoP80xVC7bgKHS1zLdWdHR8EnLGPWnOV3BJ+3Zctlqlcs2zwoKgRgxAGVwcjvM67oYM6XNaYtPFJsN++ThaBOHdpJ1+tFavyj9k6g/dYRL5NvsCvWnPYFzn8qlEMJCo8rltx1ilKcn6XcIdp3r0xHF9QgUd3fy7hxL2RVjXKJyHA7RO4qzXvyFCS8vxRQVqbc5XYL0i/yY+vyPqH376G1KlyLnvrFUTR3VLL9BeznqwCmldAB7VS5TgM+OVuXyYERkKIMjc4n87dApxFyN+mOS+XT6C5ww82+Eyay3Obri8PfgZt+dTPTcij22Rwxfthu3AsEAWzaplwe4/It1f6xjSoiamYoaEdah5XYJlcuD2T02kIp6N6yphR1VZPdGUdk1VdDP4sb8JWP/teSs3QUpwCxUdtqDUZas09ucLkHEF+lc8cs0ggcXogYbL5O9VG/wJ9q9FNnBkiq6S2ccjDCZME/Zzc71kTjyCvQ2p0ugJPfhkWO/olirJnqBK2yqOzzmkmpeLYumnzWPutNH6m1Ol8CRX4CoV/iw7/s4e7mepPah8N8quTdwKXWxHasI2/UCZ1I8V0Qvx1Km9PjtdK2laIwfo90yuW7X2bhtN1rhDQEe3OKXiVk4cbgZ3dK9WEJq8FIE0nDJPnxT9rC03p9dF3ZsuV0rcApB8Ug/8ht8ifqlSm9rugSqtzd7JtUQoipsXhqPIytHb5MMuiiq6kRF0OBrrNTYi7KnhoVlgxgUl92hY79dKnCqwUGI84pZmJMMKztknqnbIwL8OCl+G0tqA+j9dXWPF9tqCwUOL7xTXENSpTXUFHqQ6VDZdW6Xqta64iwq4YcNA+jvk9ehY79dysMyyJ8Lo9dQ+3uQ0U1vomh8OA+HLuabkqHGy+Qg8hx+ODdv09uMLkPfV8t4tfB4Jg02dj3vxVlZiecOC0PcM6kdEt1h5XapwFkwwR9PtY6oBSV6m9JlKBko8VIs/PXLAONlYnBYRFUt1ZoFX7Nr6C61Fp90jUqnjaqwjptZ71KBs3JMLWsqoxHVtXqb0iVQg4K44aSfWVFvJvoHozLsxVRl5/dahSRLPg0nDNPbnK6DquCh2lGENNZy7oe1tHElSl1QDxzjVAMDmD74N/7KjkW6iBTwkbD3j2Kseyq3bLwIU6oxKbSX2nAPgtRq3JUG6v26hMJ1l0Dz8+KK4L/o556LmhSvtzldinBTGdXRHddjO2LgFEJECSF+E0JsFUJsEULc2nT8YSFErhBifdPPqe0xpGJSHwbYstE2++Cscl29lP0pTbRiEw4a1vihlZTqbU6XQJhMFF1TQz+LG+kNgah2Y7JsL2pFNZ+VjCLGXEx9mLfe5nQZrFmlFDg6dvu2riqX+xCC4gGCz0pGEvNtxb+S6r47Uu8naJAq1lKM2fQmRHI89/T7iRNTzkB7JgS3RSv1NqnrUFPL33m9GeOdhqnKtXeX7Y+WtosFJQOR7g6E1Yqsb/9Wbt1VLptuQtzHxWR8Fw9rN3do0d2ZXt+VcnHfafRKbzjyl12FXbm8e9s5eKzLwlGwWm9ruhRacQmhDwUwJ+xc3DZv6fGih21h+8dJ+J5Uiurni6Og/ZtIuoTKJYCWktoWU7o1rfWJc/M2Eq42IbWeP5veap9UVmJduMol1Byh9X4BkA4Hcv1WrOt7tlJsW3yyl+D/LkO8ZcbRQXkeDJVLHWiLT6TD4RLddOM5aRnDL805Kp80KV52FIbKpc4Yio7NMXzSMkb9aY6hctlKeppKn6Ho2BzDJ4fEqD/NMVQuDQwMDLoDrZlV/wtotuReCOEO5B6sctnhFhoYGBh0MTpV5VJFHeYt/FseVBWN/xFCIKU85ISIF35YhFV2Z3nTg33ihR+H9Esr2Ht9d5bCNXzSMq2uP62kJ9aflnwifdyxewkwOzHtUTCV1SKdLa81OBqftEdXfQzwsJRyctPn+wCklE8d6hpv4S9bkvKsOWcU+VPrGR2TwYMRP3D9jouxzwnD8/MVLQbQFfJX9sjSbvuH359D+WQvaoA/jqRe1AVaaPBQQIJXVh3Kis3NNgr8Ir9Y0xPG9Y7kExQVNSmO6lhf3H/bgrP60DvNeopPoBV+AYTVihIVDhVVaEVFLX7HFepP6uujiFwksXsp1E8tp7zAi74PpKMVt5xAqK0+aU+Lc5/KJY2ywBcCF7e1kLrTR3LJE99zlU8G31QH8lzBScxL/IhzLrsa3z+CO2SxardECKqmjKT2snKui/+Zse47iTcpOHEyb08fnv/jFBJuWuNyGZPUAH/SZyQx/pR1XBL4Gffdc33jC9YAACU+hrT/s6E5fEm6z4Ijp5lit0uQNGs3WnoWbk4NdXlvyh/wIPvKRCJeWtkhOxOPOnBKKR1CiL0qlyrwbltVLtUAf0qvqSLGUszwl26l1//ykFYzT79/An0DdlPm5prywGpgANsf6MMrZ8xle104z6w4mV5fqnjsKEW6Wcj8P4VvTn6F6WfOwO1r19hyKEwm6k8Ygn1mCZeH/8qu2kA8hJ2qSBVPvY3rQji37yTso6HcMetDnh92KW4uGji1tF0H/N5njjcNjxQjfu6D3JDS7vLbNcYppVwALDjq6+vqsSz04f8WXE3E3NU4GuwU3TCG54Le5+L1VxOxx/Vam2qAPzvuSeCVM95j+u+XEj/XQcLS9eDU2Nu29FgwhtSBwThsXSa51b+LolJ28QhG3LKWH9YN5LcnxmL3NbPm2V2Yanr+5oC2IB0OLJUNpNuDMVe5Vm/kcKgb0kjNSCZguImADe0vT9ecXM7qagLnLAdASokyqC9Jl21DQ+D8yw+tbIee5nU+QpB2ZyK3nrqAp+69gr6/bENrIcWeuVpS5PDCUukaFUOxWbFUOUm7sQ9JmzfhrKtDmTycNZXRhP6Q5TLbL1uDMFvIPcaNV9dNJGHljh699bItOKur8d5ipvS4OgLeaX95+icz3G/yJ32qLxtj3ufEzRcRNS8VzcXG75ASvxR4N/c0Qr5bg9ZCFhdhtlA8WKAIidtvrpHIwVlTg/uXK5CApLHbXpZgYUlGPNE5m/Q2r+sgBHUnDWL46ZvJejQRZ2Wl3hZ1Key+4ObZ/sxI0BUCZxO7bxrLcxfM5Z6CMdge9UEr2nXki3ogfh+vAmhxAFuYTNSdOIiHzv2Mh76bSlyta06KKL17IU4oJfx1Y3Rzf9T43vjencWfG5JI+HGV3uZ0KRQPD+xxtZi2dUye0i4xSGbqHU3UBekkWYr4df5IlBWum1pOOhyHnPWrO2kIkf+XyjMpJ5E4K8slkn+0RPbZobhb7bhtMrLi70VYraReG4JDKiQ/nuuyz8ZeFHd3xLB+MHogwmpFeHrg4VVH2N8dM7Cjf4tTCHZdHMEfsc9x9pbLiJqXhmYkMm7OyAEE35/OAK9c8v8ThyPPNdUdTWGhiHFlaB8E4yg8qFWlqJiiwpF7qtDKmmU47LkoKuVTh3DdaYv4ePZk/JLt1B0ThWYWVMQLpCpp8HMS95kdZck6va391zHF9GLrfSH0S8zBpjawPisZ6xY33E3FeGzM65Axcf1bnKMGMOOSb0hrsFH3vxC0wt16W9T1GD0Qrxfy8TLV89Ntx2H6dY1LtigUDw+23R1DXa0Fv683HbCG1RQVSf5to3C8K6k4MVFHKzsRIVBDgim8eRQX3vcjN/pu4f9u/RDzPQWUT6miaJQTe6BGg58Tr4g9ZFwvESb920r/JsJkYtuMcBCgnbaHmjM1TDttvHPtq+zZEIAjr6BD7qO7F9MucOdS750MXHwTSZ9vRVPUxhN7K4WiIlQVNcAPAFlXBz24MSFMJpSEWOpDvSjta6V8cAOnDN5EP49cPrv7FNz/2ohz7/d8fdBKy11mEXz9mCSmTFjO0sdG4ayubuyOxUSSfkEgx568gQDnHnbOSsZ7Yc+fNFMD/Em9K5ETj1/H3f5v8EtlPwZ9PpPQpRKfdbuJKc4+YEWGGuBP3bDYHp8UW/Hxxie+DK+3fJD19SiB/tRHNvBtxRAGHJdK8clDsS1a1+5F8LoHTqeXA6swMzY+nRX3J4MExQ6BmyQNboKyvuDw1jhh2BYU4WTRlmTENLPeZv8rmKKjyLwoiosv/pU7AjajoFAvG1hd7064qZKf70pm69ghRP1ipzbITEWsQszcdBz5HfMW7cooNhuVMyv58ucxJKzJofKskWSfKrnvuB9YsHsAqz4aRPj8NDwLl/f4oAmw/YEE1p7/Ir/UhHDd/OuJf3Yr8eWNS/taCo1aSSnmRT1f8M9ZVU1Zfhxlp4PpuBH49SsmXC1l7bUDKBnkTf115dhHjST6h0qULemH3a57OFqTjzMKeB8IoXE1yFtSypeFEA8D04C9G2Lvb1oQ3yY8d1jYeLzGYxHfo174Pf6KCQWF3+q8qXNaSLIUkuvwZll1H97fOAprtqWFXE09AEUl5Y5wfjzrOXqbbDRIyYYGJxd+O5O4L+tJP8fKR2e+TtAltXxy+nDeWXkMyY8UoBW1J9dv96HqlEFMiV7MO5tPQHvPyaVh3/N53jDee/RM/JfnE7JraYsBo6cSsFEwov52IpY4iPt9HVpdnd4mdQlkfT1J/61i20x3tGAN9cMAfJfn48jYSsBGM2xMIO32Gor/U0/I7SGQmn5U99Fd5TL8uWXMTLuFBneB0yQoHSAhrA5y3fDfAp55DVj/TsFZW0e8XA9AgbPniZcpHu6gwiflI/A3VfPiyhOIf0ejz9JV4NTo87eJhz6+ktyJXnjmOkn6ahOOo3xbdjcUm42cMzSu8l2PdqrCD3n9eOv5swj6bDPeldkuuQDeb+4y/Jp+d4UWdltwbkgh4eqm1pWU+54P2WCH1ZuJu0SAUNq1Tlx/lUspcf/yn/WIfi18xRUeDGdlJX2fymFZyBCkWSVpYwrOmpp956XDASs3EdG0Nd0VfLIX6XAQ/57GuQtuxzOzBt+cIhwFy3G64ASZQSs53LMhJcj29U+6jMqlK3EonzhycqEpKYMrBUY4/HMiHQ6UP9fhQeNYkSu1MI3605yu4BND5VIHDJ80x/BJyxh+aU5X8Em7VC73Ox8DfC+l7H+EcgyVvoMwFB2bY/ikZYz605zuqHIZBgyiUR7YH8hrxf0Mlb6DMBQdm2P45JAY9ac5uvikNV31vSqXk4QQ65t+TgWeA76hcdhpOWASQiS30WYDAwODbkd7VC7LgKCDNIcMeWADA4MeT3t2DkUA2ft9zqEFeWBDpa85hqJjcwyftIxRf5rTFXzSHpXLKcDJB+uqSymnH+qagxXphNWKEhOFtJlh+y6crdz94AoqfUdDT1F0NHzSMofyixrgT83oOBrcFXzWFzXq7RymXhv1pzmdqXKZC0Tt9zmy6VirUGw2cm4ZysjzNhJi3cOnf4+h7/P5ODKy2mFSD0JREco/f0fplC6TzKMlxJB+bJ9hw+zWgFNTsW1yo9c3RWgpqXqbpiuKlxcpT8Ux/8T/EmuyM+L720i6u8h1s7+Lxl1BLdKB9Uc3eeDycwYz96aXsAmN5bW9efeUOfwwdhB/PzMan+82HrBrxpVQAwPInJbIMees46rAv/Yd/8+us6n9bzgeX612uQBqP3kE576wiNc9t1DutLC4ui+DRmUxfdDFxN/k51q5Nw+ieEp/YmNzueyjGdxx3jfcOX4hP0SNga2uFThNEeHknRVDTZgk7phMBvv+k+RaEZLfC/pQ/W0o4d9mdohksm7ywHUBCnXSxIWf3UrMd7XsHubOizPe5NjHt3N/zJVEPLvMJXNONvTrxaxr5hCsVvF03ilM8t/GQGs23yT+jynTz0asDMOR7VqZz2sDVYodnpzx9t14ZUh8t1fx+UNDeXvUPB4ecS3mRav1NlE3AteUIVbYiNm6jNl5Z3PfzI+o6uOLm4tN0VaMjmLBvc8SqLodcNyJEwWFh4LWU5xcy7FxdxJ3h46BE9onD1w+2M63FUNJmJ2LIyOLsOUm7i+9jin3LOLt61/lOscthL+yunFjvgthSStk5nvTCF7XgPuqDL4KH8/z91pYPO51Kupt+DpdLwuO7+frWP1rFL2KG58HU2gIquLJ6ppYLGX1uN7r9R+cG/9RAlDrJcfacnnRU8HtMNf0RHzWFXLsZ3fiDG6MF7LGRPhiBff8eipibdx+/6ec51lMUHLREUpqHbrl4/RItZDfzxtZVU39qSOo91HxzLUze/EJTDlrA7Fn7qThixAcmdlHLqwH4cjNI+qJxr0EclBftk/z5o9xL1CsmXG+H4wjd7nOFnY+sr5+X85RNSSYlP/EMDfxTa75/EZiVy3T2bquQ8lwjVX1wfjuqHa5l4kjPYO4OzOaHTdFRVJ6diTJ1nzATNnKELzZ2e776RY4g9bb2TkxEI9Eb0555ndO8dpEqeaOl1JHiGrhP72+46IbbyV2vjdsTUO2IJXbk1Hc3dl2kyerTn0RP8Wdy7afg1dmHWpgAFpxid7mdTqNCp9DELfvZl3fl6hxarjtFigeHkedjLYnYQoL5cqxf+Gt1JF3nBdRu6NwZOW43HCXMFtgUAK5k7zRRu7hjn6/cIbnfAIUN5xI7L3rqDl3FF6/pKBVVh61f3TTHLIu3ojPHWZM27KYs2kci6uTGGSpYqBFxSxUBlpUtlz2Gmd/9Du7HhyKGhigl6m64KytJfpbuG7X2ZQ4a/kg8SPOnfMz6f8NxxTZcVn9ugtqaAg3vfwZvyR/haewEqy6M/vm19g2K7lRzdDVsVooc7gzxFrN4pnPkfOyJ2p8b72t6nSU6AjkM+WsmPESG8bM43LvXAKUfwYuth8/h1ue/pTUN2JxHjf4qO+jW4tTNtjRtjTuzU+8w8JPQaP5LvB4GrxUnGaBZY9Gxpkqc099k68veYHT3W4n8RnVdcTcpMT6wyrqt0Zz3pDbyZ0Idx//PQtHzea0K++m19OF7dZN6U7IejvPbJ/Mpz5lpPzSB1sJeJ2Zz58nz+KRQZPJnj4AuXqzy7Ww9qLl5LH96j5MHDOCoKnZ/Dh0Dhe8dhmeF7reqoPqBguLav2ZmzeOlD9jiV5QAxJqImzkTnYya8InbB4/h9NDzsU0LQZHekab76G75pApKpKd1/bCvUAS/O5aTPt1yRMWm7hvyfWc/dAv/O+cl5m2aSb+77lI4GzCsSsT912ZJHxrYvYtZzH+1ueJmpiFOjekQ5ZVdBe0oiICzyiiGujVpNai/JnE+Ltv4a/jXmX8HTcRf4P3AQJlroR0OJAbUgjaAMqyvmz/0pv/i/uel62T9DatU9HSs/C5Kog51knI8gpiyv8ZA/cAEr828dRFl8GDH3BvzAKeiL0S81EETt3lgRuiArj9/K/5712vUXvSoAPOSYcDzy9W8ukrJxFvlnhenNfj5U1bRAiUuBhCT8+iRprY83YkWkGh3lbpjnPjNpLuzOPE1dfzyvD5VJzUV2+T9EUITL2jSbvIl3jzHuYXj2pUhXUlnBpacSk1SSHIhuY9Mulw4Letis21kTRIE+IoOyi6B06xdAOvzTkbf7WOrFOB0QOhSSJYWK1UnzeSyok1mIVKRa1NX2N1QJhM1J45AtucCub2+ZQpP9+M3w9bXaqbfjiclVVUl7hzolstpcm6P866YYqOIu+uMXh8UMWvFz/HtWkXkH1bnEu2wOWQRC54cQHbn+lP2ZVjqDl3FGLEAMquGEPxdWPwnpXHdX5ruH/r2di2H51CrO4qlwCR87YxOeE2Zp3wMenHBrO+MhKHU8VNbeDioLdJNlewsCYU8YM/UttxNLfodqgB/gibjW139uKhU74gwlzGcR/fRd9ntqHt2aO3efohBMJiQQ0MYM+oKPLOsfPFMa+T5bBj7fnqty1iCg1h2xOBLDnuWWxCMOr3W0h8tAKZtlFv03TBVFzJuqpefHraa1SeYqNOmsmwBzHOLQ07CqFqPRdtv5jQ+8CR9y8FTv5llUto1HxOuHEVs067mJwTFAL6lDAkKAdVSJ7OOJX0DRHE/NBA0B9rONqkJN0JZWASnv8t4vqwnxhk2cPlaVOpeT6C2AXL0Vzg338oTFGRZE/tRWWCg0tGL2NGwOe4C5X7C47lt/kjiPpom0tJBO9F+nkzpd860hq8mfa/60l8YafrTKK2gCM9g5yzQrn55BlU9gYtvpZh0Vn8VpLI+qwo3Na5EbWgGG3r0SeO11/l8p8bYft+JX1+NKH4eJMdHAmApbySPkWrGwe/O/ymXZOiEX48FTGPLfZwbvj8Ovq8XYA1bZXeZumKYrORc24vzrliCef7rKZA82TiyuuQa32I/qaU8K0r2iX32p2Ru7JZfccwVpmGE7dkLZqLrXluCUd+Af7vFRBgMqF4elDp7wdSklCWjlZe0e4XbJdTuZQOB1pJKZT03H7XkXwStKqMi9+5jV4/VRK7ZqVLBIQj+cRZV0foqytY8YYnK8VEpJREaduRmtajZYJbU3+cdXWYFq8BcInGRVtiinQ4Gsd5O3is11C51IEj+cS5cRtRjy9FrtrkMpmQWvWcODVkfT3OujpkfX3jBFkPDppg1J+W6Ao+MVQudcZQdGyO4ZOWMepPcwyVy1bS01T6DEXH5hg+OSRG/WmOoXJpYGBg0B0wVC4NDAwM2oihcqkDhqJjcwyftIxRf5rTFXyiq8rl0WKo9LVMT1F0NHzSMkb9aY5ePmnP5t52qVweYITNRtENYzCFhR5wXFitqP0SG5OTuhjK4GTy7xiLqXd0i+dVPz/Uvn062aquSfllY1CTE/Q2Q1dqzh0FIwfobUaXI+e+sSgDkw482JQLoz20J3DuU7kUQlhoVLn89mgKEtGRxF6SivTzPtA4L09S7vREDks6xJU9ECFQBiczbO4mrBOLoa75LhBTRDjFHway45oAFPcjbyroySg2G+6X51E+wF9vU3TDFBpC1RUV1Ac1JsERZgtqn1iXbHAcwMgBXHTRYvYk+uw7JIb3R/01BLVfYruKPurAKaV0AHtVLlOAz9qicrk/JaOCiHQvR9QeFCQ0J56+tVRFuY70lOrry447rQSaKwm8X9mntbMXYTKRcXkMAwLySXi31KVklJXBydSfMuKAFoMSFkKCz24se1xjo0BLFE2OpabOgsfmJl2m4EC2zQhCiYnU2TL9UGw2dk71pJelGPeCfwQf98R7cmXE3zh82pdprV15uKSUC6SUCVLKOCnlE0dThhjWj2G3rGPxxyPRsg/s6Qs/Hx5IXohm7RHDMa2i9LRErhv0F/Nmn4pzU/N1vXWTh/DoVR+ydOFAtJRUHSzUj7SLfMi5rAHF459Wdn10AGbhxD21PWvluy/CbKFksCT0Q9s+YUNZV4fib6ch3OcIV/dcKk8fxLzzXuehP85BXf5Pe65grOSvPQmYytrX4NA1gaHq68P2aR5kVfsR+c6WZjkmC04MY4A1D2uFUycLOxlFRb10N4GmSsJ+zG+2nVAZ1JfSaVW8mjGJ2Dd39vjthvsjzBZihudg3u6OrK3dd7yknw031Y62M0M/43TEOTKZmAF5eOz4J7eDsNkY3Cub8jjXy18LjXMjpusKWVrTh/gPHAdIjAfHl/BnbhwUtO9Fq2vgdPSN4Z7jfiD7m97NEq6aoiJJvHwbV26+As9VmTpZ2PmU7PHgVI80mFNP2oujKZwxlsIZY9kxeyTjPljHWwM/ZM9XYThcLAO8SI5joG8u4X/W73vBCrOFyt5OFuxKdqmXyF5UXx/SpqkU/BqJtuMfydvafuGE2ioJWOd6eVsVDw92PDuER+O/5oP3JqMu27TvnBoYwPCgbMpLPNutw6Rr4KyMccNLrSPyy+ba6WXjIjk/eBXVywJdJ0g4NaJeNTH255kM9ctm9dQXOfHKZYy6dB3hvYu5J2ALT2afRuhHRzWU3K3JPcGPBqlSG2xm981jyXpoLDteGsJtp/5AXaaX3ubpQt3IPkwdtIboL3cf8OKw5lWxojAapar2MFf3TKpP7MeXZ76MUyo43EFJiG1craOoaLHhjPFO65AUUroK+FgrNHbVB1HbJxjbnqomi0wILw/yj3MSZSrFf5trDforf64jaY07a8PjuTBwCGpxJcKh4TguhEUPelDyegyee5brbWanIkwmKvvZuTFwCR/cU8/Solgaqjzo7V3Jpd47UE928vkPJ2P6dY3epnYqSoOT3fVepF0ZRNC6QHy2lIHTSYOvDZNaScEJIQQGe2PalO4yqgGaWWBHIdlSyf+ueZ61l0aRY/dn/q6hjA3bQT9LHl5b27/aQNfA6Z5awrfZAzj7xb+Ynz608ZjVzpDAXJ4InE+50w2v1ApcZIRzH86aGkjbBWmgAabICIKuzuCu9VOI+Xm7y2U5l5pG2CITZxXcQdz8cqwbtxEhJYweSMrHFmbvOA5PfxOeehvayai/r6PwijgChklKkxV2D/XHEdiAV2A1ryR8xR2nTqFICSR8pxu4SOD0+Xkbt909ndK+Kk6LJGREAaOCMoj3L+bSgKWsr48i6rMs2qvYpWvg1HZmEjQ9ki9PmERDaOPMuZYl2VgVyO+PFhBjKUampOtpYpegeGIv7g7/iMe+vBStIkVvczofKfGavwKvzxSc++UnLRjtyadlowh/WCC3rHOJJL4HICXa9jS8t6fhLQRCVVF8vNHiIvji9RF4veGD9Yel7Q4S3QmtvALPz1fse4mawkLZ4h2H5uPGt28OobDeG0d2Trvvo6/WrlPDsSuTgDmZBOx32BQWytKyOJJC81F8fdCKig5ZRE9H8fBg98QG0upDCP8k1SWywbeIlCAP/LfXBUqW5MQTll2Ac7+ZU5dEyn3qCSabjQUp/UjMrnS53trBOPILIB/UPrEMdM/mvvXnkkD7h3SOODkkhIgSQvwmhNgqhNgihLi16fjDQojcg1LNdSjVTqtLB00AekfxyNiv+XjncFAEakKc3hZ1GWT7d871SGRNLYoqqYp33XWcB1MxOIjTPfKxZnfMbqrWzKrvVblMBkYDN++Xd3OWlHJw089RSQO3iJuNwd45zMk5tsOK7K5Is0qMpZiJUakUvePLrgtD9Dapy2DqU0mwZxXC5prrFQ+FVrEHJctG7iS9Lek61AYq1EkN/y0dM6DTdVQu97+nInh7/TjiZ2uIo8sb0mNQ84u58s+rUcxOQr+0ErQk1eUmhw6Fc6sXWU5vYkrW6m1K18KpEbZUI+ssV++o/0Pw8j1MWnMtkeuKOqT+/Osql0eDlraLhKuzm+0kckUcBYUkXFMCNCn26WxPVyL64ZUAB0wYGTRi+34ViQtV15swOwRy3RbCp5jQOiim/Osql0KI64QQq4UQqxtovd5zTw6abfWJdDh6tD/gKJ8Tp9bjVUCPtv7snSzqiXSFmGKoXOqMoejYHMMnLWPUn+YYKpetpKep9BmKjs0xfHJIjPrTHEPl0sDAwKA7YKhcGhgYGLQRQ+VSBwxFx+YYPmkZo/40pyv4pOuqXCoqqrcnqCrS3oCICoOiMrSiIpdS6VNDgrH3jaQmxIJUwFLpxCO9HG3bzmYzyj1F0dFQuWwZQ+WyOXr5pD0tzg5TuTwARcU5biBZJ9lwH1iGu9VOYYkPE+N3sLogipApFdDDtyUrg5PJP86X6khJ4IDdTI/9hsHWHC7fdAV1PwZiLXFDUQTSldc3C4Hi6Unl5GTyJoLi17gsRQChn1tx/2qFvvbpwO6bxmL3hcCNDtwzGrMhCXsDzswcZH0bljL1dIQAoSDMJkTfOJTiigP7zq2gPYFzn8oljQHzQuDidpSHffJwYh/Zxq2hs4lSnXgqVmqknRS7BQ3BOJ9QPlV6t+cW3YLUy7z57rzneSz3NNb8lsSsL84n+K8igvPycValNq7R09tInTCFhVJ+TDT5pzdwxeDlqGIJX2YMwvF7AL7pDvxvy2TT+N7Ef6P2+DWe+6MGBjDmqsYdVHmTfQiyVWFWNFLKQyj6dRhRr6x3KWG/fSgqipsNZ00NppBgagdGkX28Ga++pQwKzuPPnVZ6zw7qvMAppXQIIfaqXKrAu0ejcqm4u1NzfH881+ZQ0svM/WE/Ui1N3J47mT93xmPe7oZXlkS1S5QGiad95dGa3G2I+c7On6fGs25RX2IeWwFOzdgx1ETmFbHMu/4lZmy/kC8+nEDIqjpClm/FWbcdYbWy2zaEpJUFOFwoaALg54NJ7GHrXQOwrN9FTq9eSIsJW4OGbbhEWMzgYnFT8fIia/oAGgZWoxW64R9fyrS4BXyeN4zsv6LI+sOThA0ZaMUlbS67XWnlmhJ7HHVyD9XPj22PJJDQPwd5ozvBX2xj+qrrkKpAySmiT9WOxreki+nJqEvW8dSyUxGxdZiCA5E1tThranrsTpBWIwROMyypTsLjMW88ly4DKfelTpP19Xh9tsr1giaAxYwiGuuJVlYGTZo6EgjcYkFzsbR7amAAKY/Gsfi0Z1EFvF8+nHeWHcsnr52G26qdxFSsbNcWZt3ycQqzhcwb+jL39P9y939uwCd1VWPXqukP7oKP/j4UNzeiI4s5L2Idb80eh6YpqCu86fV5Do6MLL3N0w1TRDgRE7J5Z/sYolZspsWJTVcMmkDZID8irGVsbuGcdLGgCZB6dwJLTn+OL/YM5N0PTybq+1ISNq8COia26CbWVnfiIO6/cj7XfHEjfl9tdNkH/mAUm420hwZxWdRy/rv1OBzrfKktcSP6tF2M/CaN3HvGNubkFD1iUrRNOAO8OSVkC6z2MVrf+yHMFgomaFzgvYHwJ9Ow/xzNjreHoybG622aLjiPGcx/zvyce7PPYNFV44h4dgXOzds69B66tTjzjjEx2JqD0yRRggNxunBL6gAUBZ80eOeRs+n16T+ibM7AABZOPg51ahn1JzYgnhiC+ptrpVOrD/Ggvy2bwE0OTNFR4NBw5Be6/EvXPn4AT43/gjuzzmLTz4kodhhwcjqpjwfR+zq/dkvhdieE2ULa5WYsQiP7hQQsXg5KbxiFoklC/ixpcRnf0aBb4GwIbSDebGLheS8wZ8IxfLtgDIpD4JkpsVRLNAv4r9iNlupamkPOmhoC5ixrdlwrLsHnoxICfg0h/YY4Bjy2ncJnRuL243qX6YqVxVsINVUy4Ym/aXCqWBUHc5ePI3FOLXJ1S51U1yDjTBPraqLZMzOMXmsax31L0kZx5YOLWTT0OJdS/1R6RzFpYAojbNkMe2ANDqdKqd2dYFslfjfV8MnXE+j97Aac1dXtuo9ugTP+PY0JYRfydt8PeSR4BU9etfqA81XOeh4qHE/qlUkd3szuzjgKCun9moMt9kSmPL6EvytHof7uGi3P0L9Kudx0O27FTjSzoCIBRozfgddr9eTdkIxzvWvu9u3zcS3rPx4Iazbtm0j1Sqtit90LzaroLCzW+azbHcGUwmuRv/oTtrgUmbqLitAwUu4I58kLP+HF9Avxm9e8cdIWdPOp8uc6/K8O5bbYG9k91J2acVX8NW42CpDusNDHBHcG/8aZk0YQ4rqNiRbRikuI+SCLhqkqtcEWl5HFdW7cRsjGfz4HBviz1pbAk2d8wuyQJDpGTaZ7IccNJvt4d3o9uvSA47tHe3OZZybrHEN1skwftB07CbsxvHHGfPd2nE0vEkdmNlGLwnA/pZ7i4U783hftWq2j68vIkV+AyC8g5G9wbhzC1hFelDvdue23i5g96X2SLSW44kpvNSEOzCa0LS2nGRRWK3ln9iLaUYzXzkrXclHTpJianEDKrd68ffwcrv35GhIXr3ctPwAoKulnuyEOCgCmyAjczypkYckAbGvSXW6FiiO3eYZLZXAy9TeVYhEaYX/Q7iWOrcnHGQW8D4TQGMbeklK+LIR4GJgG7JWhvL89gm2mtWncvmUqvw2dy+LJs/BQBBdsu4SI73NdShdaDQpix8NeyAIbic8Eg6aBqiLcbFQOCqUi1kTd6CoiA3JY9+JgvNet0tvkTkGx2ZD94igY60NlrJPTjl3D2R65zHj7epI/yMThIuO8ByCdKA5QawXCakVYLIioMLbO9OGhmG946+Fz8SpZfuRyehJKo/Sp4mZDWCw4Y8LZdZ43p5yyijN813HD59cR/926dssmt6bFuVflcq0QwgtYI4T4uencLCnl8+20AQBnZSWh9zgZfOctnDRgC38sGELsGztxFBR2RPHdB4cD8mxceuKf7BwdhFnRqNdMxLgXkFZdTVVpIMraAKzzazGluEalUAMDSL07gc+nvoRZONnZEMBtyy9g++t9iVy+1KVerAcgJSGrnIx8YBVfhg3HM7SKKbHrqS+K5727zsZ74WrXaoULQc49o3B4SJTEKiL9y5kW9T1pdaG8u3kMG+cOJu63NTg74CXbpVQutZRU+t7tT1ZYNNHbVuJwwbV6WlkZCY+nsOLTIdj9bdi9VcxVTooremEurCC0bDda+Q40F9pNJTzccfg38GDWWez6LpbgdfUkrNyBs7JSb9N0x3PBBn6NGE382dmkZYbww6zxBP2RjyN9pWsFTQAp8Ul3UhmtoK33orTYk1eKL8Qzs5qEHeloe/Z0mE+6nMqlVlIKJaXtLaZbo5VXwOoKzIB5v+Ou9xppxJGZTeIN+diFQrij8IBtlq6Os66OkNeWIWarJDjzwKm57HMC4DV/OV57N4c0NS4kHb8TscuqXPZkDJ8050g+kQ5H43pVF2ppQyuflb2Kli6yEeCIPpHyX39ODJVLnTEUHZtj+KRljPrTHEPlspX0NJU+Q9GxOYZPDolRf5pjqFwaGBgYdAcMlUsDAwODNtJ1VC4FCETLORb3oyeq9B1R0VEIBBzSNz1B0bFVPhFN/2nFuHxP8Am0U9FRNPdVT6w/h44prX9WuqXKperrw7bHknjwxK944/Fz8fnw8Au7XUmlr/CWsUy78TvKHB78ed1IWL7xkN/tKYqOh/JJ6txhSLtC0swtrdbP6Sk+gdYrOppCQ9h2d2/OHr+SlEvj0FJSDzjf0+uPMJkouHEk51z7Oyum9kXbsfOI5bTVJ+1JZNxhKpe5V/ZjxqSfeGzhuQT8nd8Ok3oWppheHHfFKlSczPtxIg5385Ev6qGovj5cOGgVvhvMSLsLbq9sAzkXxbFs6gukVgVDQXsWJnRDFJWSy0fw9K3vAI0qn//Kbdpx7T6VSyGEhUaVy2+PpqCq4bV8mTOEpGd24diV2Q6Teg6KzcbWe0I5128Nb79wJnH/WYvp9/V6m6UbZaf2BSD0z1Ij+/thUAYmcfHVP/PM7mOpuy/EpZIYA4ihfZk84y/qpJmfHz0WR2Yb5StbyVEHTimlA9ircpkCfHZUKpdeXoyL20nRsjAcu13s7XgYlLAQRg1M4+nMUwj+39ZGXWwXWeB8MMJsoWC8RkZNAHL7Lr3N6bKo3t5sv9aHEHMFa/4zDLHs0MM6PRIh2H6TjZv9l3Hf+nPwWrjpX1sIr6vKJUDhJf2ZHvQeKVn99gUGYbbA4ESKB3tSHS7wzJYE/bATrXB3e27VrSgdHcZ/wr7k5memE1jevqSr3R2RGMtFI1fw+U/jiK3/xxemqEiyLuiF3U+iWSUJ75Q0G89zJexD47njhAU8++EUohYuc71dVqrKe+PfRRWCsDctB46DC4Hi6Ylo2o4ppWxXrgN9k0MLQXk/Jz+UDSZ4URYOGtOqpd0ez1PnfUScuYidDUEkWwo4u/cd9H603GVkIgpPseOuNOC/pVZvU3TH6Wkhwa0A74Mam2nXRbH8yue5PXcy9ZqJ5UG96XNl+xLUdlfsk4eTeaqJuelj6P1OOg4X9AGATTRQ7ZS4pZfs27NvCgsld2osVaNqsVgbxzwdDSoef3oQ9ututO1pbb6ProFTjYth+JA0flw2iD45K2g4aTji3nxmR7/F7ZvOx3ueN16bi9jxsA/+Q4tRfLyOSjy+2yEEFw9cxZO5p6LWazC8P5rNhNAk5uxiHDlHNQfXo1AG9eX6c35i/JpriLxfY/eYAG68bRG/+UQ2JklxIdSgIIqur+W8mK1smD4QR/4OhMkEQnGZhsb+5Gnu4GxMA6PYbOy8IZa5l73K4qpk5qcPpTLTByWojttmfMbO60P4/r/HwRu/tuke+maAD/JmcsAytuQnonp7o96Xx4yoX7l1zvX0emMLWnkKGmDaOYbiaEFAbWt2dfYM/E3V3B6+lv+9NYLzfVfipTTgq8D5KZfgdmcSzo2uo8MkHE7KNXccbk2rRRSVXef5klEXQNStVTgyszENGU24uZy64RMw/+I64mQA9v5RLB7xKmM/vZM+a9dRde4o8scoqHaIn53lci/a36qSoaGxvSl6RXDpOYsB+PHB8YR+s4pQKRFmC8/fcj7P3zSHL08eBG+07R666aoDOK0qUebGFmT5qclcGfE3d7xzDZGz1vzTahCCBm+JOdvabmW67kS+3YcApZ5FWUlc9epMrrthJmM/vZN7YxeS84iCKSxUbxM7DSUtmznbxpE4ZTuKuztqbC/OOH05P/w1rHHWVAhqgxR+L0/Ctrrt3a7uTsFIGwChK5w4RvQlf5zg5Alr+fHS50i9qde+rOiuwvk+a5Aebo0fTCp+pmr2OG2459b+k2quwU7oiho210VRne3V5nvoGjj3x1bcQI3TSk0vB4qvz77jzmMGc8H4pQRsdqExGyn5YvVwFlT1I3xmLWEvLsW6cBXx961i5trzGRmWhfTy0NvKTkMrr8BtoTdXh/7FjscHsmdwMOf5rsZvS9NA/9hBDLpgM39/N8jluukAyWds54G8k/BZlo3y5zri7lrFol+H4qsoaL3qUDzc9TaxU5BOyfO5kwlRFVKvCcYUFooo28OLa04AYMd0M2JYv33fd7ipaAjC/2j7vXQNnPX+ZgLUaqQJzIvX89+nz+O04RvY9n+9UQMDKL98DCHP7OLTDSPw+W6TnqZ2OpYiE4nWPBrC/fYdU2KjGRqRg7OV2w57EsGfbeHGRVfwn1O/5LYnP2G0TaUiDjIfHcOgVzewvjCC3h+5Vpd0f2o1M1gbdT6FquLwdFLqdOL/q811MuU7NQpnxbG63pPPz3+JnAtjcZaVk3h7DtO/uJaHR31L2OtZ1J82AlNYKNknWYi1FGHZ0/ZlfroGTq8tJTyWdQYDT95G3WnDCPyrgI1PDMJWqJL5ZigzHviMZSuS6Htvjkt10wHi38jmyfTTUB4rwjFpGM5jBlPwvImbw37l93V9obRcbxM7FW3PHhJnrOXDG0/n3tXnUuGsZeUlL3Ds5I18+8NoIi/PddnNE9uLg7k8+G+2P+5Lw0nDKbtwGOeMWcWzhScS/KNrrXv1+HYN0+ddT7U0c98Nn7Dj6cGUT4oj8hc7z8w7n6WZvTnv2UVY52scc9xmnnnyEsw/t31MXFeVS23HTmrvGUD4qzsY9/RP5Nj9sCk7GOORxm+VfXnp6fNJ/GorDhfsfjmyczA9OZQBL64j7rX11DitKMLJ9W9OJ/n9dBwuKC8iHQ7U39bSJyeW/3xyHDcGLmHJrjjClzrQ9uzR2zzdCH7eys03XcKboz4ge2gAvmoNZuHgkceuwi/ftdYAS4eD3q+mcF3DdCJPyOL109+j5BRPMusDAXAi+DR7GLs3hBD/SQV+G5YfVe/tiEk+mhIWh+2vcgmcDZwPVLVF5fJQSQpMUZGUjI+keDD47BCYaiFg4Y5DLj3q6UkK9keMGMCOSz0w7xH0/rYKuXrzIf/QPSWhRWuSWcixg9h1ljve6RDyawFaWsstq57iEzi8X4TJhDZmALvOsmIpV4j5XxHa9vQWd5u5Sv0RJhNK714Ujw3Zd8xUL/FdsBWtsvKAetRWn3QJlUtHdg4+H+bg8+E/x1xzc2Fz5KpN9GmSTnetUc3DI5ZuIHZp4+/Gs9LY0lL+XEfcn42fDZ80+kRLTccvNf2A4x3hmzaNcR6kcgmNKpcbhRDvCiH8Dn2lgYGBQc/BULnUAcMnzTF80jKGX5rTFXxiqFzqjKHo2BzDJy1j1J/mGCqXraSnqfQZio7NMXxySIz60xxD5dLAwMCgO2CoXBoYGBi0ka6jctlKeqJK3xFVLo9AT1B0NHzSMkb9aU5X8EmXULlsK66ygLet9JTF3oZPWsaoP83RyyddQuXSwMDAoDvRJVQuDQwMjhJFBdFCQ0kIEKIxE7yL5ePsDI56jFNK6RBC7FW5VIF326pyaYqNwRHsjVJlR+Tko1Xscbl0aXsRVivO4X2pC2pMDabUS6wldaiVdbC7cc++5oKJPQwOw+iBeD2Xx6Y/+hC67J+NhE6LoHC4gjOmlieHf8WOujD+vngwzs09XzXAFBmBFuSLWlaJdLNSHeeHNAESrKUNmCpqkdt3NarGtuc+7bm4XSqXikrOi27clvQjW2oiWJiRjH27N4r9wLenWg8Rv1Ujlm1oj6ldHiUqHP9nM/kgplH7JFer4efqeNZUxbA8PxqHU0H9OYmgddWYtme7VBBVPDwa0woKgZqcQPZpAVQlNWnpCEnvTwTmRav1NVIHlPU72PnpUK698SeuuPwfKWAFsAkVs2hsaaZZCvgl6lism3UytJNQfX3Y/kww1w/6g2WlsfhZSpkW/Cl2VIKUGv6sjWdnXTA/zh1L2OsrkQ7HkQs9BPppDkkn7vN9eNlvCvUBYBpWxqknraKywUacexENUiWr1p/Hw39kbOxMEpb37OS9zsxcsp8fQv9BfTFVgd1HoiRVYTE7uDR+Jcm2XGIGl1EtTbyYN5mKm/ri3JCit9n/OqboKLKnRBGyupb0s6xMGreJirIG6jaGoLk5+eGMWUzxnkavP2w46+r0NrdTcdbVEfLWan5bPJSFUROxe6toZoFbiQPNqjDooXU8ErKEG7dfgmfWnh6d+EP19SHtnmQWH/sc5U4Tq8pjWP/uAB5dEoqot4Oi4Aj0Iua1NNwm70b52A+tqOjIBR8CHQOnxPuT5Xg3fRRmC6kR4VBvJ98vHDQn5UMCKX36Z7w3W3p00IRGDRT3r1bQ66sDjwuTiV8HjOaHcHfyx5g47sSNvBv9I8POnkmvnt0IR/XzY+uDIVw5/Hc2nBWBsjSebU/3x3vhBjzr0lG8vDjVYwbzJrzNw8dc43IibdD43GgpqZhTwLzfcfXYIQz1zGRTgzumFwLQtvTgFrmiknFLP7666AWeLTyev+cNI+yjFALLlh3wslBLfajVzBSXeeFbntWuW+qrq74fssGOI6PpH5NfgCkslOAbK7kv8xwiv8nl6BvV3RvpcMC6LVjXQcxClU3bRuB8+ldMriC3HugHTsGXcycQ8VMxcalrkA12nE2nnZWV+Ky3wATY09tCgJ62diFMoSHk3VnLZPd0His8AUtRbY9PSajZJGd/djsJb+QTnLECrYU8pLunJHND4Dw2z+vXbtnkLhM490eYLWRcGcsrkW9y+yvXE5bnei2JllDcbJSeXoOCgtMFJkq11HQSpjXmUjxUN9M700FqfSiyR6xK7AAUlV3XxrFy2ItssLuT8n/9sWxYp7dV/y5OjZgHGjPdH6qBpfr54TyzlF8q+hH2fVa7G2JdRuVyf9TgQAacvo3H008n/O0N7Z4B6ynUjk/m+WFf8EzJEHotLNfbnC6BuUqjQnNHadDbEv1R3N3JvXsU7179KjVS4/Kvb8L6+6YWs8C7GjVj4vlg4Fz+nDuiQ3Tmu2TgLD4+mqlBq3HMDnU5kbZDYYqKpP6WUkZYd/PZV+ORKTv1NqlLUB5nwd9Uhf+WKr1N0RVhtrD70kHMv+EFRlgFDxccT+Ls3Uajg0bfZJ4nmZF2AeHfZR/5glbQ5QKnGN6f42Yu58FNZ+D1Z/qRL3ABhNXK9hlRfNlvHo8VnkDv17YbFaKJBi/BFwXDUDam6m2Krmhj+3HJjJ+INgne2RPJ9nv74czI0dss3RFWK3UnDmLWcZ9yccRKqueo5H/dF2VQ33aVe8TAKYSIEkL8JoTYKoTYIoS4ten4w0KI3INSzbULU1goJQ/bGeyRRdA77u1aLtBTEGYLhdOG8eXUWXxXncC2+/odUsTO1VBsNqpiNbatiXa5pUj7Y4rpRe4tDVzls5kni0Yy9+EzMf26pt0TID2B7NuH8Z9X32WiWxGlmgdlNW5U7vZE2Ns3ytmaFqcDuENKmQyMBm7eL+/mLCnl4Kafo1sIvxchKDk+hs8GvMtj88/H9nMPX2vTShzH9Ofxme8C8PpbZ2NZsklni7oOolcET5/4KWqN684MqUFBpF4fwaKRb5DqMPPjW+Pw/l8PXnrURqQJ7toyhbfK+/PTLeOJuKqAxJs3oKW0r4fSJVQuAcTQZK5+4FsuTbmc2Nd3ohlvS9QAf/JvreVktxoSf7+RuJdXII2B/n00hHkToFYR84OLjoMLQcYNfVh8ybMEqVYu2HoBoR9tQWvHjpieRu+306k+1saiwmTU39Z22CaALqFyqXp7s/1aD6Z6plFY4oOsctGKsB/CZCLjxiR+GjqHH2vdiX5XMWZHDyL3ODce2H4O6s58vU3RBTUwENOwMrwUlQ120D4IRtuzR2+zuhYmE4FuVexMCe/QYruEyqUI9GfCkBSW1AXj84cNae/Za0ta4xPFxxt1WDkLquN55LGrMC/p2UMXbVUuVIOCOPaMdZRsDsJZXtEJFurD4fyiFRVh+9qXwT/M4LpZt+L3tWsM47TpWXE42FXuj0dGxy587lSVy0MlHVU8PKiZ1A/hkFh/WnvElpVLJGJVVOSYATjcVCxLNrVqoL+nJO1tTXJaU2wMwR+XkH9zL+SaQyfl6ik+ASORcUscySfCbEEOS8KUuRtHfsEhv9dWnxwxcDapXM4DSqWUM/c73kzlUko56AhlGfKmB2FI4TbH8EnLGPWnOV1WHph/VC43CSHWNx27H7gYmArsolHlMlYIkSylPJxYmyFvehCGFG5zDJ8cEqP+NEcXnxgqlwYGBgZtpFPF2izCKm14HNX99qeOarqzSt/+dJRPALq7ouNeDJ+0jFF/mqOXT/717Ej7S3nacKejBre7M/+GTwB+kV9kdkhBOmD4pGWM+tOcruCT9gTOVqlcSinfAt4C2qWT3ZMwfNKco/GJGhREQ1IkTrOCbeduHJkdk8ChK2E8K83pCj5pT+Dcp3JJY8C8kMYJo6NGsdkQvaPAoaHt3K+hIJ09PgP8kRAmE2qvSGR1DbvPjKM2SBD5SyVibUq7tFO6I4rNRvk5g6m5oIIZiQvxVWt4ZOtpRN4dh7bDRbNG7VW6FApCObDHKTXNqD9D+pF6uRdqaC0N5Vb6vlaBtuXoJ+N1VbncHzUxnpS7fPni+P9SLS38UDEYgNTKILb8EU/vb6qQqze7zgMgBIqnJyIylJxTg7BOLKZicwBxH5Yx8cbl3BP0F9edfDZ110a7VLAQViuZdw3lwUs/4Zltk5n9wjnsiYe1l85i5Pm3E/W46/hC8fDAMTyB4v5ulA+zo9ocTB/0OyPc/skqVu505z8vXEXQ7GU6Wtr5qMkJ5B0fiNIgafAU9D17OwEVdsr3uPPQxK95ouI8Yu87eh0z/VQu98N57BDsD5byZ+JLvFx8LKuKo8kp9kVVnXh71PHihe/x5rETcEyLdYkgodhsVJ0yiMILahkdnYHJXkne+72J+2gdFWcNJsxSwfzKJFK/7UNUQQ+XLjyIyjMHc+8ln/Gfry6kzzPbkI5cSp9KJk/TcC9wkZdqE0UXDuSR+97jGFsZNmFCk5Jfan35unwYU/1WMsgChVoxwnnksnoaOx+ysW7cSzhxsqLegwcenIb/F+sIdHfnjY+Ow2lt37PSJaQz0i4zsaXv5wz+82b63F+Oe3kFfexFoCgonh7c+vxFfDz2Le6OuwlrDw+cwmol/7qhPDtjDlvrI5jz4anEzE0nxJbH1lcG8NT4+TyZcjKhz1mJWLHStRI6CEFZokqF5kHc/D04Y8KpfKqWuQlvcupnd5LwxdYereR4MA2egqGWYvIcCmd9MoPI3xqwFlajlFdx7ZQxzLrpTb4rH4PPrp69hbklnJkeVI1tYFuDBzd/eD2xP2zB6ZTsPi+JB/vM4+n3L29X77VrJDLWBGah4rfQHceuTLSyMpzV1Tirayia3JuXR31CgcMHtb7nvzprTxrEvTd/wua6KD5/5GQin1mBc08lKQ8H8OfJs7j/xwuIuCQL8fd6lxvbRErMlRBlKWH7dDfCZmfxU/+P+aJ0BKHLnThdLDlMxC8l3Jp1Fr6KE6/+JVgLqyE1k5xzorjwil9pkCZWPzkM67L2bqzpfiS8nsOUrZdS52zS/rRaKbl0GLfd9Rm3/nIpPt+1b19/l2hxJr5Tw63Dx9H/pk3krU2CtAxk/3i232Djk0mvUSfN3PjhDfRevblntyiEoHC4iXM8dzNk9kX0/jsD6e9L2u0JvDruPSZ+chdJT29Dc2E5EY8CJ1GmUjZNfo1CzcG9+cfz4x9DSPgr3bVa34C2dQd5s0ay/Vlvfhk8l/mf9OGr/CG8G/sSNqEx9Z076PXtapwumKLRkZWD6bmhpL8SzP0XfMajnlN48oyPuXfZefR9OKPddahLtDjl2hQ2Pz6QEd67KH+ugW0vDWTQnM18f8KrXLPucu55+Hqin1jd81NmSUnk73XM2xPNx9fMomSOOztfj+CTi17mti+uIva+lWhlZXpbqSu+G0v4b+EkPBUbqQ0BbHp0EHF3rUIr3K23aZ2PlHgt3MT1n1zPz7VhXOWTwfdJ3xCu2jl98S3EvJXmulngpcT8yxpen3sWF3jls+aCWTy6+XT63l/QIc+K7oFTWK2YYqKo91axKQ38POAjVpw2i+/T+zH9hhlEX5eP7wfLXOYBMC3dwuc3TOaiubdxSsRW1o17mxDVTvyoTMouG4ka1CM2wRwVSv8ksp80M8kvhUszJuCu1JM/TkWxWfU2TTecNTXEPrWRZ5+9mBR741CWj2IhIrwUYTbrbJ2OCAEjB9D3zO2UavXUODXkKh8cuXkdUrxugVPx8KDq/NFsf2kQJ3+3jjmPzWJRST8eKhxLuROU5T5YFq1FKynVy0RdkPX1KEvW0fvlLXyyfRiZDgfjv7iT1JXRBF6ZSc6cQMSQfnqb2fmMHojj5WpuTlzCk3MvoOxib65edhWzprxH7XgX9Md+CIuF0uPqCVEbeKeiF8VOO7fF/oI9xnVfsnLsIGJeS2NXeQDjP7qLTIcbNb0bENaOecnqEjiVwcns/jSCJ556ixWnz+LdtDFc88ht5DzXh+VFMUSqZuqG1KB4uOthXpdA2GwE+1TxQdloEp/aSey9y2GaFbvdRPo9KmpggN4mdhqqnx+7ZsLNvX5j7pNnEPX8ahwZWUR+bOIkt2oqYl24ZQUUnZPE58e+wfNFx/H15RM56b27GWHLI/N0N4SpS0xjdC6KSuq1KmbhJOgeQZ83c9lYH8WEgdtQI8I65hYdUkpbb1pcQUWlO8mWSsqd4ONWR//rN3P/8/P4Y8AXrKy34fuzG84qF9bKVlWsJgdhlgpkTQ1IiZa2C/NKL24f+CvOmI55ALoDpacm8s3oN7j9p0vwnb8W2WDH1DuarJNUFtZ44ZPuestt9qIGBpB4bQr9LYLvF4xCrt5M7Af5LKhKZNaU9xDJ8Xqb2PkMT2b2sR/yy8KhaClpaPmFvL59PI+EL8QecdQKPwegizywIyeXxLuLOPnJOzl5yS0k+hbib67m54p+HLdpCvfffx0BH65xnV1CLaDtLmbX2kjclXpk395A43hw7ZBa5uwch5rjItLJQlARrxBtMuGZrlI/aSCpr4zC44MqnjvlY+6ddyVuS456w1r3x9+XE/wbMzmqTcl9RHUtn+cNw0PYkWrHSkZ0BxS7RoY9ELdCgVAEwmRiT74X/ooJaeqYpFCtacfvlQdeK4TwAtYIIX5uOjdLSvn80dzYkZNL0Oxcgt4QZAiFvSk/PeUukOm4bshsRDbYSXwlm3kDx1D1YANec0aSN07l1sELmP/QyTgKduhtYucgJWF/1/PcuYP58/YXKNScvFB4Aou2JJM3K56or1bgdGEROy01nWc2Teaise9iG1lCzTmjyB8nmBf7X74uH4a6uwzXWqQFcvMOPs0ZwdTrf+WduImo4TV8NepVbsg6GUt2WYcsadRfHlhKkK774B8OR3YOnjP7kH+FP/LaMswNJt5/4VQCvlnjUi8Wy+8bWHr9cH6KPg7FIfFOKSMpcxtOF17Pug8p8f3Sgy8GhvLRwPfIfM6PWHMpq+p68dt/RxOQ61p71IHGjSEvBbFghsJHZ7+Ol2JneuqFWP7jAzs7RtCuTSPHB8kDj6NRHvhyYDWNrdJmiwwPzp1n0DafaCmpxN6fjlBVkE6kw9Ejg+bhfCIdDli+Ea/ljZ9d6TXbmmfF+9MVfLR+Es+dGIQ2sRx1sS/hi3YTsGN5Z5raabTGJ9YFq7AtMvGwOhYhBFZHLtLRcalZWx04D5YHFkLMBh4DZNP/XwCuPvi6rpA7r6vRZp84NWQP744az0nLtMovUqKlpBKSkgqvNB7qyU9La58V6XDAv9TQaJc8sBDiZBpVLq2ATUoZeoRyDJW+gzAUHZtj+KRljPrTnC6rctkkD/wOkHJQ0IwAXgdOBM4FHjBULtuOoejYHMMnh8SoP83pmiqXHFoe+BYgGPgayABmY6hcGhgYuACGyqXOGIqOzTF80jJG/WmOoXLZBgyVvpbpzoqOhk9axqg/zekKPmnPlstWq1xKKYdLKYebcd0sNvtj+KQ5hk9axvBLc7qCT9oTOPepXAohLDSqXH7bLmuE+Eetz8DAoG0oamNSD8X1tlm2ig70S5dQuRRmC86RyWTc4sS62pPwF5a59D71AxAC1ccbGRlGQ5A7lpxynFm5yPp6vS3TDTUkmLpBvXC4K5iqNGwbspB1ddDQgLOuTm/zOh01MICiMxKoO7OCmUmLWVoRz/r3BhD07hqXyWN7SITAFB1F4YkRuE8poOyXMKLmpoLD0a6k4F1C5VIb24+TXv+DGX7bOD3oXExfRuNIz2hvsd0eNTCA/AsSqZ+4hzPiNnOaz3re230sv28egN9qM8Fvr3I53SFTdBTbHg/kq2Nfo6/ZzEa7xtRvZyDdNQKXmvB/z7W2GCru7qQ8HseHJ73OFcuv5qV3zqUm0smAi9OpXx4HJgU2p7nmi1YI5JiB2B8r5fP45+hlcmdTQgPXHHcZZRl+9Ll1NRzlxhLdk/UJs4Wc6Q6u9dlEnsPJF4mf8cY3A/n6yePx/nSFS7Y8TRHh5J4bw5BLNjEzcA7Lqvswd81YVkTG0MeniG9OeI3FY5L4uPYU/N5f7jI+MsXGkDfLxuv9PuashTMI+VuhwUNAsuS9Se9w24Yb9DaxUxFmC/nXDuapiZ9ww+zp9HljE87KShQPD6qH9CHvNHeeumous6ZfjOWn1Xqb26mo3t7kTOvPQ9d/SJy5iO+r+nGR91YGWNxYOvQjpgUez25FII9S/1H/wJkYyyODvmXUXzdi3ujBWef/xT1BK1hxYwx1G/qgbXWRLEBNmCLCSXkylJ8nPMuCqn7c8dY0In8sJWHjagpmjmVtTRj3X+zNG7GfM+fMcfh/anGZ1sS26aG83u897njnGvrO3oJWXoEpphfDrynipZwTCf8+x6UyASm9oxhy8SZe3TWR8Bf+yRLlrK7GvCWT8bNqKXD4Yi2t75H5DQ6JEGTc0p8517zGtvpwzvvhVtTQWk4duwU/BTbYYeeLffF0rDjqW+iuOVTby4uU2gh6vaUS9dxKvvn8GCqdGueHrMbh53pJQbIuiuHXCa/wTulYPvu/k4n8sRThcDZKR7jDqGnreCT6G5yA6U8fpN01xrBKrhnDe2e/wY2LLyf69c1o5RUIs4XMCyK5KGAFaQvjcGQ1W9TRo2kI9mKy32bqvgxp1uVs6B/DMM8MPskZgZpXopOF+lA1ZSRzrnmNILWWl7ZN4vUz3uOtkR8QqKpUyXou+PFmfH7e1q576N7irPNXGeKewZeDJhDsGIBpZBkWIVCEE82m6m9gJ+NWLMnWPJnglULWXf6M8d0JwGBbJsnmOtwVMyl2yfiFt9F3Xgqai3TT98RBjKkKt0wz2p49KB4eFJ8/kAev+YisBn+iP83B0cMToRxMTbiNczx380buge1sxd2d1PMtnOiRxrPLz6F34SqdLNQBISgcqTDIYscsLHw05F0uXH0tzi3efHbFi/xZ04fEt6rRyivadRvd45L/z+n8fUcCL9/yBiWaJ79V9GXif+/i9WvfIOMqJ32WmFxqAiTg8408vv0KUi+3EhJdytLUWKLDS5iatA1vxY0fa925982r6ft2iktJBXtlQp7Djasu+olFk5Lxs1Vzos9PTHDLY8KK64nKStHbxE7HWtrAX3U28saZ6LOhMUWuM9CHnRf68OXpL7HN7kfs52U4Xaj+ICUJbxYwpvB2LHskgeur6L27gu1PmvBVHLz92hkEr2//BKLugVMr3M2KB0aw875AahwWHPcHEZ2yhavirub/xn3PRxNOw/zLGr3N7DSc1dWIZRtIaPrb+nt7s+2xZEiCORVRvP/oGYTPX+YyLc29BL2zimnetxA8OQcfSy2rM6JZtTuB8FPL8P7a86hnR7szlt83MPPN63nomvn8cUIC1Q4rDllLdF0l0SaNh7MmwY4Mvc3sdLS0XYQ/v6vxgxAUXTGaj8e8wjkbrib8y504OqDu6B44AWyL1lGbEoEQArFrI5qUxM91UDrag4yzTST+ZXPJ9XnCZCLz5v58eeYsAF78+kxi/7eGo80v0J2RDgfhL6zA9HEI9SYbfUrT6PWrg/T6YPw27+EoJ0e7NdLhIHL2Jt5dfTalfa2YqyT+m/dQmeSF57NWtv8aR6+6pXqbqStqcgKn3baEUs0Tz7d8O0xyRvfJIWh8ABy7MhvXbjYFBeWv9bz3yWQePuF/1I3vr6+BOqGNGcCz17xLX4vCxJXX0eeVdNde0OzUcOTm4cjMRusfyxjvnbz34yRkyk69LdMNZ2UlpsVrCH59KX7zliHXbKEyurFaB25yvVb4/ig2Gzuu9uM6v5XM+OJqbN933FivLiqXraX3Rzl8XzyI3dfVori71gy7YrOx83wLx9jKeLp4EFFPgqOgUG+zugSKzUbahW6EmiqI+M3hMsuxWoMa4M+Q07fyW60NW5EL+0UItKGJvHjm+zxTNIH4ecUdut65NS3OvSqXycBo4GYhRHLTuVlSysFNP+3eQdTsxpnZ5LzahxuS/qJ2fL+OLr5LUzp1CMvPfpFdDQo/vDgeudZIc7oXxc+X3v3y2FoXgXtqe5LE9zycvcI4M3Ad92w5D/O2bL3N0Q1T72jEYyWc4FbONxsHIbPyOrT8IwZOKWW+lHJt0++VQMeqXB7+5nj/bzUvr59E5vlOFC+vTrltV8B0SSEBihvn/XUDAV9udpndQa1B+vtwQcRqXl16PFraLr3N6VKUDfAmQK2ibo0/WrFrrd/cn10XhfNq/HyeLB5G74/AWVPToeW3aYzzIJVLaFS53CiEeFcI4XeIa64TQqwWQqxuoO1dB+lwkHhvEe47rAiLuc3Xd0Va4xMvSz2fVgXR5yU7zsrKTraw82nLcyJq6/mlpC/uGT3jeTgcba0/ThNcv+Iywv/qud301vjEI09yxgd3snZKfOOqnA5ueLQ6A3yTyuUS4Akp5ZdCiBAaxbD2qlyGSSmbqVzuj7fwlx2VdHSPLO0R+ecO6ZPRA7H7WrH+uqHVE0K/yC/W9ASdnSM9J8JqRRuZjCWnFMeuw+cp7ik+AaP+tIRePmmXyuV+52OA76WUh53+NlT6mmMoOjbH8EnLGPWnOd1R5TJMSpnf9PEcYHMr7meo9B2EoejYHMMnh8SoP83pdiqXFwkhBtPYVc8Arm/LjQ0MDAy6K0cMnFLKv4CW+v4dvvzIwMDAoDvQ2TuH3uoiZXQ12vtvMnzS8dd3RYz60xxdfHLUuuoGBgYGrkqX2KtuYGBg0J3otMAphDhZCLFdCJEmhLi3Fd/XbY98Z2H4pDlt9UnTNYZfmn/f8Enz73ecT6SU//oPjfLBO4FYwAJsAJKPcE0YMLTpdy9gB5AMPAzc2Rl2Gz7p+j4x/GL4RA+fdFaLcySQJqVMl1LagU+Bsw53gdRzj3znYPikOW32CRh+aQnDJ83pSJ90VuCMAPZP1ZJDGww+mj3y3QDDJ81pl0/A8EtLGD5pTnt90uUnh5r2yP8PmCml3APMBuKAwUA+8IJ+1umD4ZOWMfzSHMMnzekIn3RW4MwFovb7HNl07LA07ZH/H/CRlPJLAClloZRSk1I6gTk0Ntm7I4ZPmnNUPgHDLy1h+KQ5HeWTzgqcq4A+QojeQggLcCHw7eEuONwe+f2+1to98l0RwyfNabNPwPBLSxg+aU5H+qRTxNqklA4hxHTgJxpnw96VUm45wmU9eo+84ZPmHKVPwPBLSxg+aU6H+cTYOWRgYGDQRrr85JCBgYFBV8MInAYGBgZtxAicBgYGBm3ECJwGBgYGbcQInAYGBgZtxAicBgYGBm3ECJwGBgYGbcQInAYGBgZt5P8BFpAzPX/mrHQAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 432x288 with 50 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Display a sample of the server dataset. \n",
    "digits = {}\n",
    "for i in range(10):\n",
    "    digits[i] = np.where(y_test==i)[0][:5]\n",
    "#print(digits)\n",
    "\n",
    "fig, ax = plt.subplots(10, 5, sharex='col', sharey='row')\n",
    "for i in range(10):\n",
    "    for j in range(5):\n",
    "        ax[i, j].imshow(x_test[digits[i][j]])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CKKS Encryption Scheme\n",
    "In this section, we load the weights, deserialize the weights, decrypt and apply the averaging for the CKKS encryption scheme."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_time = time.time()\n",
    "\n",
    "# load : get the context key\n",
    "key_query_file = open('key_query', \"rb\")\n",
    "context_key = pickle.load(key_query_file)\n",
    "key_query_file.close()\n",
    "\n",
    "# recreate the key.\n",
    "context = ts.context_from(context_key[\"context\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load : get the data from file\n",
    "bob_weights_file = open('bob_enc', \"rb\")\n",
    "bob_weights = pickle.load(bob_weights_file)\n",
    "bob_weights_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Recreating the CKKS encrypted tensors from their serialized counterparts.\n",
    "#context = ts.context_from(bob_weights[\"context\"])\n",
    "enc_x1 = ts.ckks_tensor_from(context, bob_weights[\"data1\"])\n",
    "enc_x2 = ts.ckks_tensor_from(context, bob_weights[\"data2\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Decrypting the 2 encrypted arrays to form the original weight array.\n",
    "# This is done since protobuf is limited up to 2GB, and the encrpyted array is ~ 2.5GB.\n",
    "denc_x1 = enc_x1.decrypt().tolist()\n",
    "denc_x2 = enc_x2.decrypt().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Since the encrypyted array was splitted to cater for the 2GB limit, we restore the array to their original shape.\n",
    "denc_x = np.dstack((denc_x1, denc_x2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load : get the data from file\n",
    "alice_weights_file = open('alice_enc', \"rb\")\n",
    "alice_weights = pickle.load(alice_weights_file)\n",
    "alice_weights_file.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Recreating the CKKS encrypted tensors from their serialized counterparts.\n",
    "#context = ts.context_from(alice_weights[\"context\"])\n",
    "enc_y1 = ts.ckks_tensor_from(context, alice_weights[\"data1\"])\n",
    "enc_y2 = ts.ckks_tensor_from(context, alice_weights[\"data2\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Decrypting the 2 encrypted arrays to form the original weight array.\n",
    "# This is done since protobuf is limited up to 2GB, and the encrpyted array is ~ 2.5GB.\n",
    "denc_y1 = enc_y1.decrypt().tolist()\n",
    "denc_y2 = enc_y2.decrypt().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Since the encrypyted array was splitted to cater for the 2GB limit, we restore the array to their original shape.\n",
    "denc_y = np.dstack((denc_y1, denc_y2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Addition of User A weights to that of User B's.\n",
    "# The masking vector is cancelled out after addition.\n",
    "result = (denc_x + denc_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- 91.81345176696777 seconds ---\n"
     ]
    }
   ],
   "source": [
    "# Averaging the weights\n",
    "result_avg = result / 2\n",
    "print(\"--- %s seconds ---\" % (time.time() - start_time))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For FHE addition, we can add the encrypted weights together from the two users \n",
    "# and obtain nearly the same result.\n",
    "result_1 = enc_x1 + enc_y1\n",
    "result_2 = enc_x2 + enc_y2\n",
    "\n",
    "# Then, we decrypt the result and stack it together since we had to split the original weights\n",
    "# to cater for the protobuf 2GB limit.\n",
    "result_1_dec = result_1.decrypt().tolist()\n",
    "result_2_dec = result_2.decrypt().tolist()\n",
    "result_dec = np.dstack((result_1_dec, result_1_dec))\n",
    "\n",
    "# Averaging the weights\n",
    "result_dec_avg = result_dec / 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading the base_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Model's architecture.\n",
    "# model_base = tf.keras.models.Sequential([\n",
    "#   tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
    "#   tf.keras.layers.Dense(128, activation='relu', kernel_initializer='glorot_uniform'),\n",
    "#   tf.keras.layers.Dropout(0.2),\n",
    "#   tf.keras.layers.Dense(10, activation='softmax', kernel_initializer='glorot_uniform')\n",
    "# ])\n",
    "\n",
    "# model_base.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten (Flatten)            (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 10)                7840      \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 10)                0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 10)                100       \n",
      "=================================================================\n",
      "Total params: 7,940\n",
      "Trainable params: 7,940\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Loading the base_model.\n",
    "new_model = tf.keras.models.load_model('saved_model/my_model')\n",
    "\n",
    "# Check its architecture\n",
    "new_model.summary()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Updating the base_model weights for Dense layer."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading updated model.\n",
    "In this section, we uload the updated model which have the averaged weights from the server."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replacing the weights with updated weights.\n",
    "new_model.layers[1].set_weights(result_avg)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compiling the model.\n",
    "new_model.compile(\n",
    "    optimizer=tf.keras.optimizers.Adam(0.001),\n",
    "    loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
    "    metrics=[tf.keras.metrics.SparseCategoricalAccuracy()],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "e:\\UDM\\M2_privacyinAI\\code_final\\venv\\lib\\site-packages\\keras\\backend.py:4906: UserWarning: \"`sparse_categorical_crossentropy` received `from_logits=True`, but the `output` argument was produced by a sigmoid or softmax activation and thus does not represent logits. Was this intended?\"\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "157/157 [==============================] - 1s 3ms/step - loss: 0.6295 - sparse_categorical_accuracy: 0.7841 - val_loss: 0.5546 - val_sparse_categorical_accuracy: 0.8505\n",
      "Epoch 2/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.5312 - sparse_categorical_accuracy: 0.8148 - val_loss: 0.5117 - val_sparse_categorical_accuracy: 0.8655\n",
      "Epoch 3/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.5006 - sparse_categorical_accuracy: 0.8238 - val_loss: 0.4997 - val_sparse_categorical_accuracy: 0.8692\n",
      "Epoch 4/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4860 - sparse_categorical_accuracy: 0.8270 - val_loss: 0.4780 - val_sparse_categorical_accuracy: 0.8723\n",
      "Epoch 5/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4698 - sparse_categorical_accuracy: 0.8328 - val_loss: 0.4663 - val_sparse_categorical_accuracy: 0.8750\n",
      "Epoch 6/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4633 - sparse_categorical_accuracy: 0.8368 - val_loss: 0.4610 - val_sparse_categorical_accuracy: 0.8808\n",
      "Epoch 7/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4561 - sparse_categorical_accuracy: 0.8393 - val_loss: 0.4580 - val_sparse_categorical_accuracy: 0.8800\n",
      "Epoch 8/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4546 - sparse_categorical_accuracy: 0.8371 - val_loss: 0.4597 - val_sparse_categorical_accuracy: 0.8790\n",
      "Epoch 9/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4483 - sparse_categorical_accuracy: 0.8411 - val_loss: 0.4575 - val_sparse_categorical_accuracy: 0.8780\n",
      "Epoch 10/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4398 - sparse_categorical_accuracy: 0.8454 - val_loss: 0.4536 - val_sparse_categorical_accuracy: 0.8775\n",
      "Epoch 11/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4350 - sparse_categorical_accuracy: 0.8468 - val_loss: 0.4547 - val_sparse_categorical_accuracy: 0.8790\n",
      "Epoch 12/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4392 - sparse_categorical_accuracy: 0.8418 - val_loss: 0.4550 - val_sparse_categorical_accuracy: 0.8795\n",
      "Epoch 13/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4383 - sparse_categorical_accuracy: 0.8452 - val_loss: 0.4522 - val_sparse_categorical_accuracy: 0.8777\n",
      "Epoch 14/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4383 - sparse_categorical_accuracy: 0.8450 - val_loss: 0.4513 - val_sparse_categorical_accuracy: 0.8823\n",
      "Epoch 15/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4383 - sparse_categorical_accuracy: 0.8431 - val_loss: 0.4529 - val_sparse_categorical_accuracy: 0.8792\n",
      "Epoch 16/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4279 - sparse_categorical_accuracy: 0.8475 - val_loss: 0.4534 - val_sparse_categorical_accuracy: 0.8798\n",
      "Epoch 17/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4298 - sparse_categorical_accuracy: 0.8474 - val_loss: 0.4461 - val_sparse_categorical_accuracy: 0.8808\n",
      "Epoch 18/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4309 - sparse_categorical_accuracy: 0.8461 - val_loss: 0.4542 - val_sparse_categorical_accuracy: 0.8808\n",
      "Epoch 19/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4290 - sparse_categorical_accuracy: 0.8443 - val_loss: 0.4541 - val_sparse_categorical_accuracy: 0.8792\n",
      "Epoch 20/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4346 - sparse_categorical_accuracy: 0.8455 - val_loss: 0.4495 - val_sparse_categorical_accuracy: 0.8817\n",
      "Epoch 21/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4213 - sparse_categorical_accuracy: 0.8485 - val_loss: 0.4552 - val_sparse_categorical_accuracy: 0.8838\n",
      "Epoch 22/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4217 - sparse_categorical_accuracy: 0.8475 - val_loss: 0.4529 - val_sparse_categorical_accuracy: 0.8802\n",
      "Epoch 23/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4271 - sparse_categorical_accuracy: 0.8482 - val_loss: 0.4588 - val_sparse_categorical_accuracy: 0.8798\n",
      "Epoch 24/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4236 - sparse_categorical_accuracy: 0.8482 - val_loss: 0.4532 - val_sparse_categorical_accuracy: 0.8820\n",
      "Epoch 25/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4264 - sparse_categorical_accuracy: 0.8500 - val_loss: 0.4553 - val_sparse_categorical_accuracy: 0.8830\n",
      "Epoch 26/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4269 - sparse_categorical_accuracy: 0.8491 - val_loss: 0.4523 - val_sparse_categorical_accuracy: 0.8840\n",
      "Epoch 27/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4264 - sparse_categorical_accuracy: 0.8486 - val_loss: 0.4575 - val_sparse_categorical_accuracy: 0.8808\n",
      "Epoch 28/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4180 - sparse_categorical_accuracy: 0.8498 - val_loss: 0.4615 - val_sparse_categorical_accuracy: 0.8813\n",
      "Epoch 29/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4197 - sparse_categorical_accuracy: 0.8493 - val_loss: 0.4572 - val_sparse_categorical_accuracy: 0.8798\n",
      "Epoch 30/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4142 - sparse_categorical_accuracy: 0.8534 - val_loss: 0.4514 - val_sparse_categorical_accuracy: 0.8820\n",
      "Epoch 31/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4198 - sparse_categorical_accuracy: 0.8487 - val_loss: 0.4604 - val_sparse_categorical_accuracy: 0.8808\n",
      "Epoch 32/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4169 - sparse_categorical_accuracy: 0.8540 - val_loss: 0.4612 - val_sparse_categorical_accuracy: 0.8805\n",
      "Epoch 33/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4192 - sparse_categorical_accuracy: 0.8504 - val_loss: 0.4697 - val_sparse_categorical_accuracy: 0.8770\n",
      "Epoch 34/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4125 - sparse_categorical_accuracy: 0.8507 - val_loss: 0.4530 - val_sparse_categorical_accuracy: 0.8855\n",
      "Epoch 35/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4136 - sparse_categorical_accuracy: 0.8516 - val_loss: 0.4563 - val_sparse_categorical_accuracy: 0.8820\n",
      "Epoch 36/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4207 - sparse_categorical_accuracy: 0.8504 - val_loss: 0.4583 - val_sparse_categorical_accuracy: 0.8813\n",
      "Epoch 37/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4174 - sparse_categorical_accuracy: 0.8518 - val_loss: 0.4627 - val_sparse_categorical_accuracy: 0.8810\n",
      "Epoch 38/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4164 - sparse_categorical_accuracy: 0.8492 - val_loss: 0.4581 - val_sparse_categorical_accuracy: 0.8842\n",
      "Epoch 39/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4147 - sparse_categorical_accuracy: 0.8522 - val_loss: 0.4634 - val_sparse_categorical_accuracy: 0.8792\n",
      "Epoch 40/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4180 - sparse_categorical_accuracy: 0.8503 - val_loss: 0.4636 - val_sparse_categorical_accuracy: 0.8827\n",
      "Epoch 41/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4253 - sparse_categorical_accuracy: 0.8490 - val_loss: 0.4637 - val_sparse_categorical_accuracy: 0.8815\n",
      "Epoch 42/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4124 - sparse_categorical_accuracy: 0.8508 - val_loss: 0.4580 - val_sparse_categorical_accuracy: 0.8863\n",
      "Epoch 43/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4129 - sparse_categorical_accuracy: 0.8515 - val_loss: 0.4622 - val_sparse_categorical_accuracy: 0.8805\n",
      "Epoch 44/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4113 - sparse_categorical_accuracy: 0.8535 - val_loss: 0.4703 - val_sparse_categorical_accuracy: 0.8815\n",
      "Epoch 45/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4100 - sparse_categorical_accuracy: 0.8533 - val_loss: 0.4611 - val_sparse_categorical_accuracy: 0.8798\n",
      "Epoch 46/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4148 - sparse_categorical_accuracy: 0.8519 - val_loss: 0.4696 - val_sparse_categorical_accuracy: 0.8805\n",
      "Epoch 47/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4138 - sparse_categorical_accuracy: 0.8512 - val_loss: 0.4741 - val_sparse_categorical_accuracy: 0.8798\n",
      "Epoch 48/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4160 - sparse_categorical_accuracy: 0.8527 - val_loss: 0.4695 - val_sparse_categorical_accuracy: 0.8790\n",
      "Epoch 49/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4120 - sparse_categorical_accuracy: 0.8547 - val_loss: 0.4635 - val_sparse_categorical_accuracy: 0.8827\n",
      "Epoch 50/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4151 - sparse_categorical_accuracy: 0.8524 - val_loss: 0.4677 - val_sparse_categorical_accuracy: 0.8808\n",
      "Epoch 51/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4141 - sparse_categorical_accuracy: 0.8525 - val_loss: 0.4667 - val_sparse_categorical_accuracy: 0.8790\n",
      "Epoch 52/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4134 - sparse_categorical_accuracy: 0.8515 - val_loss: 0.4640 - val_sparse_categorical_accuracy: 0.8823\n",
      "Epoch 53/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4104 - sparse_categorical_accuracy: 0.8510 - val_loss: 0.4631 - val_sparse_categorical_accuracy: 0.8848\n",
      "Epoch 54/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4098 - sparse_categorical_accuracy: 0.8535 - val_loss: 0.4685 - val_sparse_categorical_accuracy: 0.8802\n",
      "Epoch 55/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4073 - sparse_categorical_accuracy: 0.8527 - val_loss: 0.4680 - val_sparse_categorical_accuracy: 0.8823\n",
      "Epoch 56/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4120 - sparse_categorical_accuracy: 0.8511 - val_loss: 0.4673 - val_sparse_categorical_accuracy: 0.8830\n",
      "Epoch 57/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4056 - sparse_categorical_accuracy: 0.8546 - val_loss: 0.4717 - val_sparse_categorical_accuracy: 0.8813\n",
      "Epoch 58/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4067 - sparse_categorical_accuracy: 0.8540 - val_loss: 0.4785 - val_sparse_categorical_accuracy: 0.8813\n",
      "Epoch 59/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4196 - sparse_categorical_accuracy: 0.8505 - val_loss: 0.4789 - val_sparse_categorical_accuracy: 0.8805\n",
      "Epoch 60/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4118 - sparse_categorical_accuracy: 0.8529 - val_loss: 0.4779 - val_sparse_categorical_accuracy: 0.8790\n",
      "Epoch 61/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4096 - sparse_categorical_accuracy: 0.8523 - val_loss: 0.4774 - val_sparse_categorical_accuracy: 0.8808\n",
      "Epoch 62/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4160 - sparse_categorical_accuracy: 0.8499 - val_loss: 0.4744 - val_sparse_categorical_accuracy: 0.8830\n",
      "Epoch 63/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4145 - sparse_categorical_accuracy: 0.8540 - val_loss: 0.4770 - val_sparse_categorical_accuracy: 0.8785\n",
      "Epoch 64/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4082 - sparse_categorical_accuracy: 0.8546 - val_loss: 0.4708 - val_sparse_categorical_accuracy: 0.8785\n",
      "Epoch 65/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4121 - sparse_categorical_accuracy: 0.8515 - val_loss: 0.4741 - val_sparse_categorical_accuracy: 0.8810\n",
      "Epoch 66/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4133 - sparse_categorical_accuracy: 0.8529 - val_loss: 0.4779 - val_sparse_categorical_accuracy: 0.8820\n",
      "Epoch 67/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4078 - sparse_categorical_accuracy: 0.8551 - val_loss: 0.4779 - val_sparse_categorical_accuracy: 0.8820\n",
      "Epoch 68/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4142 - sparse_categorical_accuracy: 0.8503 - val_loss: 0.4721 - val_sparse_categorical_accuracy: 0.8798\n",
      "Epoch 69/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4125 - sparse_categorical_accuracy: 0.8503 - val_loss: 0.4756 - val_sparse_categorical_accuracy: 0.8813\n",
      "Epoch 70/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4144 - sparse_categorical_accuracy: 0.8510 - val_loss: 0.4796 - val_sparse_categorical_accuracy: 0.8792\n",
      "Epoch 71/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4039 - sparse_categorical_accuracy: 0.8531 - val_loss: 0.4817 - val_sparse_categorical_accuracy: 0.8800\n",
      "Epoch 72/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4098 - sparse_categorical_accuracy: 0.8543 - val_loss: 0.4802 - val_sparse_categorical_accuracy: 0.8773\n",
      "Epoch 73/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.3992 - sparse_categorical_accuracy: 0.8550 - val_loss: 0.4876 - val_sparse_categorical_accuracy: 0.8780\n",
      "Epoch 74/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4042 - sparse_categorical_accuracy: 0.8532 - val_loss: 0.4816 - val_sparse_categorical_accuracy: 0.8783\n",
      "Epoch 75/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4122 - sparse_categorical_accuracy: 0.8580 - val_loss: 0.4890 - val_sparse_categorical_accuracy: 0.8798\n",
      "Epoch 76/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4125 - sparse_categorical_accuracy: 0.8518 - val_loss: 0.4935 - val_sparse_categorical_accuracy: 0.8780\n",
      "Epoch 77/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4149 - sparse_categorical_accuracy: 0.8497 - val_loss: 0.4899 - val_sparse_categorical_accuracy: 0.8792\n",
      "Epoch 78/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4055 - sparse_categorical_accuracy: 0.8551 - val_loss: 0.4850 - val_sparse_categorical_accuracy: 0.8810\n",
      "Epoch 79/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4083 - sparse_categorical_accuracy: 0.8532 - val_loss: 0.4893 - val_sparse_categorical_accuracy: 0.8767\n",
      "Epoch 80/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4105 - sparse_categorical_accuracy: 0.8526 - val_loss: 0.4856 - val_sparse_categorical_accuracy: 0.8795\n",
      "Epoch 81/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4033 - sparse_categorical_accuracy: 0.8580 - val_loss: 0.4822 - val_sparse_categorical_accuracy: 0.8783\n",
      "Epoch 82/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4168 - sparse_categorical_accuracy: 0.8490 - val_loss: 0.4936 - val_sparse_categorical_accuracy: 0.8770\n",
      "Epoch 83/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4041 - sparse_categorical_accuracy: 0.8547 - val_loss: 0.4871 - val_sparse_categorical_accuracy: 0.8790\n",
      "Epoch 84/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4107 - sparse_categorical_accuracy: 0.8480 - val_loss: 0.4908 - val_sparse_categorical_accuracy: 0.8785\n",
      "Epoch 85/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.3957 - sparse_categorical_accuracy: 0.8540 - val_loss: 0.5011 - val_sparse_categorical_accuracy: 0.8770\n",
      "Epoch 86/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4040 - sparse_categorical_accuracy: 0.8563 - val_loss: 0.4872 - val_sparse_categorical_accuracy: 0.8767\n",
      "Epoch 87/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4055 - sparse_categorical_accuracy: 0.8562 - val_loss: 0.4890 - val_sparse_categorical_accuracy: 0.8798\n",
      "Epoch 88/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4053 - sparse_categorical_accuracy: 0.8539 - val_loss: 0.4889 - val_sparse_categorical_accuracy: 0.8785\n",
      "Epoch 89/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4111 - sparse_categorical_accuracy: 0.8526 - val_loss: 0.4940 - val_sparse_categorical_accuracy: 0.8805\n",
      "Epoch 90/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4110 - sparse_categorical_accuracy: 0.8515 - val_loss: 0.4882 - val_sparse_categorical_accuracy: 0.8817\n",
      "Epoch 91/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4063 - sparse_categorical_accuracy: 0.8542 - val_loss: 0.4896 - val_sparse_categorical_accuracy: 0.8783\n",
      "Epoch 92/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4076 - sparse_categorical_accuracy: 0.8521 - val_loss: 0.4950 - val_sparse_categorical_accuracy: 0.8802\n",
      "Epoch 93/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4052 - sparse_categorical_accuracy: 0.8543 - val_loss: 0.4912 - val_sparse_categorical_accuracy: 0.8763\n",
      "Epoch 94/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4041 - sparse_categorical_accuracy: 0.8526 - val_loss: 0.5002 - val_sparse_categorical_accuracy: 0.8783\n",
      "Epoch 95/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4001 - sparse_categorical_accuracy: 0.8538 - val_loss: 0.4887 - val_sparse_categorical_accuracy: 0.8805\n",
      "Epoch 96/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.3997 - sparse_categorical_accuracy: 0.8554 - val_loss: 0.4977 - val_sparse_categorical_accuracy: 0.8775\n",
      "Epoch 97/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4081 - sparse_categorical_accuracy: 0.8559 - val_loss: 0.4989 - val_sparse_categorical_accuracy: 0.8792\n",
      "Epoch 98/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.3972 - sparse_categorical_accuracy: 0.8584 - val_loss: 0.4966 - val_sparse_categorical_accuracy: 0.8765\n",
      "Epoch 99/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4101 - sparse_categorical_accuracy: 0.8534 - val_loss: 0.5022 - val_sparse_categorical_accuracy: 0.8755\n",
      "Epoch 100/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4003 - sparse_categorical_accuracy: 0.8557 - val_loss: 0.5053 - val_sparse_categorical_accuracy: 0.8748\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2e3c4e89460>"
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Training the updated_model.\n",
    "new_model.fit(x_train,\n",
    "          y_train,\n",
    "          batch_size=128,\n",
    "          epochs=100,\n",
    "          verbose=1,\n",
    "          validation_data=(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: saved_model/updated_model\\assets\n"
     ]
    }
   ],
   "source": [
    "# Saving the updated_model\n",
    "new_model.save('saved_model/updated_model')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predictions on the base_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAANiklEQVR4nO3df4wc9XnH8c8n/kV8QGtDcF3j4ISQqE4aSHWBRNDKESUFImSiJBRLtVyJ5lALElRRW0QVBalVSlEIok0aySluHESgaQBhJTSNa6W1UKljg4yxgdaEmsau8QFOaxPAP/DTP24cHXD7vWNndmft5/2SVrs7z87Oo/F9PLMzO/t1RAjA8e9tbTcAoD8IO5AEYQeSIOxAEoQdSGJ6Pxc207PiBA31c5FAKq/qZzoYBzxRrVbYbV8s6XZJ0yT9bUTcXHr9CRrSeb6wziIBFGyIdR1rXe/G254m6auSLpG0WNIy24u7fT8AvVXnM/u5kp6OiGci4qCkeyQtbaYtAE2rE/YFkn4y7vnOatrr2B6xvcn2pkM6UGNxAOro+dH4iFgZEcMRMTxDs3q9OAAd1An7LkkLxz0/vZoGYADVCftGSWfZfpftmZKulLSmmbYANK3rU28Rcdj2tZL+SWOn3lZFxLbGOgPQqFrn2SPiQUkPNtQLgB7i67JAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJGoN2Wx7h6T9kl6TdDgihptoCkDzaoW98rGIeKGB9wHQQ+zGA0nUDXtI+oHtR2yPTPQC2yO2N9nedEgHai4OQLfq7sZfEBG7bJ8maa3tpyJi/fgXRMRKSSsl6WTPjZrLA9ClWlv2iNhV3Y9Kul/SuU00BaB5XYfd9pDtk44+lvRxSVubagxAs+rsxs+TdL/to+/zrYj4fiNdAWhc12GPiGcknd1gLwB6iFNvQBKEHUiCsANJEHYgCcIOJNHEhTApvPjZj3asvXP508V5nxqdV6wfPDCjWF9wd7k+e+dLHWtHNj9RnBd5sGUHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQ4zz5Ff/xH3+pY+9TQT8szn1lz4UvK5R2HX+5Yu/35j9Vc+LHrR6NndKwN3foLxXmnr3uk6XZax5YdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JwRP8GaTnZc+M8X9i35TXpZ58+r2PthQ+W/8+c82R5Hf/0V1ysz/zg/xbrt3zgvo61i97+SnHe7718YrH+idmdr5Wv65U4WKxvODBUrC854VDXy37P964u1t87srHr927ThlinfbF3wj8otuxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATXs0/R0Hc2FGr13vvkerPrr39pScfan5+/qLzsfy3/5v0tS97TRUdTM/2VI8X60Jbdxfop6+8t1n91Zuff25+9o/xb/MejSbfstlfZHrW9ddy0ubbX2t5e3c/pbZsA6prKbvw3JF38hmk3SFoXEWdJWlc9BzDAJg17RKyXtPcNk5dKWl09Xi3p8mbbAtC0bj+zz4uIox+onpPUcTAz2yOSRiTpBM3ucnEA6qp9ND7GrqTpeKVHRKyMiOGIGJ6hWXUXB6BL3YZ9j+35klTdjzbXEoBe6DbsayStqB6vkPRAM+0A6JVJP7Pbvltjv1x+qu2dkr4g6WZJ37Z9laRnJV3RyyZRdvi5PR1rQ/d2rknSa5O899B3Xuyio2bs+b2PFuvvn1n+8/3S3vd1rC36u2eK8x4uVo9Nk4Y9IpZ1KB2bv0IBJMXXZYEkCDuQBGEHkiDsQBKEHUiCS1zRmulnLCzWv3LjV4r1GZ5WrP/D7b/ZsXbK7oeL8x6P2LIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKcZ0drnvrDBcX6h2eVh7LedrA8HPXcJ15+yz0dz9iyA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASnGdHTx34xIc71h799G2TzF0eQej3r7uuWH/7v/1okvfPhS07kARhB5Ig7EAShB1IgrADSRB2IAnCDiTBeXb01H9f0nl7cqLL59GX/ddFxfrs7z9WrEexms+kW3bbq2yP2t46btpNtnfZ3lzdLu1tmwDqmspu/DckXTzB9Nsi4pzq9mCzbQFo2qRhj4j1kvb2oRcAPVTnAN21trdUu/lzOr3I9ojtTbY3HdKBGosDUEe3Yf+apDMlnSNpt6RbO70wIlZGxHBEDM+Y5MIGAL3TVdgjYk9EvBYRRyR9XdK5zbYFoGldhd32/HFPPylpa6fXAhgMk55nt323pCWSTrW9U9IXJC2xfY7GTmXukHR171rEIHvbSScV68t//aGOtX1HXi3OO/rFdxfrsw5sLNbxepOGPSKWTTD5jh70AqCH+LoskARhB5Ig7EAShB1IgrADSXCJK2rZftP7i/Xvnvo3HWtLt3+qOO+sBzm11iS27EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBOfZUfR/v/ORYn3Lb/9Vsf7jw4c61l76y9OL887S7mIdbw1bdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgvPsyU1f8MvF+vWf//tifZbLf0JXPra8Y+0d/8j16v3Elh1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkuA8+3HO08v/xGd/d2ex/pkTXyzW79p/WrE+7/OdtydHinOiaZNu2W0vtP1D20/Y3mb7umr6XNtrbW+v7uf0vl0A3ZrKbvxhSZ+LiMWSPiLpGtuLJd0gaV1EnCVpXfUcwICaNOwRsTsiHq0e75f0pKQFkpZKWl29bLWky3vUI4AGvKXP7LYXSfqQpA2S5kXE0R8Je07SvA7zjEgakaQTNLvrRgHUM+Wj8bZPlHSvpOsjYt/4WkSEpJhovohYGRHDETE8Q7NqNQuge1MKu+0ZGgv6XRFxXzV5j+35VX2+pNHetAigCZPuxtu2pDskPRkRXx5XWiNphaSbq/sHetIh6jn7fcXyn512Z623/+oXP1Os/+JjD9d6fzRnKp/Zz5e0XNLjtjdX027UWMi/bfsqSc9KuqInHQJoxKRhj4iHJLlD+cJm2wHQK3xdFkiCsANJEHYgCcIOJEHYgSS4xPU4MG3xezvWRu6p9/WHxauuKdYX3fnvtd4f/cOWHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeS4Dz7ceCpP+j8w76Xzd7XsTYVp//LwfILYsIfKMIAYssOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0lwnv0Y8Opl5xbr6y67tVBlyC2MYcsOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0lMZXz2hZK+KWmepJC0MiJut32TpM9Ker566Y0R8WCvGs3sf86fVqy/c3r359Lv2n9asT5jX/l6dq5mP3ZM5Us1hyV9LiIetX2SpEdsr61qt0XEl3rXHoCmTGV89t2SdleP99t+UtKCXjcGoFlv6TO77UWSPiRpQzXpWttbbK+yPeFvI9kesb3J9qZDOlCvWwBdm3LYbZ8o6V5J10fEPklfk3SmpHM0tuWf8AvaEbEyIoYjYniGZtXvGEBXphR22zM0FvS7IuI+SYqIPRHxWkQckfR1SeWrNQC0atKw27akOyQ9GRFfHjd9/riXfVLS1ubbA9CUqRyNP1/SckmP295cTbtR0jLb52js7MsOSVf3oD/U9BcvLi7WH/6tRcV67H68wW7QpqkcjX9IkicocU4dOIbwDTogCcIOJEHYgSQIO5AEYQeSIOxAEo4+Drl7sufGeb6wb8sDstkQ67Qv9k50qpwtO5AFYQeSIOxAEoQdSIKwA0kQdiAJwg4k0dfz7Lafl/TsuEmnSnqhbw28NYPa26D2JdFbt5rs7YyIeMdEhb6G/U0LtzdFxHBrDRQMam+D2pdEb93qV2/sxgNJEHYgibbDvrLl5ZcMam+D2pdEb93qS2+tfmYH0D9tb9kB9AlhB5JoJey2L7b9H7aftn1DGz10YnuH7cdtb7a9qeVeVtketb113LS5ttfa3l7dTzjGXku93WR7V7XuNtu+tKXeFtr+oe0nbG+zfV01vdV1V+irL+ut75/ZbU+T9J+SLpK0U9JGScsi4om+NtKB7R2ShiOi9S9g2P4NSS9J+mZEfKCadoukvRFxc/Uf5ZyI+JMB6e0mSS+1PYx3NVrR/PHDjEu6XNLvqsV1V+jrCvVhvbWxZT9X0tMR8UxEHJR0j6SlLfQx8CJivaS9b5i8VNLq6vFqjf2x9F2H3gZCROyOiEerx/slHR1mvNV1V+irL9oI+wJJPxn3fKcGa7z3kPQD24/YHmm7mQnMi4jd1ePnJM1rs5kJTDqMdz+9YZjxgVl33Qx/XhcH6N7sgoj4NUmXSLqm2l0dSDH2GWyQzp1OaRjvfplgmPGfa3PddTv8eV1thH2XpIXjnp9eTRsIEbGruh+VdL8GbyjqPUdH0K3uR1vu5+cGaRjviYYZ1wCsuzaHP28j7BslnWX7XbZnSrpS0poW+ngT20PVgRPZHpL0cQ3eUNRrJK2oHq+Q9ECLvbzOoAzj3WmYcbW87lof/jwi+n6TdKnGjsj/WNKfttFDh77eLemx6rat7d4k3a2x3bpDGju2cZWkUyStk7Rd0j9LmjtAvd0p6XFJWzQWrPkt9XaBxnbRt0jaXN0ubXvdFfrqy3rj67JAEhygA5Ig7EAShB1IgrADSRB2IAnCDiRB2IEk/h9BCfQTovZf9wAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "predictions = new_model.predict(x_test)\n",
    "print(np.argmax(predictions[0]))\n",
    "plt.imshow(x_test[0])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Accuracy of the base_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- 162.85454487800598 seconds ---\n"
     ]
    }
   ],
   "source": [
    "# Creating a multiclass confusion matrix before training.\n",
    "from sklearn.metrics import confusion_matrix\n",
    "confusion = confusion_matrix(y_test, np.argmax(predictions,axis=1))\n",
    "confusion\n",
    "print(\"--- %s seconds ---\" % (time.time() - start_time))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Gentry's Encryption Scheme\n",
    "In this section, we load the weights, deserialize the weights, decrypt and apply the averaging for the Gentry's encryption scheme."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [],
   "source": [
    "start_time = time.time()\n",
    "# Initial Parameters\n",
    "r = 3\n",
    "p = 29\n",
    "q = 22"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load : get the data from file\n",
    "bob_weights_file = open('bob_enc_gentry', \"rb\")\n",
    "bob_weights_dict = pickle.load(bob_weights_file)\n",
    "bob_weights_file.close()\n",
    "\n",
    "# Load Bob's weights\n",
    "bob_weights = bob_weights_dict[\"data1\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load : get the data from file\n",
    "alice_weights_file = open('Alice_enc_gentry', \"rb\")\n",
    "alice_weights_dict = pickle.load(alice_weights_file)\n",
    "alice_weights_file.close()\n",
    "\n",
    "# Load Alice's weights\n",
    "alice_weights = alice_weights_dict[\"data1\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Adding both encrypted mask w/ weights.\n",
    "total = np.add(bob_weights, alice_weights)\n",
    "\n",
    "total_decrypt = [((z % p) % 2) for z in total]\n",
    "\n",
    "# Check shape of weights first.\n",
    "for x in total_decrypt:\n",
    "    for y in range(784):\n",
    "        for z in range(10):\n",
    "            #print(total_decrypt[0][y][z])\n",
    "            if total_decrypt[0][y][z] > 1:\n",
    "                total_decrypt[0][y][z] = total_decrypt[0][y][z] - 2\n",
    "\n",
    "\n",
    "#print(\"--- %s seconds ---\" % (time.time() - start_time))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Average the weights.\n",
    "total_decrypt = np.divide(total_decrypt, 2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loading the base_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Model's architecture.\n",
    "# model_base = tf.keras.models.Sequential([\n",
    "#   tf.keras.layers.Flatten(input_shape=(28, 28)),\n",
    "#   tf.keras.layers.Dense(128, activation='relu', kernel_initializer='glorot_uniform'),\n",
    "#   tf.keras.layers.Dropout(0.2),\n",
    "#   tf.keras.layers.Dense(10, activation='softmax', kernel_initializer='glorot_uniform')\n",
    "# ])\n",
    "\n",
    "# model_base.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model: \"sequential\"\n",
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "flatten (Flatten)            (None, 784)               0         \n",
      "_________________________________________________________________\n",
      "dense (Dense)                (None, 10)                7840      \n",
      "_________________________________________________________________\n",
      "dropout (Dropout)            (None, 10)                0         \n",
      "_________________________________________________________________\n",
      "dense_1 (Dense)              (None, 10)                100       \n",
      "=================================================================\n",
      "Total params: 7,940\n",
      "Trainable params: 7,940\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "# Loading the base_model.\n",
    "new_model = tf.keras.models.load_model('saved_model/my_model')\n",
    "\n",
    "# Check its architecture\n",
    "new_model.summary()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Updating the base_model weights for Dense layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Replacing the weights with updated weights.\n",
    "new_model.layers[1].set_weights(total_decrypt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compiling the model.\n",
    "new_model.compile(\n",
    "    optimizer=tf.keras.optimizers.Adam(0.001),\n",
    "    loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),\n",
    "    metrics=[tf.keras.metrics.SparseCategoricalAccuracy()],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100\n",
      "  1/157 [..............................] - ETA: 29s - loss: 8.2570 - sparse_categorical_accuracy: 0.3281"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "e:\\UDM\\M2_privacyinAI\\code_final\\venv\\lib\\site-packages\\keras\\backend.py:4906: UserWarning: \"`sparse_categorical_crossentropy` received `from_logits=True`, but the `output` argument was produced by a sigmoid or softmax activation and thus does not represent logits. Was this intended?\"\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "157/157 [==============================] - 1s 3ms/step - loss: 2.0705 - sparse_categorical_accuracy: 0.5454 - val_loss: 0.9839 - val_sparse_categorical_accuracy: 0.6827\n",
      "Epoch 2/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.9736 - sparse_categorical_accuracy: 0.6673 - val_loss: 0.7709 - val_sparse_categorical_accuracy: 0.7555\n",
      "Epoch 3/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.8191 - sparse_categorical_accuracy: 0.7207 - val_loss: 0.6759 - val_sparse_categorical_accuracy: 0.7945\n",
      "Epoch 4/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.7437 - sparse_categorical_accuracy: 0.7430 - val_loss: 0.6168 - val_sparse_categorical_accuracy: 0.8130\n",
      "Epoch 5/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.7056 - sparse_categorical_accuracy: 0.7621 - val_loss: 0.5690 - val_sparse_categorical_accuracy: 0.8270\n",
      "Epoch 6/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.6748 - sparse_categorical_accuracy: 0.7711 - val_loss: 0.5510 - val_sparse_categorical_accuracy: 0.8340\n",
      "Epoch 7/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.6383 - sparse_categorical_accuracy: 0.7812 - val_loss: 0.5326 - val_sparse_categorical_accuracy: 0.8438\n",
      "Epoch 8/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.6174 - sparse_categorical_accuracy: 0.7886 - val_loss: 0.5100 - val_sparse_categorical_accuracy: 0.8495\n",
      "Epoch 9/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.6046 - sparse_categorical_accuracy: 0.7915 - val_loss: 0.4918 - val_sparse_categorical_accuracy: 0.8585\n",
      "Epoch 10/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.5911 - sparse_categorical_accuracy: 0.7955 - val_loss: 0.4819 - val_sparse_categorical_accuracy: 0.8550\n",
      "Epoch 11/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.5735 - sparse_categorical_accuracy: 0.7991 - val_loss: 0.4747 - val_sparse_categorical_accuracy: 0.8595\n",
      "Epoch 12/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.5645 - sparse_categorical_accuracy: 0.8067 - val_loss: 0.4740 - val_sparse_categorical_accuracy: 0.8610\n",
      "Epoch 13/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.5530 - sparse_categorical_accuracy: 0.8096 - val_loss: 0.4604 - val_sparse_categorical_accuracy: 0.8640\n",
      "Epoch 14/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.5384 - sparse_categorical_accuracy: 0.8116 - val_loss: 0.4575 - val_sparse_categorical_accuracy: 0.8660\n",
      "Epoch 15/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.5414 - sparse_categorical_accuracy: 0.8115 - val_loss: 0.4561 - val_sparse_categorical_accuracy: 0.8650\n",
      "Epoch 16/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.5312 - sparse_categorical_accuracy: 0.8115 - val_loss: 0.4527 - val_sparse_categorical_accuracy: 0.8708\n",
      "Epoch 17/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.5263 - sparse_categorical_accuracy: 0.8177 - val_loss: 0.4422 - val_sparse_categorical_accuracy: 0.8695\n",
      "Epoch 18/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.5247 - sparse_categorical_accuracy: 0.8160 - val_loss: 0.4428 - val_sparse_categorical_accuracy: 0.8683\n",
      "Epoch 19/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.5112 - sparse_categorical_accuracy: 0.8216 - val_loss: 0.4391 - val_sparse_categorical_accuracy: 0.8698\n",
      "Epoch 20/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.5053 - sparse_categorical_accuracy: 0.8235 - val_loss: 0.4458 - val_sparse_categorical_accuracy: 0.8698\n",
      "Epoch 21/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.5075 - sparse_categorical_accuracy: 0.8237 - val_loss: 0.4424 - val_sparse_categorical_accuracy: 0.8687\n",
      "Epoch 22/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.5034 - sparse_categorical_accuracy: 0.8233 - val_loss: 0.4372 - val_sparse_categorical_accuracy: 0.8695\n",
      "Epoch 23/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4949 - sparse_categorical_accuracy: 0.8270 - val_loss: 0.4359 - val_sparse_categorical_accuracy: 0.8730\n",
      "Epoch 24/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4914 - sparse_categorical_accuracy: 0.8289 - val_loss: 0.4280 - val_sparse_categorical_accuracy: 0.8740\n",
      "Epoch 25/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4961 - sparse_categorical_accuracy: 0.8247 - val_loss: 0.4335 - val_sparse_categorical_accuracy: 0.8723\n",
      "Epoch 26/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4886 - sparse_categorical_accuracy: 0.8268 - val_loss: 0.4327 - val_sparse_categorical_accuracy: 0.8715\n",
      "Epoch 27/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4926 - sparse_categorical_accuracy: 0.8240 - val_loss: 0.4344 - val_sparse_categorical_accuracy: 0.8735\n",
      "Epoch 28/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4902 - sparse_categorical_accuracy: 0.8289 - val_loss: 0.4305 - val_sparse_categorical_accuracy: 0.8742\n",
      "Epoch 29/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4865 - sparse_categorical_accuracy: 0.8299 - val_loss: 0.4300 - val_sparse_categorical_accuracy: 0.8750\n",
      "Epoch 30/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4780 - sparse_categorical_accuracy: 0.8312 - val_loss: 0.4342 - val_sparse_categorical_accuracy: 0.8740\n",
      "Epoch 31/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4736 - sparse_categorical_accuracy: 0.8339 - val_loss: 0.4278 - val_sparse_categorical_accuracy: 0.8798\n",
      "Epoch 32/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4783 - sparse_categorical_accuracy: 0.8317 - val_loss: 0.4286 - val_sparse_categorical_accuracy: 0.8777\n",
      "Epoch 33/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4781 - sparse_categorical_accuracy: 0.8304 - val_loss: 0.4297 - val_sparse_categorical_accuracy: 0.8745\n",
      "Epoch 34/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4777 - sparse_categorical_accuracy: 0.8302 - val_loss: 0.4262 - val_sparse_categorical_accuracy: 0.8785\n",
      "Epoch 35/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4723 - sparse_categorical_accuracy: 0.8333 - val_loss: 0.4266 - val_sparse_categorical_accuracy: 0.8785\n",
      "Epoch 36/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4679 - sparse_categorical_accuracy: 0.8361 - val_loss: 0.4333 - val_sparse_categorical_accuracy: 0.8745\n",
      "Epoch 37/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4685 - sparse_categorical_accuracy: 0.8345 - val_loss: 0.4291 - val_sparse_categorical_accuracy: 0.8783\n",
      "Epoch 38/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4683 - sparse_categorical_accuracy: 0.8359 - val_loss: 0.4295 - val_sparse_categorical_accuracy: 0.8763\n",
      "Epoch 39/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4658 - sparse_categorical_accuracy: 0.8365 - val_loss: 0.4329 - val_sparse_categorical_accuracy: 0.8748\n",
      "Epoch 40/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4640 - sparse_categorical_accuracy: 0.8374 - val_loss: 0.4296 - val_sparse_categorical_accuracy: 0.8777\n",
      "Epoch 41/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4640 - sparse_categorical_accuracy: 0.8359 - val_loss: 0.4330 - val_sparse_categorical_accuracy: 0.8785\n",
      "Epoch 42/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4670 - sparse_categorical_accuracy: 0.8346 - val_loss: 0.4315 - val_sparse_categorical_accuracy: 0.8777\n",
      "Epoch 43/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4713 - sparse_categorical_accuracy: 0.8357 - val_loss: 0.4286 - val_sparse_categorical_accuracy: 0.8758\n",
      "Epoch 44/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4574 - sparse_categorical_accuracy: 0.8385 - val_loss: 0.4266 - val_sparse_categorical_accuracy: 0.8755\n",
      "Epoch 45/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4567 - sparse_categorical_accuracy: 0.8397 - val_loss: 0.4292 - val_sparse_categorical_accuracy: 0.8808\n",
      "Epoch 46/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4506 - sparse_categorical_accuracy: 0.8414 - val_loss: 0.4265 - val_sparse_categorical_accuracy: 0.8817\n",
      "Epoch 47/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4565 - sparse_categorical_accuracy: 0.8367 - val_loss: 0.4232 - val_sparse_categorical_accuracy: 0.8788\n",
      "Epoch 48/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4585 - sparse_categorical_accuracy: 0.8389 - val_loss: 0.4262 - val_sparse_categorical_accuracy: 0.8808\n",
      "Epoch 49/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4560 - sparse_categorical_accuracy: 0.8401 - val_loss: 0.4268 - val_sparse_categorical_accuracy: 0.8802\n",
      "Epoch 50/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4473 - sparse_categorical_accuracy: 0.8407 - val_loss: 0.4252 - val_sparse_categorical_accuracy: 0.8785\n",
      "Epoch 51/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4475 - sparse_categorical_accuracy: 0.8407 - val_loss: 0.4289 - val_sparse_categorical_accuracy: 0.8795\n",
      "Epoch 52/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4485 - sparse_categorical_accuracy: 0.8432 - val_loss: 0.4257 - val_sparse_categorical_accuracy: 0.8792\n",
      "Epoch 53/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4567 - sparse_categorical_accuracy: 0.8422 - val_loss: 0.4350 - val_sparse_categorical_accuracy: 0.8783\n",
      "Epoch 54/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4523 - sparse_categorical_accuracy: 0.8426 - val_loss: 0.4340 - val_sparse_categorical_accuracy: 0.8777\n",
      "Epoch 55/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4438 - sparse_categorical_accuracy: 0.8448 - val_loss: 0.4256 - val_sparse_categorical_accuracy: 0.8780\n",
      "Epoch 56/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4477 - sparse_categorical_accuracy: 0.8438 - val_loss: 0.4347 - val_sparse_categorical_accuracy: 0.8767\n",
      "Epoch 57/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4527 - sparse_categorical_accuracy: 0.8393 - val_loss: 0.4321 - val_sparse_categorical_accuracy: 0.8795\n",
      "Epoch 58/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4527 - sparse_categorical_accuracy: 0.8399 - val_loss: 0.4306 - val_sparse_categorical_accuracy: 0.8773\n",
      "Epoch 59/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4473 - sparse_categorical_accuracy: 0.8418 - val_loss: 0.4255 - val_sparse_categorical_accuracy: 0.8792\n",
      "Epoch 60/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4347 - sparse_categorical_accuracy: 0.8462 - val_loss: 0.4381 - val_sparse_categorical_accuracy: 0.8802\n",
      "Epoch 61/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4478 - sparse_categorical_accuracy: 0.8417 - val_loss: 0.4385 - val_sparse_categorical_accuracy: 0.8785\n",
      "Epoch 62/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4450 - sparse_categorical_accuracy: 0.8460 - val_loss: 0.4311 - val_sparse_categorical_accuracy: 0.8810\n",
      "Epoch 63/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4445 - sparse_categorical_accuracy: 0.8450 - val_loss: 0.4376 - val_sparse_categorical_accuracy: 0.8788\n",
      "Epoch 64/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4460 - sparse_categorical_accuracy: 0.8394 - val_loss: 0.4360 - val_sparse_categorical_accuracy: 0.8773\n",
      "Epoch 65/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4515 - sparse_categorical_accuracy: 0.8381 - val_loss: 0.4336 - val_sparse_categorical_accuracy: 0.8785\n",
      "Epoch 66/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4455 - sparse_categorical_accuracy: 0.8424 - val_loss: 0.4380 - val_sparse_categorical_accuracy: 0.8795\n",
      "Epoch 67/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4418 - sparse_categorical_accuracy: 0.8408 - val_loss: 0.4364 - val_sparse_categorical_accuracy: 0.8805\n",
      "Epoch 68/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4423 - sparse_categorical_accuracy: 0.8433 - val_loss: 0.4323 - val_sparse_categorical_accuracy: 0.8805\n",
      "Epoch 69/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4380 - sparse_categorical_accuracy: 0.8444 - val_loss: 0.4361 - val_sparse_categorical_accuracy: 0.8785\n",
      "Epoch 70/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4437 - sparse_categorical_accuracy: 0.8428 - val_loss: 0.4395 - val_sparse_categorical_accuracy: 0.8795\n",
      "Epoch 71/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4303 - sparse_categorical_accuracy: 0.8482 - val_loss: 0.4361 - val_sparse_categorical_accuracy: 0.8770\n",
      "Epoch 72/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4391 - sparse_categorical_accuracy: 0.8457 - val_loss: 0.4402 - val_sparse_categorical_accuracy: 0.8800\n",
      "Epoch 73/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4410 - sparse_categorical_accuracy: 0.8426 - val_loss: 0.4374 - val_sparse_categorical_accuracy: 0.8790\n",
      "Epoch 74/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4490 - sparse_categorical_accuracy: 0.8381 - val_loss: 0.4355 - val_sparse_categorical_accuracy: 0.8792\n",
      "Epoch 75/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4434 - sparse_categorical_accuracy: 0.8414 - val_loss: 0.4366 - val_sparse_categorical_accuracy: 0.8788\n",
      "Epoch 76/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4360 - sparse_categorical_accuracy: 0.8429 - val_loss: 0.4414 - val_sparse_categorical_accuracy: 0.8805\n",
      "Epoch 77/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4352 - sparse_categorical_accuracy: 0.8449 - val_loss: 0.4383 - val_sparse_categorical_accuracy: 0.8792\n",
      "Epoch 78/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4319 - sparse_categorical_accuracy: 0.8450 - val_loss: 0.4380 - val_sparse_categorical_accuracy: 0.8780\n",
      "Epoch 79/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4315 - sparse_categorical_accuracy: 0.8472 - val_loss: 0.4383 - val_sparse_categorical_accuracy: 0.8783\n",
      "Epoch 80/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4335 - sparse_categorical_accuracy: 0.8462 - val_loss: 0.4407 - val_sparse_categorical_accuracy: 0.8808\n",
      "Epoch 81/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4295 - sparse_categorical_accuracy: 0.8504 - val_loss: 0.4425 - val_sparse_categorical_accuracy: 0.8780\n",
      "Epoch 82/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4379 - sparse_categorical_accuracy: 0.8419 - val_loss: 0.4375 - val_sparse_categorical_accuracy: 0.8820\n",
      "Epoch 83/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4385 - sparse_categorical_accuracy: 0.8429 - val_loss: 0.4401 - val_sparse_categorical_accuracy: 0.8802\n",
      "Epoch 84/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4288 - sparse_categorical_accuracy: 0.8477 - val_loss: 0.4419 - val_sparse_categorical_accuracy: 0.8777\n",
      "Epoch 85/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4373 - sparse_categorical_accuracy: 0.8444 - val_loss: 0.4444 - val_sparse_categorical_accuracy: 0.8808\n",
      "Epoch 86/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4233 - sparse_categorical_accuracy: 0.8499 - val_loss: 0.4399 - val_sparse_categorical_accuracy: 0.8813\n",
      "Epoch 87/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4320 - sparse_categorical_accuracy: 0.8475 - val_loss: 0.4407 - val_sparse_categorical_accuracy: 0.8823\n",
      "Epoch 88/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4291 - sparse_categorical_accuracy: 0.8482 - val_loss: 0.4403 - val_sparse_categorical_accuracy: 0.8792\n",
      "Epoch 89/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4237 - sparse_categorical_accuracy: 0.8482 - val_loss: 0.4394 - val_sparse_categorical_accuracy: 0.8845\n",
      "Epoch 90/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4401 - sparse_categorical_accuracy: 0.8443 - val_loss: 0.4473 - val_sparse_categorical_accuracy: 0.8808\n",
      "Epoch 91/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4279 - sparse_categorical_accuracy: 0.8483 - val_loss: 0.4462 - val_sparse_categorical_accuracy: 0.8783\n",
      "Epoch 92/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4258 - sparse_categorical_accuracy: 0.8512 - val_loss: 0.4492 - val_sparse_categorical_accuracy: 0.8792\n",
      "Epoch 93/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4317 - sparse_categorical_accuracy: 0.8490 - val_loss: 0.4461 - val_sparse_categorical_accuracy: 0.8808\n",
      "Epoch 94/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4249 - sparse_categorical_accuracy: 0.8501 - val_loss: 0.4419 - val_sparse_categorical_accuracy: 0.8800\n",
      "Epoch 95/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4398 - sparse_categorical_accuracy: 0.8443 - val_loss: 0.4558 - val_sparse_categorical_accuracy: 0.8813\n",
      "Epoch 96/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4381 - sparse_categorical_accuracy: 0.8450 - val_loss: 0.4439 - val_sparse_categorical_accuracy: 0.8792\n",
      "Epoch 97/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4230 - sparse_categorical_accuracy: 0.8530 - val_loss: 0.4479 - val_sparse_categorical_accuracy: 0.8785\n",
      "Epoch 98/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4238 - sparse_categorical_accuracy: 0.8502 - val_loss: 0.4469 - val_sparse_categorical_accuracy: 0.8792\n",
      "Epoch 99/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4287 - sparse_categorical_accuracy: 0.8458 - val_loss: 0.4496 - val_sparse_categorical_accuracy: 0.8815\n",
      "Epoch 100/100\n",
      "157/157 [==============================] - 0s 2ms/step - loss: 0.4283 - sparse_categorical_accuracy: 0.8492 - val_loss: 0.4497 - val_sparse_categorical_accuracy: 0.8825\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x2e3c5084d60>"
      ]
     },
     "execution_count": 169,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Training the updated_model.\n",
    "new_model.fit(x_train,\n",
    "          y_train,\n",
    "          batch_size=128,\n",
    "          epochs=100,\n",
    "          verbose=1,\n",
    "          validation_data=(x_test, y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Assets written to: saved_model/updated_model\\assets\n"
     ]
    }
   ],
   "source": [
    "# Saving the updated_model\n",
    "new_model.save('saved_model/updated_model')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Predictions on the base_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 171,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "7\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAAANiklEQVR4nO3df4wc9XnH8c8n/kV8QGtDcF3j4ISQqE4aSHWBRNDKESUFImSiJBRLtVyJ5lALElRRW0QVBalVSlEIok0aySluHESgaQBhJTSNa6W1UKljg4yxgdaEmsau8QFOaxPAP/DTP24cHXD7vWNndmft5/2SVrs7z87Oo/F9PLMzO/t1RAjA8e9tbTcAoD8IO5AEYQeSIOxAEoQdSGJ6Pxc207PiBA31c5FAKq/qZzoYBzxRrVbYbV8s6XZJ0yT9bUTcXHr9CRrSeb6wziIBFGyIdR1rXe/G254m6auSLpG0WNIy24u7fT8AvVXnM/u5kp6OiGci4qCkeyQtbaYtAE2rE/YFkn4y7vnOatrr2B6xvcn2pkM6UGNxAOro+dH4iFgZEcMRMTxDs3q9OAAd1An7LkkLxz0/vZoGYADVCftGSWfZfpftmZKulLSmmbYANK3rU28Rcdj2tZL+SWOn3lZFxLbGOgPQqFrn2SPiQUkPNtQLgB7i67JAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0kQdiAJwg4kQdiBJGoN2Wx7h6T9kl6TdDgihptoCkDzaoW98rGIeKGB9wHQQ+zGA0nUDXtI+oHtR2yPTPQC2yO2N9nedEgHai4OQLfq7sZfEBG7bJ8maa3tpyJi/fgXRMRKSSsl6WTPjZrLA9ClWlv2iNhV3Y9Kul/SuU00BaB5XYfd9pDtk44+lvRxSVubagxAs+rsxs+TdL/to+/zrYj4fiNdAWhc12GPiGcknd1gLwB6iFNvQBKEHUiCsANJEHYgCcIOJNHEhTApvPjZj3asvXP508V5nxqdV6wfPDCjWF9wd7k+e+dLHWtHNj9RnBd5sGUHkiDsQBKEHUiCsANJEHYgCcIOJEHYgSQ4zz5Ff/xH3+pY+9TQT8szn1lz4UvK5R2HX+5Yu/35j9Vc+LHrR6NndKwN3foLxXmnr3uk6XZax5YdSIKwA0kQdiAJwg4kQdiBJAg7kARhB5JwRP8GaTnZc+M8X9i35TXpZ58+r2PthQ+W/8+c82R5Hf/0V1ysz/zg/xbrt3zgvo61i97+SnHe7718YrH+idmdr5Wv65U4WKxvODBUrC854VDXy37P964u1t87srHr927ThlinfbF3wj8otuxAEoQdSIKwA0kQdiAJwg4kQdiBJAg7kATXs0/R0Hc2FGr13vvkerPrr39pScfan5+/qLzsfy3/5v0tS97TRUdTM/2VI8X60Jbdxfop6+8t1n91Zuff25+9o/xb/MejSbfstlfZHrW9ddy0ubbX2t5e3c/pbZsA6prKbvw3JF38hmk3SFoXEWdJWlc9BzDAJg17RKyXtPcNk5dKWl09Xi3p8mbbAtC0bj+zz4uIox+onpPUcTAz2yOSRiTpBM3ucnEA6qp9ND7GrqTpeKVHRKyMiOGIGJ6hWXUXB6BL3YZ9j+35klTdjzbXEoBe6DbsayStqB6vkPRAM+0A6JVJP7Pbvltjv1x+qu2dkr4g6WZJ37Z9laRnJV3RyyZRdvi5PR1rQ/d2rknSa5O899B3Xuyio2bs+b2PFuvvn1n+8/3S3vd1rC36u2eK8x4uVo9Nk4Y9IpZ1KB2bv0IBJMXXZYEkCDuQBGEHkiDsQBKEHUiCS1zRmulnLCzWv3LjV4r1GZ5WrP/D7b/ZsXbK7oeL8x6P2LIDSRB2IAnCDiRB2IEkCDuQBGEHkiDsQBKcZ0drnvrDBcX6h2eVh7LedrA8HPXcJ15+yz0dz9iyA0kQdiAJwg4kQdiBJAg7kARhB5Ig7EASnGdHTx34xIc71h799G2TzF0eQej3r7uuWH/7v/1okvfPhS07kARhB5Ig7EAShB1IgrADSRB2IAnCDiTBeXb01H9f0nl7cqLL59GX/ddFxfrs7z9WrEexms+kW3bbq2yP2t46btpNtnfZ3lzdLu1tmwDqmspu/DckXTzB9Nsi4pzq9mCzbQFo2qRhj4j1kvb2oRcAPVTnAN21trdUu/lzOr3I9ojtTbY3HdKBGosDUEe3Yf+apDMlnSNpt6RbO70wIlZGxHBEDM+Y5MIGAL3TVdgjYk9EvBYRRyR9XdK5zbYFoGldhd32/HFPPylpa6fXAhgMk55nt323pCWSTrW9U9IXJC2xfY7GTmXukHR171rEIHvbSScV68t//aGOtX1HXi3OO/rFdxfrsw5sLNbxepOGPSKWTTD5jh70AqCH+LoskARhB5Ig7EAShB1IgrADSXCJK2rZftP7i/Xvnvo3HWtLt3+qOO+sBzm11iS27EAShB1IgrADSRB2IAnCDiRB2IEkCDuQBOfZUfR/v/ORYn3Lb/9Vsf7jw4c61l76y9OL887S7mIdbw1bdiAJwg4kQdiBJAg7kARhB5Ig7EAShB1IgvPsyU1f8MvF+vWf//tifZbLf0JXPra8Y+0d/8j16v3Elh1IgrADSRB2IAnCDiRB2IEkCDuQBGEHkuA8+3HO08v/xGd/d2ex/pkTXyzW79p/WrE+7/OdtydHinOiaZNu2W0vtP1D20/Y3mb7umr6XNtrbW+v7uf0vl0A3ZrKbvxhSZ+LiMWSPiLpGtuLJd0gaV1EnCVpXfUcwICaNOwRsTsiHq0e75f0pKQFkpZKWl29bLWky3vUI4AGvKXP7LYXSfqQpA2S5kXE0R8Je07SvA7zjEgakaQTNLvrRgHUM+Wj8bZPlHSvpOsjYt/4WkSEpJhovohYGRHDETE8Q7NqNQuge1MKu+0ZGgv6XRFxXzV5j+35VX2+pNHetAigCZPuxtu2pDskPRkRXx5XWiNphaSbq/sHetIh6jn7fcXyn512Z623/+oXP1Os/+JjD9d6fzRnKp/Zz5e0XNLjtjdX027UWMi/bfsqSc9KuqInHQJoxKRhj4iHJLlD+cJm2wHQK3xdFkiCsANJEHYgCcIOJEHYgSS4xPU4MG3xezvWRu6p9/WHxauuKdYX3fnvtd4f/cOWHUiCsANJEHYgCcIOJEHYgSQIO5AEYQeS4Dz7ceCpP+j8w76Xzd7XsTYVp//LwfILYsIfKMIAYssOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0lwnv0Y8Opl5xbr6y67tVBlyC2MYcsOJEHYgSQIO5AEYQeSIOxAEoQdSIKwA0lMZXz2hZK+KWmepJC0MiJut32TpM9Ker566Y0R8WCvGs3sf86fVqy/c3r359Lv2n9asT5jX/l6dq5mP3ZM5Us1hyV9LiIetX2SpEdsr61qt0XEl3rXHoCmTGV89t2SdleP99t+UtKCXjcGoFlv6TO77UWSPiRpQzXpWttbbK+yPeFvI9kesb3J9qZDOlCvWwBdm3LYbZ8o6V5J10fEPklfk3SmpHM0tuWf8AvaEbEyIoYjYniGZtXvGEBXphR22zM0FvS7IuI+SYqIPRHxWkQckfR1SeWrNQC0atKw27akOyQ9GRFfHjd9/riXfVLS1ubbA9CUqRyNP1/SckmP295cTbtR0jLb52js7MsOSVf3oD/U9BcvLi7WH/6tRcV67H68wW7QpqkcjX9IkicocU4dOIbwDTogCcIOJEHYgSQIO5AEYQeSIOxAEo4+Drl7sufGeb6wb8sDstkQ67Qv9k50qpwtO5AFYQeSIOxAEoQdSIKwA0kQdiAJwg4k0dfz7Lafl/TsuEmnSnqhbw28NYPa26D2JdFbt5rs7YyIeMdEhb6G/U0LtzdFxHBrDRQMam+D2pdEb93qV2/sxgNJEHYgibbDvrLl5ZcMam+D2pdEb93qS2+tfmYH0D9tb9kB9AlhB5JoJey2L7b9H7aftn1DGz10YnuH7cdtb7a9qeVeVtketb113LS5ttfa3l7dTzjGXku93WR7V7XuNtu+tKXeFtr+oe0nbG+zfV01vdV1V+irL+ut75/ZbU+T9J+SLpK0U9JGScsi4om+NtKB7R2ShiOi9S9g2P4NSS9J+mZEfKCadoukvRFxc/Uf5ZyI+JMB6e0mSS+1PYx3NVrR/PHDjEu6XNLvqsV1V+jrCvVhvbWxZT9X0tMR8UxEHJR0j6SlLfQx8CJivaS9b5i8VNLq6vFqjf2x9F2H3gZCROyOiEerx/slHR1mvNV1V+irL9oI+wJJPxn3fKcGa7z3kPQD24/YHmm7mQnMi4jd1ePnJM1rs5kJTDqMdz+9YZjxgVl33Qx/XhcH6N7sgoj4NUmXSLqm2l0dSDH2GWyQzp1OaRjvfplgmPGfa3PddTv8eV1thH2XpIXjnp9eTRsIEbGruh+VdL8GbyjqPUdH0K3uR1vu5+cGaRjviYYZ1wCsuzaHP28j7BslnWX7XbZnSrpS0poW+ngT20PVgRPZHpL0cQ3eUNRrJK2oHq+Q9ECLvbzOoAzj3WmYcbW87lof/jwi+n6TdKnGjsj/WNKfttFDh77eLemx6rat7d4k3a2x3bpDGju2cZWkUyStk7Rd0j9LmjtAvd0p6XFJWzQWrPkt9XaBxnbRt0jaXN0ubXvdFfrqy3rj67JAEhygA5Ig7EAShB1IgrADSRB2IAnCDiRB2IEk/h9BCfQTovZf9wAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "predictions = new_model.predict(x_test)\n",
    "print(np.argmax(predictions[0]))\n",
    "plt.imshow(x_test[0])\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Accuracy of the base_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 172,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[358,   0,   3,   1,   2,   3,   1,   0,   2,   0],\n",
       "       [  0, 440,   3,   2,   0,   1,   1,   0,   3,   0],\n",
       "       [  2,   3, 375,   2,   2,   7,   6,   5,  14,   2],\n",
       "       [  0,   4,  11, 358,   0,  17,   0,  10,   6,   2],\n",
       "       [  4,   0,   3,   0, 367,   0,   6,   3,   8,  27],\n",
       "       [  7,   0,   4,  15,   7, 302,   1,   5,  25,   6],\n",
       "       [ 12,   4,  10,   0,   6,  10, 334,   0,   2,   0],\n",
       "       [  2,   7,   7,  15,   6,   0,   0, 355,   9,  10],\n",
       "       [  5,   8,   5,   9,   4,  24,   2,   4, 318,   5],\n",
       "       [  4,   0,   0,   8,  25,   5,   3,   9,  14, 323]], dtype=int64)"
      ]
     },
     "execution_count": 172,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Creating a multiclass confusion matrix before training.\n",
    "from sklearn.metrics import confusion_matrix\n",
    "confusion = confusion_matrix(y_test, np.argmax(predictions,axis=1))\n",
    "confusion\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 173,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- 30.675703525543213 seconds ---\n"
     ]
    }
   ],
   "source": [
    "print(\"--- %s seconds ---\" % (time.time() - start_time))"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "4def2c97ab8a4ac20a870d28dfe7453688818c63ec73e61b3deb2eaed7804e7f"
  },
  "kernelspec": {
   "display_name": "Python 3.9.7 64-bit ('venv': venv)",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}