RL-CartPole_A2C_online.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Remember to install the following packages if you have not done so\n",
    "import numpy as np\n",
    "import torch\n",
    "import gym\n",
    "from torch import nn\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# helper function to convert numpy arrays to tensors\n",
    "def t(x): return torch.from_numpy(x).float()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Actor module, categorical actions only\n",
    "class Actor(nn.Module):\n",
    "    def __init__(self, state_dim, n_actions):\n",
    "        super().__init__()\n",
    "        self.model = nn.Sequential(\n",
    "            nn.Linear(state_dim, 64),\n",
    "            nn.Tanh(),\n",
    "            nn.Linear(64, 32),\n",
    "            nn.Tanh(),\n",
    "            nn.Linear(32, n_actions),\n",
    "            nn.Softmax()\n",
    "        )\n",
    "    \n",
    "    def forward(self, X):\n",
    "        return self.model(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Critic module\n",
    "class Critic(nn.Module):\n",
    "    def __init__(self, state_dim):\n",
    "        super().__init__()\n",
    "        self.model = nn.Sequential(\n",
    "            nn.Linear(state_dim, 64),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(64, 32),\n",
    "            nn.ReLU(),\n",
    "            nn.Linear(32, 1)\n",
    "        )\n",
    "    \n",
    "    def forward(self, X):\n",
    "        return self.model(X)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "env = gym.make(\"CartPole-v1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# config\n",
    "state_dim = env.observation_space.shape[0]\n",
    "n_actions = env.action_space.n\n",
    "actor = Actor(state_dim, n_actions)\n",
    "critic = Critic(state_dim)\n",
    "adam_actor = torch.optim.Adam(actor.parameters(), lr=1e-3)\n",
    "adam_critic = torch.optim.Adam(critic.parameters(), lr=1e-3)\n",
    "gamma = 0.99"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/as-wanfang/anaconda3/lib/python3.7/site-packages/torch/nn/modules/container.py:117: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n",
      "  input = module(input)\n"
     ]
    }
   ],
   "source": [
    "episode_rewards = []\n",
    "\n",
    "for i in range(500):\n",
    "    done = False\n",
    "    total_reward = 0\n",
    "    state = env.reset()\n",
    "\n",
    "\n",
    "    while not done:\n",
    "        probs = actor(t(state))\n",
    "        dist = torch.distributions.Categorical(probs=probs)\n",
    "        action = dist.sample()\n",
    "        \n",
    "        next_state, reward, done, info = env.step(action.detach().data.numpy())\n",
    "        advantage = reward + (1-done)*gamma*critic(t(next_state)) - critic(t(state))\n",
    "        \n",
    "        total_reward += reward\n",
    "        state = next_state\n",
    "\n",
    "        critic_loss = advantage.pow(2).mean()\n",
    "        adam_critic.zero_grad()\n",
    "        critic_loss.backward()\n",
    "        adam_critic.step()\n",
    "\n",
    "        actor_loss = -dist.log_prob(action)*advantage.detach()\n",
    "        adam_actor.zero_grad()\n",
    "        actor_loss.backward()\n",
    "        adam_actor.step()\n",
    "            \n",
    "    episode_rewards.append(total_reward)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjMsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+AADFEAAAgAElEQVR4nO2de9hcVXX/P1+TGAiX5IWARnIDTYoQaTSBoPhDq6lEhYIVi6KQKvzoRZ/Gtg+CrdEK7VMtrYqFn5UGa6ggaFVuxSCggqhJeSMRwkUCllwkMQm8SUhC8E1cvz/OPpOTyVzOzJwzc2ZmfZ5nnjn3s/a57HX2WmuvLTPDcRzHcQBe0mkBHMdxnOLgSsFxHMcp4UrBcRzHKeFKwXEcxynhSsFxHMcp4UrBcRzHKeFKwWkYSQdIMkkTOy1Lo0haKukDnZajUSR9VdLHMj7mn0q6u4X9F0j6TAZyzJP0ZGL+KUmvz+C4H5P0d60ep98Y2WkBnGyQtD0xOwZ4EdgT5v/EzK6vse884Coze1WOIjotYGZ/3GkZkkg6ELgEmJn1sc3slRkd6mpglaQrzWwoo2P2PN5S6BHM7OD4B6wBzkgsq6oQ8kbSSyS1/TmT1NEPnk6fvw2cDSw3s42dFqQaZrYDuAd4f6dl6SZcKfQJkg6UdLWk9ZLWSbpC0ihJhwPfAY6RtD38Dpd0iqRlkrZKekbS59NWdMFEc5mkZcBO4BWSDpN0naQNktZK+lSsLMKy48P0hcE0dUyY/4ikG8N0VZkSJq0/k/QUsDIsf6ekVZK2SPpcHbk/I+nrkr4l6XlJD8RyhfWTJN0iabOkX0r607J9b5B0k6TngfdWuQdfCOXfIOlfJY0O6+ZJelLSpyU9F47/nsS+N0r6RJh+uaQloUzPSvp+YrvXSPpRWPeQpLcn1h0p6Q5J2yT9FJhSJt8MSd+XNCTpMUln1bhcbwfuLdv/3ZIeDee+W9K0xLoNkv5S0spw/66X9NIq92GDpDcmruv14b48H8o0M7Ft1XsS+CHwzhrlcMpwpdA/fBo4AXgNMAt4M/AxM3sWeBfwy0TL4llgGPgIcBjwf4AzgAsbON8HgPOBQ4ANwPXAVuAY4CTgLOC8sO19QR6AU4FfAm9KzMeVTxqZTg/le62klwPfAP4aOALYBMyuI/e7gcXhHLcA35Y0QtII4A7gJ8ArgHnA30h6U4V9xwLfqnDszwMTie7B7wDTgUsT66cCLwVeDlwELJZ0dIXjXAL8AhgPTAD+DiLFCNwO3BzKezHwzcQxrgGeA14G/BnwofiAkg4F7gKuDcc9H/iKpGomxdcEGeL9ZwBfBf4cOJLont1a9iFxNvBW4FXAHODcKscu513AV4BxRF/+XwjnTHNPHgN+N+V5HAAz81+P/YCngblly34FvCUxfybweJieBzxZ55iXAl8P0wcABkyssu1S4G8S81OAHcCoxLIPAt8N0x8GvhGmfwn8X+CrYX4DcFwDMr0hsf4i4IeJ+RHARuADVY73mbLtRwLPAicSKalVZdt/GvhSYt/v1bh+I4HfAEcllv0e8FjiHuwCDkisvxW4OEzfCHwiTP8T8E3gmLJz/D6wGlBi2XfCdToA+C0wNbHuc8DdYXo+cFfZ8RYDl1Qpz1rgzYn5fwCuK7vWm4CTE/fx7MT6LwJfqPT8hW3fmLiutyfWvQ7YEqZr3pMw/xpgZ6ffyW769brd0wEkiejrc3Vi8WrgqBr7HAf8C9FLeCBRpfbjBk67NjE9hahS2hSJAkSt1Dji5F7gk5ImEymPbwMfl3Rs2O6xBmRKnvcVyXkz2yPpV2nlNrPdkp4JxxkLTJW0JbHtCODuSvtW4BXAKOCRxDUQsDuxzSYz25WYXx32K+cfgMuAH0gaBv6fmX0ubLvGQm2YOMZRRPdfZTKuJmo9QnSPTi0r30igmoN2iKgVmCxf6flKXOvkM7YhMb2TqEWShvL9Dk7IXO+eHAIk1zt1cKXQB5iZSdpA9BI9FRZPJmo9QPSFXc6/E9lj32Nm2yVdCsxt5LSJ6bXAdmCgrMKKeYToZf5T4F4ze1ZRNNX5wH2JfdLIlDz+evaapQg+jKqKMDApsf0IosruGWAbUcvqNTX2rZVyeD2RAnilRea5SoyXdEBCMUwG7t/vJGZbgQXAAkm/S6QclgU5J5dtPhn4H6KK1UL5nk6si1lL1NI5o0YZkjxEZP6KeYaEjyJcu6PY+4zlwVrq35NXAz/PUYaew30K/cPXgU8pciIfCfwt8LWw7tfAkZIOTmx/CLA1VL7HE5l0msLM/pfIpPRPkg5RFJE0LXYmhkr/PiJ/Qew/uLdsvhmZbgVOlHS6pFFENvbD6uzzhsT2HyMyH/2MUDlL+qgip/ZISSdIel3KazBMZBe/UtJ4RUyS9PuJzUYBCyW9VNJbiMxB+/kmJP2BpKNDC3ArUejxHuBHwEuCjCPDsd8GfDMomtuATytyeJ/AvlE5NxP5Yc5RFIDwUkknS5pOZe5gr98H4CbgXZJODdfu0nDtBtNcnyZJc0/eBHw3Rxl6DlcK/cMngUeJvspXEJld/ims+zlRBbo6RI4cBvwlcGH4Yr+a6KVvhfcROQofJ3J23kTk8Iy5l6jSv6/KPI3KZGbriaKAvkBk334Z9SupbxE5YIeIHMfvNrM9oVJ/B/AGIjPJJuBL7DVlpOGjRF/Ug0SV+RIip2vM00StiQ1ECuSDZvbLCsd5NVGL6Xmi6/PPZrY0VPynEzl0nyXyGZxjZnHr8E+IrsGvgS8D/xEf0KI4/tOIfD3rg5x/T6SoKvFtYJakI8L+DwEXhONuInIon2lmu6vs3zL17omkg4hakl+rdgxnf1S5Ne84/Yei3rnjzayRKKuszt11HQgl/QXwCjO7tO7GHUDSxcAhZvbJTsvSTbhPwXGcpjCzL3ZahlqY2RWdlqEbcfOR4ziOU8LNR47jOE4Jbyk4juM4JbrapzB+/HibOnVqp8VwHMfpKpYvX77ZzI6otK6rlcLUqVMZHMwzDNpxHKf3kLS62jo3HzmO4zglXCk4juM4JVwpOI7jOCVcKTiO4zglXCk4juM4JXJVCpKelvSwpBWSBsOywyTdpWiIxLskDYTlkvRFRUMSPpQ2+6TjOI6THe1oKfyemc00s3gYxEuBe8xsGtHQenEyrbcD08LvIqJsh47jOE4b6YT56EyiYf4I/2clll9nEUuBcZImdEA+x3EKyvLVQ5x/7TKWr642IFz1/c666n7OuvrHDe/bynm7kbyVggHfk7Rc0kVh2ctCnvs43/2RYflR7DtU4DoqjJIl6SJJg5IGN23alKPojuMUjSvvfoL7Vm3myrufaGi/y29/lBXrtrJi7ZaG923lvN1I3j2aTzGzZ8JIX3dJerzGtqqwbL9sfWZ2DXANwOzZsz2bn+P0EQvmTt/nPzUh8echo0c0vm8r5+1Ccm0pmNkz4X8j8B3gJODXsVko/G8Mm68jMT4uMJFo9CfHKST9ZFIoCrOmDHDdBXOYNWWgof0WnnE8p04bz1c/1Pi+rZy3G8lNKUg6SNIh8TTRWLEriYZ9nB82mw/cEqZvBc4PUUgnE43Fuz4v+RynVfrJpNDt9FOl3ip5mo9eBnwnGluckcANZrZE0gPANyRdAKwB3hO2v4NovNUngZ1EY8U6TmHpJ5NCr7B89RBX3v0EC+ZOdwVRha4eZGf27NnmWVIdx0nL+dcu475Vmzl12niuu2BOp8XpGJKWJ7oJ7ENXp852HMdpBG/d1ceVguM4fUPsW3Cq47mPHMdxnBKuFBzHcZwSrhQcx3GcEq4UHMdxnBKuFBzHcZwSrhQcx3GcEq4UHMfpKzxnVW28n4LjOH3D8tVDXLj4AYZ2DgN4n4UKeEvBcZy+4cq7n2Bo5zADY0Z5r+YquFJwnIwoulmi6PK1gwVzp3PqtPEsmn+iJ8SrgpuPHCcj4lTaUEyzRNHlawee5qI+rhQcJyOKnmyt6PI1gqfAzg83HzlORnRqIJe0ZqFG5SuyuckHOMoPVwqOkxGdqkTzqiCLXPHGvoFeaPUUDTcfOU5GdMpmn5dZqMjmJvcN5IePvOY4GeF2bqdb8JHXHKcN+Ner0wu4T8FxHMcp4UrBcRzHKeFKwXEcxynhSsHJhSLHuDuOUx1XCk4uFDnG3XGc6rhScFLR6Jd/vc5F3pJwnGLiSsFJRaNf/vVSKnhLoj9w5d99eD8FJxVZ924tcm9ZJzs8M2v34S0FJxVZJXuLvxyBjiSPc/KjUqvAcxR1H95ScNqKfzn2LpXurffy7j5cKThtpVmzkecVKj5uEuwN3HzktJVmzVDumM6WPBzAnRpPolHc+V2b3JWCpBGSHpR0e5g/WtIySask3STppWH56DD/ZFg/NW/ZnO7BbdPZ0s9Ktp/LnoZ2mI8WAI8Bh4b5zwKfN7MbJf0bcAHwpfA/ZGavkvTesN05bZDP6QLcNp0t/Wzq6eeypyHXloKkicA7gUVhXsBbgP8KmywGzgrTZ4Z5wvq3hu0deq/J22vl6TbqmXp6+f50i5mrU+RtPvoC8DHgt2H+cGCLme0O8+uAo8L0UcBagLB+a9h+HyRdJGlQ0uCmTZvylL1QpG3ydsvL7E34YuP3p3/JzXwk6XRgo5ktl/TmeHGFTS3Fur0LzK4BroFo5LUMRO0K0jZ5uyXk05vwxcbvT/+Sp0/hFOAPJL0DOIDIp/AFYJykkaE1MBF4Jmy/DpgErJM0EhgLPJejfF1FWpt6t7zM7iMoNn5/+pfczEdm9nEzm2hmU4H3At83s/cDPwDODpvNB24J07eGecL671s3DyDdIdxe6jhOK3Sin8IlwF9JepLIZ3BtWH4tcHhY/lfApR2QzXGcjOkWP5cT0ZYezWb2Q+CHYfqXwEkVttkFvKcd8jiOkx/lvc+7xc/lRHiPZsfpIL34FV0eueQdD7sLz33kOB2kF7+iy4Md3GndXbhScJwO0i3RYo3gSqC7caXgOB3EK1CnaLhPwXEyoBt9A90os5M/rhQcJwO6MS1EN8rcDvpdWbr5yHEyoBt9A90oczvoRed/I6ibOw3Pnj3bBgcHOy2G4zg9RDtG+ev0SIKSlpvZ7ErrvKXgOI6ToB3O/yK3RlwpOI7jtJkim+7c0ey0nX535DnFph3PZ5ETV3pLwWk7RW46O/3N8tVDXLj4AYZ2DgP9+Xy6UnDaTpGbzk5/c+XdTzC0c5iBMaP69vl0peC0He/F6xSV5AdLEU077cCVguM4TsA/WNzR7DiO4yRwpeA4Tt/jEXF7caXgOD2EV27N4Xmg9uJKwWkrXmnlS9Eqt2653z463F5cKThtpWiVVq9RtMqt2fvdbmVS5M5k7cajj5y24n0U8iXL6JkskrY1e7+9g2PncKXgtBUP+esesqiYm73f/vHQOVwpOI5TkWYr5ixaGP7x0DlcKTiOU5FmK2Y3/XQ37mh2HCdT2uXs7pbIpm7DWwqO42RKu0w/3iLJB28pOI7TlWTdIvGWR4S3FBzH6UqybpF4yyPCWwqO4/QdlVoFnez4V6RWiiuFLqNID4/jdCuVelpX69Wc9zsXj/ZWlJ7+rhS6DE8T4XQDRf94KW8VxPLesGzNfnLn/c4VbbS33HwKkg4A7gNGh/P8l5l9StLRwI3AYcDPgPPM7DeSRgPXAbOAZ4FzzOzpvOTrVrynp9MJGu2QViT7fCXZy/0RsbwP/2rrfuMz5/3OFW20tzwdzS8CbzGz7ZJGAfdL+i7wV8DnzexGSf8GXAB8KfwPmdmrJL0X+CxwTo7ydSXe09PpBI1W8kX6eEkjeyznvBkTWLJy/T5y5/3OFe2dzk0pmJkB28PsqPAz4C3AuWH5YuDviJTCmWEa4L+AqyQpHMcJZJFCwHEaJW0ln3w+i1LRpZE9WTGfO2dyW+QqKrn6FCSNkLQC2AjcBTwFbDGz3WGTdcBRYfooYC1AWL8VODxP+boR9yk4WdGI3T9taul2PJ+N+iuaSYtddJ9InuSqFMxsj5nNBCYCJwGvrrRZ+FeNdSUkXSRpUNLgpk2bshO2Syhavnyne4kr8AsXP5BZ5deO57MdiiePa9MttCX6yMy2AD8ETgbGSYrNVhOBZ8L0OmASQFg/FniuwrGuMbPZZjb7iCOOyFv0wuGDgfQu7f46XTB3OgNjRjG0czizCrYdz2c7FE8e16ZbyE0pSDpC0rgwfSAwF3gM+AFwdthsPnBLmL41zBPWf9/9CU4/0W7T4KwpAyyaf2LqCrYoJpV2KJ5Gr00voVr1rqTDau1sZvt9ySf2PYHIkTyCSPl8w8wuk3QMe0NSHwQ+YGYvhhDW/wReS9RCeK+Z/bLW+WfPnm2Dg4O1NnGcrqGoQQSxXNt27WbF2i2cOm18YZzITnNIWm5msyuuq6MU/pfIri9gMjAUpscBa8zs6OzFTY8rBcfJn/OvXcZ9qzYzc+JYDj1wVOGUltM4tZRCTfORmR1tZscAdwJnmNl4MzscOB34dvaiOo5TNGIb/sIzjq9qtlm+eoizrrqfs67+8X7mpVrrGiUPE1anzGJFMceVk9ancKKZ3RHPmNl3gTflI5LjOO2mVgWVxoZ/5d1PsGLdVlas3bKfT6TWukbJw+/SqTDvooaXp+28tlnSJ4CvEZmTPkCUisJxnB6g1bQUC+ZOZ9sLwyDt55itta6Z80DU8/j8a5dlYsrqVO/rIvX6TlLTp1DaKHI4fwo4lUgp3AdcVsvR3A7cp+A42VBUJ3c1Yj+HO72bo5ZPoW5LQdII4ONmtiBzyRzHKQRFy79Tj6J+ZfcCdZWCme2RNKsdwjiO46Sh25RYN5HW0fygpFslnSfpD+NfrpI5jtNxPDKn/0irFA4jciy/BTgj/E7PSyjH6ReKUPnVkqFIkTlFuFb9QKroIzP7YN6COE4/UoTBaGrJ0KztvlXHdaXzFuFa9QOplEJIQXEBcDxwQLzczD6Uk1yO0xcUwWFaS4bYdh9/pScHoanXb6GVCrySz6AI16ofSGs++k/g5cBpwL1E2U2fz0sox+kXipD1Nm3ntPtWbeaKOx9PZU6qlcm0GTNQHiGzRTBHFUGGctIqhVeZ2UJgh5ktBt4JvCY/sZwi0soDXMSH30lPXMlffNqxqTKH1lI0zfgpeqknc9FkKCdtj+bh8L9F0gxgAzA1F4mcwtKKScDtwdW5YdkarrjzcS4+7djCDgXZzHCV1b7u582YwMO/2sq8GRNSHyPeNkvTURHMUUWQoZy0LYVrJA0AC4nGPXgU+GxuUjmFpJXBTXzEuOpccefjDO0c5oo7H8/keHm1yho9brWv4CUr1zO0c5glK9enPsaSleszN7N1i+mu3aSNPloUJu8FjslPHKfItNJhyDsbVefi044ttRSyoNVWWbWWS3zcbS8Mp0qhXe0ruN7XcbKF0UiropsoclqRtNFHTwFLgR8B95nZo7lK5Th9xLlzJmdqNmrVJJFsuSTlio+3bdfu/ZROpUqu2odAvQ+EpFIDSq2KZq9RESvgIptT0/oUjgPmAP8H+GdJxwI/N7N35SaZ4/Qg7aigWm2VVWu5JMNT4zLEZFnJVVJqrZgdi1gBF9GXEJM2S+pI4ESiMRTeCBwOPGRmf5KveLXxLKmVKeKXkRPdlwsXP8DQzuH9snt24z0rVw5Flb/atU06sdP0vejGe1SNlrKkBrYBDwOfA/7dzHwshQLT7i+jXnpZ8uTKu59gaOcwA2NG7feFWMSv2XqUy1xUuau1nGL5H/7VVoZ2RgGWac1aRS1rFqRVCu8jaiH8OXChpJ8Q+RbuyU0ypymWrx5i2wvDzJw0rm1N0355WVolaTIoV55FNidUoxWZi/AhkRywJ24ppNk+y3tUhOtQTirzUWnjyJfwduCjwJFmdmBegqXBzUf704nBR4r4YPc7Rb8neT+nacuf9XaN0qnBglo2H0n6FjATeJIoAul8YFlmEjqZ0YkvTg83LR5Fb71l/ZyWV9ppy5/1do1SyBaimdX9ETmZR6TZtp2/WbNmmeN0gsGnn7PzFi21waefS7W83dSSoygyZinLeYuW2pRLbrfzFi1t6LhZb9ctAINWpV5N26P5EeDjkq4BkDRNko+n0EfU6s3aj3mNqvXWbTWXTVbXMm3uoXbeu0rnauV6JY9X3mM+bU/hetvF5wAK1/M4L9I6mv8DWA68IcyvA74J3J6HUE7xqNV8LrqpIg+a7a1bj3Zcy6SM7bx3lc7VyvWqF/2UhR/g8tseYcW6rWx7YZibP/LGpo7RbaRVCq80s3MkvQ/AzF6QpBzlcgpGrZc37YtddOdnI/I121u3Hu2wMSdlbKdNO3mu5LWud72q3Zd6sidDThfNP7G5Zy6u5vqoukvbee0nwFuBH5vZ6yS9Evi6mZ2Ut4C18Oij7qJTkRZpKbp85dTrlNWI8m2Xwo7Ps23Xblas3ZLqWp919Y9ZsXYLMyeN4+YPn5Lq+LGiqNZZsFF5i/oh0yy1oo/q+hRCi+DfgCXAJEnXA/cAH8tUSqcu3W67L3qm1Kzka9d9ytKv0axtv9nMqZjVvdbxsXe8uDtaUOUDNilDshyzpgywaP6JLd3TdmQxLdp7Xdd8ZGYmaQHwNuBkQMACM9tce08na7rddl/E0NXyL8Es5GvXfSo3n7Qy9kA1U0y9L+VGy1qrA1+1Y8+cOLZmxZ6UYcHc6Wx7YZhtu3azfPVQps9cXq2Gor3XaX0KS4FjzOy/8xTGqU0tG2qvNnPzJo8XMtlT9vxrl+V2T8orvFbKUi8VRPKYyWetUZ9EvUo6mbY7rQIp3+7QA0dx36rNXLj4geZ9CRVod1+FTr3TaX0KjwLTgdXADqLWgpnZCfmKVxv3KeylaPbw5AP9iw3P5zKyWBYvTZ4vXnxPBsaMyrRyqkZe4xiXHzPrZy2pCOK03QNjRvHgJ9/WtMyt+hKqHbfW9c36+uf5TtfyKaRVClMqLTez1S3K1hKuFPZStJZC8oGOE4618qKXk9eLX+k8zV7XdsnYbrJ+1l572fdKz0cybXcjHxDLVw9x+W2PgMTC048DWs/a2mg5s67E4zLt+M0eDho9koWnH5fZu92Soxmiyr/Sr85JJ0n6gaTHJD0S/BJIOkzSXZJWhf+BsFySvijpSUkPSXpdowXtRdI6oYo2rF/stJ03YwLjD3oph4weWXdksUYcbrUyjjZDfO4blq3ZR4ZWOldl4egsIlk+a8tXD+3zfJw7ZzIPfvJtDbcor7z7CVas28qKtVtKTuZWZWz03tcKVGjGmRybwlZt3F4qVztI26O5GXYDf21mryZyUH9Y0nHApcA9ZjaNKIrp0rD924Fp4XcR8KUcZesaWu0h2ylmTRlgwdzpXHHn46zatIPXTh5X90VvpKzxC5iVWeby2x/lvlWbWXjzw/vIMG/GBAbGjGpoOMhkBVA0ZZ0lWUTNXHn3E6mfj1osmDudmRPHZpoduFolX63cefQiT5Yr9lHlHaWU1tHcMGa2Hlgfpp+X9BhwFHAm8Oaw2WLgh8AlYfl1IS/HUknjJE0Ix+lbCpkwKyWNfs03UtbMI5mCGXWPsY+8yUHm01ZaRYsmyYtWy5llmvdZUwYy73HciPO9Hq30Io/Hw27Xc5WbUkgiaSrwWqLMqi+LK3ozWy/pyLDZUcDaxG7rwrJ9lIKki4haEkyenJ3TsqgUIYyzWRtyI+GH0NmyLjzj+H1s0rG8zSjlblbkjdBqlFVs8jl12nhmTRloutNdpfuWJ9VCgWvJ3Wwv8vJw27T7tUJD4yk0dQLpYOBe4B/M7NuStpjZuMT6ITMbkPTfwD+a2f1h+T3Ax8xsebVju6O5PRQtsqlXKK9MihYskJZmn4/y8jZznHgfoGTqafc1zDPKLC9nc8uO5hZOPAr4FnC9mX07LP61pAlh/QRgY1i+DpiU2H0i8Eye8jnpKHpP5LzJq8dpuQ+lm/xHScd8syagcht8M89ZuS+hExlgF8ydzsCYUQztHM7s3iWzs7bb2Zyb+Sikx7gWeMzMPpdYdSswH/hM+L8lsfwjkm4E5gBb+92fUBRaMet069dvknoduJotV7k5oJvMTuXjG8cmoFao95xVuublvoRWbPetyL1o/on75FxqlWR21oVnHM+2F4ZBasuzkadP4RTgPOBhSSvCsr8hUgbfkHQBsAZ4T1h3B/AOotHddgIfzFE2p02U20S7UUFUqqybqXDKK7XySrAI/qO0JP0JacY3zoI017xTGWDj88Zf+Mnr0mp21jyc6DVPnbdPIU/cp1B8khVh/FL3gm+imZaC+2ZaowitzvIsrNV6e498idj9Wyvd60Zlz7usLY/R7DiNUC3JXDeZR+pR/mWY5uXNuvydiLzJi0YjeDpFsrUCVBw0KNmDP6k8GmlZdrKsrhSczKn2AhThpc6aRl72rMsfh3TG0918bbulb0e5Yi/PyFruX2gltLlT5Bp95PQn/RStlEzn0e6c+Hn0ds0zYqfWseOyxBVsUUlGTMVpKMqjgpLbJKOI5s2YwIWLH+CGZWsqHrtaqpV240qhiyna4BwxvZzaoZy4rEtWrs80nDTNvY0dkDd/+JTU56933DzDYmsdu1oFW0SSlff6LS9wyOiRHDfh0IrX9fLbHuG+VZu5/LZHShlgr7jz8YrHja/PFXc+3tHQZDcfdTFFbXKndZJl5UwrggOyUfNAvaE04+EqofHBa2qRDCWt1NEqTzNHvWN3i4mlPBwX4KbBtaXpZNjyU5u2A/DUph28f85kbhpcWzUxZCciuirh0UddTBEqw0qypI0yKo/GabY8nYzqadbZW03mePnMiWNLOW+yykYaj8oWf7F6FFRzJK/lNx5YAxJ/NHvSfiGolSKRihKW7dFHPUq547KTSqKZHC3l2zXb8mnXF2al69uss3fejAk8/Kut+2VfbTRfVDW5ykle26w7WvUbyfcumSQxni4fFjX55Z/mGe/0x54rhR6ik+akBXP3jo2b9vzlSq3Zyr1dUU2Vrm9c7lq9TSu95NWyrzZTlnomIdhXCfViFFi7qVVxV3pO4nuc5hnvtFnYHc09RCejfrJwFDbioG53lEy1NM9JZ2+9r/TkdUlzr8rlqFbmNLl3kkqokXI7lal0T5O9mavd21rPeL39l68e4qyr7uesq3+c6z1ypVBgGu8A2OAAABO1SURBVH1JOx31006llFWUTKVrnIwYSZ5vxbqtHHpA1Lhu5L5Uui5p7lV5crcLFz9QsczJEd6qhaamuTfdlJCv01S6nvH1W7Jyfc17W+29rrd/+ehyeeHmow5Sz3bY6WZko7TTLJGVH6HiNU7knal0vnb1Ti0/Z60Bi+JzJFNJN9pxsFuif4pApeuZvH6NmpfK969EGlNlJphZ1/5mzZpl3cx5i5balEtut/MWLa24fvDp5+y8RUtt8Onn2ixZ/1DpGg8+/Zyd+a8/sjOvur/ite/EfUl7zuR2/vx0jlrvdq370q57BgxalXrVQ1I7SCeiDDod2VBEKl2TXkhe10wZ/PnIhrzCq7O6Px6SWlA6EQWStUkqi4e00xVRtaii5H830kwZus1kWVSyMBlWoh33x5VCn5F1ZdfKQxorg/Vbd7Fq43a2vTDcUN74rLKEVromnQzbzEpJNlOGXlCG3Uy9e9aO++PRR31G1hFKrUQcxQplw9YXogUJx27a/bOIxsgzaquZMMI8I6vq0ekINqc2s6YMlAIP8gpLdaXg1KRexdJKJRJnxnz52AOZduTBpfM1uv+0Iw9m/dZducdvN0MziquZPgzVzl0tlr5o18lJT96hw64UCkwRXuA8H8C4w9uqjdvZvP3Fhr/4445jE8Ye0NaBzRu5L8n01mlbU2kUbaW+FJXOXS2W3vsidC959wdyn0KBKYLTrxEbZjO28CwyQzabaqKevNXWNzqwTivj61Yb/rFSX4pK564VS+90J3n7uzwktcCkqWQ7HbmTJA6nGxgzqmoOnk5RKdSvXvhftfWtXvNG9k/KAJQyqAKpHOxFej6c4uAhqV1Kmi+CIrQmYhbM3Ts+bdGGh6z0hVzvq7laJtM096WZHq1p5Y7HWjh12vi6FX2aZHmOk8R9Cl1Ou5Pg1bKnJ3PwFM08UclOX892nyaJXDVq2fwbuWflwz8umDsdzFL7KNIky3OcJK4UupTk2K/tDCGs56jsVEhjHk75ZOXd8PHLbP7J/Vu5RsnEfGn2L7KidoqJm4+6lE6ZjZKO4TjNb/mIU50gvh7bXhjObMSypJmoWqK5aiw8/bj9HMSN7F/NwVzNpJW2HI5TD1cKXUraKJJy23arjsfybJzJcWo7WfHE12Hbrt25KMs4winuD1HPwdvqAEJJJQLsM11pcJ5auLPZaQRXCjmS58uY9uuv/Au13hdrcijBWi2AuJLc8Zs9TDn8oI6bJuLrkbzmWV7/0iBCTQy9mZQvLbUc4+XT9ShSMIKTDXnWLa4UcqQIL2N55ZI24Va9FkCykkwTBdMukmkA4iidpEkJKOVLqjTYei3als++rBwL5k7f5x54PiMnz7rFlUIFstLCRXgZy79Q0ybcqtSZrPy6FKF8SWL5YmUwc+JYTp02fh+TElD62l/97I6GTF+tdkRrlKz8JO5T6D3yfPe881oFss6l3ys23aKPMRDLN+2Ig9i84zdcfNqxnDtn8n5O28tve4Qdv9kDwEGjR7aUYTVPypVcUa+7033U6rzmIakVyDr2vx35ZrIMyaw1QHyRQxvjPEMbtr3I0M5hrrjzcW5YtobLb3uEbbt2A9FX88Izjmfz9hdZtXF76tDOThB/4S88/bhCX3ent/CWQhtoR0shy6/4orcIahHLPvIlYvdvrdRxCyil34iVdBHTcThOO+hIS0HSVyRtlLQysewwSXdJWhX+B8JySfqipCclPSTpdXnJ1Qna0aGr2ld8pa/+eq2KeTMmMDBmVNVY+E5nb611/vg6XHbmDE6dNp6LTzuWmRPHcsjokQztHOaPv7KM9Vt3MXPSuI4ohE5fO8epR57mo68C88qWXQrcY2bTgHvCPMDbgWnhdxHwpRzlagtZv/zNjmtQyXRVz5xVL71Dp9Mv1zp/fB3OnTO59H/zR97IVz90EgNjRvH8i3s6ajbq9LVznHrkFn1kZvdJmlq2+EzgzWF6MfBD4JKw/DqLbFlLJY2TNMHMGk86kwPNmH+yDhmrdbxa8jWTCK7V9XnTbMe9RfNPLIWjFl12x+kYZpbbD5gKrEzMbylbPxT+bwfemFh+DzC7yjEvAgaBwcmTJ1s7OG/RUptyye123qKlZmY2+PRzdt6ipTb49HNV90mzTSPUOl65fM3K1IjMWZcvD9JeF8fpN4BBq1JvF6WfQqWRQip6wM3sGuAaiBzNeQoVU/51l6YVkHVseK3jpf36rBf3nixX3HGqUs/m5auHuHDxA4VIb1EL/yp3nMZpt1L4dWwWkjQB2BiWrwMmJbabCDzTZtmq0moem7xJm99/2wvDzJw0DqCiUkuWq1bP5ivvfoKhncMMjBmV+TXIOjVFURWW4xSVdvdTuBWYH6bnA7cklp8fopBOBrZaQfwJlehUeuhWSKZcLo97r5SGO47iufi0Y/eLaorX5RG9445Yx+ksufVTkPR1IqfyeODXwKeAm4FvAJOBNcB7zOw5SQKuIopW2gl80MzqdkDoln4KRaDWF3iR+iX0Su9vxykytfopeOe1HOi2iq3b5HUcpzU8zUWb6TYTSDVzmHe0cpz+w5VCDsQ5eLbt2l2qUPOoYPOutLtNuTmO0zquFHKgNNbA2i2lCjVZwWZVmeddaRc9AZ7jONlTlH4KPUetwW2y6u2cd2ish3Q6Tv/hjuY20Mg4ye70dRwnb9zRnDGNmn/KzTy1+jmkMQm5A9hxnLxw81ETNGr+acTMk2bbIoz97DhOb+JKoQkateU3YptPs22987sJynGcZnGfQg9SpB7KjuMUD/cpFIR2+QI8lNRxnGZx81EbaZcvwENJHcdpFm8ptJF6PZ2rTTuO47QLbynkzA3L1nDFnY9z8WnHcu6cyRx64KhSyOl1F8zZp/UAVJz2r37HcdqFK4WcueLOxxnaOcwVdz7O77z8kNJAN8kIom0vDLNt127+aPak0rJfbHieh3+1lXkzJng0keM4bcPNRzlz8WnHMjBmFBefduw+A90ApYFt4jxJS1auL3VqW7JyPUM7h1mycn0mOY7cHOU4Thq8pVCBLL/Mz50zmXPnTAbgd15+CLB//qNK/Q5qLWtGXu/w5jhOGlwpVCCvCjQe5vLKu59g3owJAKWKvPw8yWX1vu7TyFutw1usUObNmMCSles7bqJyU5njdBZXChXIM/torQq8WgVdr9JPI2+54onPtW3Xblas3cLDv9rK0M7hqudoF96icZzO4kqhAq3E+df70o0r7nkzJnDWVfeDxMLTj9un8i+voOtV+s3IG59r5sSxnDpt/D6KqJPknQ7ccZzauFLImHpfunEFfv61y1ixbmtpn2TlX15B59EZLVn5xum8l6xcn+k5msE73jlOZ3GlkDFpv3TjUFQk5s2YwPnXLmPB3OmlCjF2TudFeeXrZhvHccAT4qUib+dnnMAuDl3NwuGblBmoK383OHiXrx7i8tse2cfk5jhO49RKiOcthRSk/YputtJaMHd6yY8Qd3ard65GZIbKvaPLFUHRWwhxP494uujyOk430ped15avHuKsq+7nrKt/vE+45w3L1vDay77HDcvW7LN92qyjcaW1Yu2WhjqazZoywKL5J3LqtPFcfNqxdc+VpiNaMs/SvBkTKh4zVhwXLn6gK/ItLZg7nWlHHMQho0eWQnodx8mWvmwplH9xxn0HHlyzhedf3F1KSdHoV3TST9Bo9EzyHPX8CWlaLrOmDJTyLB16wMiK282bMYGfPPUsQzuHS0qsyH6FWVMGmDDuQFZt2sGSletz97s4Tj/Sl0qhvPKOK9kxo17CmFEjGH/waC6//VFWrN0CRBVkLZt7ct3NH3ljW+RP/je73ZKV69n9W2NgzKhUPaeLgIesOk6+9LWjOdlZLLblD4wZxdDOYWZOHBttFPwDybj+Qw/cW4leftsjPPHr7ewc3sPMiWPbohSyohucy47jZI87mqsQtwa27drNovkn7tObOKkoklE823bt5r5Vm9n2wjCrn9tZcgoD7PjNnlJoaREq2XqVfjc4lx3HaS996WguEVpJD6/bwi82PM91F8zh3DmTue6COaUspYeMHsG2XbuBYLIwY+akcSCV1k878mCmHXkwG7buajmbaZZkkV21KBTdCe44vUJftxQWnnE853z5p+z+rfGPdzy6T/+AuGWwfusuVqzdwuW3PwpmrFi3lZkTx/JHJ05m9bM7OGf2JB5dv41tu3bz/Iu797PPd5Jutb/Hob07frMHgINGR49p0sfjOE4+9LVPAfaOjDb+4NGs2ridU6eN36fSOeuq+0uKAIkVa7cwc9I4Dj1gZKnDWeyDiH0NRTAddTNxZ74kIwTHHHEwn3n3CX59HadFusanIGkecCUwAlhkZp/J+5zxeAflPYBjFp5xfMWewTFFSTndSyyYO50H1wzx/It7Ssv2GGze/qJfY8fJmcK0FCSNAJ4Afh9YBzwAvM/MHq22T7vSXDjtJ9k7/OSjD+OmwbWlca4dx2mNbmkpnAQ8aWa/BJB0I3AmUFUpOL3LrCkD+4T3XvqOV3dQGsfpH4oUfXQUsDYxvy4s2wdJF0kalDS4adOmtgnnOI7TDxRJKajCsv1sW2Z2jZnNNrPZRxxxRBvEchzH6R+KpBTWAZMS8xOBZzoki+M4Tl9SJKXwADBN0tGSXgq8F7i1wzI5juP0FYVxNJvZbkkfAe4kCkn9ipk90mGxHMdx+orCKAUAM7sDuKPTcjiO4/QrRTIfOY7jOB2mMJ3XmkHSJmB1k7uPBzbX3aq38DL3B17m/qCVMk8xs4rhm12tFFpB0mC1Hn29ipe5P/Ay9wd5ldnNR47jOE4JVwqO4zhOiX5WCtd0WoAO4GXuD7zM/UEuZe5bn4LjOI6zP/3cUnAcx3HKcKXgOI7jlOhLpSBpnqRfSHpS0qWdlicrJH1F0kZJKxPLDpN0l6RV4X8gLJekL4Zr8JCk13VO8uaRNEnSDyQ9JukRSQvC8p4tt6QDJP2PpJ+HMn86LD9a0rJQ5ptCDjEkjQ7zT4b1Uzspf7NIGiHpQUm3h/meLi+ApKclPSxphaTBsCzXZ7vvlEIY4e1q4O3AccD7JB3XWaky46vAvLJllwL3mNk04J4wD1H5p4XfRcCX2iRj1uwG/trMXg2cDHw43M9eLveLwFvM7HeBmcA8SScDnwU+H8o8BFwQtr8AGDKzVwGfD9t1IwuAxxLzvV7emN8zs5mJPgn5Pttm1lc/4PXAnYn5jwMf77RcGZZvKrAyMf8LYEKYngD8Ikx/mWi40/226+YfcAvRkK59UW5gDPAzYA5R79aRYXnpOSdKMvn6MD0ybKdOy95gOSeGCvAtwO1E46/0bHkT5X4aGF+2LNdnu+9aCqQc4a2HeJmZrQcI/0eG5T13HYKZ4LXAMnq83MGUsgLYCNwFPAVsMbPdYZNkuUplDuu3Aoe3V+KW+QLwMeC3Yf5weru8MQZ8T9JySReFZbk+24XKktomUo3w1gf01HWQdDDwLeCjZrZNqlS8aNMKy7qu3Ga2B5gpaRzwHaDSINZxubq6zJJOBzaa2XJJb44XV9i0J8pbxilm9oykI4G7JD1eY9tMyt2PLYV+G+Ht15ImAIT/jWF5z1wHSaOIFML1ZvbtsLjnyw1gZluAHxL5U8ZJij/0kuUqlTmsHws8115JW+IU4A8kPQ3cSGRC+gK9W94SZvZM+N9IpPxPIudnux+VQr+N8HYrMD9MzyeyucfLzw8RCycDW+MmaTehqElwLfCYmX0usapnyy3piNBCQNKBwFwiB+wPgLPDZuVljq/F2cD3LRiduwEz+7iZTTSzqUTv6/fN7P30aHljJB0k6ZB4GngbsJK8n+1OO1I65Lx5B/AEkR32bzstT4bl+jqwHhgm+mq4gMiWeg+wKvwfFrYVURTWU8DDwOxOy99kmd9I1ER+CFgRfu/o5XIDJwAPhjKvBD4Zlh8D/A/wJPBNYHRYfkCYfzKsP6bTZWih7G8Gbu+H8oby/Tz8HonrqryfbU9z4TiO45ToR/OR4ziOUwVXCo7jOE4JVwqO4zhOCVcKjuM4TglXCo7jOE4JVwqO0ySSLpM0N4PjbM9CHsfJAg9JdZwOI2m7mR3caTkcB7yl4Dj7IOkDYayCFZK+HBLPbZf0L5J+JukeSUeEbb8q6eww/RlJj4Y89v8clk0J2z8U/ieH5UdL+qmkByRdXnb+i8PyhxTGSXCcduJKwXECkl4NnEOUhGwmsAd4P3AQ8DMzex1wL/Cpsv0OA94FHG9mJwB/H1ZdBVwXll0PfDEsvxL4kpmdCGxIHOdtRLnwTyIaJ2GWpFPzKKvjVMOVguPs5a3ALOCBkJb6rUSpBn4L3BS2+RpRao0k24BdwCJJfwjsDMtfD9wQpv8zsd8pRClJ4uUxbwu/B4nGSDiWSEk4Ttvox9TZjlMNAYvN7OP7LJQWlm23jyPOzHZLOolIibwX+AhRJs9yrMp08vz/aGZfblRwx8kKbyk4zl7uAc4OuevjsXCnEL0ncTbOc4H7kzuFsRzGmtkdwEeJTD8APyFSEhCZoeL9fly2POZO4EPheEg6KpbFcdqFtxQcJ2Bmj0r6BNFIVy8hyjb7YWAHcLyk5USjeJ1TtushwC2SDiD62v/LsPwvgK9IuhjYBHwwLF8A3CBpAdE4EPH5vxf8Gj8NgwRtBz7A3nz5jpM7HpLqOHXwkFGnn3DzkeM4jlPCWwqO4zhOCW8pOI7jOCVcKTiO4zglXCk4juM4JVwpOI7jOCVcKTiO4zgl/j/B2oCEMGOp7AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(np.arange(len(episode_rewards)), episode_rewards, s=2)\n",
    "plt.title(\"Total reward per episode (online)\")\n",
    "plt.ylabel(\"reward\")\n",
    "plt.xlabel(\"episode\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}