Implementing_ResNet.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Implementing_ResNet.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyP81e8WLivsax7qVfNxn6IG",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "534116b477d24846a6090108ae372280": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_61c5678e446d49a3a1608c60c4b2130c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_647d407aaabf4f99b28b161de14d788d",
              "IPY_MODEL_b708d613e047433d9594f20da60b1d4c",
              "IPY_MODEL_e66eb7c58ef54456871f20670af22a17"
            ]
          }
        },
        "61c5678e446d49a3a1608c60c4b2130c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "647d407aaabf4f99b28b161de14d788d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_05112ac805cf4d3a87ccc49e2c197542",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Dl Completed...: 100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_86ce0d9a10df4710a73aaa1c646e6362"
          }
        },
        "b708d613e047433d9594f20da60b1d4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6d5662d17f4f4c5197ae24dd8d6d3a27",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 4,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 4,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7d66794a843c4eed947c8dab09a76c86"
          }
        },
        "e66eb7c58ef54456871f20670af22a17": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_bfed275927af4dd49af51a33ccd039ba",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 4/4 [00:01&lt;00:00,  2.45 file/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_923e61853f3c43219446f8eb3d69e1ed"
          }
        },
        "05112ac805cf4d3a87ccc49e2c197542": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "86ce0d9a10df4710a73aaa1c646e6362": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6d5662d17f4f4c5197ae24dd8d6d3a27": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7d66794a843c4eed947c8dab09a76c86": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "bfed275927af4dd49af51a33ccd039ba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "923e61853f3c43219446f8eb3d69e1ed": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/armeh429/A-journey-for-learning-advanced-tensorflow/blob/main/Implementing_ResNet.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Cf5f_ByiM9i6"
      },
      "source": [
        "Residual Networks make use of skip connections to make deep models easier to train.\n",
        "\n",
        "###Import"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AQJp9WqdMyv4"
      },
      "source": [
        "try:\n",
        "  %tensorflow_version 2.x\n",
        "except Exception :\n",
        "  pass\n",
        "\n",
        "import tensorflow as tf\n",
        "import tensorflow_datasets as tfds\n",
        "from tensorflow.keras.layers import Layer"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xzVpRPeKOj6Q"
      },
      "source": [
        "###**Implement Model subclasses**\n",
        "\n",
        "As shown in the lectures, you will first implement the Identity Block which contains the skip connections (i.e. the add() operation below. This will also inherit the Model class and implement the __init__() and call() methods"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "d9C3Ddg4OJ1I"
      },
      "source": [
        "class IdentityBlock(tf.keras.Model):\n",
        "    def __init__(self, filters, kernel_size):\n",
        "        super(IdentityBlock, self).__init__(name='')\n",
        "\n",
        "        self.conv1 = tf.keras.layers.Conv2D(filters, kernel_size, padding='same')\n",
        "        self.bn1 = tf.keras.layers.BatchNormalization()\n",
        "\n",
        "        self.conv2 = tf.keras.layers.Conv2D(filters, kernel_size, padding='same')\n",
        "        self.bn2 = tf.keras.layers.BatchNormalization()\n",
        "\n",
        "        self.act = tf.keras.layers.Activation('relu')\n",
        "        self.add = tf.keras.layers.Add()\n",
        "    \n",
        "    def call(self, input_tensor):\n",
        "        x = self.conv1(input_tensor)\n",
        "        x = self.bn1(x)\n",
        "        x = self.act(x)\n",
        "\n",
        "        x = self.conv2(x)\n",
        "        x = self.bn2(x)\n",
        "        x = self.act(x)\n",
        "\n",
        "        x = self.add([x, input_tensor])\n",
        "        x = self.act(x)\n",
        "        return x"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CYG9OqGYSFY3"
      },
      "source": [
        "From there, you can build the rest of the ResNet model.\n",
        "\n",
        "\n",
        "*   You will call your IdentityBlock class two times below and that takes care of inserting those blocks of layers into this network\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o2rL9KVQSEd8"
      },
      "source": [
        "class ResNet(tf.keras.Model):\n",
        "    def __init__(self, num_classes):\n",
        "        super(ResNet, self).__init__()\n",
        "        self.conv = tf.keras.layers.Conv2D(64, 7, padding='same')\n",
        "        self.bn = tf.keras.layers.BatchNormalization()\n",
        "        self.act = tf.keras.layers.Activation('relu')\n",
        "        self.max_pool = tf.keras.layers.MaxPool2D((3, 3))\n",
        "\n",
        "        # Use the Identity blocks that you just defined\n",
        "        self.id1a = IdentityBlock(64, 3)\n",
        "        self.id1b = IdentityBlock(64, 3)\n",
        "\n",
        "        self.global_pool = tf.keras.layers.GlobalAveragePooling2D()\n",
        "        self.classifier = tf.keras.layers.Dense(num_classes, activation='softmax')\n",
        "\n",
        "    def call(self, inputs):\n",
        "        x = self.conv(inputs)\n",
        "        x = self.bn(x)\n",
        "        x = self.act(x)\n",
        "        x = self.max_pool(x)\n",
        "\n",
        "        # insert the identity blocks in the middle of the network\n",
        "        x = self.id1a(x)\n",
        "        x = self.id1b(x)\n",
        "\n",
        "        x = self.global_pool(x)\n",
        "        return self.classifier(x)"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5YkOMnoPSVNN"
      },
      "source": [
        "###**Training the Model**\n",
        "\n",
        "As mentioned before, inheriting the Model class allows you to make use of the other APIs that Keras provides, such as:\n",
        "\n",
        "* training\n",
        "* serialization\n",
        "* evaluation\n",
        "\n",
        "You can instantiate a Resnet object and train it as usual like below:\n",
        "\n",
        "**Note:** If you have issues with training in the Coursera lab environment, you can also run this in Colab using the \"open in colab\" badge link."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 223,
          "referenced_widgets": [
            "534116b477d24846a6090108ae372280",
            "61c5678e446d49a3a1608c60c4b2130c",
            "647d407aaabf4f99b28b161de14d788d",
            "b708d613e047433d9594f20da60b1d4c",
            "e66eb7c58ef54456871f20670af22a17",
            "05112ac805cf4d3a87ccc49e2c197542",
            "86ce0d9a10df4710a73aaa1c646e6362",
            "6d5662d17f4f4c5197ae24dd8d6d3a27",
            "7d66794a843c4eed947c8dab09a76c86",
            "bfed275927af4dd49af51a33ccd039ba",
            "923e61853f3c43219446f8eb3d69e1ed"
          ]
        },
        "id": "u4ECLcf9SSvU",
        "outputId": "ec8afd67-000e-46b6-df53-bf1d49d372de"
      },
      "source": [
        "# utility function to normalize the images and return (image, label) pairs.\n",
        "def preprocess(features):\n",
        "    return tf.cast(features['image'], tf.float32) / 255., features['label']\n",
        "\n",
        "# create a ResNet instance with 10 output units for MNIST\n",
        "resnet = ResNet(10)\n",
        "resnet.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n",
        "\n",
        "# load and preprocess the dataset\n",
        "dataset = tfds.load('mnist', split=tfds.Split.TRAIN, data_dir='./data')\n",
        "dataset = dataset.map(preprocess).batch(32)\n",
        "\n",
        "# train the model.\n",
        "resnet.fit(dataset, epochs=1)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\u001b[1mDownloading and preparing dataset mnist/3.0.1 (download: 11.06 MiB, generated: 21.00 MiB, total: 32.06 MiB) to ./data/mnist/3.0.1...\u001b[0m\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:absl:Dataset mnist is hosted on GCS. It will automatically be downloaded to your\n",
            "local data directory. If you'd instead prefer to read directly from our public\n",
            "GCS bucket (recommended if you're running on GCP), you can instead pass\n",
            "`try_gcs=True` to `tfds.load` or set `data_dir=gs://tfds-data/datasets`.\n",
            "\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "534116b477d24846a6090108ae372280",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Dl Completed...:   0%|          | 0/4 [00:00<?, ? file/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "\u001b[1mDataset mnist downloaded and prepared to ./data/mnist/3.0.1. Subsequent calls will reuse this data.\u001b[0m\n",
            "1875/1875 [==============================] - 222s 118ms/step - loss: 0.1608 - accuracy: 0.9605\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<keras.callbacks.History at 0x7f481489e150>"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R2TsE7_6Sn0s"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}