1M_replication.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Fjallripa/TinyStories/blob/main/1M_replication.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1DUsTR2g9aPu",
        "outputId": "a21a324a-ed1a-4c23-8b0b-6811d5dce65d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "# Use the Colab UI instead. Select the folder button on the left and then the drive button.\n",
        "# This way you will not be asked every execution for permissions\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xQl6jP9K_Lb_",
        "outputId": "6a849dbd-b927-4489-a775-ab48b276cb59"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting datasets\n",
            "  Downloading datasets-2.18.0-py3-none-any.whl (510 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m510.5/510.5 kB\u001b[0m \u001b[31m3.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from datasets) (3.13.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets) (1.25.2)\n",
            "Requirement already satisfied: pyarrow>=12.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (14.0.2)\n",
            "Requirement already satisfied: pyarrow-hotfix in /usr/local/lib/python3.10/dist-packages (from datasets) (0.6)\n",
            "Collecting dill<0.3.9,>=0.3.0 (from datasets)\n",
            "  Downloading dill-0.3.8-py3-none-any.whl (116 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m116.3/116.3 kB\u001b[0m \u001b[31m11.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (1.5.3)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (4.66.2)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets) (3.4.1)\n",
            "Collecting multiprocess (from datasets)\n",
            "  Downloading multiprocess-0.70.16-py310-none-any.whl (134 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m13.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]<=2024.2.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.9.3)\n",
            "Requirement already satisfied: huggingface-hub>=0.19.4 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.20.3)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (6.0.1)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.2.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.1)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.5)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.19.4->datasets) (4.10.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2024.2.2)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.4)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n",
            "Installing collected packages: dill, multiprocess, datasets\n",
            "Successfully installed datasets-2.18.0 dill-0.3.8 multiprocess-0.70.16\n"
          ]
        }
      ],
      "source": [
        "# install dataset\n",
        "import os\n",
        "\n",
        "!pip install datasets"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "l6KOfFH8MmGK"
      },
      "outputs": [],
      "source": [
        "if \"COLAB_TPU_ADDR\" in os.environ:\n",
        "    !pip install --upgrade jax jaxlib\n",
        "    !pip install torch~=2.2.0 torch_xla[tpu]~=2.2.0 -f https://storage.googleapis.com/libtpu-releases/index.html"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EyolnRlpqflI",
        "outputId": "feac355c-dd20-4d31-bc40-3d322e3ee7c9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Selected device: cpu\n"
          ]
        }
      ],
      "source": [
        "from transformers import GPTNeoConfig, GPTNeoForCausalLM, get_scheduler, DataCollatorForSeq2Seq, AutoTokenizer, DataCollatorForLanguageModeling, AutoModelForCausalLM\n",
        "from datasets import load_dataset, DatasetDict\n",
        "import torch\n",
        "from torch.utils.data import DataLoader\n",
        "from datetime import datetime\n",
        "from copy import deepcopy\n",
        "from tqdm.auto import tqdm\n",
        "\n",
        "# Check for TPU availability\n",
        "if \"COLAB_TPU_ADDR\" in os.environ:\n",
        "    import torch_xla.core.xla_model as xm\n",
        "    import torch_xla.distributed.parallel_loader as pl\n",
        "    import torch_xla.distributed.xla_multiprocessing as xmp\n",
        "    is_tpu = True\n",
        "    device = \"xla\"\n",
        "elif torch.cuda.is_available():\n",
        "    is_tpu = False\n",
        "    device = torch.device(\"cuda\")  # GPU\n",
        "else:\n",
        "    is_tpu = False\n",
        "    device = torch.device(\"cpu\")  # CPU\n",
        "\n",
        "print(\"Selected device:\", device)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "WUQVhO1A-3DM"
      },
      "outputs": [],
      "source": [
        "config = GPTNeoConfig(\n",
        "    activation_function=\"gelu_new\",\n",
        "    architectures=[\"GPTNeoForCausalLM\"],\n",
        "    attention_dropout=0,\n",
        "    attention_layers=[\"global\", \"local\", \"global\", \"local\", \"global\", \"local\", \"global\", \"local\"],\n",
        "    attention_types=[[[\"global\", \"local\"], 4]],\n",
        "    bos_token_id=50256,\n",
        "    context_length=512,\n",
        "    embed_dropout=0,\n",
        "    eos_token_id=50256,\n",
        "    gradient_checkpointing=False,\n",
        "    hidden_size=64,\n",
        "    initializer_range=0.02,\n",
        "    intermediate_size=None,\n",
        "    layer_norm_epsilon=1e-05,\n",
        "    max_position_embeddings=2048,\n",
        "    model_type=\"gpt_neo\",\n",
        "    num_heads=16,\n",
        "    num_layers=8,\n",
        "    resid_dropout=0,\n",
        "    summary_activation=None,\n",
        "    summary_first_dropout=0.1,\n",
        "    summary_proj_to_labels=True,\n",
        "    summary_type=\"cls_index\",\n",
        "    summary_use_proj=True,\n",
        "    torch_dtype=\"float32\",\n",
        "    transformers_version=\"4.28.0\",\n",
        "    use_cache=True,\n",
        "    vocab_size=50257,\n",
        "    window_size=256\n",
        ")\n",
        "\n",
        "model_directory = \"/content/drive/My Drive/genAI project/model\"\n",
        "\n",
        "if os.path.exists(model_directory):\n",
        "    model = GPTNeoForCausalLM.from_pretrained(model_directory)\n",
        "else:\n",
        "    model = GPTNeoForCausalLM(config)\n",
        "#model = AutoModelForCausalLM.from_pretrained(\"EleutherAI/gpt-neo-125M\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "pEh0auhooKBJ"
      },
      "outputs": [],
      "source": [
        "tokenizer = AutoTokenizer.from_pretrained(\"EleutherAI/gpt-neo-125M\")\n",
        "#tokenizer.add_tokens([\"<|pad|>]\"])\n",
        "tokenizer.pad_token = tokenizer.eos_token\n",
        "\n",
        "# Create a collator for language modeling\n",
        "data_collator = DataCollatorForLanguageModeling(\n",
        "    tokenizer=tokenizer,\n",
        "    mlm=False,  # False means Causal Language Modeling\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "b928fea6408b4f139bb2fc380c6d628d",
            "2fe30fc829cc46748bfa2655eea0e13e",
            "e6928eaa648d478a973efb2b38607814",
            "7f44f9a1140d49cfa2955cece33112c0",
            "2654d8d8fec8482ab1216f3211962016",
            "85960a7633904c81836f9d2a780ed0b6",
            "9a84f879173b46cba28cfcb85dee2cba",
            "9136a9bd2ac542a4ae55c993d17a7071",
            "2e6f8f97d8ff404aac8c676f73a79699",
            "f6200de3f9b148d081b94a6b06f3cd08",
            "9849f0221d6a48f2ab8c3b5c0a5ce442"
          ]
        },
        "id": "_KcaTYiwUIEq",
        "outputId": "b3797cf9-af18-4007-f7c3-102333ca395d"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Loading dataset from disk:   0%|          | 0/22 [00:00<?, ?it/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b928fea6408b4f139bb2fc380c6d628d"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "dataset_directory = \"/content/drive/My Drive/genAI project/dataset_prepared\"\n",
        "\n",
        "if os.path.exists(dataset_directory):\n",
        "    tokenized_datasets = DatasetDict.load_from_disk(dataset_directory)\n",
        "else:\n",
        "    raw_dataset = load_dataset(\"roneneldan/TinyStories\")\n",
        "\n",
        "\n",
        "    # Tokenize text\n",
        "    tokenized_datasets = raw_dataset.map(\n",
        "        lambda examples: tokenizer(examples[\"text\"], truncation=True, padding=\"max_length\", max_length=1024),\n",
        "        batched=True)\n",
        "    tokenized_datasets = tokenized_datasets.remove_columns([\"text\"])\n",
        "    tokenized_datasets.set_format(\"torch\")\n",
        "\n",
        "    tokenized_datasets.save_to_disk(dataset_directory)\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "raw_dataset = load_dataset(\"roneneldan/TinyStories\")\n",
        "tokenized_datasets = raw_dataset.map(lambda examples: tokenizer(examples[\"text\"]), batched=True)\n",
        "tokenized_datasets = tokenized_datasets.remove_columns([\"text\"])\n",
        "tokenized_datasets.set_format(\"torch\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81,
          "referenced_widgets": [
            "e66844862ae844aeac57e40c75e5b6d2",
            "839f08916bfa4c85b0fbb1fed2f3e7d0",
            "42190f8873f54f7b868f3a2174d5b4fd",
            "d3ef292a8e24458ab7525003e080e9d1",
            "2092e4ad0beb4d39af1fe5b3ca52a72b",
            "e5fdcb2eb4b14e63b7cd3460056f031e",
            "6e7243be334c4dedb85b713cb219be71",
            "cdea874551884b7e932966fc096b56db",
            "2985305fb2f54432b1b77fb505759d57",
            "7466871e2fa340c1ada59be9f5dd6bdc",
            "7ce032dcefe246f1a588dccaaeccb760",
            "04e749e75b784d699f172d277d9167a4",
            "d6cdf9cb4eaf423d97e144dfdfb614f2",
            "8f47e91d73cd4a0fbf14345ef26157bc",
            "f74472e476ce400db2b0bf0460ac2b5e",
            "2aebb49b83104cd9a046599d7b2cab68",
            "9e3dac97989a49b39faa5fa8aed52a03",
            "4fbee1d85a0c485a8936502f95c4ec82",
            "1b5d3365bdfa4c259b08112ba42ae0fb",
            "2ce23b5c525b44a5add350b53f5e5661",
            "be91168371d740469b764b8e2f11811f",
            "bf2316f72e5946bf94c548ae3804e3d4"
          ]
        },
        "id": "W2PzVbTZohnp",
        "outputId": "73677bcb-6ee7-42a6-c154-d62cd314a599"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/2119719 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e66844862ae844aeac57e40c75e5b6d2"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Map:   0%|          | 0/21990 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "04e749e75b784d699f172d277d9167a4"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_dataset_without_padding = tokenized_datasets"
      ],
      "metadata": {
        "id": "bM-AW6ISN2Yz"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer.pad_token = tokenizer.eos_token\n",
        "#tokenizer.pad_token = None\n",
        "#tokenizer.padding_strategy = \"DO_NOT_PAD\""
      ],
      "metadata": {
        "id": "YURVyTrUOCcH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#for batch in train_dataloader:\n",
        "#    batch[\"diff\"] = batch[\"input_ids\"] - batch[\"labels\"]\n",
        "#    mask = batch[\"diff\"] == 5035\n",
        "#    for i in range(4):\n",
        "#        print(torch.sum(batch[\"attention_mask\"][i]))\n",
        "#        for j, value in enumerate(batch[\"input_ids\"][i]):\n",
        "#            if value == 50256:\n",
        "#                print(j)\n",
        "#                break\n",
        "#    #batch[\"diff\"][mask] = 0\n",
        "#    for i, example_attention_mask in enumerate(batch[\"attention_mask\"]):\n",
        "#        for j in range(len(example_attention_mask)):\n",
        "#            if batch[\"attention_mask\"][i][j]:\n",
        "#                if batch[\"input_ids\"][i][j] == 50256:\n",
        "#                    print(f\"error at {i},{j}: {example_attention_mask[j]} != {not mask[i][j]}, number: {batch['input_ids'][i][j]}\")\n",
        "#    #has_diff = torch.sum(batch[\"diff\"])\n",
        "#    #if has_diff:\n",
        "#    #    print(\"has diff\")\n",
        "#    break"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZRqP3QjUOIs7",
        "outputId": "6f3aee2f-0795-44ed-f52e-d11ad8634f57"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor(190)\n",
            "190\n",
            "tensor(224)\n",
            "224\n",
            "tensor(457)\n",
            "457\n",
            "tensor(311)\n",
            "311\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "#tokenized_dataset = tokenized_dataset_without_padding\n",
        "#tokenized_dataset.flatten()\n",
        "#additional_column = torch.full((tokenized_dataset[\"input_ids\"].shape[0], 1), 50256)\n",
        "#data = torch.cat(tokenized_dataset[\"input_ids\"], additional_column)\n",
        "tokenized_dataset_without_padding.save_to_disk(\"/content/drive/My Drive/genAI project/dataset_without_padding\")\n",
        "\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81,
          "referenced_widgets": [
            "d70acc399a69462dae0bdbffe7ed1625",
            "7b182723ca864fb284a2f2f08f4c3c80",
            "84f3bcc10e5b4847a39ba1d5225bace9",
            "3b6fffa36cd04f0da4d13636e7911542",
            "3c39c7467fb84212942baf72d496dd42",
            "cc6bb120fbcd43868371ca6953385b78",
            "47d05641c7b24907b3ef100b0a5e1ece",
            "b18f3b10f4ac474fa5fd599cfb1af458",
            "14a47d6a22014171956a1b022d5bb4f6",
            "208b6270177941d0a346c627df2dd247",
            "5538ca2dcca344bd89cd0b0d14e8e39f",
            "8ce297a708c141dfbb2f6b9547164d66",
            "e21b2dc88b6847f8a326dc02237a905e",
            "cda51e0eef1d444995a5d3f38ca3c9c0",
            "79e11919df8a4c1c959d46ac1b730b7b",
            "4ea18c70d3354da6af49ca8600d376ec",
            "fcbfd9315ccb4cc289f3ec421ead2da8",
            "0f083f1b1fee47ac9ff2b7ce877a08a3",
            "5ea9a4635d824e18962aa771a69d9ae0",
            "241703f379ad47da90bf2fc7c0c47462",
            "a2041eae069242d2a208129cf8d2f94b",
            "971b00c3bdf848dc9e51f4a9e5d6f639"
          ]
        },
        "id": "jTLmHB9fds1x",
        "outputId": "ad5947bf-d360-4b3a-a66b-d8857e57154b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Saving the dataset (0/5 shards):   0%|          | 0/2119719 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d70acc399a69462dae0bdbffe7ed1625"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Saving the dataset (0/1 shards):   0%|          | 0/21990 [00:00<?, ? examples/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "8ce297a708c141dfbb2f6b9547164d66"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer.encode(\"<|endoftext|>\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6-s0353Rgtd5",
        "outputId": "203419e7-aa3b-4c17-f408-1f116dc997f5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[50256]"
            ]
          },
          "metadata": {},
          "execution_count": 84
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "class TinyStoriesConcatinated(torch.utils.data.Dataset):\n",
        "  def __init__(self, tokenized_dataset):\n",
        "    self.data = self.convert_into_1d(tokenized_dataset)\n",
        "\n",
        "  def __len__(self):\n",
        "    return len(self.sequences)\n",
        "\n",
        "  def __getitem__(self, idx):\n",
        "    input_ids = self.sequences[idx]\n",
        "    labels = self.generate_shifted_labels(input_ids)  # Implement label generation logic\n",
        "    return input_ids, labels\n",
        "\n",
        "  def convert_into_1d(self, input_ids):\n",
        "    pass\n",
        "\n",
        "  def generate_shifted_labels(self, input_ids):\n",
        "    # Implement logic to generate shifted labels for the sequence\n",
        "    pass\n"
      ],
      "metadata": {
        "id": "J0nYxpBXfC_V"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PY-WbPLHWBBl",
        "outputId": "a791178b-440a-4c6c-8b92-57f648a048c7"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input_ids': torch.Size([4, 1024]),\n",
              " 'attention_mask': torch.Size([4, 1024]),\n",
              " 'labels': torch.Size([4, 1024])}"
            ]
          },
          "metadata": {},
          "execution_count": 38
        }
      ],
      "source": [
        "batch_size = 4\n",
        "#batch_size = 16\n",
        "\n",
        "\n",
        "train_dataloader = DataLoader(\n",
        "    #tokenized_datasets[\"train\"].select(range(700)), shuffle=True, batch_size=batch_size, collate_fn=data_collator\n",
        "    tokenized_datasets[\"train\"], shuffle=True, batch_size=batch_size, collate_fn=data_collator\n",
        ")\n",
        "eval_dataloader = DataLoader(\n",
        "    tokenized_datasets[\"validation\"], batch_size=batch_size, collate_fn=data_collator\n",
        ")\n",
        "\n",
        "for batch in train_dataloader:\n",
        "    break\n",
        "{k: v.shape for k, v in batch.items()}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "PfyyQ8GkW51p"
      },
      "outputs": [],
      "source": [
        "optimizer = torch.optim.AdamW(model.parameters(), lr=5e-4, betas=(0.9, 0.95), weight_decay=0.1)\n",
        "original_batch_size = 80\n",
        "num_epochs = 1\n",
        "num_training_steps = num_epochs * len(train_dataloader)\n",
        "gradient_accumulation_steps=16 * int(original_batch_size / batch_size)\n",
        "lr_scheduler = get_scheduler(\n",
        "    \"constant\",\n",
        "    optimizer=optimizer,\n",
        "    num_warmup_steps=0,\n",
        "    num_training_steps=num_training_steps,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HkFHFNRKItDB",
        "outputId": "a100c43d-3633-4ddf-8837-c38005e255f4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "80\n"
          ]
        }
      ],
      "source": [
        "print(gradient_accumulation_steps)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/",
          "height": 67,
          "referenced_widgets": [
            "2caad1e4ffef406eb2c4b097824f2427",
            "80070a844f2f4f7a81db3a71f82a2e9f",
            "00a2f493e4e546e89b06cf80b137a728",
            "46fae3303b99416b9f7f3daa01523afb",
            "6ae0465fcae2481db2667407bb82259b",
            "56bb104332fd4119a4c5c27669d512d1",
            "1d0e13a48a7f408fa2c07b56ee370ae2",
            "63d5423b24da49a3bf24b42d0be2a824",
            "bd05364f55884ba4a21044a312e64f11",
            "bb0822a7367c425298a966362bd87b7b",
            "f5a2abd917fc44a393e03db2eeb6233e"
          ]
        },
        "id": "Bl_zfV6WQfVc",
        "outputId": "4c77b503-e4b6-49cf-d14d-418c948e4bfb"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Training GPT-Neo type model with 3.7M parameters\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2caad1e4ffef406eb2c4b097824f2427",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/132483 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "\n",
        "model_size = sum(t.numel() for t in model.parameters())\n",
        "print(f\"Training GPT-Neo type model with {model_size/1000**2:.1f}M parameters\")\n",
        "\n",
        "losses = []\n",
        "\n",
        "if not is_tpu:\n",
        "    # CUDA and CPU training loop\n",
        "    progress_bar = tqdm(range(num_training_steps))\n",
        "    model.train().to(device)\n",
        "    for epoch in range(num_epochs):\n",
        "        for index, batch in enumerate(train_dataloader):\n",
        "            batch = {k: v.to(device) for k, v in batch.items()}\n",
        "            outputs = model(**batch)\n",
        "            loss = outputs.loss\n",
        "            loss = loss / gradient_accumulation_steps\n",
        "            loss.backward()\n",
        "            losses.append(loss)\n",
        "\n",
        "            if (index + 1) % gradient_accumulation_steps == 0:\n",
        "                optimizer.step()\n",
        "                lr_scheduler.step()\n",
        "                optimizer.zero_grad()\n",
        "\n",
        "            progress_bar.update(1)\n",
        "else:\n",
        "    # TPU training loop\n",
        "    def _mp_fn(index):\n",
        "        progress_bar = tqdm(range(num_training_steps))\n",
        "        device = xm.xla_device()\n",
        "        mp_device_loader = pl.MpDeviceLoader(train_dataloader, device)\n",
        "        this_proc_model = model.train().to(device)\n",
        "        for epoch in range(num_epochs):\n",
        "            #for data, target in mp_device_loader:\n",
        "            for example in mp_device_loader:\n",
        "                optimizer.zero_grad()\n",
        "                output = this_proc_model(input_ids=example.input_ids, attention_mask=example.attention_mask, labels=example.labels)\n",
        "                loss = output.loss\n",
        "                loss.backward()\n",
        "\n",
        "                if (index + 1) % gradient_accumulation_steps == 0:\n",
        "                    xm.optimizer_step(optimizer)\n",
        "                    lr_scheduler.step()\n",
        "                    optimizer.zero_grad()\n",
        "\n",
        "            progress_bar.update(1)\n",
        "\n",
        "    if __name__ == '__main__':\n",
        "        xmp.spawn(_mp_fn, args=(), nprocs=1)\n",
        "model.save_pretrained(f\"/content/drive/My Drive/genAI project/model-{datetime.now().isoformat()}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5LJ-shR_rMWH",
        "outputId": "e1dc7075-541b-45b4-9925-51c94ef879e7"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Once upon a time Tom ate apples and his friends were very happy. They played together every day and played together every day.\n",
            "\n",
            "One day, Tom and his friends were playing together. They were playing together and they were playing together. They were having fun and having fun.\n",
            "\n",
            "They were playing together and they were playing together. They were having fun and having fun.\n",
            "\n",
            "They were having fun and having fun. They had fun and having fun.\n",
            "\n",
            "They were having fun and having fun. They had fun and having fun.\n",
            "\n",
            "They were having fun and having fun. They had fun and having fun.\n",
            "\n",
            "They had a great time together. They were happy and happy. They had a great time together.\n",
            "\n",
            "They had a great time together. They had a great time together. They had a great time together. They had a great time together.\n",
            "\n",
            "They had a great time together. They had a great time together. They were happy and happy. They had a great time together. They had a great time together. They had a great time together. They had a great time together. They were happy. They had a great time together. They had a great time. They had a great time together. They were happy. They had a great time. They had a great time. They had a great time together. They had a great time. They had a great time. They had a good time. They had a good time. They had a good time. They were happy. They had a good time. They had a good time. They had a new friend. They had a good time. They had a new friend. They had a new friend. They had a new friend. They had a new friend. They had a new friend. They had a new friend. They had a new friend. They had a new friend. They had a new friend. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy and happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy and happy. They were happy. They were happy. They were happy and happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy and happy and happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy and happy. They were happy. They were happy. They were happy. They were happy and happy. They were happy. They were happy and happy. They were happy and happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy. They were happy and happy. They were happy and happy. They were happy. They were happy. They were happy. They were happy and happy. They were happy. They were happy. They were happy and happy. They were happy. They were happy and happy. They were happy and happy. They were happy. They were happy. They were happy and happy and happy. They were happy. They were happy and happy.\n"
          ]
        }
      ],
      "source": [
        "\n",
        "#model.to(device)\n",
        "input_ids = tokenizer.encode(f\"Once upon a time Tom ate apples and\", return_tensors=\"pt\").to(device)\n",
        "attention_mask = torch.ones(input_ids.shape, device=device)\n",
        "output = model.generate(input_ids, max_length = 1024, num_beams=1, attention_mask=attention_mask, eos_token_id = 50256)\n",
        "output_text = tokenizer.decode(output[0], skip_special_tokens=True)\n",
        "print(output_text)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Hgl2mq_2U0xA"
      },
      "outputs": [],
      "source": [
        "import pickle\n",
        "\n",
        "with open('/content/drive/My Drive/genAI project/losses.pickle', 'wb') as handle:\n",
        "    pickle.dump(losses, handle, protocol=pickle.HIGHEST_PROTOCOL)\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(len(losses))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XOws1eP3sLIh",
        "outputId": "88d2bbda-ae82-4a88-d209-d44518a38485"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "132483\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "losses_tensor = torch.tensor(losses)"
      ],
      "metadata": {
        "id": "L3F5IEH9sRVU"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "losses_tensor.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dEFHYPEUs2uO",
        "outputId": "210f3277-83da-4d6f-c22f-f48a6165d216"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([132483])"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "torch.save(losses_tensor, \"/content/drive/My Drive/genAI project/losses_tensor.pt\")"
      ],
      "metadata": {
        "id": "YHSfsHcYs6eK"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "4DEboDpztIDR"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "00a2f493e4e546e89b06cf80b137a728": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_63d5423b24da49a3bf24b42d0be2a824",
            "max": 132483,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_bd05364f55884ba4a21044a312e64f11",
            "value": 113073
          }
        },
        "1d0e13a48a7f408fa2c07b56ee370ae2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2caad1e4ffef406eb2c4b097824f2427": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_80070a844f2f4f7a81db3a71f82a2e9f",
              "IPY_MODEL_00a2f493e4e546e89b06cf80b137a728",
              "IPY_MODEL_46fae3303b99416b9f7f3daa01523afb"
            ],
            "layout": "IPY_MODEL_6ae0465fcae2481db2667407bb82259b"
          }
        },
        "46fae3303b99416b9f7f3daa01523afb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bb0822a7367c425298a966362bd87b7b",
            "placeholder": "​",
            "style": "IPY_MODEL_f5a2abd917fc44a393e03db2eeb6233e",
            "value": " 113073/132483 [9:05:23&lt;1:33:44,  3.45it/s]"
          }
        },
        "56bb104332fd4119a4c5c27669d512d1": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "63d5423b24da49a3bf24b42d0be2a824": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6ae0465fcae2481db2667407bb82259b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "80070a844f2f4f7a81db3a71f82a2e9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_56bb104332fd4119a4c5c27669d512d1",
            "placeholder": "​",
            "style": "IPY_MODEL_1d0e13a48a7f408fa2c07b56ee370ae2",
            "value": " 85%"
          }
        },
        "bb0822a7367c425298a966362bd87b7b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bd05364f55884ba4a21044a312e64f11": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f5a2abd917fc44a393e03db2eeb6233e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b928fea6408b4f139bb2fc380c6d628d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2fe30fc829cc46748bfa2655eea0e13e",
              "IPY_MODEL_e6928eaa648d478a973efb2b38607814",
              "IPY_MODEL_7f44f9a1140d49cfa2955cece33112c0"
            ],
            "layout": "IPY_MODEL_2654d8d8fec8482ab1216f3211962016"
          }
        },
        "2fe30fc829cc46748bfa2655eea0e13e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_85960a7633904c81836f9d2a780ed0b6",
            "placeholder": "​",
            "style": "IPY_MODEL_9a84f879173b46cba28cfcb85dee2cba",
            "value": "Loading dataset from disk: 100%"
          }
        },
        "e6928eaa648d478a973efb2b38607814": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9136a9bd2ac542a4ae55c993d17a7071",
            "max": 22,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2e6f8f97d8ff404aac8c676f73a79699",
            "value": 22
          }
        },
        "7f44f9a1140d49cfa2955cece33112c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f6200de3f9b148d081b94a6b06f3cd08",
            "placeholder": "​",
            "style": "IPY_MODEL_9849f0221d6a48f2ab8c3b5c0a5ce442",
            "value": " 22/22 [00:08&lt;00:00,  2.61it/s]"
          }
        },
        "2654d8d8fec8482ab1216f3211962016": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "85960a7633904c81836f9d2a780ed0b6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9a84f879173b46cba28cfcb85dee2cba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9136a9bd2ac542a4ae55c993d17a7071": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2e6f8f97d8ff404aac8c676f73a79699": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f6200de3f9b148d081b94a6b06f3cd08": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9849f0221d6a48f2ab8c3b5c0a5ce442": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e66844862ae844aeac57e40c75e5b6d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_839f08916bfa4c85b0fbb1fed2f3e7d0",
              "IPY_MODEL_42190f8873f54f7b868f3a2174d5b4fd",
              "IPY_MODEL_d3ef292a8e24458ab7525003e080e9d1"
            ],
            "layout": "IPY_MODEL_2092e4ad0beb4d39af1fe5b3ca52a72b"
          }
        },
        "839f08916bfa4c85b0fbb1fed2f3e7d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e5fdcb2eb4b14e63b7cd3460056f031e",
            "placeholder": "​",
            "style": "IPY_MODEL_6e7243be334c4dedb85b713cb219be71",
            "value": "Map: 100%"
          }
        },
        "42190f8873f54f7b868f3a2174d5b4fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cdea874551884b7e932966fc096b56db",
            "max": 2119719,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2985305fb2f54432b1b77fb505759d57",
            "value": 2119719
          }
        },
        "d3ef292a8e24458ab7525003e080e9d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7466871e2fa340c1ada59be9f5dd6bdc",
            "placeholder": "​",
            "style": "IPY_MODEL_7ce032dcefe246f1a588dccaaeccb760",
            "value": " 2119719/2119719 [24:58&lt;00:00, 977.17 examples/s]"
          }
        },
        "2092e4ad0beb4d39af1fe5b3ca52a72b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e5fdcb2eb4b14e63b7cd3460056f031e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6e7243be334c4dedb85b713cb219be71": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cdea874551884b7e932966fc096b56db": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2985305fb2f54432b1b77fb505759d57": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7466871e2fa340c1ada59be9f5dd6bdc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7ce032dcefe246f1a588dccaaeccb760": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "04e749e75b784d699f172d277d9167a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d6cdf9cb4eaf423d97e144dfdfb614f2",
              "IPY_MODEL_8f47e91d73cd4a0fbf14345ef26157bc",
              "IPY_MODEL_f74472e476ce400db2b0bf0460ac2b5e"
            ],
            "layout": "IPY_MODEL_2aebb49b83104cd9a046599d7b2cab68"
          }
        },
        "d6cdf9cb4eaf423d97e144dfdfb614f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9e3dac97989a49b39faa5fa8aed52a03",
            "placeholder": "​",
            "style": "IPY_MODEL_4fbee1d85a0c485a8936502f95c4ec82",
            "value": "Map: 100%"
          }
        },
        "8f47e91d73cd4a0fbf14345ef26157bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1b5d3365bdfa4c259b08112ba42ae0fb",
            "max": 21990,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_2ce23b5c525b44a5add350b53f5e5661",
            "value": 21990
          }
        },
        "f74472e476ce400db2b0bf0460ac2b5e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_be91168371d740469b764b8e2f11811f",
            "placeholder": "​",
            "style": "IPY_MODEL_bf2316f72e5946bf94c548ae3804e3d4",
            "value": " 21990/21990 [00:15&lt;00:00, 1513.42 examples/s]"
          }
        },
        "2aebb49b83104cd9a046599d7b2cab68": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9e3dac97989a49b39faa5fa8aed52a03": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4fbee1d85a0c485a8936502f95c4ec82": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1b5d3365bdfa4c259b08112ba42ae0fb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2ce23b5c525b44a5add350b53f5e5661": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "be91168371d740469b764b8e2f11811f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bf2316f72e5946bf94c548ae3804e3d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d70acc399a69462dae0bdbffe7ed1625": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7b182723ca864fb284a2f2f08f4c3c80",
              "IPY_MODEL_84f3bcc10e5b4847a39ba1d5225bace9",
              "IPY_MODEL_3b6fffa36cd04f0da4d13636e7911542"
            ],
            "layout": "IPY_MODEL_3c39c7467fb84212942baf72d496dd42"
          }
        },
        "7b182723ca864fb284a2f2f08f4c3c80": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cc6bb120fbcd43868371ca6953385b78",
            "placeholder": "​",
            "style": "IPY_MODEL_47d05641c7b24907b3ef100b0a5e1ece",
            "value": "Saving the dataset (5/5 shards): 100%"
          }
        },
        "84f3bcc10e5b4847a39ba1d5225bace9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b18f3b10f4ac474fa5fd599cfb1af458",
            "max": 2119719,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_14a47d6a22014171956a1b022d5bb4f6",
            "value": 2119719
          }
        },
        "3b6fffa36cd04f0da4d13636e7911542": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_208b6270177941d0a346c627df2dd247",
            "placeholder": "​",
            "style": "IPY_MODEL_5538ca2dcca344bd89cd0b0d14e8e39f",
            "value": " 2119719/2119719 [00:50&lt;00:00, 40514.20 examples/s]"
          }
        },
        "3c39c7467fb84212942baf72d496dd42": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cc6bb120fbcd43868371ca6953385b78": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "47d05641c7b24907b3ef100b0a5e1ece": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b18f3b10f4ac474fa5fd599cfb1af458": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "14a47d6a22014171956a1b022d5bb4f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "208b6270177941d0a346c627df2dd247": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5538ca2dcca344bd89cd0b0d14e8e39f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8ce297a708c141dfbb2f6b9547164d66": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e21b2dc88b6847f8a326dc02237a905e",
              "IPY_MODEL_cda51e0eef1d444995a5d3f38ca3c9c0",
              "IPY_MODEL_79e11919df8a4c1c959d46ac1b730b7b"
            ],
            "layout": "IPY_MODEL_4ea18c70d3354da6af49ca8600d376ec"
          }
        },
        "e21b2dc88b6847f8a326dc02237a905e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fcbfd9315ccb4cc289f3ec421ead2da8",
            "placeholder": "​",
            "style": "IPY_MODEL_0f083f1b1fee47ac9ff2b7ce877a08a3",
            "value": "Saving the dataset (1/1 shards): 100%"
          }
        },
        "cda51e0eef1d444995a5d3f38ca3c9c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5ea9a4635d824e18962aa771a69d9ae0",
            "max": 21990,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_241703f379ad47da90bf2fc7c0c47462",
            "value": 21990
          }
        },
        "79e11919df8a4c1c959d46ac1b730b7b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a2041eae069242d2a208129cf8d2f94b",
            "placeholder": "​",
            "style": "IPY_MODEL_971b00c3bdf848dc9e51f4a9e5d6f639",
            "value": " 21990/21990 [00:00&lt;00:00, 70028.93 examples/s]"
          }
        },
        "4ea18c70d3354da6af49ca8600d376ec": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fcbfd9315ccb4cc289f3ec421ead2da8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0f083f1b1fee47ac9ff2b7ce877a08a3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5ea9a4635d824e18962aa771a69d9ae0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "241703f379ad47da90bf2fc7c0c47462": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "a2041eae069242d2a208129cf8d2f94b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "971b00c3bdf848dc9e51f4a9e5d6f639": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}