nb_qlora.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "d8ce0797-4461-49d5-ad44-c45fbf9bec87",
   "metadata": {},
   "source": [
    "# Prerequisites"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "037585b2-09e1-4cb7-ac7c-ed5d937e9653",
   "metadata": {},
   "source": [
    "## Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cc2db9ba-041c-45ab-8db2-73295ccd6f5a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defaulting to user installation because normal site-packages is not writeable\n",
      "Requirement already satisfied: accelerate in /home/g/.local/lib/python3.10/site-packages (0.26.1)\n",
      "Requirement already satisfied: peft in /home/g/.local/lib/python3.10/site-packages (0.7.1)\n",
      "Collecting peft\n",
      "  Downloading peft-0.8.1-py3-none-any.whl.metadata (25 kB)\n",
      "Requirement already satisfied: transformers in /home/g/.local/lib/python3.10/site-packages (4.37.2)\n",
      "Requirement already satisfied: einops in /home/g/.local/lib/python3.10/site-packages (0.4.1)\n",
      "Collecting einops\n",
      "  Downloading einops-0.7.0-py3-none-any.whl.metadata (13 kB)\n",
      "Requirement already satisfied: datasets in /home/g/.local/lib/python3.10/site-packages (2.16.1)\n",
      "Requirement already satisfied: bitsandbytes in /home/g/.local/lib/python3.10/site-packages (0.42.0)\n",
      "Requirement already satisfied: numpy>=1.17 in /home/g/.local/lib/python3.10/site-packages (from accelerate) (1.26.3)\n",
      "Requirement already satisfied: packaging>=20.0 in /home/g/.local/lib/python3.10/site-packages (from accelerate) (23.2)\n",
      "Requirement already satisfied: psutil in /home/g/.local/lib/python3.10/site-packages (from accelerate) (5.9.5)\n",
      "Requirement already satisfied: pyyaml in /home/g/.local/lib/python3.10/site-packages (from accelerate) (6.0.1)\n",
      "Requirement already satisfied: torch>=1.10.0 in /home/g/.local/lib/python3.10/site-packages (from accelerate) (2.1.0)\n",
      "Requirement already satisfied: huggingface-hub in /home/g/.local/lib/python3.10/site-packages (from accelerate) (0.19.4)\n",
      "Requirement already satisfied: safetensors>=0.3.1 in /home/g/.local/lib/python3.10/site-packages (from accelerate) (0.4.2)\n",
      "Requirement already satisfied: tqdm in /home/g/.local/lib/python3.10/site-packages (from peft) (4.66.1)\n",
      "Requirement already satisfied: filelock in /home/g/.local/lib/python3.10/site-packages (from transformers) (3.13.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /home/g/.local/lib/python3.10/site-packages (from transformers) (2023.8.8)\n",
      "Requirement already satisfied: requests in /home/g/.local/lib/python3.10/site-packages (from transformers) (2.31.0)\n",
      "Requirement already satisfied: tokenizers<0.19,>=0.14 in /home/g/.local/lib/python3.10/site-packages (from transformers) (0.15.1)\n",
      "Requirement already satisfied: pyarrow>=8.0.0 in /home/g/.local/lib/python3.10/site-packages (from datasets) (11.0.0)\n",
      "Requirement already satisfied: pyarrow-hotfix in /home/g/.local/lib/python3.10/site-packages (from datasets) (0.6)\n",
      "Requirement already satisfied: dill<0.3.8,>=0.3.0 in /home/g/.local/lib/python3.10/site-packages (from datasets) (0.3.7)\n",
      "Requirement already satisfied: pandas in /home/g/.local/lib/python3.10/site-packages (from datasets) (2.1.4)\n",
      "Requirement already satisfied: xxhash in /home/g/.local/lib/python3.10/site-packages (from datasets) (3.3.0)\n",
      "Requirement already satisfied: multiprocess in /home/g/.local/lib/python3.10/site-packages (from datasets) (0.70.15)\n",
      "Requirement already satisfied: fsspec<=2023.10.0,>=2023.1.0 in /home/g/.local/lib/python3.10/site-packages (from fsspec[http]<=2023.10.0,>=2023.1.0->datasets) (2023.6.0)\n",
      "Requirement already satisfied: aiohttp in /home/g/.local/lib/python3.10/site-packages (from datasets) (3.8.5)\n",
      "Requirement already satisfied: scipy in /home/g/.local/lib/python3.10/site-packages (from bitsandbytes) (1.12.0)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /home/g/.local/lib/python3.10/site-packages (from aiohttp->datasets) (23.1.0)\n",
      "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /home/g/.local/lib/python3.10/site-packages (from aiohttp->datasets) (3.2.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /home/g/.local/lib/python3.10/site-packages (from aiohttp->datasets) (6.0.4)\n",
      "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /home/g/.local/lib/python3.10/site-packages (from aiohttp->datasets) (4.0.3)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /home/g/.local/lib/python3.10/site-packages (from aiohttp->datasets) (1.9.2)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /home/g/.local/lib/python3.10/site-packages (from aiohttp->datasets) (1.4.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /home/g/.local/lib/python3.10/site-packages (from aiohttp->datasets) (1.3.1)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/g/.local/lib/python3.10/site-packages (from huggingface-hub->accelerate) (4.8.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /usr/lib/python3/dist-packages (from requests->transformers) (3.3)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/g/.local/lib/python3.10/site-packages (from requests->transformers) (1.26.16)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/g/.local/lib/python3.10/site-packages (from requests->transformers) (2023.11.17)\n",
      "Requirement already satisfied: sympy in /usr/lib/python3/dist-packages (from torch>=1.10.0->accelerate) (1.9)\n",
      "Requirement already satisfied: networkx in /home/g/.local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.1)\n",
      "Requirement already satisfied: jinja2 in /home/g/.local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (3.1.2)\n",
      "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /home/g/.local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /home/g/.local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /home/g/.local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (12.1.105)\n",
      "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /home/g/.local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (8.9.2.26)\n",
      "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /home/g/.local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (12.1.3.1)\n",
      "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /home/g/.local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (11.0.2.54)\n",
      "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /home/g/.local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (10.3.2.106)\n",
      "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /home/g/.local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (11.4.5.107)\n",
      "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /home/g/.local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (12.1.0.106)\n",
      "Requirement already satisfied: nvidia-nccl-cu12==2.18.1 in /home/g/.local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (2.18.1)\n",
      "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /home/g/.local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (12.1.105)\n",
      "Requirement already satisfied: triton==2.1.0 in /home/g/.local/lib/python3.10/site-packages (from torch>=1.10.0->accelerate) (2.1.0)\n",
      "Requirement already satisfied: nvidia-nvjitlink-cu12 in /home/g/.local/lib/python3.10/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.10.0->accelerate) (12.2.140)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /home/g/.local/lib/python3.10/site-packages (from pandas->datasets) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /usr/lib/python3/dist-packages (from pandas->datasets) (2022.1)\n",
      "Requirement already satisfied: tzdata>=2022.1 in /home/g/.local/lib/python3.10/site-packages (from pandas->datasets) (2023.3)\n",
      "Requirement already satisfied: six>=1.5 in /usr/lib/python3/dist-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /home/g/.local/lib/python3.10/site-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.3)\n",
      "Downloading peft-0.8.1-py3-none-any.whl (183 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m183.4/183.4 kB\u001b[0m \u001b[31m1.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hDownloading einops-0.7.0-py3-none-any.whl (44 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m44.6/44.6 kB\u001b[0m \u001b[31m1.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: einops, peft\n",
      "  Attempting uninstall: einops\n",
      "    Found existing installation: einops 0.4.1\n",
      "    Uninstalling einops-0.4.1:\n",
      "      Successfully uninstalled einops-0.4.1\n",
      "  Attempting uninstall: peft\n",
      "    Found existing installation: peft 0.7.1\n",
      "    Uninstalling peft-0.7.1:\n",
      "      Successfully uninstalled peft-0.7.1\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "trlx 0.7.0 requires deepspeed>=0.8.1, which is not installed.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed einops-0.7.0 peft-0.8.1\n"
     ]
    }
   ],
   "source": [
    "!pip install -U accelerate peft transformers einops datasets bitsandbytes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d5a2bfb3-45e1-4fc7-bc7e-a4ceadf4cf4b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "accelerate==0.26.1\n",
      "bitsandbytes==0.42.0\n",
      "datasets==2.16.1\n",
      "einops==0.7.0\n",
      "peft==0.8.1\n",
      "torch==2.1.0\n",
      "transformers==4.37.2\n"
     ]
    }
   ],
   "source": [
    "!pip freeze | egrep \"accelerate|peft|transformers|einops|datasets|^torch=|bitsandbytes\" "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7d43e101-bcf0-4e9d-b6fe-0a88f11ee35c",
   "metadata": {},
   "source": [
    "## Set a seed for reproducibility of the training process"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "06979abb-be2f-42a8-8c63-0f28b901dc60",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import set_seed\n",
    "\n",
    "set_seed(42)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "81cc2af6-2d62-4cc6-9077-3870085d2535",
   "metadata": {
    "id": "81cc2af6-2d62-4cc6-9077-3870085d2535"
   },
   "source": [
    "# Load model and tokenizer"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6377948-532c-4f66-801c-ee9874d5c6e5",
   "metadata": {
    "id": "a6377948-532c-4f66-801c-ee9874d5c6e5"
   },
   "source": [
    "## Load model in 4 bit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "25884709-1cb8-420d-8946-e6abe2d5f499",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 209,
     "referenced_widgets": [
      "f9428a8ed0bc4a02829416d5f4dc0add",
      "14f76d243a6c4e2d9d2ab097f8679183",
      "65722027d7ac4c89a698875db248c0ff",
      "49fb73c7b6b04f6baea866d68cf2f7ea",
      "11cdc86b257b43aeac774f9e4071f329",
      "a4672f7b7dc14add931058afedde1a16",
      "a7cae549b7a643dda513ad0ac5eec950",
      "8e29afb281fe46b68e50b137a7e93fa2",
      "50942dc4e2444918850fe32139035ed4",
      "1ac88c4d9d6e498f9cc19b38a6db4556",
      "b625b2b63f7e4a27a1054cb557df52f1",
      "ad012b1891424905b7f18ff2d3ca8702",
      "11ef5cd3633a41d2a016db4afd74ff67",
      "d7e1f1088e6f4422a977d89e4f2fd212",
      "23239ab0077c45538ffcb5eb2ad97324",
      "afbcdd6f1f864744a426d80dc6658fd8",
      "5e97ff18273448c5b50a9564617b0e96",
      "77643d235c9543eb8346710077c7500c",
      "9431ae1271844e709a27bcd19252bdcb",
      "d339ab1734fb4f38bd5149959795dd77",
      "75ebb53d003b41ebb299ca5c6388ba59",
      "b5bfa027a895443ab3f6fbdf688acf94",
      "e0ca89b3129b4344913ccf5a99bb51d1",
      "3e6dd8f5317440279b9878061c8f7b08",
      "a7ea6206547d4cdda1a4b6cb560ed7af",
      "dfacf3507ed4481b8ff57aba3903f409",
      "e6e893f6b07243b69e7cd4a4c5f446fe",
      "ab3270871c61444caaa89678fd9127aa",
      "5b633ee953554c8aa810dd13c474ed1e",
      "e6f1b170de014e6dbe9bc382c11a500d",
      "639355e013c04b43a068c3f8686c22e1",
      "83312456189b4576a2062fdb9055cf49",
      "11e89b669aa64cc4970cf0534d0437c5",
      "fe2014d38632490aa3d34964b4f56292",
      "89ac160bdb66433c9e07dec69ef9daf5",
      "3b29845189c44e64aff348a56e6ed9ec",
      "c0df8e1d80304432b14bf2fb8acaf2e2",
      "6ed3e52e2495498c90a39252d89bae30",
      "1c1414766d694a179ec5957718b7da40",
      "d773d2e8dcdb4982839470462880d172",
      "d1b3b6b0e17f4c1bbad206a4115ded8d",
      "78e4588c0f5a43d3a19516d0349dff08",
      "d63e23a364ef433b80155e9dcb447e6d",
      "9b9bd64660dc45eabd21a23bb9c00502",
      "b8d5a4d111e248008be34e8859de2539",
      "da950b8eef6e435982f6f667dd7f7fc5",
      "2109ffc0430e4392b17a4e7da4ff2224",
      "795ff049af254445866f3a6f9098a4c5",
      "7c15a12341584fc99ce7fff3983909dc",
      "8960ee82549c4841b0d8ad1f64167e2c",
      "48a92287432f441f990f3a737c02adf1",
      "799d8da4136c4ccb90727c076a67b619",
      "66c53415d0f4412fb62c5c8ca256feb5",
      "1631398755bb4bb6ad5d288320d31fd0",
      "ba4fc49b2bf14b69b1a430665510f848",
      "31af8b40703d488882f7b009decc28f4",
      "ac8869fb8ede4c9aa506f039a7c88b8e",
      "e793f51d5cc14dcfabda21ab93b1cf34",
      "655d90789ce6431d8184ccb0c071d96e",
      "791cd85ddea14577a87d5d6a51d1c284",
      "53ae7440dcc84e21ab09ce375c743b58",
      "bf49c49372a84c07a77d9bdd732c2f0d",
      "e09bb49f89e141579c1ebb3fb7eac98e",
      "407e2720459d4a47a61264de01e4cd05",
      "5b3bedeb6cfe40d8bd4c6623ac3fac43",
      "1d9185f34cd44483a332f3a08d8995aa"
     ]
    },
    "executionInfo": {
     "elapsed": 88750,
     "status": "ok",
     "timestamp": 1703794288995,
     "user": {
      "displayName": "Gerald Stampfel",
      "userId": "16044924958378286623"
     },
     "user_tz": -60
    },
    "id": "25884709-1cb8-420d-8946-e6abe2d5f499",
    "outputId": "99897cf8-4396-462f-d823-08ecb2dbf63b"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bc6bf934887145d387c18550239066c1",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import AutoModelForCausalLM, BitsAndBytesConfig\n",
    "\n",
    "# Load model\n",
    "modelpath = \"microsoft/phi-2\"\n",
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "    modelpath,    \n",
    "    device_map=\"auto\",\n",
    "    quantization_config=BitsAndBytesConfig(\n",
    "        load_in_4bit=True,\n",
    "        bnb_4bit_compute_dtype=torch.bfloat16,\n",
    "        bnb_4bit_quant_type=\"nf4\",\n",
    "    ),\n",
    "    torch_dtype=torch.bfloat16,\n",
    "    # FA2 does not work yet\n",
    "    # attn_implementation=\"flash_attention_2\",          \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "910e3f4a-5c8e-4974-87e0-9afc349ecea8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "PhiForCausalLM(\n",
       "  (model): PhiModel(\n",
       "    (embed_tokens): Embedding(51200, 2560)\n",
       "    (embed_dropout): Dropout(p=0.0, inplace=False)\n",
       "    (layers): ModuleList(\n",
       "      (0-31): 32 x PhiDecoderLayer(\n",
       "        (self_attn): PhiAttention(\n",
       "          (q_proj): Linear4bit(in_features=2560, out_features=2560, bias=True)\n",
       "          (k_proj): Linear4bit(in_features=2560, out_features=2560, bias=True)\n",
       "          (v_proj): Linear4bit(in_features=2560, out_features=2560, bias=True)\n",
       "          (dense): Linear4bit(in_features=2560, out_features=2560, bias=True)\n",
       "          (rotary_emb): PhiRotaryEmbedding()\n",
       "        )\n",
       "        (mlp): PhiMLP(\n",
       "          (activation_fn): NewGELUActivation()\n",
       "          (fc1): Linear4bit(in_features=2560, out_features=10240, bias=True)\n",
       "          (fc2): Linear4bit(in_features=10240, out_features=2560, bias=True)\n",
       "        )\n",
       "        (input_layernorm): LayerNorm((2560,), eps=1e-05, elementwise_affine=True)\n",
       "        (resid_dropout): Dropout(p=0.1, inplace=False)\n",
       "      )\n",
       "    )\n",
       "    (final_layernorm): LayerNorm((2560,), eps=1e-05, elementwise_affine=True)\n",
       "  )\n",
       "  (lm_head): Linear(in_features=2560, out_features=51200, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ebd3fc1-3b90-4f1e-b5cc-972930f7f462",
   "metadata": {
    "id": "6ebd3fc1-3b90-4f1e-b5cc-972930f7f462"
   },
   "source": [
    "## Load tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "782ae955-64d9-4e57-a793-a79c12b487e0",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 227,
     "referenced_widgets": [
      "dc6a36ff57f54fc3b8b4152b668bf55b",
      "9f5c7f8cb87e4025873c2a39b1ddb93a",
      "156918fac00741cd8e53f36aa43c7b6c",
      "af24999c54c44fa49799569fa85ad72c",
      "8917e9e09611490b8e326471b6d3d040",
      "c93626da431a463db581c12b5669cda9",
      "e3b7a45dc2cc4ea188966f76674d7de1",
      "1b02f5ef26fa4a6bbd384e104802a350",
      "ca2eef6af4f447ccbae0da87223f992e",
      "56716551ece24c599ae71074905e5e90",
      "b06ce6f27066467591cce3b512fad9d7",
      "f8818688f7484c65ac575959f7035840",
      "3a2ae78f09a54411ab19cfdcf1aca0b8",
      "6d81157c81384b51819335b6d7022d53",
      "598378cadf4245b1a60bc85a6fd4c1a2",
      "d951753bb05e45d584ff7699f9c415bd",
      "dfa9bc6f87e34aa4aabd0d8d2bb91df3",
      "d39c73b183a54f64a48e786ff9dcfb8c",
      "9dc37f312cc24806980118893311d200",
      "520213b9c4004b559a392ebd4396739b",
      "39ab404653ac454dbf06414f00c98b84",
      "519ff4745e184522adc93983dcb0e3df",
      "327411b689bb4eedaeb5dd802fb2cf56",
      "2e8b611b3c814acaa190ab9104ecb8f1",
      "4b4c5e2231bf4625a16dfb7cbc25fd57",
      "295662b9afd5401da453e7cc52705464",
      "e27059389ac045dd9426a76f3ebe0a6d",
      "a02825051f2f47f0b8ff4dba7ed94ab6",
      "cbef4f92c1ad464cbef0167a22ad05cd",
      "744dab8be6a5432fbf5526c661b82425",
      "8d6f4662173342f68010079f8751bc62",
      "55e4b78d734c4c6796f66848f844e386",
      "ab55b9070723401889e4b8941bfd9c7f",
      "f6a60332a4294512868b9d8bf0bfd8d5",
      "780db1e0d21c4e7f920e93b79da7e8c3",
      "18e1e9d387024d4aaa70c594e1f7c0a2",
      "fcdf3351a7214fd796626a5d0b7c3d08",
      "ca7221018dc446c1b9e0549b91eda07a",
      "9e616d74e28941d1bca1421dbb88cc37",
      "411a300862f948ffb2d8d5e3fe7ae543",
      "bd031d553cba483c8c080c43bf71b4be",
      "e5bf775c779247278b9e32403e5d992b",
      "6e470b8541204e63b79794b33ee081b6",
      "0468fe85167642ed830f43fdf3235503",
      "582874d489bd4398824e4c6ec01eae7d",
      "05a8c12b4e6f47428baf6cd7929ab314",
      "d5a4a0095968417d831db8f2bfa1ad25",
      "35177e251c974b9b9280916dc18c76f0",
      "0ee67c97a76c453397dbd04966a19681",
      "9abdbe87d2fd4be49ec86d64828deacf",
      "3a3ec4d9fa3b462cb314c08d68daf5c5",
      "57ac9644d750448aaf24fbf41c1fdfa8",
      "329733174bab4cdd875614bffe5da0d2",
      "0c94a07bbcae4cbfa7fc800236b79a65",
      "a07857b000c64de39609847731041940",
      "47332cf01eeb409bb107dee0c3c3c536",
      "cf9114aedcaa42699e0535925370c218",
      "0118c3dfb3f9410dbaedd2bc39326b8f",
      "2606f25c83b34cb5b30a7a1e6d03d3db",
      "4749e6e10f834d25a67acf94b3051481",
      "eaeed24afb9b47fb864d5e8776947465",
      "88d9fb7556a343278b09e72ab3e7d39a",
      "86ce00dc99ff42789a512c49ab2ee7cb",
      "d9d02387630f46fc96c25ec0cecaf410",
      "4f7daddffe864bd8b611726df3cac0cb",
      "032ab17b00a1470eb7d8d9a2489e30bc"
     ]
    },
    "executionInfo": {
     "elapsed": 1684,
     "status": "ok",
     "timestamp": 1703794329852,
     "user": {
      "displayName": "Gerald Stampfel",
      "userId": "16044924958378286623"
     },
     "user_tz": -60
    },
    "id": "782ae955-64d9-4e57-a793-a79c12b487e0",
    "outputId": "23ed9aad-cf7e-4368-d2b5-dc6f761437aa"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoTokenizer\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(modelpath, use_fast=False)    # fast tokenizer sometimes ignores the added tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "4046ebba-c598-40b0-9924-5d7a7705d3b9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "50295"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(tokenizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b47d5c1c-e72d-4dc6-848e-76ef08698b8f",
   "metadata": {
    "id": "b47d5c1c-e72d-4dc6-848e-76ef08698b8f"
   },
   "source": [
    "## Add ChatML tokens to tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "af4f17f8-08d2-47bd-8c3a-d8ca59bf9e24",
   "metadata": {
    "executionInfo": {
     "elapsed": 324,
     "status": "ok",
     "timestamp": 1703794343429,
     "user": {
      "displayName": "Gerald Stampfel",
      "userId": "16044924958378286623"
     },
     "user_tz": -60
    },
    "id": "af4f17f8-08d2-47bd-8c3a-d8ca59bf9e24"
   },
   "outputs": [],
   "source": [
    "tokenizer.add_tokens([\"<|im_start|>\", \"<PAD>\"])\n",
    "tokenizer.pad_token = \"<PAD>\"\n",
    "tokenizer.add_special_tokens(dict(eos_token=\"<|im_end|>\"))\n",
    "model.config.eos_token_id = tokenizer.eos_token_id"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a9c705ab-4396-4dbd-abb6-5feae9117000",
   "metadata": {},
   "source": [
    "Note: there is no need to rezise the token embeddings, phi-2 already has embeddings sized for additional tokens. The model's vocab. size is 51200, this means you can add ~700 tokens to the tokenizer without having to resize the embeddings. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "a21bce8e-2bf5-4855-b6b9-85663debcce0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Embedding(51200, 2560)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.model.embed_tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7676aa68-dca3-4beb-a41d-446f635a34ca",
   "metadata": {
    "id": "7676aa68-dca3-4beb-a41d-446f635a34ca"
   },
   "source": [
    "## Prepare LoRA adapters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "7b992cae-a004-4713-b3ad-05abaf3a60fb",
   "metadata": {
    "executionInfo": {
     "elapsed": 623,
     "status": "ok",
     "timestamp": 1703794419845,
     "user": {
      "displayName": "Gerald Stampfel",
      "userId": "16044924958378286623"
     },
     "user_tz": -60
    },
    "id": "7b992cae-a004-4713-b3ad-05abaf3a60fb"
   },
   "outputs": [],
   "source": [
    "from peft import prepare_model_for_kbit_training, LoraConfig, get_peft_model\n",
    "\n",
    "model = prepare_model_for_kbit_training(model, use_gradient_checkpointing=True) \n",
    "\n",
    "lora_config = LoraConfig(\n",
    "    r=32, \n",
    "    lora_alpha=32, \n",
    "    target_modules = [ \"q_proj\", \"k_proj\", \"v_proj\", \"dense\" ],\n",
    "    modules_to_save = [\"lm_head\", \"embed_tokens\"],\n",
    "    lora_dropout=0.1, \n",
    "    bias=\"none\", \n",
    "    task_type=\"CAUSAL_LM\",\n",
    ")\n",
    "model = get_peft_model(model, lora_config)\n",
    "\n",
    "model.config.use_cache = False"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "36206aee-05a5-4c34-9084-e36f2d10ee73",
   "metadata": {
    "id": "36206aee-05a5-4c34-9084-e36f2d10ee73"
   },
   "source": [
    "# Load and preprocess dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "uwHLm-BqEOsT",
   "metadata": {
    "executionInfo": {
     "elapsed": 1410,
     "status": "ok",
     "timestamp": 1703794612404,
     "user": {
      "displayName": "Gerald Stampfel",
      "userId": "16044924958378286623"
     },
     "user_tz": -60
    },
    "id": "uwHLm-BqEOsT"
   },
   "outputs": [],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "dataset = load_dataset(\"g-ronimo/riddles_evolved\")\n",
    "dataset = dataset[\"train\"].train_test_split(test_size=0.1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "A_B62I90EczT",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 7,
     "status": "ok",
     "timestamp": 1703794616497,
     "user": {
      "displayName": "Gerald Stampfel",
      "userId": "16044924958378286623"
     },
     "user_tz": -60
    },
    "id": "A_B62I90EczT",
    "outputId": "8ebca8d2-f208-4cfd-c296-70e47a363194"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DatasetDict({\n",
       "    train: Dataset({\n",
       "        features: ['number', 'messages'],\n",
       "        num_rows: 1513\n",
       "    })\n",
       "    test: Dataset({\n",
       "        features: ['number', 'messages'],\n",
       "        num_rows: 169\n",
       "    })\n",
       "})"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "1LANgRVxEeDd",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 268,
     "status": "ok",
     "timestamp": 1703794620265,
     "user": {
      "displayName": "Gerald Stampfel",
      "userId": "16044924958378286623"
     },
     "user_tz": -60
    },
    "id": "1LANgRVxEeDd",
    "outputId": "47148858-269a-452f-b27b-a10b700e79a8"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'number': 1509,\n",
       " 'messages': [\"What starts with a 'C' is covered with keys, but we use only one. What is it?\",\n",
       "  'The answer to the riddle is a \"coconut.\" While a coconut is covered with many small \"keys\" or eyes, we typically open it with just one tool, such as a coconut crusher or a large nail.',\n",
       "  \"Cool, so what's that thing we eat that looks like a big brown ball and has lots of little bumps on it, but we only use one thing to open it? I guess it's not a basketball?\",\n",
       "  'I believe you\\'re referring to an artichoke. While an artichoke looks like a large brown ball with small bumps on it, we typically use a tool called a \"dip tool\" or a \"lemon tool\" with a sharp tip or a pair of scissors to remove the small, sharp thorns covering the artichoke\\'s bracts, allowing us to eat the tender heart inside.']}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset[\"train\"][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b52840a7-b6ae-48b3-bfed-7e4e526c7ee7",
   "metadata": {
    "id": "b52840a7-b6ae-48b3-bfed-7e4e526c7ee7"
   },
   "source": [
    "## Apply ChatML format and tokenize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "431e1e03-938e-4403-a692-50987f787790",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 81,
     "referenced_widgets": [
      "ed399e0a352a4a789963a6c367f8c8c2",
      "7240723e9f68485b939d2a3c17df4d28",
      "18bfc50ea9a24949a92ca09c4dd3e22c",
      "5b4b59d0ec8842068576af45165d9645",
      "9cee4195fc014b84b10bf84ea2268457",
      "512f9eaa94bc49828ce88423342f6086",
      "4dedc2cb743746468e81d266c06d6eb6",
      "e06a6d8b98d94d378bf1049a1abbcff9",
      "dc092383151a42faa5cb76c6ccb1787d",
      "811a542292624b6caf5ab9c5e6660d29",
      "ed25603c0dc5462c90b50625a37c5c97",
      "762fb2dea76b4adebc53be598e904b99",
      "cec6322f32e44a0c8ff445e0d8d33e1b",
      "4dfbea42c59b405885383178618ef4de",
      "f20f02632b2a4d41b9b1f7159aff336e",
      "99d94f8392c242f084bd6e1c4eb6542a",
      "a7efe1c651c144d3b7d964dd4313c35b",
      "664a68ed4e3640acaa22c65051f8da68",
      "198fbb4d1f384105be84d84661eed44d",
      "267cdeec38644c3db346920c0030221e",
      "f98f7e87395544089fbe9823c1a0fe6c",
      "4d7fd6e5584d42f38fd59f4f25c6ec16"
     ]
    },
    "executionInfo": {
     "elapsed": 13868,
     "status": "ok",
     "timestamp": 1703794746768,
     "user": {
      "displayName": "Gerald Stampfel",
      "userId": "16044924958378286623"
     },
     "user_tz": -60
    },
    "id": "431e1e03-938e-4403-a692-50987f787790",
    "outputId": "fda174fd-2968-4776-e33c-eefcfafd9cca"
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9b20477dcbb9443cb415271e63c71226",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map (num_proc=32):   0%|          | 0/1513 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "28081226d13c470486ead45876964479",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map (num_proc=32):   0%|          | 0/169 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import os\n",
    "from functools import partial\n",
    "\n",
    "# Format (chatML) and tokenize dataset\n",
    "templates=[\n",
    "    \"<|im_start|>assistant\\n{msg}<|im_end|>\",\n",
    "    \"<|im_start|>user\\n{msg}<|im_end|>\"\n",
    "]\n",
    "IGNORE_INDEX=-100\n",
    "\n",
    "def tokenize(input, max_length):\n",
    "    input_ids, attention_mask, labels = [], [], []\n",
    "\n",
    "    for i,msg in enumerate(input[\"messages\"]):\n",
    "        isHuman = i%2==0\n",
    "        msg_chatml=templates[isHuman].format(msg=msg)\n",
    "        msg_tokenized=tokenizer(msg_chatml, truncation=False, add_special_tokens=False)\n",
    "\n",
    "        input_ids+=msg_tokenized[\"input_ids\"]\n",
    "        attention_mask+=msg_tokenized[\"attention_mask\"]\n",
    "        labels+=[IGNORE_INDEX]*len(msg_tokenized[\"input_ids\"]) if isHuman else msg_tokenized[\"input_ids\"]\n",
    "\n",
    "    return {\n",
    "        \"input_ids\": input_ids[:max_length],\n",
    "        \"attention_mask\": attention_mask[:max_length],\n",
    "        \"labels\": labels[:max_length],\n",
    "    }\n",
    "\n",
    "dataset_tokenized = dataset.map(\n",
    "    partial(tokenize, max_length=1024), # max sample length 1024 tokens, enough for this dataset\n",
    "    batched=False, \n",
    "    num_proc=os.cpu_count(),    # multithreaded\n",
    "    remove_columns=dataset[\"train\"].column_names  # don't need this anymore, we have tokens from here on\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "75bab8eb-b7be-4d11-a17f-a5a1cac5cbb7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "longest sample: 952 tokens\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAigAAAGdCAYAAAA44ojeAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/YYfK9AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAeyElEQVR4nO3dfWyV9f3/8dexpz20tT2jrZzjkSIla+ZNi3PFdHTMViklrMgMy0DxBiN/oEDHGTBuZIlotK0sA2bIWDRGHIzVLIPNDecoU7s1jbOWdZZu8SYWaaVnndqd02o9xfL5/eHXK7/TgnqgeD4tz0dy/dHrep/2c3mF9OnVc+MyxhgBAABY5KJELwAAAGA4AgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAddyJXsDZOHXqlE6cOKGMjAy5XK5ELwcAAHwBxhj19fUpEAjooos++x7JmAyUEydOKDc3N9HLAAAAZ6Gzs1OTJ0/+zJkxGSgZGRmSPjnBzMzMBK8GAAB8EZFIRLm5uc7v8c8yJgPl0z/rZGZmEigAAIwxX+TpGTxJFgAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1nEnegEYHVM3Hkz0EuJ2rLYy0UsAAFiKOygAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrxBUoW7Zskcvlitn8fr9z3BijLVu2KBAIKDU1VWVlZWpvb4/5HtFoVFVVVcrJyVF6eroWLFigrq6u0TkbAAAwLsR9B+Xqq69Wd3e3s7W1tTnHtm7dqm3btmnnzp1qbm6W3+/XnDlz1NfX58wEg0EdOHBAdXV1amxsVH9/v+bPn6+hoaHROSMAADDmueN+gNsdc9fkU8YY7dixQ5s3b9bChQslSU899ZR8Pp/27dun5cuXKxwO64knntCePXtUXl4uSdq7d69yc3N1+PBhzZ079xxPBwAAjAdx30F54403FAgElJeXp1tuuUVvvfWWJKmjo0OhUEgVFRXOrMfjUWlpqZqamiRJLS0tOnnyZMxMIBBQQUGBM3M60WhUkUgkZgMAAONXXIFSXFysX/7yl/rzn/+sxx9/XKFQSCUlJXrvvfcUCoUkST6fL+YxPp/PORYKhZSSkqKJEyeeceZ0ampq5PV6nS03NzeeZQMAgDEmrkCZN2+evve976mwsFDl5eU6ePCgpE/+lPMpl8sV8xhjzIh9w33ezKZNmxQOh52ts7MznmUDAIAx5pxeZpyenq7CwkK98cYbzvNSht8J6enpce6q+P1+DQ4Oqre394wzp+PxeJSZmRmzAQCA8eucAiUajerf//63Lr30UuXl5cnv96u+vt45Pjg4qIaGBpWUlEiSioqKlJycHDPT3d2to0ePOjMAAABxvYpn3bp1uummmzRlyhT19PTooYceUiQS0dKlS+VyuRQMBlVdXa38/Hzl5+erurpaaWlpWrJkiSTJ6/Vq2bJlWrt2rbKzs5WVlaV169Y5fzICAACQ4gyUrq4u3XrrrXr33Xd1ySWX6Jvf/KZeeuklXX755ZKk9evXa2BgQCtWrFBvb6+Ki4t16NAhZWRkON9j+/btcrvdWrRokQYGBjR79mzt3r1bSUlJo3tmAABgzHIZY0yiFxGvSCQir9ercDjM81H+z9SNBxO9hLgdq61M9BIAAF+ieH5/81k8AADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArONO9AJw4Zq68WCilxC3Y7WViV4CAFwQuIMCAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOucU6DU1NTI5XIpGAw6+4wx2rJliwKBgFJTU1VWVqb29vaYx0WjUVVVVSknJ0fp6elasGCBurq6zmUpAABgHDnrQGlubtZjjz2m6dOnx+zfunWrtm3bpp07d6q5uVl+v19z5sxRX1+fMxMMBnXgwAHV1dWpsbFR/f39mj9/voaGhs7+TAAAwLhxVoHS39+v2267TY8//rgmTpzo7DfGaMeOHdq8ebMWLlyogoICPfXUU/rwww+1b98+SVI4HNYTTzyhn/70pyovL9e1116rvXv3qq2tTYcPHx6dswIAAGPaWQXKypUrVVlZqfLy8pj9HR0dCoVCqqiocPZ5PB6VlpaqqalJktTS0qKTJ0/GzAQCARUUFDgzw0WjUUUikZgNAACMX+54H1BXV6cjR46oubl5xLFQKCRJ8vl8Mft9Pp/efvttZyYlJSXmzsunM58+friamho98MAD8S4VAACMUXHdQens7NTq1au1d+9eTZgw4YxzLpcr5mtjzIh9w33WzKZNmxQOh52ts7MznmUDAIAxJq5AaWlpUU9Pj4qKiuR2u+V2u9XQ0KBHH31UbrfbuXMy/E5IT0+Pc8zv92twcFC9vb1nnBnO4/EoMzMzZgMAAONXXIEye/ZstbW1qbW11dlmzJih2267Ta2trZo2bZr8fr/q6+udxwwODqqhoUElJSWSpKKiIiUnJ8fMdHd36+jRo84MAAC4sMX1HJSMjAwVFBTE7EtPT1d2drazPxgMqrq6Wvn5+crPz1d1dbXS0tK0ZMkSSZLX69WyZcu0du1aZWdnKysrS+vWrVNhYeGIJ90CAIALU9xPkv0869ev18DAgFasWKHe3l4VFxfr0KFDysjIcGa2b98ut9utRYsWaWBgQLNnz9bu3buVlJQ02ssBAABjkMsYYxK9iHhFIhF5vV6Fw2Gej/J/pm48mOglXBCO1VYmegkAMGbF8/ubz+IBAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANaJK1B27dql6dOnKzMzU5mZmZo5c6b+9Kc/OceNMdqyZYsCgYBSU1NVVlam9vb2mO8RjUZVVVWlnJwcpaena8GCBerq6hqdswEAAONCXIEyefJk1dbW6pVXXtErr7yiG2+8Ud/97nedCNm6dau2bdumnTt3qrm5WX6/X3PmzFFfX5/zPYLBoA4cOKC6ujo1Njaqv79f8+fP19DQ0OieGQAAGLNcxhhzLt8gKytLP/nJT3T33XcrEAgoGAxqw4YNkj65W+Lz+fTII49o+fLlCofDuuSSS7Rnzx4tXrxYknTixAnl5ubq2Wef1dy5c7/Qz4xEIvJ6vQqHw8rMzDyX5Y8bUzceTPQSLgjHaisTvQQAGLPi+f191s9BGRoaUl1dnT744APNnDlTHR0dCoVCqqiocGY8Ho9KS0vV1NQkSWppadHJkydjZgKBgAoKCpwZAAAAd7wPaGtr08yZM/XRRx/p4osv1oEDB3TVVVc5geHz+WLmfT6f3n77bUlSKBRSSkqKJk6cOGImFAqd8WdGo1FFo1Hn60gkEu+yAQDAGBL3HZSvfe1ram1t1UsvvaR7771XS5cu1b/+9S/nuMvlipk3xozYN9znzdTU1Mjr9Tpbbm5uvMsGAABjSNyBkpKSoq9+9auaMWOGampqdM011+hnP/uZ/H6/JI24E9LT0+PcVfH7/RocHFRvb+8ZZ05n06ZNCofDztbZ2RnvsgEAwBhyzu+DYoxRNBpVXl6e/H6/6uvrnWODg4NqaGhQSUmJJKmoqEjJyckxM93d3Tp69Kgzczoej8d5afOnGwAAGL/ieg7Kfffdp3nz5ik3N1d9fX2qq6vTiy++qOeee04ul0vBYFDV1dXKz89Xfn6+qqurlZaWpiVLlkiSvF6vli1bprVr1yo7O1tZWVlat26dCgsLVV5efl5OEAAAjD1xBcp//vMf3XHHHeru7pbX69X06dP13HPPac6cOZKk9evXa2BgQCtWrFBvb6+Ki4t16NAhZWRkON9j+/btcrvdWrRokQYGBjR79mzt3r1bSUlJo3tmAABgzDrn90FJBN4HZSTeB+XLwfugAMDZ+1LeBwUAAOB8IVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYx53oBQBjydSNBxO9hLgdq61M9BIAIG7cQQEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdPovnNMbi560AADCecAcFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFgnrkCpqanRddddp4yMDE2aNEk333yzXnvttZgZY4y2bNmiQCCg1NRUlZWVqb29PWYmGo2qqqpKOTk5Sk9P14IFC9TV1XXuZwMAAMaFuAKloaFBK1eu1EsvvaT6+np9/PHHqqio0AcffODMbN26Vdu2bdPOnTvV3Nwsv9+vOXPmqK+vz5kJBoM6cOCA6urq1NjYqP7+fs2fP19DQ0Ojd2YAAGDMchljzNk++L///a8mTZqkhoYGXX/99TLGKBAIKBgMasOGDZI+uVvi8/n0yCOPaPny5QqHw7rkkku0Z88eLV68WJJ04sQJ5ebm6tlnn9XcuXM/9+dGIhF5vV6Fw2FlZmae7fLPiPdBwXhyrLYy0UsAAEnx/f4+p+eghMNhSVJWVpYkqaOjQ6FQSBUVFc6Mx+NRaWmpmpqaJEktLS06efJkzEwgEFBBQYEzM1w0GlUkEonZAADA+HXWgWKM0Zo1azRr1iwVFBRIkkKhkCTJ5/PFzPp8PudYKBRSSkqKJk6ceMaZ4WpqauT1ep0tNzf3bJcNAADGgLMOlFWrVunVV1/Vr3/96xHHXC5XzNfGmBH7hvusmU2bNikcDjtbZ2fn2S4bAACMAWcVKFVVVXrmmWf0wgsvaPLkyc5+v98vSSPuhPT09Dh3Vfx+vwYHB9Xb23vGmeE8Ho8yMzNjNgAAMH7FFSjGGK1atUr79+/X888/r7y8vJjjeXl58vv9qq+vd/YNDg6qoaFBJSUlkqSioiIlJyfHzHR3d+vo0aPODAAAuLDF9WnGK1eu1L59+/T73/9eGRkZzp0Sr9er1NRUuVwuBYNBVVdXKz8/X/n5+aqurlZaWpqWLFnizC5btkxr165Vdna2srKytG7dOhUWFqq8vHz0zxAAAIw5cQXKrl27JEllZWUx+5988kndddddkqT169drYGBAK1asUG9vr4qLi3Xo0CFlZGQ489u3b5fb7daiRYs0MDCg2bNna/fu3UpKSjq3swEAAOPCOb0PSqLwPijAF8f7oACwxZf2PigAAADnA4ECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOu5ELwDA+TV148FELyFux2orE70EAAnGHRQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYJ24A+Wvf/2rbrrpJgUCAblcLv3ud7+LOW6M0ZYtWxQIBJSamqqysjK1t7fHzESjUVVVVSknJ0fp6elasGCBurq6zulEAADA+BF3oHzwwQe65pprtHPnztMe37p1q7Zt26adO3equblZfr9fc+bMUV9fnzMTDAZ14MAB1dXVqbGxUf39/Zo/f76GhobO/kwAAMC44Y73AfPmzdO8efNOe8wYox07dmjz5s1auHChJOmpp56Sz+fTvn37tHz5coXDYT3xxBPas2ePysvLJUl79+5Vbm6uDh8+rLlz557D6QAAgPFgVJ+D0tHRoVAopIqKCmefx+NRaWmpmpqaJEktLS06efJkzEwgEFBBQYEzM1w0GlUkEonZAADA+DWqgRIKhSRJPp8vZr/P53OOhUIhpaSkaOLEiWecGa6mpkZer9fZcnNzR3PZAADAMuflVTwulyvma2PMiH3DfdbMpk2bFA6Hna2zs3PU1goAAOwzqoHi9/slacSdkJ6eHueuit/v1+DgoHp7e884M5zH41FmZmbMBgAAxq9RDZS8vDz5/X7V19c7+wYHB9XQ0KCSkhJJUlFRkZKTk2Nmuru7dfToUWcGAABc2OJ+FU9/f7/efPNN5+uOjg61trYqKytLU6ZMUTAYVHV1tfLz85Wfn6/q6mqlpaVpyZIlkiSv16tly5Zp7dq1ys7OVlZWltatW6fCwkLnVT0AAODCFnegvPLKK7rhhhucr9esWSNJWrp0qXbv3q3169drYGBAK1asUG9vr4qLi3Xo0CFlZGQ4j9m+fbvcbrcWLVqkgYEBzZ49W7t371ZSUtIonBIAABjrXMYYk+hFxCsSicjr9SocDp+X56NM3Xhw1L8ngC/uWG1lopcA4DyI5/c3n8UDAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADrECgAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOgQKAACwDoECAACsQ6AAAADruBO9AAAYburGg4leQtyO1VYmegnAuMIdFAAAYB0CBQAAWIdAAQAA1iFQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANbhs3gAYBSMxc8PkvgMIdiLOygAAMA6BAoAALAOgQIAAKxDoAAAAOsQKAAAwDoECgAAsA6BAgAArEOgAAAA6xAoAADAOryTLABcwMbiO+Dy7rcXBu6gAAAA6xAoAADAOgQKAACwDoECAACsk9BA+fnPf668vDxNmDBBRUVF+tvf/pbI5QAAAEskLFCefvppBYNBbd68Wf/4xz/07W9/W/PmzdPx48cTtSQAAGAJlzHGJOIHFxcX6xvf+IZ27drl7Lvyyit18803q6am5jMfG4lE5PV6FQ6HlZmZOeprG4svuwMAYDSdj5dzx/P7OyHvgzI4OKiWlhZt3LgxZn9FRYWamppGzEejUUWjUefrcDgs6ZMTPR9ORT88L98XAICx4nz8jv30e36ReyMJCZR3331XQ0ND8vl8Mft9Pp9CodCI+ZqaGj3wwAMj9ufm5p63NQIAcCHz7jh/37uvr09er/czZxL6TrIulyvma2PMiH2StGnTJq1Zs8b5+tSpU3r//feVnZ192nmMFIlElJubq87OzvPyZzGMDq6T/bhGYwPXyU7GGPX19SkQCHzubEICJScnR0lJSSPulvT09Iy4qyJJHo9HHo8nZt9XvvKV87nEcSszM5N/rGMA18l+XKOxgetkn8+7c/KphLyKJyUlRUVFRaqvr4/ZX19fr5KSkkQsCQAAWCRhf+JZs2aN7rjjDs2YMUMzZ87UY489puPHj+uee+5J1JIAAIAlEhYoixcv1nvvvacHH3xQ3d3dKigo0LPPPqvLL788UUsa1zwej+6///4RfyqDXbhO9uMajQ1cp7EvYe+DAgAAcCZ8Fg8AALAOgQIAAKxDoAAAAOsQKAAAwDoEyhhWU1Oj6667ThkZGZo0aZJuvvlmvfbaazEzxhht2bJFgUBAqampKisrU3t7e8xMNBpVVVWVcnJylJ6ergULFqirq+vLPJULRk1NjVwul4LBoLOPa2SHd955R7fffruys7OVlpamr3/962ppaXGOc50S6+OPP9aPf/xj5eXlKTU1VdOmTdODDz6oU6dOOTNco3HGYMyaO3euefLJJ83Ro0dNa2urqaysNFOmTDH9/f3OTG1trcnIyDC//e1vTVtbm1m8eLG59NJLTSQScWbuuecec9lll5n6+npz5MgRc8MNN5hrrrnGfPzxx4k4rXHr5ZdfNlOnTjXTp083q1evdvZzjRLv/fffN5dffrm56667zN///nfT0dFhDh8+bN58801nhuuUWA899JDJzs42f/zjH01HR4f5zW9+Yy6++GKzY8cOZ4ZrNL4QKONIT0+PkWQaGhqMMcacOnXK+P1+U1tb68x89NFHxuv1ml/84hfGGGP+97//meTkZFNXV+fMvPPOO+aiiy4yzz333Jd7AuNYX1+fyc/PN/X19aa0tNQJFK6RHTZs2GBmzZp1xuNcp8SrrKw0d999d8y+hQsXmttvv90YwzUaj/gTzzgSDoclSVlZWZKkjo4OhUIhVVRUODMej0elpaVqamqSJLW0tOjkyZMxM4FAQAUFBc4Mzt3KlStVWVmp8vLymP1cIzs888wzmjFjhr7//e9r0qRJuvbaa/X44487x7lOiTdr1iz95S9/0euvvy5J+uc//6nGxkZ95zvfkcQ1Go8S+mnGGD3GGK1Zs0azZs1SQUGBJDkfxjj8Axh9Pp/efvttZyYlJUUTJ04cMTP8wxxxdurq6nTkyBE1NzePOMY1ssNbb72lXbt2ac2aNbrvvvv08ssv6wc/+IE8Ho/uvPNOrpMFNmzYoHA4rCuuuEJJSUkaGhrSww8/rFtvvVUS/5bGIwJlnFi1apVeffVVNTY2jjjmcrlivjbGjNg33BeZwefr7OzU6tWrdejQIU2YMOGMc1yjxDp16pRmzJih6upqSdK1116r9vZ27dq1S3feeaczx3VKnKefflp79+7Vvn37dPXVV6u1tVXBYFCBQEBLly515rhG4wd/4hkHqqqq9Mwzz+iFF17Q5MmTnf1+v1+SRvyfQU9Pj/N/GX6/X4ODg+rt7T3jDM5eS0uLenp6VFRUJLfbLbfbrYaGBj366KNyu93Of2OuUWJdeumluuqqq2L2XXnllTp+/Lgk/i3Z4Ec/+pE2btyoW265RYWFhbrjjjv0wx/+UDU1NZK4RuMRgTKGGWO0atUq7d+/X88//7zy8vJijufl5cnv96u+vt7ZNzg4qIaGBpWUlEiSioqKlJycHDPT3d2to0ePOjM4e7Nnz1ZbW5taW1udbcaMGbrtttvU2tqqadOmcY0s8K1vfWvES/Rff/1158NL+beUeB9++KEuuij2V1ZSUpLzMmOu0TiUqGfn4tzde++9xuv1mhdffNF0d3c724cffujM1NbWGq/Xa/bv32/a2trMrbfeetqX3U2ePNkcPnzYHDlyxNx444287O48+v9fxWMM18gGL7/8snG73ebhhx82b7zxhvnVr35l0tLSzN69e50ZrlNiLV261Fx22WXOy4z3799vcnJyzPr1650ZrtH4QqCMYZJOuz355JPOzKlTp8z9999v/H6/8Xg85vrrrzdtbW0x32dgYMCsWrXKZGVlmdTUVDN//nxz/PjxL/lsLhzDA4VrZIc//OEPpqCgwHg8HnPFFVeYxx57LOY41ymxIpGIWb16tZkyZYqZMGGCmTZtmtm8ebOJRqPODNdofHEZY0wi7+AAAAAMx3NQAACAdQgUAABgHQIFAABYh0ABAADWIVAAAIB1CBQAAGAdAgUAAFiHQAEAANYhUAAAgHUIFAAAYB0CBQAAWIdAAQAA1vl/8k4lg3MtzZUAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Sample size distribution\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "data = [len(tok) for tok in (dataset_tokenized[\"train\"][\"input_ids\"]+dataset_tokenized[\"test\"][\"input_ids\"])] \n",
    "print(f\"longest sample: {max(data)} tokens\")\n",
    "\n",
    "plt.hist(data, bins=10)  \n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "91d98033-b61d-43ad-94b1-03bdbd2b1b87",
   "metadata": {
    "id": "91d98033-b61d-43ad-94b1-03bdbd2b1b87"
   },
   "source": [
    "## Define a collate function, train on answers only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "6039dd03-6e1c-4b21-b16e-1694b2d69c2b",
   "metadata": {
    "executionInfo": {
     "elapsed": 446,
     "status": "ok",
     "timestamp": 1703794789126,
     "user": {
      "displayName": "Gerald Stampfel",
      "userId": "16044924958378286623"
     },
     "user_tz": -60
    },
    "id": "6039dd03-6e1c-4b21-b16e-1694b2d69c2b"
   },
   "outputs": [],
   "source": [
    "# collate function - to transform list of dictionaries [ {input_ids: [123, ..]}, {.. ] to single batch dictionary { input_ids: [..], labels: [..], attention_mask: [..] }\n",
    "def collate(elements):\n",
    "    tokens=[e[\"input_ids\"] for e in elements]\n",
    "    tokens_maxlen=max([len(t) for t in tokens])\n",
    "\n",
    "    for i,sample in enumerate(elements):\n",
    "        input_ids=sample[\"input_ids\"]\n",
    "        labels=sample[\"labels\"]\n",
    "        attention_mask=sample[\"attention_mask\"]\n",
    "\n",
    "        pad_len=tokens_maxlen-len(input_ids)\n",
    "\n",
    "        input_ids.extend( pad_len * [tokenizer.pad_token_id] )   \n",
    "        labels.extend( pad_len * [IGNORE_INDEX] )    \n",
    "        attention_mask.extend( pad_len * [0] ) \n",
    "\n",
    "    batch={\n",
    "        \"input_ids\": torch.tensor( [e[\"input_ids\"] for e in elements] ),\n",
    "        \"labels\": torch.tensor( [e[\"labels\"] for e in elements] ),\n",
    "        \"attention_mask\": torch.tensor( [e[\"attention_mask\"] for e in elements] ),\n",
    "    }\n",
    "\n",
    "    return batch"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93877bc2-7d12-45b5-b3d3-cbd5c39d2e9d",
   "metadata": {
    "id": "93877bc2-7d12-45b5-b3d3-cbd5c39d2e9d"
   },
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2c7dc69d-fa8f-437d-a75d-a61a239af2c0",
   "metadata": {
    "id": "2c7dc69d-fa8f-437d-a75d-a61a239af2c0"
   },
   "source": [
    "## Set hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "11fd3c0e-4c4f-4333-bddf-11a28b47d8ec",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 429
    },
    "executionInfo": {
     "elapsed": 7,
     "status": "error",
     "timestamp": 1703794942550,
     "user": {
      "displayName": "Gerald Stampfel",
      "userId": "16044924958378286623"
     },
     "user_tz": -60
    },
    "id": "11fd3c0e-4c4f-4333-bddf-11a28b47d8ec",
    "outputId": "df253027-2128-48a0-b4e9-d7f9ed4c5bd8"
   },
   "outputs": [],
   "source": [
    "from transformers import TrainingArguments, Trainer\n",
    "\n",
    "# dataset-specific parameters\n",
    "bs=1     # batch size for training\n",
    "bs_eval=16    # batch size for evaluation\n",
    "ga_steps=16  # gradient accumulation steps\n",
    "lr=0.00002  # learning rate\n",
    "epochs=20\n",
    "\n",
    "steps_per_epoch=len(dataset_tokenized[\"train\"])//(bs*ga_steps)\n",
    "\n",
    "args = TrainingArguments(\n",
    "    output_dir=\"out\",\n",
    "    per_device_train_batch_size=bs,\n",
    "    per_device_eval_batch_size=bs_eval,\n",
    "    evaluation_strategy=\"steps\",\n",
    "    logging_steps=1,\n",
    "    eval_steps=steps_per_epoch//2,    # 2 evals per epoch\n",
    "    save_steps=steps_per_epoch,     # save once per epoch\n",
    "    gradient_accumulation_steps=ga_steps,\n",
    "    num_train_epochs=epochs,\n",
    "    lr_scheduler_type=\"constant\",\n",
    "    optim=\"paged_adamw_32bit\",      # val_loss will go nan with paged_adamw_8bit\n",
    "    learning_rate=lr,\n",
    "    group_by_length=False,\n",
    "    bf16=True,        \n",
    "    ddp_find_unused_parameters=False,\n",
    ")\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    tokenizer=tokenizer,\n",
    "    args=args,\n",
    "    data_collator=collate,\n",
    "    train_dataset=dataset_tokenized[\"train\"],\n",
    "    eval_dataset=dataset_tokenized[\"test\"],\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9719ac8a-f20d-48a0-a4b6-bdb8778ebc4b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# log to a specific wandb project\n",
    "import wandb \n",
    "run = wandb.init(\n",
    "    project=\"phi2\",\n",
    "    name=\"testrun\",\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7e5c5798-747b-434a-9fac-08cf241420ac",
   "metadata": {
    "id": "7e5c5798-747b-434a-9fac-08cf241420ac"
   },
   "source": [
    "## Start training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9a482f3-eec8-40f7-9382-af6538ab51c0",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 552,
     "status": "ok",
     "timestamp": 1703794975434,
     "user": {
      "displayName": "Gerald Stampfel",
      "userId": "16044924958378286623"
     },
     "user_tz": -60
    },
    "id": "c9a482f3-eec8-40f7-9382-af6538ab51c0",
    "outputId": "e75ab87d-f0dc-4d94-fecf-e5e963cae1a7"
   },
   "outputs": [],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "88faaf3c-5910-4c36-bc75-f65e2fdf39ea",
   "metadata": {},
   "source": [
    "# After training: Merge LoRA adapters with base model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6e8a208e-12a3-4b2c-8d02-793a9023b585",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a8508edc33f0469b8abde04cee7c9545",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModelForCausalLM, AutoTokenizer, GenerationConfig\n",
    "from peft import PeftModel\n",
    "import torch\n",
    "\n",
    "# base model\n",
    "base_path=\"microsoft/phi-2\"  \n",
    "\n",
    "# adapters: path to folder with adapter_model.safetensors\n",
    "adapter_path=\"out/checkpoint-1880\" \n",
    "\n",
    "# where to save merged model\n",
    "save_to=\"trained_model\"       \n",
    "\n",
    "# Load model and tokenizer\n",
    "base_model = AutoModelForCausalLM.from_pretrained(\n",
    "    base_path,\n",
    "    torch_dtype=torch.bfloat16,\n",
    "    device_map=\"auto\",\n",
    ")\n",
    "tokenizer = AutoTokenizer.from_pretrained(base_path)\n",
    "\n",
    "# Add/set tokens same tokens to base model before merging, like we did before training  \n",
    "tokenizer.add_tokens([\"<|im_start|>\", \"<PAD>\"])\n",
    "tokenizer.pad_token = \"<PAD>\"\n",
    "tokenizer.add_special_tokens(dict(eos_token=\"<|im_end|>\"))\n",
    "\n",
    "# Add ChatML template to tokenizer\n",
    "tokenizer.chat_template=\"{% if not add_generation_prompt is defined %}{% set add_generation_prompt = false %}{% endif %}{% for message in messages %}{{'<|im_start|>' + message['role'] + '\\n' + message['content'] + '<|im_end|>' + '\\n'}}{% endfor %}{% if add_generation_prompt %}{{ '<|im_start|>assistant\\n' }}{% endif %}\"\n",
    "\n",
    "base_model.config.eos_token_id = tokenizer.eos_token_id\n",
    "\n",
    "# Set a default Generation configuration: Llama precise\n",
    "generation_config = GenerationConfig(\n",
    "    max_new_tokens=100, \n",
    "    temperature=0.7,\n",
    "    top_p=0.1,\n",
    "    top_k=40,\n",
    "    repetition_penalty=1.18,\n",
    "    do_sample=True,\n",
    "    pad_token_id=tokenizer.pad_token_id,\n",
    "    eos_token_id=tokenizer.eos_token_id,\n",
    ")\n",
    "\n",
    "# Load LoRA and merge\n",
    "model = PeftModel.from_pretrained(base_model, adapter_path)\n",
    "model = model.merge_and_unload()\n",
    "\n",
    "model.save_pretrained(save_to, safe_serialization=True, max_shard_size='4GB')\n",
    "tokenizer.save_pretrained(save_to)\n",
    "generation_config.save_pretrained(save_to)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0e4e8796-3a99-43b0-8f96-7b4cc74c223e",
   "metadata": {},
   "source": [
    "# Inference of trained model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c4b5e41a-638c-4f59-9cfb-e36bcb0d6d68",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a00ac372b1e4420095baa6f2749697ba",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I am an artificial intelligence language model designed to help answer questions and provide information. I don't have a specific identity or creator like humans do. If there's any particular question you'd like me to help with, feel free to ask!\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModelForCausalLM, AutoTokenizer, GenerationConfig\n",
    "from peft import PeftModel\n",
    "import torch\n",
    "\n",
    "model_path=\"trained_model\"    \n",
    "question=\"Hello, who are you?\"\n",
    "\n",
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "        model_path,    \n",
    "        torch_dtype=torch.bfloat16,\n",
    "        device_map=\"auto\"\n",
    "    )\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_path) \n",
    "\n",
    "messages=[\n",
    "    {\"role\": \"user\", \"content\": question}\n",
    "]\n",
    "        \n",
    "input_tokens = tokenizer.apply_chat_template(\n",
    "    messages, \n",
    "    add_generation_prompt=True,\n",
    "    return_tensors=\"pt\"\n",
    ").to(\"cuda\")\n",
    "\n",
    "output_tokens = model.generate(input_tokens)\n",
    "output = tokenizer.decode(\n",
    "    output_tokens[0][len(input_tokens[0]):],\n",
    "    skip_special_tokens=True\n",
    "    )               \n",
    "\n",
    "print(output)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4ea9aeaf-2b9f-4249-abcb-07a7b9fdd3a4",
   "metadata": {},
   "source": [
    "# Upload model to hub"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "960101ae-92ae-46de-bf9c-91ff4c437708",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "7492fd20abb64e318681425bbdb25085",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import torch\n",
    "from transformers import AutoModelForCausalLM, AutoTokenizer\n",
    "from transformers import \n",
    "\n",
    "# Load model and tokenizer\n",
    "modelpath=\"trained_model\"\n",
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "    modelpath,\n",
    "    device_map=\"auto\",\n",
    "    torch_dtype=torch.bfloat16,\n",
    ")\n",
    "tokenizer = AutoTokenizer.from_pretrained(modelpath) \n",
    "\n",
    "# Upload\n",
    "model.push_to_hub(\"g-ronimo/phi-2_riddles-evolved\")\n",
    "tokenizer.push_to_hub(\"g-ronimo/phi-2_riddles-evolved\")"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "provenance": [
    {
     "file_id": "1M5C-KyytevRhvLynyce05PmvhO4CZ6d8",
     "timestamp": 1703795350307
    }
   ]
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "0118c3dfb3f9410dbaedd2bc39326b8f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_86ce00dc99ff42789a512c49ab2ee7cb",
      "max": 2114924,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_d9d02387630f46fc96c25ec0cecaf410",
      "value": 2114924
     }
    },
    "032ab17b00a1470eb7d8d9a2489e30bc": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "0468fe85167642ed830f43fdf3235503": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "05a8c12b4e6f47428baf6cd7929ab314": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9abdbe87d2fd4be49ec86d64828deacf",
      "placeholder": "​",
      "style": "IPY_MODEL_3a3ec4d9fa3b462cb314c08d68daf5c5",
      "value": "special_tokens_map.json: 100%"
     }
    },
    "0c94a07bbcae4cbfa7fc800236b79a65": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "0ee67c97a76c453397dbd04966a19681": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "11cdc86b257b43aeac774f9e4071f329": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "11e89b669aa64cc4970cf0534d0437c5": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "11ef5cd3633a41d2a016db4afd74ff67": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_5e97ff18273448c5b50a9564617b0e96",
      "placeholder": "​",
      "style": "IPY_MODEL_77643d235c9543eb8346710077c7500c",
      "value": "Downloading shards: 100%"
     }
    },
    "14f76d243a6c4e2d9d2ab097f8679183": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a4672f7b7dc14add931058afedde1a16",
      "placeholder": "​",
      "style": "IPY_MODEL_a7cae549b7a643dda513ad0ac5eec950",
      "value": "model.safetensors.index.json: 100%"
     }
    },
    "156918fac00741cd8e53f36aa43c7b6c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1b02f5ef26fa4a6bbd384e104802a350",
      "max": 7339,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_ca2eef6af4f447ccbae0da87223f992e",
      "value": 7339
     }
    },
    "1631398755bb4bb6ad5d288320d31fd0": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "18bfc50ea9a24949a92ca09c4dd3e22c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e06a6d8b98d94d378bf1049a1abbcff9",
      "max": 1513,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_dc092383151a42faa5cb76c6ccb1787d",
      "value": 1513
     }
    },
    "18e1e9d387024d4aaa70c594e1f7c0a2": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_bd031d553cba483c8c080c43bf71b4be",
      "max": 1080,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_e5bf775c779247278b9e32403e5d992b",
      "value": 1080
     }
    },
    "198fbb4d1f384105be84d84661eed44d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1ac88c4d9d6e498f9cc19b38a6db4556": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1b02f5ef26fa4a6bbd384e104802a350": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1c1414766d694a179ec5957718b7da40": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "1d9185f34cd44483a332f3a08d8995aa": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "2109ffc0430e4392b17a4e7da4ff2224": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_799d8da4136c4ccb90727c076a67b619",
      "max": 2,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_66c53415d0f4412fb62c5c8ca256feb5",
      "value": 2
     }
    },
    "23239ab0077c45538ffcb5eb2ad97324": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_75ebb53d003b41ebb299ca5c6388ba59",
      "placeholder": "​",
      "style": "IPY_MODEL_b5bfa027a895443ab3f6fbdf688acf94",
      "value": " 2/2 [00:47&lt;00:00, 20.08s/it]"
     }
    },
    "2606f25c83b34cb5b30a7a1e6d03d3db": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_4f7daddffe864bd8b611726df3cac0cb",
      "placeholder": "​",
      "style": "IPY_MODEL_032ab17b00a1470eb7d8d9a2489e30bc",
      "value": " 2.11M/2.11M [00:00&lt;00:00, 29.5MB/s]"
     }
    },
    "267cdeec38644c3db346920c0030221e": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "295662b9afd5401da453e7cc52705464": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_55e4b78d734c4c6796f66848f844e386",
      "placeholder": "​",
      "style": "IPY_MODEL_ab55b9070723401889e4b8941bfd9c7f",
      "value": " 456k/456k [00:00&lt;00:00, 9.04MB/s]"
     }
    },
    "2e8b611b3c814acaa190ab9104ecb8f1": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a02825051f2f47f0b8ff4dba7ed94ab6",
      "placeholder": "​",
      "style": "IPY_MODEL_cbef4f92c1ad464cbef0167a22ad05cd",
      "value": "merges.txt: 100%"
     }
    },
    "31af8b40703d488882f7b009decc28f4": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_ac8869fb8ede4c9aa506f039a7c88b8e",
       "IPY_MODEL_e793f51d5cc14dcfabda21ab93b1cf34",
       "IPY_MODEL_655d90789ce6431d8184ccb0c071d96e"
      ],
      "layout": "IPY_MODEL_791cd85ddea14577a87d5d6a51d1c284"
     }
    },
    "327411b689bb4eedaeb5dd802fb2cf56": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_2e8b611b3c814acaa190ab9104ecb8f1",
       "IPY_MODEL_4b4c5e2231bf4625a16dfb7cbc25fd57",
       "IPY_MODEL_295662b9afd5401da453e7cc52705464"
      ],
      "layout": "IPY_MODEL_e27059389ac045dd9426a76f3ebe0a6d"
     }
    },
    "329733174bab4cdd875614bffe5da0d2": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "35177e251c974b9b9280916dc18c76f0": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_0c94a07bbcae4cbfa7fc800236b79a65",
      "placeholder": "​",
      "style": "IPY_MODEL_a07857b000c64de39609847731041940",
      "value": " 99.0/99.0 [00:00&lt;00:00, 2.48kB/s]"
     }
    },
    "39ab404653ac454dbf06414f00c98b84": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "3a2ae78f09a54411ab19cfdcf1aca0b8": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_dfa9bc6f87e34aa4aabd0d8d2bb91df3",
      "placeholder": "​",
      "style": "IPY_MODEL_d39c73b183a54f64a48e786ff9dcfb8c",
      "value": "vocab.json: 100%"
     }
    },
    "3a3ec4d9fa3b462cb314c08d68daf5c5": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "3b29845189c44e64aff348a56e6ed9ec": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d1b3b6b0e17f4c1bbad206a4115ded8d",
      "max": 576934128,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_78e4588c0f5a43d3a19516d0349dff08",
      "value": 576934128
     }
    },
    "3e6dd8f5317440279b9878061c8f7b08": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_ab3270871c61444caaa89678fd9127aa",
      "placeholder": "​",
      "style": "IPY_MODEL_5b633ee953554c8aa810dd13c474ed1e",
      "value": "model-00001-of-00002.safetensors: 100%"
     }
    },
    "407e2720459d4a47a61264de01e4cd05": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "411a300862f948ffb2d8d5e3fe7ae543": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "47332cf01eeb409bb107dee0c3c3c536": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_cf9114aedcaa42699e0535925370c218",
       "IPY_MODEL_0118c3dfb3f9410dbaedd2bc39326b8f",
       "IPY_MODEL_2606f25c83b34cb5b30a7a1e6d03d3db"
      ],
      "layout": "IPY_MODEL_4749e6e10f834d25a67acf94b3051481"
     }
    },
    "4749e6e10f834d25a67acf94b3051481": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "48a92287432f441f990f3a737c02adf1": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "49fb73c7b6b04f6baea866d68cf2f7ea": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1ac88c4d9d6e498f9cc19b38a6db4556",
      "placeholder": "​",
      "style": "IPY_MODEL_b625b2b63f7e4a27a1054cb557df52f1",
      "value": " 24.3k/24.3k [00:00&lt;00:00, 928kB/s]"
     }
    },
    "4b4c5e2231bf4625a16dfb7cbc25fd57": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_744dab8be6a5432fbf5526c661b82425",
      "max": 456318,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_8d6f4662173342f68010079f8751bc62",
      "value": 456318
     }
    },
    "4d7fd6e5584d42f38fd59f4f25c6ec16": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "4dedc2cb743746468e81d266c06d6eb6": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "4dfbea42c59b405885383178618ef4de": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_198fbb4d1f384105be84d84661eed44d",
      "max": 169,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_267cdeec38644c3db346920c0030221e",
      "value": 169
     }
    },
    "4f7daddffe864bd8b611726df3cac0cb": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "50942dc4e2444918850fe32139035ed4": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "512f9eaa94bc49828ce88423342f6086": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "519ff4745e184522adc93983dcb0e3df": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "520213b9c4004b559a392ebd4396739b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "53ae7440dcc84e21ab09ce375c743b58": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "55e4b78d734c4c6796f66848f844e386": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "56716551ece24c599ae71074905e5e90": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "57ac9644d750448aaf24fbf41c1fdfa8": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "582874d489bd4398824e4c6ec01eae7d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_05a8c12b4e6f47428baf6cd7929ab314",
       "IPY_MODEL_d5a4a0095968417d831db8f2bfa1ad25",
       "IPY_MODEL_35177e251c974b9b9280916dc18c76f0"
      ],
      "layout": "IPY_MODEL_0ee67c97a76c453397dbd04966a19681"
     }
    },
    "598378cadf4245b1a60bc85a6fd4c1a2": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_39ab404653ac454dbf06414f00c98b84",
      "placeholder": "​",
      "style": "IPY_MODEL_519ff4745e184522adc93983dcb0e3df",
      "value": " 798k/798k [00:00&lt;00:00, 9.90MB/s]"
     }
    },
    "5b3bedeb6cfe40d8bd4c6623ac3fac43": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "5b4b59d0ec8842068576af45165d9645": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_811a542292624b6caf5ab9c5e6660d29",
      "placeholder": "​",
      "style": "IPY_MODEL_ed25603c0dc5462c90b50625a37c5c97",
      "value": " 1513/1513 [00:06&lt;00:00, 138.23 examples/s]"
     }
    },
    "5b633ee953554c8aa810dd13c474ed1e": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "5e97ff18273448c5b50a9564617b0e96": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "639355e013c04b43a068c3f8686c22e1": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "655d90789ce6431d8184ccb0c071d96e": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_5b3bedeb6cfe40d8bd4c6623ac3fac43",
      "placeholder": "​",
      "style": "IPY_MODEL_1d9185f34cd44483a332f3a08d8995aa",
      "value": " 69.0/69.0 [00:00&lt;00:00, 5.01kB/s]"
     }
    },
    "65722027d7ac4c89a698875db248c0ff": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_8e29afb281fe46b68e50b137a7e93fa2",
      "max": 24294,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_50942dc4e2444918850fe32139035ed4",
      "value": 24294
     }
    },
    "664a68ed4e3640acaa22c65051f8da68": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "66c53415d0f4412fb62c5c8ca256feb5": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "6d81157c81384b51819335b6d7022d53": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9dc37f312cc24806980118893311d200",
      "max": 798156,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_520213b9c4004b559a392ebd4396739b",
      "value": 798156
     }
    },
    "6e470b8541204e63b79794b33ee081b6": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "6ed3e52e2495498c90a39252d89bae30": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7240723e9f68485b939d2a3c17df4d28": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_512f9eaa94bc49828ce88423342f6086",
      "placeholder": "​",
      "style": "IPY_MODEL_4dedc2cb743746468e81d266c06d6eb6",
      "value": "Map (num_proc=2): 100%"
     }
    },
    "744dab8be6a5432fbf5526c661b82425": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "75ebb53d003b41ebb299ca5c6388ba59": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "762fb2dea76b4adebc53be598e904b99": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_cec6322f32e44a0c8ff445e0d8d33e1b",
       "IPY_MODEL_4dfbea42c59b405885383178618ef4de",
       "IPY_MODEL_f20f02632b2a4d41b9b1f7159aff336e"
      ],
      "layout": "IPY_MODEL_99d94f8392c242f084bd6e1c4eb6542a"
     }
    },
    "77643d235c9543eb8346710077c7500c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "780db1e0d21c4e7f920e93b79da7e8c3": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9e616d74e28941d1bca1421dbb88cc37",
      "placeholder": "​",
      "style": "IPY_MODEL_411a300862f948ffb2d8d5e3fe7ae543",
      "value": "added_tokens.json: 100%"
     }
    },
    "78e4588c0f5a43d3a19516d0349dff08": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "791cd85ddea14577a87d5d6a51d1c284": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "795ff049af254445866f3a6f9098a4c5": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1631398755bb4bb6ad5d288320d31fd0",
      "placeholder": "​",
      "style": "IPY_MODEL_ba4fc49b2bf14b69b1a430665510f848",
      "value": " 2/2 [00:29&lt;00:00, 12.39s/it]"
     }
    },
    "799d8da4136c4ccb90727c076a67b619": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "7c15a12341584fc99ce7fff3983909dc": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "811a542292624b6caf5ab9c5e6660d29": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "83312456189b4576a2062fdb9055cf49": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "86ce00dc99ff42789a512c49ab2ee7cb": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "88d9fb7556a343278b09e72ab3e7d39a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "8917e9e09611490b8e326471b6d3d040": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "8960ee82549c4841b0d8ad1f64167e2c": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "89ac160bdb66433c9e07dec69ef9daf5": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_1c1414766d694a179ec5957718b7da40",
      "placeholder": "​",
      "style": "IPY_MODEL_d773d2e8dcdb4982839470462880d172",
      "value": "model-00002-of-00002.safetensors: 100%"
     }
    },
    "8d6f4662173342f68010079f8751bc62": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "8e29afb281fe46b68e50b137a7e93fa2": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9431ae1271844e709a27bcd19252bdcb": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "99d94f8392c242f084bd6e1c4eb6542a": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9abdbe87d2fd4be49ec86d64828deacf": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9b9bd64660dc45eabd21a23bb9c00502": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "9cee4195fc014b84b10bf84ea2268457": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9dc37f312cc24806980118893311d200": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9e616d74e28941d1bca1421dbb88cc37": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "9f5c7f8cb87e4025873c2a39b1ddb93a": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_c93626da431a463db581c12b5669cda9",
      "placeholder": "​",
      "style": "IPY_MODEL_e3b7a45dc2cc4ea188966f76674d7de1",
      "value": "tokenizer_config.json: 100%"
     }
    },
    "a02825051f2f47f0b8ff4dba7ed94ab6": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a07857b000c64de39609847731041940": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "a4672f7b7dc14add931058afedde1a16": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a7cae549b7a643dda513ad0ac5eec950": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "a7ea6206547d4cdda1a4b6cb560ed7af": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e6f1b170de014e6dbe9bc382c11a500d",
      "max": 4982467864,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_639355e013c04b43a068c3f8686c22e1",
      "value": 4982467864
     }
    },
    "a7efe1c651c144d3b7d964dd4313c35b": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ab3270871c61444caaa89678fd9127aa": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ab55b9070723401889e4b8941bfd9c7f": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "ac8869fb8ede4c9aa506f039a7c88b8e": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_53ae7440dcc84e21ab09ce375c743b58",
      "placeholder": "​",
      "style": "IPY_MODEL_bf49c49372a84c07a77d9bdd732c2f0d",
      "value": "generation_config.json: 100%"
     }
    },
    "ad012b1891424905b7f18ff2d3ca8702": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_11ef5cd3633a41d2a016db4afd74ff67",
       "IPY_MODEL_d7e1f1088e6f4422a977d89e4f2fd212",
       "IPY_MODEL_23239ab0077c45538ffcb5eb2ad97324"
      ],
      "layout": "IPY_MODEL_afbcdd6f1f864744a426d80dc6658fd8"
     }
    },
    "af24999c54c44fa49799569fa85ad72c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_56716551ece24c599ae71074905e5e90",
      "placeholder": "​",
      "style": "IPY_MODEL_b06ce6f27066467591cce3b512fad9d7",
      "value": " 7.34k/7.34k [00:00&lt;00:00, 152kB/s]"
     }
    },
    "afbcdd6f1f864744a426d80dc6658fd8": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "b06ce6f27066467591cce3b512fad9d7": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "b5bfa027a895443ab3f6fbdf688acf94": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "b625b2b63f7e4a27a1054cb557df52f1": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "b8d5a4d111e248008be34e8859de2539": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_da950b8eef6e435982f6f667dd7f7fc5",
       "IPY_MODEL_2109ffc0430e4392b17a4e7da4ff2224",
       "IPY_MODEL_795ff049af254445866f3a6f9098a4c5"
      ],
      "layout": "IPY_MODEL_7c15a12341584fc99ce7fff3983909dc"
     }
    },
    "ba4fc49b2bf14b69b1a430665510f848": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "bd031d553cba483c8c080c43bf71b4be": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "bf49c49372a84c07a77d9bdd732c2f0d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "c0df8e1d80304432b14bf2fb8acaf2e2": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d63e23a364ef433b80155e9dcb447e6d",
      "placeholder": "​",
      "style": "IPY_MODEL_9b9bd64660dc45eabd21a23bb9c00502",
      "value": " 577M/577M [00:03&lt;00:00, 160MB/s]"
     }
    },
    "c93626da431a463db581c12b5669cda9": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ca2eef6af4f447ccbae0da87223f992e": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "ca7221018dc446c1b9e0549b91eda07a": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "cbef4f92c1ad464cbef0167a22ad05cd": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "cec6322f32e44a0c8ff445e0d8d33e1b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_a7efe1c651c144d3b7d964dd4313c35b",
      "placeholder": "​",
      "style": "IPY_MODEL_664a68ed4e3640acaa22c65051f8da68",
      "value": "Map (num_proc=2): 100%"
     }
    },
    "cf9114aedcaa42699e0535925370c218": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_eaeed24afb9b47fb864d5e8776947465",
      "placeholder": "​",
      "style": "IPY_MODEL_88d9fb7556a343278b09e72ab3e7d39a",
      "value": "tokenizer.json: 100%"
     }
    },
    "d1b3b6b0e17f4c1bbad206a4115ded8d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d339ab1734fb4f38bd5149959795dd77": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "d39c73b183a54f64a48e786ff9dcfb8c": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "d5a4a0095968417d831db8f2bfa1ad25": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_57ac9644d750448aaf24fbf41c1fdfa8",
      "max": 99,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_329733174bab4cdd875614bffe5da0d2",
      "value": 99
     }
    },
    "d63e23a364ef433b80155e9dcb447e6d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d773d2e8dcdb4982839470462880d172": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "d7e1f1088e6f4422a977d89e4f2fd212": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9431ae1271844e709a27bcd19252bdcb",
      "max": 2,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_d339ab1734fb4f38bd5149959795dd77",
      "value": 2
     }
    },
    "d951753bb05e45d584ff7699f9c415bd": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "d9d02387630f46fc96c25ec0cecaf410": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "da950b8eef6e435982f6f667dd7f7fc5": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_8960ee82549c4841b0d8ad1f64167e2c",
      "placeholder": "​",
      "style": "IPY_MODEL_48a92287432f441f990f3a737c02adf1",
      "value": "Loading checkpoint shards: 100%"
     }
    },
    "dc092383151a42faa5cb76c6ccb1787d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "dc6a36ff57f54fc3b8b4152b668bf55b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_9f5c7f8cb87e4025873c2a39b1ddb93a",
       "IPY_MODEL_156918fac00741cd8e53f36aa43c7b6c",
       "IPY_MODEL_af24999c54c44fa49799569fa85ad72c"
      ],
      "layout": "IPY_MODEL_8917e9e09611490b8e326471b6d3d040"
     }
    },
    "dfa9bc6f87e34aa4aabd0d8d2bb91df3": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "dfacf3507ed4481b8ff57aba3903f409": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_83312456189b4576a2062fdb9055cf49",
      "placeholder": "​",
      "style": "IPY_MODEL_11e89b669aa64cc4970cf0534d0437c5",
      "value": " 4.98G/4.98G [00:43&lt;00:00, 181MB/s]"
     }
    },
    "e06a6d8b98d94d378bf1049a1abbcff9": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e09bb49f89e141579c1ebb3fb7eac98e": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e0ca89b3129b4344913ccf5a99bb51d1": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_3e6dd8f5317440279b9878061c8f7b08",
       "IPY_MODEL_a7ea6206547d4cdda1a4b6cb560ed7af",
       "IPY_MODEL_dfacf3507ed4481b8ff57aba3903f409"
      ],
      "layout": "IPY_MODEL_e6e893f6b07243b69e7cd4a4c5f446fe"
     }
    },
    "e27059389ac045dd9426a76f3ebe0a6d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e3b7a45dc2cc4ea188966f76674d7de1": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "e5bf775c779247278b9e32403e5d992b": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "e6e893f6b07243b69e7cd4a4c5f446fe": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e6f1b170de014e6dbe9bc382c11a500d": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e793f51d5cc14dcfabda21ab93b1cf34": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_e09bb49f89e141579c1ebb3fb7eac98e",
      "max": 69,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_407e2720459d4a47a61264de01e4cd05",
      "value": 69
     }
    },
    "eaeed24afb9b47fb864d5e8776947465": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ed25603c0dc5462c90b50625a37c5c97": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "ed399e0a352a4a789963a6c367f8c8c2": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_7240723e9f68485b939d2a3c17df4d28",
       "IPY_MODEL_18bfc50ea9a24949a92ca09c4dd3e22c",
       "IPY_MODEL_5b4b59d0ec8842068576af45165d9645"
      ],
      "layout": "IPY_MODEL_9cee4195fc014b84b10bf84ea2268457"
     }
    },
    "f20f02632b2a4d41b9b1f7159aff336e": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_f98f7e87395544089fbe9823c1a0fe6c",
      "placeholder": "​",
      "style": "IPY_MODEL_4d7fd6e5584d42f38fd59f4f25c6ec16",
      "value": " 169/169 [00:00&lt;00:00, 220.66 examples/s]"
     }
    },
    "f6a60332a4294512868b9d8bf0bfd8d5": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_780db1e0d21c4e7f920e93b79da7e8c3",
       "IPY_MODEL_18e1e9d387024d4aaa70c594e1f7c0a2",
       "IPY_MODEL_fcdf3351a7214fd796626a5d0b7c3d08"
      ],
      "layout": "IPY_MODEL_ca7221018dc446c1b9e0549b91eda07a"
     }
    },
    "f8818688f7484c65ac575959f7035840": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_3a2ae78f09a54411ab19cfdcf1aca0b8",
       "IPY_MODEL_6d81157c81384b51819335b6d7022d53",
       "IPY_MODEL_598378cadf4245b1a60bc85a6fd4c1a2"
      ],
      "layout": "IPY_MODEL_d951753bb05e45d584ff7699f9c415bd"
     }
    },
    "f9428a8ed0bc4a02829416d5f4dc0add": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_14f76d243a6c4e2d9d2ab097f8679183",
       "IPY_MODEL_65722027d7ac4c89a698875db248c0ff",
       "IPY_MODEL_49fb73c7b6b04f6baea866d68cf2f7ea"
      ],
      "layout": "IPY_MODEL_11cdc86b257b43aeac774f9e4071f329"
     }
    },
    "f98f7e87395544089fbe9823c1a0fe6c": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "fcdf3351a7214fd796626a5d0b7c3d08": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_6e470b8541204e63b79794b33ee081b6",
      "placeholder": "​",
      "style": "IPY_MODEL_0468fe85167642ed830f43fdf3235503",
      "value": " 1.08k/1.08k [00:00&lt;00:00, 27.7kB/s]"
     }
    },
    "fe2014d38632490aa3d34964b4f56292": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_89ac160bdb66433c9e07dec69ef9daf5",
       "IPY_MODEL_3b29845189c44e64aff348a56e6ed9ec",
       "IPY_MODEL_c0df8e1d80304432b14bf2fb8acaf2e2"
      ],
      "layout": "IPY_MODEL_6ed3e52e2495498c90a39252d89bae30"
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}