T5-fine_tuning_with_IMDB_wikisql.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7eea1eae",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sun Jul 16 08:51:26 2023       \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| NVIDIA-SMI 520.61.05    Driver Version: 520.61.05    CUDA Version: 11.8     |\r\n",
      "|-------------------------------+----------------------+----------------------+\r\n",
      "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\r\n",
      "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\r\n",
      "|                               |                      |               MIG M. |\r\n",
      "|===============================+======================+======================|\r\n",
      "|   0  Tesla V100-SXM2...  On   | 00000000:3D:00.0 Off |                    0 |\r\n",
      "| N/A   35C    P0    42W / 300W |      0MiB / 32768MiB |      0%      Default |\r\n",
      "|                               |                      |                  N/A |\r\n",
      "+-------------------------------+----------------------+----------------------+\r\n",
      "                                                                               \r\n",
      "+-----------------------------------------------------------------------------+\r\n",
      "| Processes:                                                                  |\r\n",
      "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\r\n",
      "|        ID   ID                                                   Usage      |\r\n",
      "|=============================================================================|\r\n",
      "|  No running processes found                                                 |\r\n",
      "+-----------------------------------------------------------------------------+\r\n"
     ]
    }
   ],
   "source": [
    "!nvidia-smi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e45abbc4",
   "metadata": {},
   "outputs": [],
   "source": [
    "from pynvml import *\n",
    "import torch.optim as optim\n",
    "\n",
    "\n",
    "def print_gpu_utilization():\n",
    "    nvmlInit()\n",
    "    handle = nvmlDeviceGetHandleByIndex(0)\n",
    "    info = nvmlDeviceGetMemoryInfo(handle)\n",
    "    print(f\"GPU memory occupied: {info.used//1024**2} MB.\")\n",
    "\n",
    "\n",
    "def print_summary(result):\n",
    "    print(f\"Time: {result.metrics['train_runtime']:.2f}\")\n",
    "    print(f\"Samples/second: {result.metrics['train_samples_per_second']:.2f}\")\n",
    "    print_gpu_utilization()\n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4f5f49ea",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GPU memory occupied: 257 MB.\n"
     ]
    }
   ],
   "source": [
    "print_gpu_utilization()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e82c105a",
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"content/t5-small-finetuned-wikisql\"\n",
    "from transformers import AutoTokenizer, T5ForConditionalGeneration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "19d0d613",
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
    "model = T5ForConditionalGeneration.from_pretrained(model_name)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "4ab4ba10",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f8c10ed9",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from datasets import Dataset\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f7083684",
   "metadata": {},
   "outputs": [],
   "source": [
    "df1 = pd.read_csv('Training_set_IMDB/training_set_sample_no_target.csv')\n",
    "df1 = df1.dropna()\n",
    "\n",
    "\n",
    "shuffled_df = df1.sample(frac=1, random_state=42)  # Set random_state for reproducibility\n",
    "\n",
    "train_df, eval_df = train_test_split(shuffled_df, test_size=0.1, random_state=42)\n",
    "\n",
    "\n",
    "text = []\n",
    "sql = []\n",
    "\n",
    "for index, row in train_df.iterrows():\n",
    "    \n",
    "    text_input = \"Translate to SQL: \" + row['Text'],\n",
    "    sql_input = row['SQL']\n",
    "    text.append(text_input)\n",
    "    sql.append(sql_input)\n",
    "    \n",
    "inputs = {\"inputs\": text,\n",
    "        \"target\": sql}\n",
    "\n",
    "train_dataset = Dataset.from_dict(inputs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "b594ea2a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"SELECT r.role AS recent_role FROM actors a INNER JOIN roles r ON a.id = r.actor_id INNER JOIN movies m ON r.movie_id = m.id WHERE a.first_name = 'Pete (I)' AND a.last_name = 'Wilson' ORDER BY m.year DESC LIMIT 1\""
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_dataset['target'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "bbc786cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "text_2 = []\n",
    "sql_2 = []\n",
    "\n",
    "for index, row in eval_df.iterrows():\n",
    "    \n",
    "    text_input = \"Translate to SQL: \" + row['Text'],\n",
    "    sql_input = row['SQL']\n",
    "    text_2.append(text_input)\n",
    "    sql_2.append(sql_input)\n",
    "    \n",
    "inputs_2 = {\"inputs\": text_2,\n",
    "        \"target\": sql_2}\n",
    "eval_dataset = Dataset.from_dict(inputs_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "62665c0e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'inputs': ['Translate to SQL: what is actor Tom (XVI) Moore recent role?'],\n",
       " 'target': \"SELECT r.role AS recent_role FROM actors a INNER JOIN roles r ON a.id = r.actor_id INNER JOIN movies m ON r.movie_id = m.id WHERE a.first_name = 'Tom (XVI)' AND a.last_name = 'Moore' ORDER BY m.year DESC LIMIT 1\"}"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_dataset[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "e24fc70a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map (num_proc=4):   0%|          | 0/10000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def map_to_lenght(x):\n",
    "    x[\"input_len\"] = len(tokenizer(x[\"inputs\"]).input_ids)\n",
    "    x[\"input_longer_256\"] = int(x[\"input_len\"]>256)\n",
    "    x[\"input_longer_128\"] = int(x[\"input_len\"]>128)\n",
    "    x[\"input_longet_64\"] = int(x[\"input_len\"]>64)\n",
    "    x[\"output_len\"] = len(tokenizer(x[\"target\"]).input_ids)\n",
    "    x[\"output_longet_256\"] = int(x[\"output_len\"]>256)\n",
    "    x[\"output_longet_128\"] = int(x[\"output_len\"]>128)\n",
    "    x[\"output_longet_64\"] = int(x[\"output_len\"]>64)\n",
    "    return x\n",
    "\n",
    "sample_size = 10000\n",
    "data_stats = train_dataset.select(range(sample_size)).map(map_to_lenght, num_proc=4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "62d76548",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/10000 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Input mean: 1.0 \n",
      " % of input len > 256: 0.0, \n",
      " % of input len > 128: 0.0, \n",
      " % of input len > 64: 0.0, \n",
      " Ouput mean: 106.4648,\n",
      "% of output len > 256: 0.0, \n",
      "% of output len > 128: 0.2934, \n",
      "% of output len > 64: 0.7241\n"
     ]
    }
   ],
   "source": [
    "def compute_and_print(x):\n",
    "    if len(x[\"input_len\"])==sample_size:\n",
    "        print(\n",
    "            f\"Input mean: {sum(x['input_len'])/sample_size} \\n % of input len > 256: {sum(x['input_longer_256'])/sample_size}, \\n % of input len > 128: {sum(x['input_longer_128'])/sample_size}, \\n % of input len > 64: {sum(x['input_longet_64'])/sample_size}, \\n Ouput mean: {sum(x['output_len'])/sample_size},\\n% of output len > 256: {sum(x['output_longet_256'])/sample_size}, \\n% of output len > 128: {sum(x['output_longet_128'])/sample_size}, \\n% of output len > 64: {sum(x['output_longet_64'])/sample_size}\")\n",
    "\n",
    "output = data_stats.map(compute_and_print, batched=True, batch_size=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a96d88e9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/38756 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/4307 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def convert_to_features(example_batch, padding = \"max_length\"):\n",
    "    inputs = tokenizer.batch_encode_plus(example_batch[\"inputs\"],is_split_into_words=True, max_length=64, padding=\"max_length\", truncation=True)\n",
    "    \n",
    "    targets = tokenizer.batch_encode_plus(example_batch[\"target\"], max_length=256,padding = 'max_length',truncation = True)\n",
    "    if padding == \"max_length\":\n",
    "        targets[\"inputs_ids\"] = [\n",
    "            [(l if l != tokenizer.pad_token_id else -100) for l in target] for target in targets[\"input_ids\"]\n",
    "        ]\n",
    "    \n",
    "    inputs[\"labels\"] = targets['input_ids']\n",
    "    return inputs\n",
    "\n",
    "train_data = train_dataset.map(convert_to_features, batched=True, remove_columns=train_dataset.column_names)\n",
    "test_data = eval_dataset.map(convert_to_features, batched=True, remove_columns=eval_dataset.column_names)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "35546b6f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['input_ids', 'attention_mask', 'labels'],\n",
       "    num_rows: 4307\n",
       "})"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "dd753dfc",
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = ['input_ids', 'attention_mask', 'labels']\n",
    "\n",
    "train_data.set_format(type='torch', columns=columns)\n",
    "test_data.set_format(type='torch', columns=columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "8b9cfd32",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4307"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "f74670fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def sequence_check(list_of_seq):\n",
    "    len_seq = len(list_of_seq[0])\n",
    "    for seq in list_of_seq:\n",
    "        if len(seq)!=len_seq:\n",
    "            print(len(seq))\n",
    "            return False\n",
    "        \n",
    "    return True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "c85fe554",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sequence_check(test_data['input_ids'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "375cbb22",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import Seq2SeqTrainer, Seq2SeqTrainingArguments, DataCollatorForSeq2Seq\n",
    "import os\n",
    "output_dir = 'T5-fine-tuned-with-IMDB-wikisql'\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "8b9a87fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "training_args = Seq2SeqTrainingArguments(\n",
    "    output_dir=output_dir,\n",
    "    per_device_train_batch_size=16,\n",
    "    num_train_epochs=10,\n",
    "    per_device_eval_batch_size=16,\n",
    "    predict_with_generate=True,\n",
    "    evaluation_strategy=\"epoch\",\n",
    "    do_train=True,\n",
    "    learning_rate=2e-4,\n",
    "    weight_decay=0.1,\n",
    "    do_eval=True,\n",
    "    logging_strategy=\"epoch\",\n",
    "    save_strategy=\"epoch\",\n",
    "    overwrite_output_dir=True,\n",
    "    save_total_limit=3,\n",
    "    load_best_model_at_end=True,\n",
    "    push_to_hub=False,\n",
    "    fp16=True \n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "84447030",
   "metadata": {},
   "outputs": [],
   "source": [
    "from evaluate import load\n",
    "rouge = load(\"rouge\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "570e9596",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def compute_metrics(pred):\n",
    "    predictions, labels = pred\n",
    "    \n",
    "    pred_str = tokenizer.batch_decode(predictions, skip_special_tokens=True)\n",
    "    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n",
    "    label_str = tokenizer.batch_decode(labels, skip_special_tokens=True)\n",
    "    \n",
    "    rouge_output = rouge.compute(predictions=pred_str, references=label_str)\n",
    "    return {\n",
    "        \"rouge1\": round(rouge_output['rouge1']* 100,4),\n",
    "        \"rouge2\": round(rouge_output['rouge2']*100, 4),\n",
    "        \"rougeL\": round(rouge_output['rougeL']*100, 4)\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "194641d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_collator = DataCollatorForSeq2Seq(tokenizer, model=model, label_pad_token_id=-100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "c836f916",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer = Seq2SeqTrainer(\n",
    "    model = model,\n",
    "    args=training_args,\n",
    "    compute_metrics=compute_metrics,\n",
    "    train_dataset=train_data,\n",
    "    eval_dataset= test_data,\n",
    "    tokenizer=tokenizer,\n",
    "    data_collator=data_collator\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "64966203",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/toibazd/miniconda3/lib/python3.9/site-packages/transformers/optimization.py:411: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
      "  warnings.warn(\n",
      "You're using a T5TokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='24230' max='24230' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [24230/24230 1:14:36, Epoch 10/10]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Rouge1</th>\n",
       "      <th>Rouge2</th>\n",
       "      <th>Rougel</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.047800</td>\n",
       "      <td>0.006899</td>\n",
       "      <td>39.035500</td>\n",
       "      <td>36.503100</td>\n",
       "      <td>39.045800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.009100</td>\n",
       "      <td>0.005094</td>\n",
       "      <td>39.035500</td>\n",
       "      <td>36.503700</td>\n",
       "      <td>39.044700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.006600</td>\n",
       "      <td>0.003901</td>\n",
       "      <td>39.035500</td>\n",
       "      <td>36.503700</td>\n",
       "      <td>39.044700</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.005200</td>\n",
       "      <td>0.003373</td>\n",
       "      <td>39.035200</td>\n",
       "      <td>36.504000</td>\n",
       "      <td>39.045400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.004200</td>\n",
       "      <td>0.003175</td>\n",
       "      <td>39.035500</td>\n",
       "      <td>36.503100</td>\n",
       "      <td>39.045400</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.003600</td>\n",
       "      <td>0.002929</td>\n",
       "      <td>39.035600</td>\n",
       "      <td>36.504400</td>\n",
       "      <td>39.045800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.003100</td>\n",
       "      <td>0.002774</td>\n",
       "      <td>39.035600</td>\n",
       "      <td>36.504400</td>\n",
       "      <td>39.045800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.002700</td>\n",
       "      <td>0.002746</td>\n",
       "      <td>39.035600</td>\n",
       "      <td>36.504400</td>\n",
       "      <td>39.045800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.002500</td>\n",
       "      <td>0.002736</td>\n",
       "      <td>39.035600</td>\n",
       "      <td>36.504400</td>\n",
       "      <td>39.045800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>0.002300</td>\n",
       "      <td>0.002705</td>\n",
       "      <td>39.035600</td>\n",
       "      <td>36.504400</td>\n",
       "      <td>39.045800</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=24230, training_loss=0.00870360885333582, metrics={'train_runtime': 4481.7871, 'train_samples_per_second': 86.474, 'train_steps_per_second': 5.406, 'total_flos': 6556633572311040.0, 'train_loss': 0.00870360885333582, 'epoch': 10.0})"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "b1e7ff2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "trainer.save_model(output_dir)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "831e1de5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('T5-fine-tuned-with-IMDB-wikisql/tokenizer_config.json',\n",
       " 'T5-fine-tuned-with-IMDB-wikisql/special_tokens_map.json',\n",
       " 'T5-fine-tuned-with-IMDB-wikisql/tokenizer.json')"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.save_pretrained(output_dir)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}