Model_evaluation_standalone.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a9538a02",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2023-07-13 13:50:07,047] [INFO] [real_accelerator.py:110:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "2023-07-13 13:50:20.426990: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 AVX512F FMA\n",
      "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
      "2023-07-13 13:50:20.720251: E tensorflow/stream_executor/cuda/cuda_blas.cc:2981] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
      "2023-07-13 13:50:29.335391: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory\n",
      "2023-07-13 13:50:29.335611: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory\n",
      "2023-07-13 13:50:29.335638: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "PeftModelForSeq2SeqLM(\n",
       "  (base_model): LoraModel(\n",
       "    (model): T5ForConditionalGeneration(\n",
       "      (shared): Embedding(32128, 512)\n",
       "      (encoder): T5Stack(\n",
       "        (embed_tokens): Embedding(32128, 512)\n",
       "        (block): ModuleList(\n",
       "          (0): T5Block(\n",
       "            (layer): ModuleList(\n",
       "              (0): T5LayerSelfAttention(\n",
       "                (SelfAttention): T5Attention(\n",
       "                  (q): Linear(\n",
       "                    in_features=512, out_features=512, bias=False\n",
       "                    (lora_dropout): ModuleDict(\n",
       "                      (default): Dropout(p=0.05, inplace=False)\n",
       "                    )\n",
       "                    (lora_A): ModuleDict(\n",
       "                      (default): Linear(in_features=512, out_features=8, bias=False)\n",
       "                    )\n",
       "                    (lora_B): ModuleDict(\n",
       "                      (default): Linear(in_features=8, out_features=512, bias=False)\n",
       "                    )\n",
       "                    (lora_embedding_A): ParameterDict()\n",
       "                    (lora_embedding_B): ParameterDict()\n",
       "                  )\n",
       "                  (k): Linear(in_features=512, out_features=512, bias=False)\n",
       "                  (v): Linear(\n",
       "                    in_features=512, out_features=512, bias=False\n",
       "                    (lora_dropout): ModuleDict(\n",
       "                      (default): Dropout(p=0.05, inplace=False)\n",
       "                    )\n",
       "                    (lora_A): ModuleDict(\n",
       "                      (default): Linear(in_features=512, out_features=8, bias=False)\n",
       "                    )\n",
       "                    (lora_B): ModuleDict(\n",
       "                      (default): Linear(in_features=8, out_features=512, bias=False)\n",
       "                    )\n",
       "                    (lora_embedding_A): ParameterDict()\n",
       "                    (lora_embedding_B): ParameterDict()\n",
       "                  )\n",
       "                  (o): Linear(in_features=512, out_features=512, bias=False)\n",
       "                  (relative_attention_bias): Embedding(32, 8)\n",
       "                )\n",
       "                (layer_norm): T5LayerNorm()\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (1): T5LayerFF(\n",
       "                (DenseReluDense): T5DenseActDense(\n",
       "                  (wi): Linear(in_features=512, out_features=2048, bias=False)\n",
       "                  (wo): Linear(in_features=2048, out_features=512, bias=False)\n",
       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
       "                  (act): ReLU()\n",
       "                )\n",
       "                (layer_norm): T5LayerNorm()\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (1-5): 5 x T5Block(\n",
       "            (layer): ModuleList(\n",
       "              (0): T5LayerSelfAttention(\n",
       "                (SelfAttention): T5Attention(\n",
       "                  (q): Linear(\n",
       "                    in_features=512, out_features=512, bias=False\n",
       "                    (lora_dropout): ModuleDict(\n",
       "                      (default): Dropout(p=0.05, inplace=False)\n",
       "                    )\n",
       "                    (lora_A): ModuleDict(\n",
       "                      (default): Linear(in_features=512, out_features=8, bias=False)\n",
       "                    )\n",
       "                    (lora_B): ModuleDict(\n",
       "                      (default): Linear(in_features=8, out_features=512, bias=False)\n",
       "                    )\n",
       "                    (lora_embedding_A): ParameterDict()\n",
       "                    (lora_embedding_B): ParameterDict()\n",
       "                  )\n",
       "                  (k): Linear(in_features=512, out_features=512, bias=False)\n",
       "                  (v): Linear(\n",
       "                    in_features=512, out_features=512, bias=False\n",
       "                    (lora_dropout): ModuleDict(\n",
       "                      (default): Dropout(p=0.05, inplace=False)\n",
       "                    )\n",
       "                    (lora_A): ModuleDict(\n",
       "                      (default): Linear(in_features=512, out_features=8, bias=False)\n",
       "                    )\n",
       "                    (lora_B): ModuleDict(\n",
       "                      (default): Linear(in_features=8, out_features=512, bias=False)\n",
       "                    )\n",
       "                    (lora_embedding_A): ParameterDict()\n",
       "                    (lora_embedding_B): ParameterDict()\n",
       "                  )\n",
       "                  (o): Linear(in_features=512, out_features=512, bias=False)\n",
       "                )\n",
       "                (layer_norm): T5LayerNorm()\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (1): T5LayerFF(\n",
       "                (DenseReluDense): T5DenseActDense(\n",
       "                  (wi): Linear(in_features=512, out_features=2048, bias=False)\n",
       "                  (wo): Linear(in_features=2048, out_features=512, bias=False)\n",
       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
       "                  (act): ReLU()\n",
       "                )\n",
       "                (layer_norm): T5LayerNorm()\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "        (final_layer_norm): T5LayerNorm()\n",
       "        (dropout): Dropout(p=0.1, inplace=False)\n",
       "      )\n",
       "      (decoder): T5Stack(\n",
       "        (embed_tokens): Embedding(32128, 512)\n",
       "        (block): ModuleList(\n",
       "          (0): T5Block(\n",
       "            (layer): ModuleList(\n",
       "              (0): T5LayerSelfAttention(\n",
       "                (SelfAttention): T5Attention(\n",
       "                  (q): Linear(\n",
       "                    in_features=512, out_features=512, bias=False\n",
       "                    (lora_dropout): ModuleDict(\n",
       "                      (default): Dropout(p=0.05, inplace=False)\n",
       "                    )\n",
       "                    (lora_A): ModuleDict(\n",
       "                      (default): Linear(in_features=512, out_features=8, bias=False)\n",
       "                    )\n",
       "                    (lora_B): ModuleDict(\n",
       "                      (default): Linear(in_features=8, out_features=512, bias=False)\n",
       "                    )\n",
       "                    (lora_embedding_A): ParameterDict()\n",
       "                    (lora_embedding_B): ParameterDict()\n",
       "                  )\n",
       "                  (k): Linear(in_features=512, out_features=512, bias=False)\n",
       "                  (v): Linear(\n",
       "                    in_features=512, out_features=512, bias=False\n",
       "                    (lora_dropout): ModuleDict(\n",
       "                      (default): Dropout(p=0.05, inplace=False)\n",
       "                    )\n",
       "                    (lora_A): ModuleDict(\n",
       "                      (default): Linear(in_features=512, out_features=8, bias=False)\n",
       "                    )\n",
       "                    (lora_B): ModuleDict(\n",
       "                      (default): Linear(in_features=8, out_features=512, bias=False)\n",
       "                    )\n",
       "                    (lora_embedding_A): ParameterDict()\n",
       "                    (lora_embedding_B): ParameterDict()\n",
       "                  )\n",
       "                  (o): Linear(in_features=512, out_features=512, bias=False)\n",
       "                  (relative_attention_bias): Embedding(32, 8)\n",
       "                )\n",
       "                (layer_norm): T5LayerNorm()\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (1): T5LayerCrossAttention(\n",
       "                (EncDecAttention): T5Attention(\n",
       "                  (q): Linear(\n",
       "                    in_features=512, out_features=512, bias=False\n",
       "                    (lora_dropout): ModuleDict(\n",
       "                      (default): Dropout(p=0.05, inplace=False)\n",
       "                    )\n",
       "                    (lora_A): ModuleDict(\n",
       "                      (default): Linear(in_features=512, out_features=8, bias=False)\n",
       "                    )\n",
       "                    (lora_B): ModuleDict(\n",
       "                      (default): Linear(in_features=8, out_features=512, bias=False)\n",
       "                    )\n",
       "                    (lora_embedding_A): ParameterDict()\n",
       "                    (lora_embedding_B): ParameterDict()\n",
       "                  )\n",
       "                  (k): Linear(in_features=512, out_features=512, bias=False)\n",
       "                  (v): Linear(\n",
       "                    in_features=512, out_features=512, bias=False\n",
       "                    (lora_dropout): ModuleDict(\n",
       "                      (default): Dropout(p=0.05, inplace=False)\n",
       "                    )\n",
       "                    (lora_A): ModuleDict(\n",
       "                      (default): Linear(in_features=512, out_features=8, bias=False)\n",
       "                    )\n",
       "                    (lora_B): ModuleDict(\n",
       "                      (default): Linear(in_features=8, out_features=512, bias=False)\n",
       "                    )\n",
       "                    (lora_embedding_A): ParameterDict()\n",
       "                    (lora_embedding_B): ParameterDict()\n",
       "                  )\n",
       "                  (o): Linear(in_features=512, out_features=512, bias=False)\n",
       "                )\n",
       "                (layer_norm): T5LayerNorm()\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (2): T5LayerFF(\n",
       "                (DenseReluDense): T5DenseActDense(\n",
       "                  (wi): Linear(in_features=512, out_features=2048, bias=False)\n",
       "                  (wo): Linear(in_features=2048, out_features=512, bias=False)\n",
       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
       "                  (act): ReLU()\n",
       "                )\n",
       "                (layer_norm): T5LayerNorm()\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "          (1-5): 5 x T5Block(\n",
       "            (layer): ModuleList(\n",
       "              (0): T5LayerSelfAttention(\n",
       "                (SelfAttention): T5Attention(\n",
       "                  (q): Linear(\n",
       "                    in_features=512, out_features=512, bias=False\n",
       "                    (lora_dropout): ModuleDict(\n",
       "                      (default): Dropout(p=0.05, inplace=False)\n",
       "                    )\n",
       "                    (lora_A): ModuleDict(\n",
       "                      (default): Linear(in_features=512, out_features=8, bias=False)\n",
       "                    )\n",
       "                    (lora_B): ModuleDict(\n",
       "                      (default): Linear(in_features=8, out_features=512, bias=False)\n",
       "                    )\n",
       "                    (lora_embedding_A): ParameterDict()\n",
       "                    (lora_embedding_B): ParameterDict()\n",
       "                  )\n",
       "                  (k): Linear(in_features=512, out_features=512, bias=False)\n",
       "                  (v): Linear(\n",
       "                    in_features=512, out_features=512, bias=False\n",
       "                    (lora_dropout): ModuleDict(\n",
       "                      (default): Dropout(p=0.05, inplace=False)\n",
       "                    )\n",
       "                    (lora_A): ModuleDict(\n",
       "                      (default): Linear(in_features=512, out_features=8, bias=False)\n",
       "                    )\n",
       "                    (lora_B): ModuleDict(\n",
       "                      (default): Linear(in_features=8, out_features=512, bias=False)\n",
       "                    )\n",
       "                    (lora_embedding_A): ParameterDict()\n",
       "                    (lora_embedding_B): ParameterDict()\n",
       "                  )\n",
       "                  (o): Linear(in_features=512, out_features=512, bias=False)\n",
       "                )\n",
       "                (layer_norm): T5LayerNorm()\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (1): T5LayerCrossAttention(\n",
       "                (EncDecAttention): T5Attention(\n",
       "                  (q): Linear(\n",
       "                    in_features=512, out_features=512, bias=False\n",
       "                    (lora_dropout): ModuleDict(\n",
       "                      (default): Dropout(p=0.05, inplace=False)\n",
       "                    )\n",
       "                    (lora_A): ModuleDict(\n",
       "                      (default): Linear(in_features=512, out_features=8, bias=False)\n",
       "                    )\n",
       "                    (lora_B): ModuleDict(\n",
       "                      (default): Linear(in_features=8, out_features=512, bias=False)\n",
       "                    )\n",
       "                    (lora_embedding_A): ParameterDict()\n",
       "                    (lora_embedding_B): ParameterDict()\n",
       "                  )\n",
       "                  (k): Linear(in_features=512, out_features=512, bias=False)\n",
       "                  (v): Linear(\n",
       "                    in_features=512, out_features=512, bias=False\n",
       "                    (lora_dropout): ModuleDict(\n",
       "                      (default): Dropout(p=0.05, inplace=False)\n",
       "                    )\n",
       "                    (lora_A): ModuleDict(\n",
       "                      (default): Linear(in_features=512, out_features=8, bias=False)\n",
       "                    )\n",
       "                    (lora_B): ModuleDict(\n",
       "                      (default): Linear(in_features=8, out_features=512, bias=False)\n",
       "                    )\n",
       "                    (lora_embedding_A): ParameterDict()\n",
       "                    (lora_embedding_B): ParameterDict()\n",
       "                  )\n",
       "                  (o): Linear(in_features=512, out_features=512, bias=False)\n",
       "                )\n",
       "                (layer_norm): T5LayerNorm()\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "              (2): T5LayerFF(\n",
       "                (DenseReluDense): T5DenseActDense(\n",
       "                  (wi): Linear(in_features=512, out_features=2048, bias=False)\n",
       "                  (wo): Linear(in_features=2048, out_features=512, bias=False)\n",
       "                  (dropout): Dropout(p=0.1, inplace=False)\n",
       "                  (act): ReLU()\n",
       "                )\n",
       "                (layer_norm): T5LayerNorm()\n",
       "                (dropout): Dropout(p=0.1, inplace=False)\n",
       "              )\n",
       "            )\n",
       "          )\n",
       "        )\n",
       "        (final_layer_norm): T5LayerNorm()\n",
       "        (dropout): Dropout(p=0.1, inplace=False)\n",
       "      )\n",
       "      (lm_head): Linear(in_features=512, out_features=32128, bias=False)\n",
       "    )\n",
       "  )\n",
       ")"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "from peft import PeftModel, PeftConfig\n",
    "from transformers import AutoModelForSeq2SeqLM, AutoTokenizer, TextStreamer\n",
    "\n",
    "hf_peft_repo = \"LoRA_adapted_T5\"\n",
    "peft_config = PeftConfig.from_pretrained(hf_peft_repo)\n",
    "model = AutoModelForSeq2SeqLM.from_pretrained(peft_config.base_model_name_or_path, return_dict=True, device_map='auto')\n",
    "tokenizer = AutoTokenizer.from_pretrained(peft_config.base_model_name_or_path)\n",
    "streamer = TextStreamer(tokenizer)\n",
    "\n",
    "# Load the finetuned Lora PEFT model\n",
    "model = PeftModel.from_pretrained(model, hf_peft_repo)\n",
    "model.eval()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b75984ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "from datasets import load_from_disk\n",
    "\n",
    "test_set = load_from_disk('Training_set_IMDB/testing_dataset_sample/').with_format('torch')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d4b1699b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['inputs', 'target'],\n",
       "    num_rows: 4307\n",
       "})"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "6ca8dd64",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "val_dataset = test_set.select(range(200))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "1349a023",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dataset({\n",
       "    features: ['inputs', 'target'],\n",
       "    num_rows: 200\n",
       "})"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "val_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "d1cbcbcb",
   "metadata": {},
   "outputs": [],
   "source": [
    "def convert_to_features(example_batch, padding = \"max_length\",input_max = 64, output_max = 256):\n",
    "    inputs = tokenizer.batch_encode_plus(example_batch[\"inputs\"],is_split_into_words=True, padding='max_length', max_length=input_max, truncation=True)\n",
    "    \n",
    "    targets = tokenizer.batch_encode_plus(example_batch[\"target\"],padding = \"max_length\", max_length=output_max,truncation = True)\n",
    "    if padding == \"max_length\":\n",
    "        targets[\"inputs_ids\"] = [\n",
    "            [(l if l != tokenizer.pad_token_id else -100) for l in target] for target in targets[\"input_ids\"]\n",
    "        ]\n",
    "    \n",
    "    inputs[\"labels\"] = targets['input_ids']\n",
    "    return inputs\n",
    "\n",
    "def evaluate_peft_model(sample):\n",
    "    outputs = model.generate(input_ids=sample[\"input_ids\"].unsqueeze(0).cuda(),max_new_tokens = 200, top_p=0.9)\n",
    "    prediction = tokenizer.decode(outputs[0].detach().cpu().numpy(), skip_special_tokens=True)\n",
    "    labels = np.where(sample['labels'] != -100, sample['labels'], tokenizer.pad_token_id)\n",
    "    labels = tokenizer.decode(labels, skip_special_tokens=True)\n",
    "    # Some simple post-processing\n",
    "    _ = execution_accuracy(prediction, labels)\n",
    "    \n",
    "    return prediction, labels\n",
    "\n",
    "def execution_accuracy(prediction, label):\n",
    "    all_executions.append('1')\n",
    "    try:\n",
    "        cursor.execute(prediction)\n",
    "        \n",
    "        \n",
    "        result_pred = cursor.fetchall()\n",
    "        print(result_pred)\n",
    "        \n",
    "        cursor.execute(label)\n",
    "        \n",
    "        result_label = cursor.fetchall()\n",
    "        print(result_label)\n",
    "        if result_label == result_pred:\n",
    "            accurate_executions.append(\"1\")\n",
    "    except:\n",
    "            failed_executions.append(\"f\")\n",
    "    return None\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "080bb7d4",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /home/toibazd/Data/Text2SQL/Training_set_IMDB/testing_dataset_sample/cache-787cdf1e9cf375a9_*_of_00004.arrow\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mapping\n",
      "mapped, generating SQL\n",
      "[]\n",
      "[(50516, 'Julio César', 'Marmol', 1)]\n",
      "[(\"Bernie's mother\",)]\n",
      "[(\"Bernie's mother\",)]\n",
      "[(43417, 'Akira', 'Kurosawa')]\n",
      "[(43417, 'Akira', 'Kurosawa')]\n",
      "[(40043, 'Anwar', 'Kawadri')]\n",
      "[(40043, 'Anwar', 'Kawadri')]\n",
      "[('Short',)]\n",
      "[('Short',)]\n",
      "[]\n",
      "[]\n",
      "[('Adult',)]\n",
      "[('Adult',)]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[]\n",
      "[(70093, 'Alberto Seixas', 'Santos')]\n",
      "[(2591,)]\n",
      "[(2591,)]\n",
      "[(7.049999833106995,)]\n",
      "[(7.049999833106995,)]\n",
      "[]\n",
      "[('Professor Marr',)]\n",
      "[('Comedy',), ('Music',)]\n",
      "[('Comedy',), ('Music',)]\n",
      "[(18,)]\n",
      "[(18,)]\n",
      "[(9.300000190734863,)]\n",
      "[(9.300000190734863,)]\n",
      "[(0,)]\n",
      "[(0,)]\n",
      "[]\n",
      "[(53662, 'Jacob Adrian', 'Mikkelsen', 5)]\n",
      "[(6,)]\n",
      "[(6,)]\n",
      "[(1,)]\n",
      "[(1,)]\n",
      "[(53793, 'Chris (XIII)', 'Miller', 2)]\n",
      "[(53793, 'Chris (XIII)', 'Miller', 2)]\n",
      "[(110609, 'Fei teng de qun shan', 1976, None, 'Wei (I)', 'Li'), (300640, 'Si ge xiao huo ban', 1981, None, 'Wei (I)', 'Li')]\n",
      "[(110609, 'Fei teng de qun shan', 1976, None, 'Wei (I)', 'Li'), (300640, 'Si ge xiao huo ban', 1981, None, 'Wei (I)', 'Li')]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[]\n",
      "[(303128, 'Sirr abi', 1946, None, 'Wali Eddine', 'Sameh'), (189151, \"Li'bat al sitt\", 1946, None, 'Wali Eddine', 'Sameh'), (375923, 'Zoul wijhain', 1949, None, 'Wali Eddine', 'Sameh'), (8694, 'Ahmar shafayef', 1950, None, 'Wali Eddine', 'Sameh')]\n",
      "[(5.0,)]\n",
      "[(5.0,)]\n",
      "[('Comedy',), ('Documentary',), ('Short',)]\n",
      "[('Comedy',), ('Documentary',), ('Short',)]\n",
      "[(11,)]\n",
      "[(11,)]\n",
      "[]\n",
      "[]\n",
      "[('Drama',)]\n",
      "[('Drama',)]\n",
      "[(97779, 'Eenrichting', 2000, None)]\n",
      "[(97779, 'Eenrichting', 2000, None)]\n",
      "[(0,)]\n",
      "[(0,)]\n",
      "[(2166,)]\n",
      "[(2166,)]\n",
      "[('Music',), ('Short',)]\n",
      "[('Music',), ('Short',)]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[(63853, 'Igor', 'Pretnar')]\n",
      "[(63853, 'Igor', 'Pretnar')]\n",
      "[(86830, 'Jim', 'Wynorski', 3)]\n",
      "[(86830, 'Jim', 'Wynorski', 3)]\n",
      "[(5.800000190734863,)]\n",
      "[(5.800000190734863,)]\n",
      "[]\n",
      "[(70825, 'Zachary Quinn', 'Scheuren', 2)]\n",
      "[]\n",
      "[(139978, 'Hateman', 1987, None), (90882, 'Doomsday Machine, The', 1972, 4.5)]\n",
      "[]\n",
      "[(50902, 'Marco A.', 'Martinez-Galarce', 1)]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[('Dave',)]\n",
      "[('Documentary',)]\n",
      "[('Documentary',)]\n",
      "[(387309, '\"Edge of Night, The\"', 1956, None)]\n",
      "[(387309, '\"Edge of Night, The\"', 1956, None)]\n",
      "[(10,)]\n",
      "[(10,)]\n",
      "[('Action',), ('Adventure',)]\n",
      "[('Action',), ('Adventure',)]\n",
      "[(0,)]\n",
      "[(0,)]\n",
      "[('Short',)]\n",
      "[('Short',)]\n",
      "[]\n",
      "[(26755, 'Ram Narayan', 'Gabale', 3)]\n",
      "[]\n",
      "[(108303, 'Family Honor, The', 1920, None), (244351, 'Other Half, The', 1919, None), (260247, 'Poor Relations', 1919, None), (331687, 'Three Wise Fools', 1923, None), (365053, 'Wife of the Centaur, The', 1924, None), (367549, 'Woman of Bronze, The', 1923, None), (35265, 'Better Times', 1919, None), (138682, 'Happiness', 1924, None), (266102, 'Proud Flesh', 1925, None), (341930, 'Turn in the Road, The', 1919, None), (5534, 'Accusing Toe, The', 1918, None), (132833, 'Grand Military Parade, The', 1913, None), (145999, 'His Hour', 1924, None), (190500, \"Light's Diamond Jubilee\", 1954, None), (273263, 'Real Adventure', 1922, None), (366121, 'Wine of Youth', 1924, None), (48663, \"Bud's Recruit\", 1918, None), (69667, 'Conquering the Woman', 1922, None), (95458, 'Dusk to Dawn', 1922, None), (153133, 'Hurricane in Galveston', 1913, None), (251827, \"Peg o' My Heart\", 1922, None), (29862, 'Bardelys the Magnificent', 1926, 5.0), (304310, 'Sky Pilot, The', 1921, 5.3), (240610, 'On Our Merry Way', 1948, 5.3), (165695, 'Japanese War Bride', 1952, 5.4), (195912, 'Love Never Dies', 1921, 5.7), (68879, 'Comrade X', 1940, 5.7), (14163, 'American Romance, An', 1944, 5.7), (307688, 'Solomon and Sheba', 1959, 5.7), (360705, 'Wedding Night, The', 1935, 5.8), (38041, 'Billy the Kid', 1930, 5.9), (38541, 'Bird of Paradise', 1932, 5.9), (316273, \"Stranger's Return, The\", 1933, 5.9), (190588, 'Lightning Strikes Twice', 1951, 5.9), (35745, 'Beyond the Forest', 1949, 6.0), (75884, 'Cynara', 1932, 6.2), (42935, 'Boheme, La', 1926, 6.3), (365379, 'Wild Oranges', 1924, 6.4), (359174, 'War and Peace', 1956, 6.5), (137619, 'Hallelujah', 1929, 6.6), (235494, 'Not So Dumb', 1930, 6.6), (282927, 'Ruby Gentry', 1952, 6.6), (203691, 'Man Without a Star', 1955, 6.9), (306468, 'So Red the Rose', 1935, 6.9), (164541, 'Jack-Knife Man, The', 1920, 7.0), (118236, 'Fountainhead, The', 1949, 7.0), (244797, 'Our Daily Bread', 1934, 7.0), (94766, 'Duel in the Sun', 1946, 7.0), (314354, 'Stella Dallas', 1937, 7.1), (329116, 'Texas Rangers, The', 1936, 7.1), (58797, 'Champ, The (1931/I)', 1931, 7.2), (136816, 'H.M. Pulham, Esq.', 1941, 7.3), (64712, 'Citadel, The', 1938, 7.4), (316692, 'Street Scene', 1931, 7.8), (250810, 'Patsy, The', 1928, 7.9), (300106, 'Show People', 1928, 8.0), (73935, 'Crowd, The', 1928, 8.1), (367073, 'Wizard of Oz, The', 1939, 8.3), (37256, 'Big Parade, The', 1925, 8.8)]\n",
      "[]\n",
      "[(61324, 'Gordon (II)', 'Payne', 1)]\n",
      "[]\n",
      "[('',)]\n",
      "[(316875, 'Stricken', 1998, 2.6)]\n",
      "[(316875, 'Stricken', 1998, 2.6)]\n",
      "[(1,)]\n",
      "[(1,)]\n",
      "[(74322, 'Crystalis', 1990, None, 'K.', 'Kôno')]\n",
      "[(74322, 'Crystalis', 1990, None, 'K.', 'Kôno')]\n",
      "[]\n",
      "[(68452, 'Justin Thomas', 'Rowe', 1)]\n",
      "[(399717, '\"Not Fade Away\"', 1996, None)]\n",
      "[(399717, '\"Not Fade Away\"', 1996, None)]\n",
      "[('Family',)]\n",
      "[('Family',)]\n",
      "[(5,)]\n",
      "[(5,)]\n",
      "[]\n",
      "[(57903, 'Centaur', 1972, None), (128763, 'Girls of Mountain Street', 1970, None), (377035, ' qui appartient ce gage?', 1973, None)]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[(40549, 'Blessing Way, The', 2000, None)]\n",
      "[(40549, 'Blessing Way, The', 2000, None)]\n",
      "[('Action',), ('Adventure',), ('Comedy',), ('Crime',), ('Drama',), ('Horror',), ('Romance',), ('Thriller',), ('War',), ('Western',)]\n",
      "[('Action',), ('Adventure',), ('Comedy',), ('Crime',), ('Drama',), ('Horror',), ('Romance',), ('Thriller',), ('War',), ('Western',)]\n",
      "[(51521, 'Mario', 'Mattoli', 82)]\n",
      "[(51521, 'Mario', 'Mattoli', 82)]\n",
      "[]\n",
      "[]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[(140994, 'Headhunter (2002/II)', 2002, None)]\n",
      "[(140994, 'Headhunter (2002/II)', 2002, None)]\n",
      "[('Drama',)]\n",
      "[('Drama',)]\n",
      "[]\n",
      "[(29560, 'Philip', 'Goodhew')]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[(28430, 'Arek', 'Gielnik', 1)]\n",
      "[(28430, 'Arek', 'Gielnik', 1)]\n",
      "[(75422, 'Andrés', 'Spinoza', 1)]\n",
      "[(75422, 'Andrés', 'Spinoza', 1)]\n",
      "[(1,)]\n",
      "[(1,)]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[(123234, 'Galop, galop', 1983, None, 'Youlia', 'Kantcheva'), (35874, 'Bez upoyka', 1987, None, 'Youlia', 'Kantcheva'), (374862, 'Zhivotat e pred nas', 1988, None, 'Youlia', 'Kantcheva'), (354031, 'Viensko kolelo', 1991, None, 'Youlia', 'Kantcheva'), (199486, 'Made In Bulgaria', 1992, None, 'Youlia', 'Kantcheva'), (60726, 'Cherniyat pat na hristovite nevesti', 2001, None, 'Youlia', 'Kantcheva'), (35859, 'Bez semeyna prilika', 2004, None, 'Youlia', 'Kantcheva')]\n",
      "[(123234, 'Galop, galop', 1983, None, 'Youlia', 'Kantcheva'), (35874, 'Bez upoyka', 1987, None, 'Youlia', 'Kantcheva'), (374862, 'Zhivotat e pred nas', 1988, None, 'Youlia', 'Kantcheva'), (354031, 'Viensko kolelo', 1991, None, 'Youlia', 'Kantcheva'), (199486, 'Made In Bulgaria', 1992, None, 'Youlia', 'Kantcheva'), (60726, 'Cherniyat pat na hristovite nevesti', 2001, None, 'Youlia', 'Kantcheva'), (35859, 'Bez semeyna prilika', 2004, None, 'Youlia', 'Kantcheva')]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[(4947, 'Alfredo', 'Barrientos', 1)]\n",
      "[(4947, 'Alfredo', 'Barrientos', 1)]\n",
      "[(125,)]\n",
      "[(125,)]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[]\n",
      "[(244312, 'Othello (1981/II)', 1981, None, 'Franklin', 'Melton')]\n",
      "[(406478, '\"Stawka wieksza niz zycie\"', 1968, None, 'Andrzej', 'Konic'), (220144, 'Motodrama', 1971, None, 'Andrzej', 'Konic'), (385477, '\"Czarne chmury\"', 1973, None, 'Andrzej', 'Konic'), (398927, '\"Najwazniejszy dzien zycia\"', 1974, None, 'Andrzej', 'Konic'), (357981, 'W te dni przedwiosenne', 1975, None, 'Andrzej', 'Konic'), (412116, '\"Zycie na goraco\"', 1979, None, 'Andrzej', 'Konic'), (404010, '\"Rcher, Retter und Rapiere\"', 1982, None, 'Andrzej', 'Konic'), (322041, 'Szczesliwy brzeg', 1983, None, 'Andrzej', 'Konic'), (196939, 'Lucky Edge', 1983, None, 'Andrzej', 'Konic'), (226004, 'Na wolnosc', 1985, None, 'Andrzej', 'Konic'), (401714, '\"Pogranicze w ogniu\"', 1988, None, 'Andrzej', 'Konic')]\n",
      "[(406478, '\"Stawka wieksza niz zycie\"', 1968, None, 'Andrzej', 'Konic'), (220144, 'Motodrama', 1971, None, 'Andrzej', 'Konic'), (385477, '\"Czarne chmury\"', 1973, None, 'Andrzej', 'Konic'), (398927, '\"Najwazniejszy dzien zycia\"', 1974, None, 'Andrzej', 'Konic'), (357981, 'W te dni przedwiosenne', 1975, None, 'Andrzej', 'Konic'), (412116, '\"Zycie na goraco\"', 1979, None, 'Andrzej', 'Konic'), (404010, '\"Rcher, Retter und Rapiere\"', 1982, None, 'Andrzej', 'Konic'), (322041, 'Szczesliwy brzeg', 1983, None, 'Andrzej', 'Konic'), (196939, 'Lucky Edge', 1983, None, 'Andrzej', 'Konic'), (226004, 'Na wolnosc', 1985, None, 'Andrzej', 'Konic'), (401714, '\"Pogranicze w ogniu\"', 1988, None, 'Andrzej', 'Konic')]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[]\n",
      "[(185134, 'Last Chance Detectives: Legend of the Desert Bigfoot, The', 1995, None), (279424, 'Road to Redemption', 2001, 4.9)]\n",
      "[('Himself',)]\n",
      "[('Himself',)]\n",
      "[(3467, 'Phillip', 'Avalon', 1)]\n",
      "[(3467, 'Phillip', 'Avalon', 1)]\n",
      "[(63149, 'Harry A.', 'Pollard')]\n",
      "[(63149, 'Harry A.', 'Pollard')]\n",
      "[(42400, 'Nadejda', 'Koseva', 2)]\n",
      "[(42400, 'Nadejda', 'Koseva', 2)]\n",
      "[]\n",
      "[]\n",
      "[(14,)]\n",
      "[(14,)]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[('Documentary',)]\n",
      "[('Documentary',)]\n",
      "[('Farnk',)]\n",
      "[('Farnk',)]\n",
      "[(626,)]\n",
      "[(626,)]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[]\n",
      "[(3182, 'Manabu', 'Asou')]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[(133834, 'Great Train Robbery, The', 1903, 7.5)]\n",
      "[(133834, 'Great Train Robbery, The', 1903, 7.5)]\n",
      "[(7252, 'Aedena', 2002, None), (24877, 'Avatars', 2001, None)]\n",
      "[(24877, 'Avatars', 2001, None), (7252, 'Aedena', 2002, None)]\n",
      "[('Short',)]\n",
      "[('Short',)]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[]\n",
      "[]\n",
      "[('',)]\n",
      "[('',)]\n",
      "[(0,)]\n",
      "[(0,)]\n",
      "[(76725, 'Dag Jan', 1993, None, 'Jan', 'van den Berg'), (324248, 'Taming the Floods', 1999, None, 'Jan', 'van den Berg'), (295314, 'Seven Dreams of Tibet', 2001, None, 'Jan', 'van den Berg')]\n",
      "[(76725, 'Dag Jan', 1993, None, 'Jan', 'van den Berg'), (324248, 'Taming the Floods', 1999, None, 'Jan', 'van den Berg'), (295314, 'Seven Dreams of Tibet', 2001, None, 'Jan', 'van den Berg')]\n",
      "[(61507, 'Anthony', 'Pelissier', 2)]\n",
      "[(61507, 'Anthony', 'Pelissier', 2)]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[]\n",
      "[('Gabrielle (6 ans)',)]\n",
      "[(9970, 'Richard (I)', 'Brooks')]\n",
      "[(9970, 'Richard (I)', 'Brooks')]\n",
      "[('Drama',)]\n",
      "[('Drama',)]\n",
      "[]\n",
      "[(88375, 'Manuel', 'Zimmer', 1)]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[(9,)]\n",
      "[(9,)]\n",
      "[]\n",
      "[(5914, 'Craig', 'Belknap', 1)]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[(218794, 'Moonlight Sonata', 1990, None)]\n",
      "[(218794, 'Moonlight Sonata', 1990, None)]\n",
      "[]\n",
      "[]\n",
      "[(3859, 'Aly', 'Badrakhan', 1)]\n",
      "[(3859, 'Aly', 'Badrakhan', 1)]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[]\n",
      "[(110283, 'Fear of Falling', 1993, None, 'Gregory Allen', 'Black')]\n",
      "[(36146, 'Jimmy', 'Huston', 6)]\n",
      "[(36146, 'Jimmy', 'Huston', 6)]\n",
      "[(373712, 'Zakhmi Aurat', 1988, None, 'Avtar', 'Bhogal'), (136886, 'Haar Jeet', 1990, None, 'Avtar', 'Bhogal'), (4172, 'Aaj Ki Aurat', 1993, None, 'Avtar', 'Bhogal')]\n",
      "[(373712, 'Zakhmi Aurat', 1988, None, 'Avtar', 'Bhogal'), (136886, 'Haar Jeet', 1990, None, 'Avtar', 'Bhogal'), (4172, 'Aaj Ki Aurat', 1993, None, 'Avtar', 'Bhogal')]\n",
      "[(4,)]\n",
      "[(4,)]\n",
      "[]\n",
      "[(25458, 'Francis', 'Ford')]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[(2.8,)]\n",
      "[(2.8,)]\n",
      "[]\n",
      "[(24768, 'Sean', 'Findley')]\n",
      "[(559,)]\n",
      "[(559,)]\n",
      "[(65512, 'Fred Olen', 'Ray')]\n",
      "[(65512, 'Fred Olen', 'Ray')]\n",
      "[(74432, 'David (I)', 'Smith', 55)]\n",
      "[(74432, 'David (I)', 'Smith', 55)]\n",
      "[(17457, 'Anne-Laure', 'Daffis', 2)]\n",
      "[(17457, 'Anne-Laure', 'Daffis', 2)]\n",
      "[]\n",
      "[(367753, 'Woman, Wake Up', 1922, None, 'Marcus', 'Harrison')]\n",
      "[('Animation',), ('Comedy',), ('Romance',)]\n",
      "[('Animation',), ('Comedy',), ('Romance',)]\n",
      "[(4,)]\n",
      "[(4,)]\n",
      "[('Animation',), ('Musical',), ('Short',)]\n",
      "[('Animation',), ('Musical',), ('Short',)]\n",
      "[(16,)]\n",
      "[(16,)]\n",
      "[('',)]\n",
      "[('',)]\n",
      "[('Jennifer Baker',)]\n",
      "[('Jennifer Baker',)]\n",
      "[(45516, 'Kang-cheon', 'Lee', 2)]\n",
      "[(45516, 'Kang-cheon', 'Lee', 2)]\n",
      "[(42,)]\n",
      "[(42,)]\n",
      "[('Black woman',)]\n",
      "[('Black woman',)]\n",
      "[(9,)]\n",
      "[(9,)]\n",
      "[(323507, 'Taken Away', 2002, 2.9)]\n",
      "[(323507, 'Taken Away', 2002, 2.9)]\n",
      "[(384822, '\"Comdiens, Les\"', 1965, None, 'Joseph', 'Drimal'), (393488, '\"Jeunes annes, Les\"', 1965, None, 'Joseph', 'Drimal'), (410307, '\"Vive la vie\"', 1966, None, 'Joseph', 'Drimal'), (408000, '\"Thibaud\"', 1968, None, 'Joseph', 'Drimal'), (384009, '\"Chemins de pierre, Les\"', 1972, None, 'Joseph', 'Drimal'), (409448, '\"Un homme, une ville\"', 1973, None, 'Joseph', 'Drimal'), (396597, '\"Malaventure\"', 1974, None, 'Joseph', 'Drimal'), (67074, 'Collectionneur des cerveaux, Le', 1976, None, 'Joseph', 'Drimal'), (382472, '\"Bonjour Paris\"', 1976, None, 'Joseph', 'Drimal'), (386224, '\"Deux auteurs en folie\"', 1977, None, 'Joseph', 'Drimal'), (383487, '\"Carnets de mdecins\"', 1977, None, 'Joseph', 'Drimal'), (248251, 'Par devant notaire', 1979, None, 'Joseph', 'Drimal'), (201989, 'Mal bleu, Le', 1979, None, 'Joseph', 'Drimal'), (118797, 'Francis', 1990, None, 'Joseph', 'Drimal')]\n",
      "[(393488, '\"Jeunes annes, Les\"', 1965, None, 'Joseph', 'Drimal'), (384822, '\"Comdiens, Les\"', 1965, None, 'Joseph', 'Drimal'), (410307, '\"Vive la vie\"', 1966, None, 'Joseph', 'Drimal'), (408000, '\"Thibaud\"', 1968, None, 'Joseph', 'Drimal'), (384009, '\"Chemins de pierre, Les\"', 1972, None, 'Joseph', 'Drimal'), (409448, '\"Un homme, une ville\"', 1973, None, 'Joseph', 'Drimal'), (396597, '\"Malaventure\"', 1974, None, 'Joseph', 'Drimal'), (67074, 'Collectionneur des cerveaux, Le', 1976, None, 'Joseph', 'Drimal'), (382472, '\"Bonjour Paris\"', 1976, None, 'Joseph', 'Drimal'), (386224, '\"Deux auteurs en folie\"', 1977, None, 'Joseph', 'Drimal'), (383487, '\"Carnets de mdecins\"', 1977, None, 'Joseph', 'Drimal'), (248251, 'Par devant notaire', 1979, None, 'Joseph', 'Drimal'), (201989, 'Mal bleu, Le', 1979, None, 'Joseph', 'Drimal'), (118797, 'Francis', 1990, None, 'Joseph', 'Drimal')]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[]\n",
      "[]\n",
      "[('Harriet',)]\n",
      "[('Harriet',)]\n",
      "[]\n",
      "[('Comedy',), ('Crime',), ('Documentary',), ('Fantasy',), ('Music',), ('Short',), ('Thriller',)]\n",
      "[(182,)]\n",
      "[(182,)]\n",
      "[(32414, 'Becoming Marty', 2003, None, 'Rodney Anthony', 'Beckwith')]\n",
      "[(32414, 'Becoming Marty', 2003, None, 'Rodney Anthony', 'Beckwith')]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[(209129, 'Mean Anything', 2000, None)]\n",
      "[(209129, 'Mean Anything', 2000, None)]\n",
      "[(7.400000095367432,)]\n",
      "[(7.400000095367432,)]\n",
      "[(7.199999809265137,)]\n",
      "[(7.199999809265137,)]\n",
      "[(12,)]\n",
      "[(12,)]\n",
      "[(6725, 'Curtis', 'Bernhardt', 42)]\n",
      "[(6725, 'Curtis', 'Bernhardt', 42)]\n",
      "[(22508, 'Alice (I)', 'Eklund', 1)]\n",
      "[(22508, 'Alice (I)', 'Eklund', 1)]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[(0,)]\n",
      "[(0,)]\n",
      "[(51257, 'S.', 'Mastan', 1)]\n",
      "[(51257, 'S.', 'Mastan', 1)]\n",
      "[]\n",
      "[]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[(204,)]\n",
      "[(204,)]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[(1.9,)]\n",
      "[(1.9,)]\n",
      "[]\n",
      "[(183032, 'La Divina', 1989, None)]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[]\n",
      "[]\n",
      "[('Comedy',), ('Crime',), ('Drama',), ('War',)]\n",
      "[('Comedy',), ('Crime',), ('Drama',), ('War',)]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[]\n",
      "[(125954, 'Geologia', 1967, None)]\n",
      "[(548,)]\n",
      "[(548,)]\n",
      "[]\n",
      "[(47039, 'Nick', 'Lindsay', 1)]\n",
      "[('Bartender',)]\n",
      "[('Bartender',)]\n",
      "[(306219, 'Snow Cure, The', 1916, None, 'Arvid E.', 'Gillstrom'), (26770, 'Back Stage', 1917, None, 'Arvid E.', 'Gillstrom'), (92000, 'Dough Nuts', 1917, None, 'Arvid E.', 'Gillstrom'), (214157, 'Millionaire, The', 1917, None, 'Arvid E.', 'Gillstrom'), (53787, 'Candy Kid, The', 1917, None, 'Arvid E.', 'Gillstrom'), (253769, 'Pest, The (1917/II)', 1917, None, 'Arvid E.', 'Gillstrom'), (304622, 'Slave, The (1917/II)', 1917, None, 'Arvid E.', 'Gillstrom'), (61373, 'Chief Cook, The', 1917, None, 'Arvid E.', 'Gillstrom'), (129904, 'Goat, The', 1917, None, 'Arvid E.', 'Gillstrom'), (28926, 'Band Master, The', 1917, None, 'Arvid E.', 'Gillstrom'), (116088, 'Fly Cop, The', 1917, None, 'Arvid E.', 'Gillstrom'), (147113, 'Hobo, The', 1917, 5.0, 'Arvid E.', 'Gillstrom'), (143887, 'Hero, The', 1917, None, 'Arvid E.', 'Gillstrom'), (354559, 'Villain, The', 1917, None, 'Arvid E.', 'Gillstrom'), (280488, 'Rogue, The', 1918, None, 'Arvid E.', 'Gillstrom'), (320898, 'Swat the Spy', 1918, None, 'Arvid E.', 'Gillstrom'), (145870, 'His Day Out', 1918, None, 'Arvid E.', 'Gillstrom'), (211819, 'Messenger, The', 1918, None, 'Arvid E.', 'Gillstrom'), (316276, 'Stranger, The', 1918, None, 'Arvid E.', 'Gillstrom'), (243259, 'Orderly, The', 1918, None, 'Arvid E.', 'Gillstrom'), (290295, 'Scholar, The', 1918, None, 'Arvid E.', 'Gillstrom'), (327222, 'Tell It to the Marines', 1918, None, 'Arvid E.', 'Gillstrom'), (305585, 'Smiles', 1919, None, 'Arvid E.', 'Gillstrom'), (209583, \"Meet Betty's Husband\", 1920, None, 'Arvid E.', 'Gillstrom'), (35380, \"Betty's Green-Eyed Monster\", 1920, None, 'Arvid E.', 'Gillstrom'), (35365, 'Betty Sets the Pace', 1920, None, 'Arvid E.', 'Gillstrom'), (142605, 'Hello, Judge', 1922, None, 'Arvid E.', 'Gillstrom'), (307879, 'Some Family', 1922, None, 'Arvid E.', 'Gillstrom'), (116759, 'Foolish Lives', 1922, None, 'Arvid E.', 'Gillstrom'), (192186, 'Little Rascal, The', 1922, None, 'Arvid E.', 'Gillstrom'), (155702, 'Idle Roomer, An', 1922, None, 'Arvid E.', 'Gillstrom'), (333680, 'Tips', 1922, None, 'Arvid E.', 'Gillstrom'), (192050, 'Little Miss Mischief', 1922, None, 'Arvid E.', 'Gillstrom'), (251871, 'Peggy, Behave!', 1922, None, 'Arvid E.', 'Gillstrom'), (270636, 'Radio Hound, The', 1922, None, 'Arvid E.', 'Gillstrom'), (48696, 'Budding Youth', 1924, None, 'Arvid E.', 'Gillstrom'), (260216, 'Poor Kid', 1924, None, 'Arvid E.', 'Gillstrom'), (30199, 'Barnum Junior', 1924, None, 'Arvid E.', 'Gillstrom'), (5016, 'About Face', 1924, None, 'Arvid E.', 'Gillstrom'), (90291, \"Don't Fall\", 1924, None, 'Arvid E.', 'Gillstrom'), (186805, 'Leave It to Gerry', 1924, None, 'Arvid E.', 'Gillstrom'), (196570, 'Low Bridges', 1924, None, 'Arvid E.', 'Gillstrom'), (314488, 'Stepping Some', 1924, None, 'Arvid E.', 'Gillstrom'), (51798, 'Caddy, The', 1925, None, 'Arvid E.', 'Gillstrom'), (48171, 'Bruisers and Losers', 1926, None, 'Arvid E.', 'Gillstrom'), (40845, 'Blisters Under the Skin', 1926, None, 'Arvid E.', 'Gillstrom'), (187483, 'Legionnaires in Paris', 1927, None, 'Arvid E.', 'Gillstrom'), (65129, \"Clancy's Kosher Wedding\", 1927, None, 'Arvid E.', 'Gillstrom'), (56269, 'Cash and Carry', 1927, None, 'Arvid E.', 'Gillstrom'), (210226, 'Melancholy Dame', 1928, None, 'Arvid E.', 'Gillstrom'), (238775, 'Oft in the Silly Night', 1929, None, 'Arvid E.', 'Gillstrom'), (118738, 'Framing of the Shrew, The', 1929, None, 'Arvid E.', 'Gillstrom'), (150219, 'Hot Lemonade', 1929, None, 'Arvid E.', 'Gillstrom'), (244890, 'Our Nagging Wives', 1930, None, 'Arvid E.', 'Gillstrom'), (195466, 'Love Bargain, The', 1931, None, 'Arvid E.', 'Gillstrom'), (37065, 'Big Flash, The', 1932, None, 'Arvid E.', 'Gillstrom'), (332545, 'Tied for Life', 1933, None, 'Arvid E.', 'Gillstrom'), (146823, 'Hitchhiker, The', 1933, None, 'Arvid E.', 'Gillstrom'), (333726, 'Tired Feet', 1933, None, 'Arvid E.', 'Gillstrom'), (149338, 'Hooks and Jabs', 1933, None, 'Arvid E.', 'Gillstrom'), (331452, 'Three Little Swigs', 1933, None, 'Arvid E.', 'Gillstrom'), (279528, 'Roaming Romeo', 1933, None, 'Arvid E.', 'Gillstrom'), (257923, 'Please', 1933, None, 'Arvid E.', 'Gillstrom'), (178898, 'Knight Duty', 1933, None, 'Arvid E.', 'Gillstrom'), (240532, 'On Ice', 1933, None, 'Arvid E.', 'Gillstrom'), (254278, 'Petting Preferred', 1934, None, 'Arvid E.', 'Gillstrom'), (64541, 'Circus Hoodoo', 1934, None, 'Arvid E.', 'Gillstrom'), (171054, 'Just an Echo', 1934, None, 'Arvid E.', 'Gillstrom'), (233783, 'No More Bridge!', 1934, None, 'Arvid E.', 'Gillstrom')]\n",
      "[(306219, 'Snow Cure, The', 1916, None, 'Arvid E.', 'Gillstrom'), (354559, 'Villain, The', 1917, None, 'Arvid E.', 'Gillstrom'), (26770, 'Back Stage', 1917, None, 'Arvid E.', 'Gillstrom'), (92000, 'Dough Nuts', 1917, None, 'Arvid E.', 'Gillstrom'), (214157, 'Millionaire, The', 1917, None, 'Arvid E.', 'Gillstrom'), (53787, 'Candy Kid, The', 1917, None, 'Arvid E.', 'Gillstrom'), (253769, 'Pest, The (1917/II)', 1917, None, 'Arvid E.', 'Gillstrom'), (304622, 'Slave, The (1917/II)', 1917, None, 'Arvid E.', 'Gillstrom'), (61373, 'Chief Cook, The', 1917, None, 'Arvid E.', 'Gillstrom'), (129904, 'Goat, The', 1917, None, 'Arvid E.', 'Gillstrom'), (28926, 'Band Master, The', 1917, None, 'Arvid E.', 'Gillstrom'), (116088, 'Fly Cop, The', 1917, None, 'Arvid E.', 'Gillstrom'), (147113, 'Hobo, The', 1917, 5.0, 'Arvid E.', 'Gillstrom'), (143887, 'Hero, The', 1917, None, 'Arvid E.', 'Gillstrom'), (280488, 'Rogue, The', 1918, None, 'Arvid E.', 'Gillstrom'), (320898, 'Swat the Spy', 1918, None, 'Arvid E.', 'Gillstrom'), (145870, 'His Day Out', 1918, None, 'Arvid E.', 'Gillstrom'), (211819, 'Messenger, The', 1918, None, 'Arvid E.', 'Gillstrom'), (316276, 'Stranger, The', 1918, None, 'Arvid E.', 'Gillstrom'), (243259, 'Orderly, The', 1918, None, 'Arvid E.', 'Gillstrom'), (290295, 'Scholar, The', 1918, None, 'Arvid E.', 'Gillstrom'), (327222, 'Tell It to the Marines', 1918, None, 'Arvid E.', 'Gillstrom'), (305585, 'Smiles', 1919, None, 'Arvid E.', 'Gillstrom'), (35365, 'Betty Sets the Pace', 1920, None, 'Arvid E.', 'Gillstrom'), (209583, \"Meet Betty's Husband\", 1920, None, 'Arvid E.', 'Gillstrom'), (35380, \"Betty's Green-Eyed Monster\", 1920, None, 'Arvid E.', 'Gillstrom'), (270636, 'Radio Hound, The', 1922, None, 'Arvid E.', 'Gillstrom'), (142605, 'Hello, Judge', 1922, None, 'Arvid E.', 'Gillstrom'), (307879, 'Some Family', 1922, None, 'Arvid E.', 'Gillstrom'), (116759, 'Foolish Lives', 1922, None, 'Arvid E.', 'Gillstrom'), (192186, 'Little Rascal, The', 1922, None, 'Arvid E.', 'Gillstrom'), (155702, 'Idle Roomer, An', 1922, None, 'Arvid E.', 'Gillstrom'), (333680, 'Tips', 1922, None, 'Arvid E.', 'Gillstrom'), (192050, 'Little Miss Mischief', 1922, None, 'Arvid E.', 'Gillstrom'), (251871, 'Peggy, Behave!', 1922, None, 'Arvid E.', 'Gillstrom'), (314488, 'Stepping Some', 1924, None, 'Arvid E.', 'Gillstrom'), (48696, 'Budding Youth', 1924, None, 'Arvid E.', 'Gillstrom'), (260216, 'Poor Kid', 1924, None, 'Arvid E.', 'Gillstrom'), (30199, 'Barnum Junior', 1924, None, 'Arvid E.', 'Gillstrom'), (5016, 'About Face', 1924, None, 'Arvid E.', 'Gillstrom'), (90291, \"Don't Fall\", 1924, None, 'Arvid E.', 'Gillstrom'), (186805, 'Leave It to Gerry', 1924, None, 'Arvid E.', 'Gillstrom'), (196570, 'Low Bridges', 1924, None, 'Arvid E.', 'Gillstrom'), (51798, 'Caddy, The', 1925, None, 'Arvid E.', 'Gillstrom'), (48171, 'Bruisers and Losers', 1926, None, 'Arvid E.', 'Gillstrom'), (40845, 'Blisters Under the Skin', 1926, None, 'Arvid E.', 'Gillstrom'), (56269, 'Cash and Carry', 1927, None, 'Arvid E.', 'Gillstrom'), (187483, 'Legionnaires in Paris', 1927, None, 'Arvid E.', 'Gillstrom'), (65129, \"Clancy's Kosher Wedding\", 1927, None, 'Arvid E.', 'Gillstrom'), (210226, 'Melancholy Dame', 1928, None, 'Arvid E.', 'Gillstrom'), (118738, 'Framing of the Shrew, The', 1929, None, 'Arvid E.', 'Gillstrom'), (150219, 'Hot Lemonade', 1929, None, 'Arvid E.', 'Gillstrom'), (238775, 'Oft in the Silly Night', 1929, None, 'Arvid E.', 'Gillstrom'), (244890, 'Our Nagging Wives', 1930, None, 'Arvid E.', 'Gillstrom'), (195466, 'Love Bargain, The', 1931, None, 'Arvid E.', 'Gillstrom'), (37065, 'Big Flash, The', 1932, None, 'Arvid E.', 'Gillstrom'), (240532, 'On Ice', 1933, None, 'Arvid E.', 'Gillstrom'), (332545, 'Tied for Life', 1933, None, 'Arvid E.', 'Gillstrom'), (146823, 'Hitchhiker, The', 1933, None, 'Arvid E.', 'Gillstrom'), (333726, 'Tired Feet', 1933, None, 'Arvid E.', 'Gillstrom'), (149338, 'Hooks and Jabs', 1933, None, 'Arvid E.', 'Gillstrom'), (331452, 'Three Little Swigs', 1933, None, 'Arvid E.', 'Gillstrom'), (279528, 'Roaming Romeo', 1933, None, 'Arvid E.', 'Gillstrom'), (257923, 'Please', 1933, None, 'Arvid E.', 'Gillstrom'), (178898, 'Knight Duty', 1933, None, 'Arvid E.', 'Gillstrom'), (254278, 'Petting Preferred', 1934, None, 'Arvid E.', 'Gillstrom'), (64541, 'Circus Hoodoo', 1934, None, 'Arvid E.', 'Gillstrom'), (171054, 'Just an Echo', 1934, None, 'Arvid E.', 'Gillstrom'), (233783, 'No More Bridge!', 1934, None, 'Arvid E.', 'Gillstrom')]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(230354, 'Never Leave Nevada', 1990, None)]\n",
      "[(230354, 'Never Leave Nevada', 1990, None)]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[(None,)]\n",
      "[]\n",
      "[]\n",
      "[(243025, 'Opration Magali', 1953, None)]\n",
      "[(243025, 'Opration Magali', 1953, None)]\n",
      "[(32,)]\n",
      "[(32,)]\n",
      "[(11,)]\n",
      "[(11,)]\n",
      "Execution rate: 91.0%\n",
      "Execution accuracy: 74.5%\n"
     ]
    }
   ],
   "source": [
    "import evaluate\n",
    "import numpy as np\n",
    "from tqdm import tqdm\n",
    "import pandas as pd\n",
    "import mysql.connector\n",
    "\n",
    "\n",
    "\n",
    "connection = mysql.connector.connect(\n",
    "    host=\"relational.fit.cvut.cz\",\n",
    "    user=\"guest\",\n",
    "    password=\"relational\",\n",
    "    database=\"imdb_ijs\"\n",
    ")\n",
    "cursor = connection.cursor()\n",
    "\n",
    "\n",
    "\n",
    "print(\"mapping\")\n",
    "tokenized_dataset = val_dataset.map(convert_to_features, batched=True, num_proc=4)\n",
    "print(\"mapped, generating SQL\")\n",
    "\n",
    "all_executions = []\n",
    "failed_executions = []\n",
    "accurate_executions = []\n",
    "\n",
    "predictions, references = [] , []\n",
    "for sample in tokenized_dataset:\n",
    "    p,l = evaluate_peft_model(sample)\n",
    "    predictions.append(p)\n",
    "    references.append(l)\n",
    "    \n",
    "\n",
    "cursor.close()\n",
    "connection.close()\n",
    "\n",
    "\n",
    "print(f\"Execution rate: {round(100 - len(failed_executions)/len(all_executions)*100, 2)}%\")\n",
    "print(f\"Execution accuracy: {round(len(accurate_executions)/len(all_executions)*100, 2)}%\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "63d89262",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(len(failed_executions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "c4076606",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SELECT COUNT(distinct r.role) AS role_count FROM roles r JOIN movies m ON r.movie_id = m.id WHERE m.name = '\"Gaffer, The\"'\n"
     ]
    }
   ],
   "source": [
    "print(val_dataset['target'][-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "9ed0a031",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\"SELECT a.first_name as first_name, a.last_name as last_name, r.role AS first_role FROM actors a INNER JOIN ( SELECT r.actor_id, MIN(m.year) AS min_movie_year FROM roles r INNER JOIN movies m ON r.movie_id = m.id GROUP BY r.actor_id ) t ON a.id = t.actor_id INNER JOIN roles r ON t.actor_id = r.actor_id INNER JOIN movies m ON r.movie_id = m.id AND t.min_movie_year = m.year WHERE a.first_name = 'Jack (VI)' AND a.last_name = 'Lewis'\", \"SELECT d.id AS director_id, d.first_name, d.last_name, COUNT(m.id) AS movie_count FROM directors d JOIN movies_directors md ON d.id = md.director_id JOIN movies m ON md.movie_id = m.id WHERE d.first_name = 'Julio César' AND d.last_name = 'Marmol' AND m.year = 1975 GROUP BY d.id, d.first_name, d.last_name\"]\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "641c0dee",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}