i2b2-ast-preprocessing.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "beth ast path C:\\Users\\kcaro\\Documents\\GitHub\\clinical-adapter\\Data/concept_assertion_relation_training_data\\beth\\ast\n",
      "beth txt path C:\\Users\\kcaro\\Documents\\GitHub\\clinical-adapter\\Data/concept_assertion_relation_training_data\\beth\\txt\n",
      "partners ast path C:\\Users\\kcaro\\Documents\\GitHub\\clinical-adapter\\Data/concept_assertion_relation_training_data\\partners\\ast\n",
      "partners txt path C:\\Users\\kcaro\\Documents\\GitHub\\clinical-adapter\\Data/concept_assertion_relation_training_data\\partners\\txt\n",
      "test ast path C:\\Users\\kcaro\\Documents\\GitHub\\clinical-adapter\\Data/reference_standard_for_test_data\\ast\n",
      "test txt path C:\\Users\\kcaro\\Documents\\GitHub\\clinical-adapter\\Data/test_data\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import re\n",
    "import en_ner_bc5cdr_md\n",
    "import spacy\n",
    "import copy\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import evaluate\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.metrics import accuracy_score, precision_score, recall_score, f1_score\n",
    "from transformers import AutoTokenizer, DataCollatorWithPadding\n",
    "from datasets import Dataset, DatasetDict\n",
    "import torch \n",
    "from transformers import TrainingArguments, Trainer\n",
    "from torch.optim import AdamW\n",
    "\n",
    "\n",
    "# Set a random seed for reproducibility\n",
    "seed = 42\n",
    "torch.manual_seed(seed)\n",
    "np.random.seed(seed)\n",
    "spacy.util.fix_random_seed(seed)\n",
    "\n",
    "# SET DATA TO TEST\n",
    "# \"Line Data\" or \"Sentencizer Data\"\n",
    "data_to_test = \"Line Data\"\n",
    "\n",
    "\n",
    "# Get paths for data and labels\n",
    "cwd  = os.getcwd()\n",
    "processed_path = os.path.join(cwd, \"Data/preprocessed-data\")\n",
    "labels_path_beth = os.path.join(cwd,\"Data/concept_assertion_relation_training_data\",\"beth\",\"ast\")\n",
    "data_path_beth = os.path.join(cwd,\"Data/concept_assertion_relation_training_data\",\"beth\",\"txt\")\n",
    "labels_path_partners = os.path.join(cwd,\"Data/concept_assertion_relation_training_data\",\"partners\",\"ast\")\n",
    "data_path_partners = os.path.join(cwd,\"Data/concept_assertion_relation_training_data\",\"partners\",\"txt\")\n",
    "labels_path_test = os.path.join(cwd,\"Data/reference_standard_for_test_data\",\"ast\")\n",
    "data_path_test = os.path.join(cwd,\"Data/test_data\")\n",
    "print('beth ast path', labels_path_beth)\n",
    "print('beth txt path', data_path_beth)\n",
    "print('partners ast path', labels_path_partners)\n",
    "print('partners txt path', data_path_partners)\n",
    "print('test ast path', labels_path_test)\n",
    "print('test txt path', data_path_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_text(text):\n",
    "    \"\"\"\n",
    "    Applies some pre-processing on the given text.\n",
    "    Returns clean text\n",
    "    \n",
    "    Steps :\n",
    "    - Removes HTML tags\n",
    "    - Removes punctuation\n",
    "    - lowers text\n",
    "    \"\"\"\n",
    "    text = re.sub(r'<.*?>', '', text)\n",
    "    text = re.sub(r\"\\\\\", \"\", text)    \n",
    "    text = re.sub(r\"\\'\", \"\", text)    \n",
    "    text = re.sub(r\"\\\"\", \"\", text)    \n",
    "    text = text.strip().lower()\n",
    "    text = re.sub(r'[^\\x00-\\x7f]',r'', text) \n",
    "    filters='!\"\\'#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n'\n",
    "    translate_dict = dict((c, \" \") for c in filters)\n",
    "    translate_map = str.maketrans(translate_dict)\n",
    "    text = text.translate(translate_map)\n",
    "    text = \" \".join(text.split())\n",
    "    \n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_adjacent_periods(text):\n",
    "    \"\"\"\n",
    "    Takes a text string\n",
    "    Returns the text string without adjacent periods (see 'Dr.' to 'Dr' below while keeping end period)\n",
    "    \n",
    "    For example:\n",
    "    input: 'Admitted directly to OR from ambulance transfer and underwent cabg x3 with Dr. Howard on 06-13 .'\n",
    "    output: 'Admitted directly to OR from ambulance transfer and underwent cabg x3 with Dr Howard on 06-13 .'\n",
    "\n",
    "    This is needed for preventing the sentencizer from splitting on periods that are inside sentences.\n",
    "    \n",
    "    \"\"\"\n",
    "    pattern = re.compile(r'(?<=\\S)\\.')\n",
    "    modified_text = re.sub(pattern, '', text)\n",
    "    \n",
    "    return modified_text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def list_files_in_directory(directory_path):\n",
    "    \"\"\"\n",
    "    Takes the path to the records.txt files\n",
    "    Returns a list of the names of the records in each path\n",
    "    \"\"\"\n",
    "    ignore = ['.DS_Store']\n",
    "    files = os.listdir(directory_path)\n",
    "    files = [file for file in files if os.path.isfile(os.path.join(directory_path, file)) and file not in ignore]\n",
    "    files = [file[:-4] for file in files]\n",
    "    \n",
    "    return files\n",
    "\n",
    "\n",
    "notes_beth = list_files_in_directory(data_path_beth)\n",
    "notes_partners = list_files_in_directory(data_path_partners)\n",
    "notes_test = list_files_in_directory(data_path_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of beth training records: 73\n",
      "number of partners training records: 97\n",
      "number of all test records: 256\n",
      "number of combined beth and partners records: 170\n",
      "number of all combined records: 426\n"
     ]
    }
   ],
   "source": [
    "def load_record_content_dict(notes_list, data_path):\n",
    "    \"\"\"\n",
    "    Takes the list of names of the records and the path to the data records.txt files\n",
    "    \n",
    "    Returns a dictionary containing the record_name associated with all the text in the record.txt file:\n",
    "    content_dict = {record: <all text in the record.txt file>}\n",
    "    \n",
    "    eg. \n",
    "    content_dict = {'record-105': 'Admission Date :\\n2017-06-13\\nDischarge Date :\\n2017...( End of Report )'}\n",
    "    \n",
    "    \"\"\"\n",
    "    content_notes = {}\n",
    "    for note in notes_list:\n",
    "        _file = os.path.join(data_path, note + '.txt')\n",
    "        with open(_file) as f:\n",
    "            content = f.read()\n",
    "            content_notes[note] = content\n",
    "    return content_notes\n",
    "\n",
    "\n",
    "beth_content_dict = load_record_content_dict(notes_beth, data_path_beth)\n",
    "partners_content_dict = load_record_content_dict(notes_partners, data_path_partners)\n",
    "test_content_dict = load_record_content_dict(notes_test, data_path_test)\n",
    "\n",
    "# Merge the content dictionaries into one\n",
    "training_content_dict = {**beth_content_dict, **partners_content_dict}\n",
    "all_content_dict = {**training_content_dict, **test_content_dict}\n",
    "\n",
    "# Print the number of records for each dictionary\n",
    "print(\"number of beth training records:\", len(beth_content_dict))\n",
    "print(\"number of partners training records:\", len(partners_content_dict))\n",
    "print(\"number of all test records:\", len(test_content_dict))\n",
    "print(\"number of combined beth and partners records:\", len(training_content_dict))\n",
    "print(\"number of all combined records:\", len(all_content_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "number of beth records with labels: 72\n",
      "number of partners records with labels: 96\n",
      "number of test records with labels: 255\n",
      "number of combined beth and partners records with labels: 168\n",
      "number of all combined records with labels: 423\n"
     ]
    }
   ],
   "source": [
    "def load_record_labels_dict(notes_list, labels_path):\n",
    "    \"\"\"\n",
    "    Takes the list of names of the records and the path to the data records.txt files\n",
    "    \n",
    "    Returns a dictionary containing the record_name and all the text in the record.ast file:\n",
    "    labels_dict = {record: <all text in the record.ast file>}\n",
    "    \n",
    "    For example:\n",
    "    labels_dict = {'record-105': 'c=\"left basilar atelectasis\" 55:6 55:8||t=\"problem\"||a=\"present\"\\nc=\"ventral ...'}\n",
    "    \n",
    "    \"\"\"\n",
    "    labels_notes = {}\n",
    "    for note in notes_list:\n",
    "        _file = os.path.join(labels_path, note + '.ast')\n",
    "        with open(_file) as f:\n",
    "            content = f.read()\n",
    "            line_dict = {}\n",
    "            for line in content:\n",
    "                \n",
    "                \n",
    "                labels_notes[note] = content\n",
    "            \n",
    "    f.close()\n",
    "    return labels_notes\n",
    "\n",
    "\n",
    "beth_labels_dict  = load_record_labels_dict(notes_beth, labels_path_beth)\n",
    "partners_labels_dict = load_record_labels_dict(notes_partners, labels_path_partners)\n",
    "test_labels_dict = load_record_labels_dict(notes_test, labels_path_test)\n",
    "\n",
    "# # Merge the labels dictionaries into one\n",
    "training_labels_dict = {**beth_labels_dict, **partners_labels_dict}\n",
    "all_labels_dict = {**training_labels_dict, **test_labels_dict}\n",
    "\n",
    "# Print the number of records conatining labels for each dictionary\n",
    "print(\"number of beth records with labels:\", len(beth_labels_dict))\n",
    "print(\"number of partners records with labels:\", len(partners_labels_dict))\n",
    "print(\"number of test records with labels:\", len(test_labels_dict))\n",
    "print(\"number of combined beth and partners records with labels:\", len(training_labels_dict))\n",
    "print(\"number of all combined records with labels:\", len(all_labels_dict))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "the following records have no cooresponding labels (the .ast files are blank):\n",
      "\n",
      "records in beth data without any labels: {'record-58'}\n",
      "records in partners data without any labels: {'262182942'}\n",
      "records in test data without any labels: {'0305'}\n"
     ]
    }
   ],
   "source": [
    "# Print the records that don't have any labels because they are blank records\n",
    "keys_unique_to_beth_dict = set(beth_content_dict.keys()) - set(beth_labels_dict.keys())\n",
    "keys_unique_to_partners_dict = set(partners_content_dict.keys()) - set(partners_labels_dict.keys())\n",
    "keys_unique_to_test_dict = set(test_content_dict.keys()) - set(test_labels_dict.keys())\n",
    "print('the following records have no cooresponding labels (the .ast files are blank):')\n",
    "print()\n",
    "print(\"records in beth data without any labels:\", keys_unique_to_beth_dict)\n",
    "print(\"records in partners data without any labels:\", keys_unique_to_partners_dict)\n",
    "print(\"records in test data without any labels:\", keys_unique_to_test_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total number of beth/partners assertion labels for data: 7073\n",
      "total number of test assertion labels for data: 12592\n",
      "total number of all assertion labels for data: 19665\n"
     ]
    }
   ],
   "source": [
    "def create_labels_dict(labels_dict):\n",
    "    \"\"\"\n",
    "    Takes labels dictionary:\n",
    "    labels_dict = {record: <all text in the record.ast file>}\n",
    "    \n",
    "    Returns a new dictionary of the labels for each record on each line of the record:\n",
    "    new_labels_dict = {record: {ast_line: {'location': [<list of locations of entity in record.txt file>],\n",
    "                                           'problem': '<the entity that is the target for assertion classification>',\n",
    "                                           'assertion:' 'the assertion label'}}}\n",
    "\n",
    "    For example:\n",
    "    new_labels_dict = {'record-105': {1 : {'location': [55, 6, 55, 8],\n",
    "                                           'problem': 'left basilar atelectasis',\n",
    "                                           'assertion': 'present'}, \n",
    "                                      ...\n",
    "                                      n :  {...}}\n",
    "    \"\"\"\n",
    "    labels_dict_new = {}\n",
    "    for key in labels_dict:\n",
    "        lines = labels_dict[key].split(\"\\n\")\n",
    "        line_dict = {}\n",
    "        i = 1\n",
    "        for line in lines:\n",
    "            if line != '':\n",
    "                ast = line.strip().split('||')\n",
    "                assertion = re.findall('\"([^\"]*)\"', ast[2])[0]\n",
    "                entity = re.split(r'\" \\d+', ast[0])[0][3:]\n",
    "                loc_section = re.split(r'\" (\\d)', ast[0])\n",
    "                loc = \"\".join([loc_section[1], loc_section[2]])\n",
    "                start, end = loc.split(' ')\n",
    "                s_line, s_tok = start.split(':')\n",
    "                e_line, e_tok = end.split(':')\n",
    "                line_info = {'location': [int(s_line), int(s_tok), int(e_line), int(e_tok)], 'problem': entity, 'assertion': assertion}\n",
    "                line_dict[i] = line_info\n",
    "                i+= 1\n",
    "        labels_dict_new[key] = line_dict\n",
    "            \n",
    "    return labels_dict_new\n",
    "\n",
    "\n",
    "training_labels = create_labels_dict(training_labels_dict)\n",
    "test_labels = create_labels_dict(test_labels_dict)\n",
    "all_labels = create_labels_dict(all_labels_dict)\n",
    "\n",
    "# Print the numbet of examples in each group of data (combined beth and partaners labels, test labels)\n",
    "total_training_labels = 0\n",
    "for record in training_labels:\n",
    "    for entry in training_labels[record]:\n",
    "        if 'assertion' in training_labels[record][entry]:\n",
    "            total_training_labels += 1\n",
    "    \n",
    "total_test_labels = 0\n",
    "for record in test_labels:\n",
    "    for entry in test_labels[record]:\n",
    "        if 'assertion' in test_labels[record][entry]:\n",
    "            total_test_labels += 1\n",
    "\n",
    "total_labels_all = 0\n",
    "for record in all_labels:\n",
    "    for entry in all_labels[record]:\n",
    "        if 'assertion' in all_labels[record][entry]:\n",
    "            total_labels_all += 1\n",
    "    \n",
    "print(\"total number of beth/partners assertion labels for data:\", total_training_labels)\n",
    "print(\"total number of test assertion labels for data:\", total_test_labels)\n",
    "print(\"total number of all assertion labels for data:\", total_labels_all)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_lined_content_dict(content_dict):\n",
    "    \"\"\"\n",
    "    Takes the content dictionary:\n",
    "    content_dict = {record: <all text in the record.txt file>}\n",
    "\n",
    "    Returns a new dictionary of the content line-by-line, with a key for each line\n",
    "    new_content_dict = {'record': {1: <all the text on line 1 of record.txt>,\n",
    "                                   2: <all the text on line 2 of record.txt>,\n",
    "                                   ...,\n",
    "                                   n: <all the text on last line (n) of record.txt>}}\n",
    "\n",
    "    For example:\n",
    "    new_content_dict = {'record-105': {1: 'Admission Date :',\n",
    "                                       2: '2017-06-13',\n",
    "                                       3: 'Discharge Date :',\n",
    "                                       ...\n",
    "                                       153: '( End of Report )'}}\n",
    "    \"\"\"\n",
    "    new_content = {}\n",
    "    for key in content_dict:\n",
    "        lines = content_dict[key].split(\"\\n\")\n",
    "        line_dict = {}\n",
    "        i = 1\n",
    "        for line in lines:\n",
    "            line_dict[i] = line\n",
    "            i += 1\n",
    "            \n",
    "        new_content[key] = line_dict\n",
    "    return new_content\n",
    "\n",
    "training_content = create_lined_content_dict(training_content_dict)\n",
    "test_content = create_lined_content_dict(test_content_dict)\n",
    "all_content = create_lined_content_dict(all_content_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_entity_and_assertion(new_labels, new_content):\n",
    "    \"\"\"\n",
    "    Takes the new_content dictionary and the new_label dictionary:\n",
    "    new_content_dict = {'record': {1: <all the text on line 1 of record.txt>,\n",
    "                               2: <all the text on line 2 of record.txt>,\n",
    "                               ...,\n",
    "                               n: <all the text on last line (n) of record.txt>}}\n",
    "\n",
    "    Returns a data dictionary that contains the record and each line of the record with infomration for the\n",
    "    ast_line number, txt_line number, entity, old_line text, new_line text, and ast label.\n",
    "    data = {'record': {1: {'ast_line': <line number the ast file (matches the key)>,\n",
    "                           'txt_line': <line number of the txt file where the target entity is located>,\n",
    "                           'entity': <the target entity (problem to be classified) from the line in the record.ast file>,\n",
    "                           'old_line': <all the text on the line in record.txt where the target entity is located>,\n",
    "                           'new_line': <all the text on the line in record.txt where the target entity is located \n",
    "                                       with [entity] tokens inserted around the target entity>,\n",
    "                           'label': <the label of the entity in the text (present, absent, possible, etc.)>},\n",
    "                           ...\n",
    "                           n: {...}}\n",
    "\n",
    "    For example:\n",
    "    data = {'record-105': {1: {'ast_line': 1,\n",
    "                               'txt_line': 55,\n",
    "                               'entity': 'left basilar atelectasis',\n",
    "                               'old_line': 'There has been interval improvement in left basilar atelectasis .',\n",
    "                               'new_line': 'There has been interval improvement in [entity] left basilar atelectasis [entity] .',\n",
    "                               'label': 'present'},\n",
    "                           ...\n",
    "                           n: {...}}\n",
    "    \"\"\"\n",
    "    data = {}\n",
    "    for record in new_labels:\n",
    "        i = 1\n",
    "        data[record] = {}\n",
    "        for key in new_labels[record]:\n",
    "            s_line = new_labels[record][key]['location'][0]\n",
    "            s_tok = new_labels[record][key]['location'][1]\n",
    "            e_line = new_labels[record][key]['location'][2]\n",
    "            e_tok = new_labels[record][key]['location'][3]\n",
    "            assertion = new_labels[record][key]['assertion']\n",
    "            entity = new_labels[record][key]['problem']\n",
    "            old_line = new_content[record][s_line]\n",
    "            \n",
    "#             entity = clean_text(entity)\n",
    "#             old_line = clean_text(old_line)\n",
    "            entity = remove_adjacent_periods(entity)\n",
    "            old_line = remove_adjacent_periods(old_line)\n",
    "            \n",
    "            words = old_line.split()\n",
    "            words.insert(s_tok, '[entity]')\n",
    "            words.insert(e_tok + 2, '[entity]')\n",
    "            new_line = ' '.join(words)\n",
    "            \n",
    "            data[record][i] = {'ast_line': i, 'txt_line': s_line, \n",
    "                               'entity': entity,'old_line': old_line, \n",
    "                               'new_line': new_line, 'label': assertion}\n",
    "            i += 1\n",
    "        \n",
    "    return data\n",
    "\n",
    "training_line_data = add_entity_and_assertion(training_labels, training_content)\n",
    "test_line_data = add_entity_and_assertion(test_labels, test_content)\n",
    "all_line_data = add_entity_and_assertion(all_labels, all_content)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "total number new_line in training data: 7073\n",
      "total number new_line in test data: 12592\n",
      "total number of new_line in all data: 19665\n"
     ]
    }
   ],
   "source": [
    "# Print the number of examples in each group of data (combined beth and partaners labels, test labels)\n",
    "total_training_sents = 0\n",
    "for record in training_line_data:\n",
    "    for entry in training_line_data[record]:\n",
    "        if 'new_line' in training_line_data[record][entry]:\n",
    "            total_training_sents += 1\n",
    "    \n",
    "total_test_sents = 0\n",
    "for record in test_line_data:\n",
    "    for entry in test_line_data[record]:\n",
    "        if 'new_line' in test_line_data[record][entry]:\n",
    "            total_test_sents += 1\n",
    "            \n",
    "total_sents = 0\n",
    "for record in all_line_data:\n",
    "    for entry in all_line_data[record]:\n",
    "        if 'new_line' in all_line_data[record][entry]:\n",
    "            total_sents += 1\n",
    "    \n",
    "print(\"total number new_line in training data:\", total_training_sents)  \n",
    "print(\"total number new_line in test data:\", total_test_sents)\n",
    "print(\"total number of new_line in all data:\", total_sents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data_by_line(data):\n",
    "    \"\"\"\n",
    "    Takes line_data dictionary\n",
    "    Returns line data dataframe\n",
    "    \"\"\"\n",
    "    record_dfs = []\n",
    "    for record, info in data.items():\n",
    "        df = pd.DataFrame.from_dict(info, orient='index')\n",
    "        df['record'] = record\n",
    "        record_dfs.append(df)\n",
    "    final_df = pd.concat(record_dfs).reset_index(drop=True)\n",
    "    final_df = final_df[['record', 'ast_line', 'txt_line', 'entity', 'new_line', 'label']]\n",
    "\n",
    "    return final_df\n",
    "\n",
    "training_line_data_df = get_data_by_line(training_line_data)\n",
    "test_line_data_df = get_data_by_line(test_line_data)\n",
    "all_line_data_df = get_data_by_line(all_line_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def filter_data(data_df):\n",
    "    \"\"\"\n",
    "    Takes dataframe\n",
    "    Returns filtered dataframe with only present, absent, possible and dropped null rows\n",
    "    \"\"\"\n",
    "    allowed_labels = ['present', 'absent', 'possible']\n",
    "    filtered_df = data_df[data_df['label'].isin(allowed_labels)]\n",
    "    filtered_df = filtered_df.dropna()\n",
    "    filtered_df.reset_index(inplace=True, drop=True)\n",
    "    \n",
    "    return filtered_df\n",
    "\n",
    "training_line_data_filtered_df = filter_data(training_line_data_df)\n",
    "test_line_data_filtered_df = filter_data(test_line_data_df)\n",
    "all_line_data_filtered_df = filter_data(all_line_data_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>record</th>\n",
       "      <th>ast_line</th>\n",
       "      <th>txt_line</th>\n",
       "      <th>entity</th>\n",
       "      <th>new_line</th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>record-105</td>\n",
       "      <td>1</td>\n",
       "      <td>55</td>\n",
       "      <td>left basilar atelectasis</td>\n",
       "      <td>There has been interval improvement in [entity...</td>\n",
       "      <td>present</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>record-105</td>\n",
       "      <td>2</td>\n",
       "      <td>143</td>\n",
       "      <td>ventral hernia</td>\n",
       "      <td>HTN [entity] ventral hernia [entity] right nep...</td>\n",
       "      <td>present</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>record-105</td>\n",
       "      <td>3</td>\n",
       "      <td>26</td>\n",
       "      <td>htn</td>\n",
       "      <td>[entity] HTN [entity] elev chol right facial d...</td>\n",
       "      <td>present</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>record-105</td>\n",
       "      <td>4</td>\n",
       "      <td>68</td>\n",
       "      <td>spontaneous echo contrast</td>\n",
       "      <td>No [entity] spontaneous echo contrast [entity]...</td>\n",
       "      <td>absent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>record-105</td>\n",
       "      <td>5</td>\n",
       "      <td>21</td>\n",
       "      <td>80% lm lesion</td>\n",
       "      <td>Cath at Kindred/North Shore today showed [enti...</td>\n",
       "      <td>present</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18392</th>\n",
       "      <td>0477</td>\n",
       "      <td>57</td>\n",
       "      <td>38</td>\n",
       "      <td>constrictive pericarditis</td>\n",
       "      <td>Most recently he was admitted to an outside ho...</td>\n",
       "      <td>present</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18393</th>\n",
       "      <td>0477</td>\n",
       "      <td>58</td>\n",
       "      <td>74</td>\n",
       "      <td>communication with the intestines</td>\n",
       "      <td>This showed no [entity] communication with the...</td>\n",
       "      <td>absent</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18394</th>\n",
       "      <td>0477</td>\n",
       "      <td>59</td>\n",
       "      <td>38</td>\n",
       "      <td>adult respiratory distress syndrome</td>\n",
       "      <td>Most recently he was admitted to an outside ho...</td>\n",
       "      <td>present</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18395</th>\n",
       "      <td>0477</td>\n",
       "      <td>60</td>\n",
       "      <td>41</td>\n",
       "      <td>his gunshot wound</td>\n",
       "      <td>He had previous substernal gastric bypass beca...</td>\n",
       "      <td>present</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18396</th>\n",
       "      <td>0477</td>\n",
       "      <td>61</td>\n",
       "      <td>41</td>\n",
       "      <td>an interruption of his esophagus</td>\n",
       "      <td>He had previous substernal gastric bypass beca...</td>\n",
       "      <td>present</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>18397 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           record  ast_line  txt_line                               entity  \\\n",
       "0      record-105         1        55             left basilar atelectasis   \n",
       "1      record-105         2       143                       ventral hernia   \n",
       "2      record-105         3        26                                  htn   \n",
       "3      record-105         4        68            spontaneous echo contrast   \n",
       "4      record-105         5        21                        80% lm lesion   \n",
       "...           ...       ...       ...                                  ...   \n",
       "18392        0477        57        38            constrictive pericarditis   \n",
       "18393        0477        58        74    communication with the intestines   \n",
       "18394        0477        59        38  adult respiratory distress syndrome   \n",
       "18395        0477        60        41                    his gunshot wound   \n",
       "18396        0477        61        41     an interruption of his esophagus   \n",
       "\n",
       "                                                new_line    label  \n",
       "0      There has been interval improvement in [entity...  present  \n",
       "1      HTN [entity] ventral hernia [entity] right nep...  present  \n",
       "2      [entity] HTN [entity] elev chol right facial d...  present  \n",
       "3      No [entity] spontaneous echo contrast [entity]...   absent  \n",
       "4      Cath at Kindred/North Shore today showed [enti...  present  \n",
       "...                                                  ...      ...  \n",
       "18392  Most recently he was admitted to an outside ho...  present  \n",
       "18393  This showed no [entity] communication with the...   absent  \n",
       "18394  Most recently he was admitted to an outside ho...  present  \n",
       "18395  He had previous substernal gastric bypass beca...  present  \n",
       "18396  He had previous substernal gastric bypass beca...  present  \n",
       "\n",
       "[18397 rows x 6 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# OPTION 1: Use line data\n",
    "\"\"\"\n",
    "Pros: Clean approach, uses all the data, no duplicates created\n",
    "Cons: Lines are extremely varaible in size, some entities on lines by themselves (no context for assertion task)\n",
    "\"\"\"\n",
    "# all_line_data_filtered_df.to_csv(processed_path + '/ast_line_data.csv', index=False)\n",
    "all_line_data_filtered_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAkQAAAHFCAYAAAAT5Oa6AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8pXeV/AAAACXBIWXMAAA9hAAAPYQGoP6dpAABM2UlEQVR4nO3dfVxUZf4//tcIzAATDHI7TqKS4k3immKZdgOk4h1quau1pGlrZpoYinmz1UatYeFH5ROmWR9Ty8h296Nun22jMMFyvQlR8iZXXcURFQQJGUCcO67fH/44X4d7EJkZzuv5eJzHeq5znZn3xRnYV+dc54xCCCFAREREJGOd7F0AERERkb0xEBEREZHsMRARERGR7DEQERERkewxEBEREZHsMRARERGR7DEQERERkewxEBEREZHsMRARERGR7DEQETmQv/3tb1AoFPjyyy/rbBs4cCAUCgW+/fbbOtt69uyJwYMH39XasrKyoFAokJWV1az+x44dw/PPP4+QkBC4u7vjnnvuweDBg5GcnIxff/31rtbaXGlpaUhJSWl2/8jISISFhdW77dq1a1AoFEhMTJTaWvozq7F+/Xps2bKlRfsQ0Z1hICJyIJGRkVAoFMjMzLRp//XXX3H8+HGo1eo62y5duoTz588jKiqqPUtt1Mcff4zw8HBkZ2fj1VdfRXp6Onbu3IkpU6bgww8/xKxZs+xdIoCWB6KWGjx4MA4cONDisMpARNT+XO1dABH9P/7+/ggLC6tzRmHv3r1wdXXFrFmz6gSimvW2CERVVVXw8PC4o9c4cOAA5s6di1GjRmHXrl1QqVTStlGjRiEhIQHp6el3WqpT8Pb2xsMPP2zvMlrsxo0b8PT0tHcZRO2KZ4iIHExUVBROnz6NgoICqS0rKwsPPvggxo0bh5ycHJSXl9tsc3FxwWOPPQYAuHnzJpYvX46QkBAolUrce++9ePnll3H9+nWb9+nRowdiYmKwY8cODBo0CO7u7njrrbcAAP/+978xZswYeHp6wt/fHy+99JLNezYmKSkJCoUCH330kU0YqqFUKjFx4kRpvbq6GsnJyejbty9UKhUCAwPx3HPP4dKlS3XqnTlzZp3Xi4yMRGRkpM3PQ6FQ4IsvvsBrr70GnU4Hb29vjBw5EqdPn7bZ7+uvv4Zer4dCoZCWtlTfJbPz58/jmWeegU6ng0qlQlBQEEaMGIHc3FxpnCdPnsTevXulmnr06CHtf/HiRUybNg2BgYFQqVTo168fVq9ejerqapv3vnTpEn73u9/By8sLPj4+ePbZZ5GdnQ2FQmFz9mnmzJm45557cPz4cURHR8PLywsjRowAAGRkZGDSpEno2rUr3N3d0atXL8yZMwfXrl2zea/ExEQoFAocO3YMU6ZMgUajga+vLxYtWgSLxYLTp09jzJgx8PLyQo8ePZCcnNymP2eitsAzREQOJioqCu+//z6ysrLw+9//HsCts0AxMTF45JFHoFAo8OOPP2LcuHHStsGDB0Oj0UAIgSeffBLff/89li9fjsceewzHjh3Dm2++iQMHDuDAgQM2IeXIkSM4deoUXn/9dYSEhECtVuPq1auIiIiAm5sb1q9fj6CgIHz++eeYP39+k7VbrVbs2bMH4eHhCA4ObtZ4586di48++gjz589HTEwMLly4gDfeeANZWVk4cuQI/P39W/FTBP74xz/ikUcewf/8z//AYDBg6dKlmDBhAk6dOgUXFxesX78eL774Is6dO4edO3e26LUtFkudNqvV2qx9x40bB6vViuTkZHTr1g3Xrl3D/v37pcC6c+dO/O53v4NGo8H69esBQDpmxcXFGD58OEwmE/785z+jR48e+Mc//oHFixfj3LlzUv/KykpERUXh119/xXvvvYdevXohPT0dTz/9dL01mUwmTJw4EXPmzMGyZcuk8Z07dw7Dhg3DCy+8AI1GgwsXLmDNmjV49NFHcfz4cbi5udm8ztSpUzFt2jTMmTMHGRkZSE5Ohtlsxu7duzFv3jwsXrwYaWlpWLp0KXr16oXJkyc362dG1C4EETmUX3/9VXTq1Em8+OKLQgghrl27JhQKhUhPTxdCCPHQQw+JxYsXCyGEuHjxogAglixZIoQQIj09XQAQycnJNq/55ZdfCgDio48+ktq6d+8uXFxcxOnTp236Ll26VCgUCpGbm2vTPmrUKAFAZGZmNlh7YWGhACCeeeaZZo311KlTAoCYN2+eTfuhQ4cEAPHHP/7Rpt4ZM2bUeY2IiAgREREhrWdmZgoAYty4cTb9/vKXvwgA4sCBA1Lb+PHjRffu3ZtVa817AWh0efPNN+vUUvMzu3btmgAgUlJSGn2f/v3724ypxrJlywQAcejQIZv2uXPnCoVCIR3LDz74QAAQ33zzjU2/OXPmCABi8+bNUtuMGTMEAPHJJ580WlN1dbUwm81Cr9cLAOLvf/+7tO3NN98UAMTq1att9nnggQcEALFjxw6pzWw2i4CAADF58uRG34+ovfGSGZGD6dy5MwYOHChdZtm7dy9cXFzwyCOPAAAiIiKkeUO15w/t2bMHAOpcWpoyZQrUajW+//57m/bf/OY36N27t01bZmYm+vfvj4EDB9q0x8bG3vngaqmpv3a9Dz30EPr161en3pa4/bIccGusAKDX61v9msCtO/qys7PrLLt3725yX19fX/Ts2ROrVq3CmjVrcPTo0TqXuhqzZ88e3H///XjooYds2mfOnAkhhHT89+7dCy8vL4wZM8amX80Zx/r89re/rdNWVFSEl156CcHBwXB1dYWbmxu6d+8OADh16lSd/jExMTbr/fr1g0KhwNixY6U2V1dX9OrV646PA1FbYyAickBRUVE4c+YMrly5gszMTISHh+Oee+4BcCsQHT16FGVlZcjMzISrqyseffRRAEBJSQlcXV0REBBg83oKhQJarRYlJSU27V26dKnz3iUlJdBqtXXa62urzd/fH56ensjLy2vWOGvqqa8OnU5Xp96W8PPzs1mvuexUVVXV6tcEAHd3dwwZMqTOUjtA1kehUOD777/H6NGjkZycjMGDByMgIAALFixo1hytkpKSBn9WNdtr/jcoKKhOv/raAMDT0xPe3t42bdXV1YiOjsaOHTuwZMkSfP/99/jpp59w8OBBAPX/HH19fW3WlUolPD094e7uXqf95s2bDQ2TyC4YiIgcUM0Zn6ysLGRlZSEiIkLaVhN+fvjhB2mydU1Y8vPzg8ViQXFxsc3rCSFQWFhYZz5OfZOI/fz8UFhYWKe9vrbaXFxcMGLECOTk5NSZFF2fmtBy+wTyGleuXLGp193dHUajsU6/2hN8HV337t2xadMmFBYW4vTp01i4cCHWr1+PV199tcl9/fz8GvxZAZB+Xn5+frh69Wqdfg0dw/o+BydOnMDPP/+MVatWIS4uDpGRkXjwwQfrBE2ijoKBiMgBPf7443BxccHf/vY3nDx50uYuKo1GgwceeABbt27FhQsXbG63r7k7aNu2bTav97//+7+orKyUtjcmKioKJ0+exM8//2zTnpaW1qzaly9fDiEEZs+eDZPJVGe72WzG//3f/wEAnnjiiXrrzc7OxqlTp2zq7dGjB44dO2bT78yZMzZ3jrWUSqW64zNGd6J37954/fXXMWDAABw5cqTJukaMGIFffvnFpi8AfPrpp1AoFNJnISIiAuXl5fjmm29s+m3fvr3ZtdWEpNp3Cm7cuLHZr0HkTHiXGZED8vb2xuDBg7Fr1y506tRJmj9UIyIiQnqg4O2BaNSoURg9ejSWLl0Kg8GARx55RLrLbNCgQZg+fXqT7x0fH49PPvkE48ePx4oVK6S7zP797383q/Zhw4Zhw4YNmDdvHsLDwzF37lz0798fZrMZR48exUcffYSwsDBMmDABffr0wYsvvojU1FR06tQJY8eOle4yCw4OxsKFC6XXnT59OqZNm4Z58+bht7/9LfR6PZKTk+tcHmyJAQMGYMeOHdiwYQPCw8PRqVMnDBkypNWv15Rjx45h/vz5mDJlCkJDQ6FUKrFnzx4cO3YMy5Yts6lr+/bt+PLLL3HffffB3d0dAwYMwMKFC/Hpp59i/PjxePvtt9G9e3d8/fXXWL9+PebOnSvNB5sxYwbWrl2LadOmYcWKFejVqxe++eYb6SnnnTo1/d/Cffv2Rc+ePbFs2TIIIeDr64v/+7//Q0ZGxt354RDZm33ndBNRQ5YsWSIAiCFDhtTZtmvXLgFAKJVKUVlZabOtqqpKLF26VHTv3l24ubmJLl26iLlz54rS0lKbft27dxfjx4+v971/+eUXMWrUKOHu7i58fX3FrFmzxN///vcm7zK7XW5urpgxY4bo1q2bUCqVQq1Wi0GDBok//elPoqioSOpntVrFe++9J3r37i3c3NyEv7+/mDZtmsjPz7d5verqapGcnCzuu+8+4e7uLoYMGSL27NnT4F1mf/3rX232z8vLq3OH1a+//ip+97vfCR8fH6FQKERTfxIjIiJE//79691WXFzc5F1mV69eFTNnzhR9+/YVarVa3HPPPeI3v/mNWLt2rbBYLNJ+Fy5cENHR0cLLy0sAsLkTTq/Xi9jYWOHn5yfc3NxEnz59xKpVq4TVarWp5+LFi2Ly5MninnvuEV5eXuK3v/2t+Oc//1nnDrEZM2YItVpd75hqPgdeXl6ic+fOYsqUKdKdjbePs+Yus+LiYpv9G3rtxn6ORPaiEEII+0QxIiJqT0lJSXj99ddx8eJFdO3a1d7lEDkUXjIjIuqA1q1bB+DWpS+z2Yw9e/bg/fffx7Rp0xiGiOrBQERE1AF5enpi7dq1uHDhAoxGI7p164alS5fi9ddft3dpRA6Jl8yIiIhI9njbPREREckeAxERERHJHgMRERERyR4nVTdTdXU1rly5Ai8vr3ofc09ERESORwiB8vJy6HS6Rh9KykDUTFeuXEFwcLC9yyAiIqJWyM/Pb/SREwxEzeTl5QXg1g+09rdCExERkWMyGAwIDg6W/n+8IQxEzVRzmczb25uBiIiIyMk0Nd2Fk6qJiIhI9hiIiIiISPYYiIiIiEj2GIiIiIhI9hiIiIiISPYYiIiIiEj2GIiIiIhI9hiIiIiISPbsGoh++OEHTJgwATqdDgqFArt27Wqw75w5c6BQKJCSkmLTbjQaERcXB39/f6jVakycOBGXLl2y6VNaWorp06dDo9FAo9Fg+vTpuH79etsPiIiIiJySXQNRZWUlBg4ciHXr1jXab9euXTh06BB0Ol2dbfHx8di5cye2b9+Offv2oaKiAjExMbBarVKf2NhY5ObmIj09Henp6cjNzcX06dPbfDxERETknOz61R1jx47F2LFjG+1z+fJlzJ8/H99++y3Gjx9vs62srAybNm3CZ599hpEjRwIAtm3bhuDgYOzevRujR4/GqVOnkJ6ejoMHD2Lo0KEAgI8//hjDhg3D6dOn0adPn7szOCIiInIaDj2HqLq6GtOnT8err76K/v3719mek5MDs9mM6OhoqU2n0yEsLAz79+8HABw4cAAajUYKQwDw8MMPQ6PRSH3qYzQaYTAYbBYiIiLqmBw6EL333ntwdXXFggUL6t1eWFgIpVKJzp0727QHBQWhsLBQ6hMYGFhn38DAQKlPfVauXCnNOdJoNAgODr6DkRAREZEjc9hAlJOTg//+7//Gli1bmvyG2tqEEDb71Ld/7T61LV++HGVlZdKSn5/fohqIiIjIeThsIPrxxx9RVFSEbt26wdXVFa6urtDr9UhISECPHj0AAFqtFiaTCaWlpTb7FhUVISgoSOpz9erVOq9fXFws9amPSqWCt7e3zeKMiouLce7cOWkpLi62d0lEREQOx66Tqhszffp0aaJ0jdGjR2P69Ol4/vnnAQDh4eFwc3NDRkYGpk6dCgAoKCjAiRMnkJycDAAYNmwYysrK8NNPP+Ghhx4CABw6dAhlZWUYPnx4O46o/RUXFyM2di5KSoxSm5+fCmlpGxAQEGDHyoiIiByLXQNRRUUF/vOf/0jreXl5yM3Nha+vL7p16wY/Pz+b/m5ubtBqtdKdYRqNBrNmzUJCQgL8/Pzg6+uLxYsXY8CAAVKY6tevH8aMGYPZs2dj48aNAIAXX3wRMTExHf4OM4PBgJISI1SqBHh4BKOqKh8lJathMBgYiIiIiG5j10B0+PBhREVFSeuLFi0CAMyYMQNbtmxp1musXbsWrq6umDp1KqqqqjBixAhs2bIFLi4uUp/PP/8cCxYskO5GmzhxYpPPPupIPDyCoVb3BAAYjU10JiIikiG7BqLIyEgIIZrd/8KFC3Xa3N3dkZqaitTU1Ab38/X1xbZt21pTIhEREcmAw06qJiIiImovDEREREQkewxEREREJHsMRERERCR7DEREREQkewxEREREJHsMRERERCR7DEREREQkewxEREREJHsMRERERCR7DEREREQkewxEREREJHsMRERERCR7DEREREQkewxEREREJHsMRERERCR7rvYugNpOcXExDAaDtK7X62GxWOxYERERkXNgIOogiouLERs7FyUlRqnNaKxEfv5VaDTGRvYkIiIiBiIndvsZIb1ej6tXK6FWL4WHRzAAoLT0ICyWd2CxWKV9zGYj9Hq9zeuYTCYolUpp3dvbGwEBAe0wAiIiIsfAQOSkap8RqjkbNHBgINTqngCAqqrawacEev15xMW9C5VKBeBWQLpyJQ/33tsLrq63Pg5+fiqkpW1gKCIiItlgIHJSBoMBJSVGqFQJ8PAIrvdsUG1WawUsFiWUyoXw8ekN4NZZpKqqd+DisgA+Pr1RVZWPkpLVMBgMDERERCQbDEROzsMjGGp1zzpngxrj7t61zlmk29uMnHJEREQyw9vuiYiISPYYiIiIiEj2GIiIiIhI9hiIiIiISPYYiIiIiEj2GIiIiIhI9hiIiIiISPYYiIiIiEj2GIiIiIhI9hiIiIiISPYYiIiIiEj2GIiIiIhI9hiIiIiISPYYiIiIiEj2GIiIiIhI9hiIiIiISPZc7V0A1a+4uBgGg0Fa9/b2RkBAgB0rIiIi6rgYiBxQcXExYmPnoqTEKLX5+amQlraBoYiIiOgusOslsx9++AETJkyATqeDQqHArl27pG1msxlLly7FgAEDoFarodPp8Nxzz+HKlSs2r2E0GhEXFwd/f3+o1WpMnDgRly5dsulTWlqK6dOnQ6PRQKPRYPr06bh+/Xo7jLB1DAYDSkqMUKkS4OOTApUqASUlRpszRneT2WyEXq/HuXPnpKW4uLhd3puIiMge7BqIKisrMXDgQKxbt67Oths3buDIkSN44403cOTIEezYsQNnzpzBxIkTbfrFx8dj586d2L59O/bt24eKigrExMTAarVKfWJjY5Gbm4v09HSkp6cjNzcX06dPv+vju1MeHsFQq3vCwyO43d7TZCqBXn8ecXHvYsqUeGmJjZ3LUERERB2WXS+ZjR07FmPHjq13m0ajQUZGhk1bamoqHnroIVy8eBHdunVDWVkZNm3ahM8++wwjR44EAGzbtg3BwcHYvXs3Ro8ejVOnTiE9PR0HDx7E0KFDAQAff/wxhg0bhtOnT6NPnz53d5BOxmqtgMWihFK5ED4+vQEAVVX5KClZDYPBwEt2RETUITnVXWZlZWVQKBTw8fEBAOTk5MBsNiM6Olrqo9PpEBYWhv379wMADhw4AI1GI4UhAHj44Yeh0WikPvUxGm9dorp9kRN3965Qq3u2+xkqIiIie3CaQHTz5k0sW7YMsbGx8Pb2BgAUFhZCqVSic+fONn2DgoJQWFgo9QkMDKzzeoGBgVKf+qxcuVKac6TRaBAczFBARETUUTlFIDKbzXjmmWdQXV2N9evXN9lfCAGFQiGt3/7vhvrUtnz5cpSVlUlLfn5+64onIiIih+fwgchsNmPq1KnIy8tDRkaGdHYIALRaLUwmE0pLS232KSoqQlBQkNTn6tWrdV63uLhY6lMflUoFb29vm4WIiIg6JocORDVh6OzZs9i9ezf8/PxstoeHh8PNzc1m8nVBQQFOnDiB4cOHAwCGDRuGsrIy/PTTT1KfQ4cOoaysTOpDRERE8mbXu8wqKirwn//8R1rPy8tDbm4ufH19odPp8Lvf/Q5HjhzBP/7xD1itVmnOj6+vL5RKJTQaDWbNmoWEhAT4+fnB19cXixcvxoABA6S7zvr164cxY8Zg9uzZ2LhxIwDgxRdfRExMjFPdYVbzbKAaer0eFovFjhURERF1HHYNRIcPH0ZUVJS0vmjRIgDAjBkzkJiYiK+++goA8MADD9jsl5mZicjISADA2rVr4erqiqlTp6KqqgojRozAli1b4OLiIvX//PPPsWDBAulutIkTJ9b77CNHdfuzgVQqFQDAaKxEfv5VaDTGJvYmIiKiptg1EEVGRkII0eD2xrbVcHd3R2pqKlJTUxvs4+vri23btrWqRkdQ37OBSksPwmJ5BxaLtYm9iYiIqCn8LjMnUvNsIACoqtI30ZuIiIiay6EnVRMRERG1BwYiIiIikj0GIiIiIpI9BiIiIiKSPQYiIiIikj0GIiIiIpI9BiIiIiKSPQYiIiIikj0GIiIiIpI9BiIiIiKSPQYiIiIikj0GIiIiIpI9BiIiIiKSPQYiIiIikj0GIiIiIpI9BiIiIiKSPQYiIiIikj0GIiIiIpI9BiIiIiKSPVd7F0BAcXExDAaDtK7X62GxWOxYERERkbwwENlZcXExYmPnoqTEKLUZjZXIz78KjcbYyJ5ERETUVhiI7MxgMKCkxAiVKgEeHsEAgNLSg7BY3oHFYrVzdURERPLAQOQgPDyCoVb3BABUVentXA0REZG8cFI1ERERyR4DEREREckeAxERERHJHgMRERERyR4DEREREckeAxERERHJHgMRERERyR4DEREREckeAxERERHJHgMRERERyR4DEREREckeAxERERHJHgMRERERyR4DEREREckeAxERERHJnl0D0Q8//IAJEyZAp9NBoVBg165dNtuFEEhMTIROp4OHhwciIyNx8uRJmz5GoxFxcXHw9/eHWq3GxIkTcenSJZs+paWlmD59OjQaDTQaDaZPn47r16/f5dERERGRs7BrIKqsrMTAgQOxbt26ercnJydjzZo1WLduHbKzs6HVajFq1CiUl5dLfeLj47Fz505s374d+/btQ0VFBWJiYmC1WqU+sbGxyM3NRXp6OtLT05Gbm4vp06ff9fERERGRc3C155uPHTsWY8eOrXebEAIpKSl47bXXMHnyZADA1q1bERQUhLS0NMyZMwdlZWXYtGkTPvvsM4wcORIAsG3bNgQHB2P37t0YPXo0Tp06hfT0dBw8eBBDhw4FAHz88ccYNmwYTp8+jT59+rTPYImIiMhhOewcory8PBQWFiI6OlpqU6lUiIiIwP79+wEAOTk5MJvNNn10Oh3CwsKkPgcOHIBGo5HCEAA8/PDD0Gg0Uh8iIiKSN7ueIWpMYWEhACAoKMimPSgoCHq9XuqjVCrRuXPnOn1q9i8sLERgYGCd1w8MDJT61MdoNMJoNErrBoOhdQMhIiIih+ewZ4hqKBQKm3UhRJ222mr3qa9/U6+zcuVKaRK2RqNBcHBwCysnIiIiZ+GwgUir1QJAnbM4RUVF0lkjrVYLk8mE0tLSRvtcvXq1zusXFxfXOft0u+XLl6OsrExa8vPz72g8RERE5LgcNhCFhIRAq9UiIyNDajOZTNi7dy+GDx8OAAgPD4ebm5tNn4KCApw4cULqM2zYMJSVleGnn36S+hw6dAhlZWVSn/qoVCp4e3vbLHJmNhuh1+tx7tw5nDt3DsXFxfYuiYiIqM3YdQ5RRUUF/vOf/0jreXl5yM3Nha+vL7p164b4+HgkJSUhNDQUoaGhSEpKgqenJ2JjYwEAGo0Gs2bNQkJCAvz8/ODr64vFixdjwIAB0l1n/fr1w5gxYzB79mxs3LgRAPDiiy8iJiaGd5g1k8lUAr3+POLi3oVKpQIA+PmpkJa2AQEBAXaujoiI6M7ZNRAdPnwYUVFR0vqiRYsAADNmzMCWLVuwZMkSVFVVYd68eSgtLcXQoUPx3XffwcvLS9pn7dq1cHV1xdSpU1FVVYURI0Zgy5YtcHFxkfp8/vnnWLBggXQ32sSJExt89hHVZbVWwGJRQqlcCB+f3qiqykdJyWoYDAYGIiIi6hDsGogiIyMhhGhwu0KhQGJiIhITExvs4+7ujtTUVKSmpjbYx9fXF9u2bbuTUgmAu3tXqNU9AQC33YBHRETk9Bx2DhERERFRe3HY5xCRY6uZZH07b29vXkIjIiKnxEBELVbfJGuAE62JiMh5MRBRi9WeZA2AE62JiMipMRBRq90+yRrgRGsiInJenFRNREREssdARERERLLHQERERESyx0BEREREssdARERERLLHQERERESyx0BEREREssdARERERLLHQERERESyx0BEREREssdARERERLLHQERERESyx0BEREREssdARERERLLHQERERESyx0BEREREssdARERERLLHQERERESyx0BEREREssdARERERLLHQERERESyx0BEREREssdARERERLLHQERERESy16pAlJeX19Z1EBEREdlNqwJRr169EBUVhW3btuHmzZttXRMRERFRu2pVIPr5558xaNAgJCQkQKvVYs6cOfjpp5/aujYiIiKidtGqQBQWFoY1a9bg8uXL2Lx5MwoLC/Hoo4+if//+WLNmDYqLi9u6TiIiIqK75o4mVbu6uuKpp57CX/7yF7z33ns4d+4cFi9ejK5du+K5555DQUFBW9VJREREdNfcUSA6fPgw5s2bhy5dumDNmjVYvHgxzp07hz179uDy5cuYNGlSW9VJREREdNe4tmanNWvWYPPmzTh9+jTGjRuHTz/9FOPGjUOnTrfyVUhICDZu3Ii+ffu2abFEREREd0OrAtGGDRvwhz/8Ac8//zy0Wm29fbp164ZNmzbdUXFERERE7aFVgejs2bNN9lEqlZgxY0ZrXp6IiIioXbVqDtHmzZvx17/+tU77X//6V2zduvWOiyIiIiJqT60KRO+++y78/f3rtAcGBiIpKemOiyIiIiJqT60KRHq9HiEhIXXau3fvjosXL95xUTUsFgtef/11hISEwMPDA/fddx/efvttVFdXS32EEEhMTIROp4OHhwciIyNx8uRJm9cxGo2Ii4uDv78/1Go1Jk6ciEuXLrVZnUREROTcWhWIAgMDcezYsTrtP//8M/z8/O64qBrvvfcePvzwQ6xbtw6nTp1CcnIyVq1ahdTUVKlPcnIy1qxZg3Xr1iE7OxtarRajRo1CeXm51Cc+Ph47d+7E9u3bsW/fPlRUVCAmJgZWq7XNaiXAbDZCr9fj3LlzOHfuHB/QSURETqNVk6qfeeYZLFiwAF5eXnj88ccBAHv37sUrr7yCZ555ps2KO3DgACZNmoTx48cDAHr06IEvvvgChw8fBnDr7FBKSgpee+01TJ48GQCwdetWBAUFIS0tDXPmzEFZWRk2bdqEzz77DCNHjgQAbNu2DcHBwdi9ezdGjx7dZvXKmclUAr3+POLi3oVKpQIA+PmpkJa2AQEBAXaujoiIqHGtOkO0YsUKDB06FCNGjICHhwc8PDwQHR2NJ554ok3nED366KP4/vvvcebMGQC3zkDt27cP48aNAwDk5eWhsLAQ0dHR0j4qlQoRERHYv38/ACAnJwdms9mmj06nQ1hYmNSnPkajEQaDwWahhlmtFbBYlFAqF8LHJwUqVQJKSoz8uRERkVNo1RkipVKJL7/8En/+85/x888/w8PDAwMGDED37t3btLilS5eirKwMffv2hYuLC6xWK9555x38/ve/BwAUFhYCAIKCgmz2CwoKgl6vl/oolUp07ty5Tp+a/euzcuVKvPXWW205HFlwd+8KtbonAMBotHMxREREzdSqQFSjd+/e6N27d1vVUseXX36Jbdu2IS0tDf3790dubi7i4+Oh0+lsnnGkUChs9hNC1Gmrrak+y5cvx6JFi6R1g8GA4ODgVo6EiIiIHFmrApHVasWWLVvw/fffo6ioyOauLwDYs2dPmxT36quvYtmyZdK8pAEDBkCv12PlypWYMWOG9JTswsJCdOnSRdqvqKhIOmuk1WphMplQWlpqc5aoqKgIw4cPb/C9VSqVNBeGiIiIOrZWzSF65ZVX8Morr8BqtSIsLAwDBw60WdrKjRs3pO9Hq+Hi4iIFsJCQEGi1WmRkZEjbTSYT9u7dK4Wd8PBwuLm52fQpKCjAiRMnGg1EREREJB+tOkO0fft2/OUvf5EmN98tEyZMwDvvvINu3bqhf//+OHr0KNasWYM//OEPAG5dKouPj0dSUhJCQ0MRGhqKpKQkeHp6IjY2FgCg0Wgwa9YsJCQkwM/PD76+vli8eDEGDBgg3XVGRERE8tbqSdW9evVq61rqSE1NxRtvvIF58+ahqKgIOp0Oc+bMwZ/+9Cepz5IlS1BVVYV58+ahtLQUQ4cOxXfffQcvLy+pz9q1a+Hq6oqpU6eiqqoKI0aMwJYtW+Di4nLXx0BERESOr1WBKCEhAf/93/+NdevWNTl5+U54eXkhJSUFKSkpDfZRKBRITExEYmJig33c3d2Rmppq80BHIiIiohqtCkT79u1DZmYmvvnmG/Tv3x9ubm4223fs2NEmxRERERG1h1YFIh8fHzz11FNtXQsRERGRXbQqEG3evLmt6yAiIiKym1bddg/c+ib63bt3Y+PGjdIXqV65cgUVFRVtVhwRERFRe2jVGSK9Xo8xY8bg4sWLMBqNGDVqFLy8vJCcnIybN2/iww8/bOs6iYiIiO6aVj+YcciQISgtLYWHh4fU/tRTT+H7779vs+KIiIiI2kOr7zL717/+BaVSadPevXt3XL58uU0KIyIiImovrTpDVF1dDavVWqf90qVLNg9EJCIiInIGrQpEo0aNsnlYokKhQEVFBd588827/nUeRERERG2tVZfM1q5di6ioKNx///24efMmYmNjcfbsWfj7++OLL75o6xqJiIiI7qpWBSKdTofc3Fx88cUXOHLkCKqrqzFr1iw8++yzNpOsiYiIiJxBqwIRAHh4eOAPf/iD9M3zRERERM6qVYHo008/bXT7c88916piiIiIiOyhVYHolVdesVk3m824ceMGlEolPD09GYiIiIjIqbTqLrPS0lKbpaKiAqdPn8ajjz7KSdVERETkdFr9XWa1hYaG4t13361z9oiIiIjI0bVZIAIAFxcXXLlypS1fkoiIiOiua9Ucoq+++spmXQiBgoICrFu3Do888kibFEZERETUXloViJ588kmbdYVCgYCAADzxxBNYvXp1W9RFRERE1G5aFYiqq6vbug4iIiIiu2nTOUREREREzqhVZ4gWLVrU7L5r1qxpzVsQERERtZtWBaKjR4/iyJEjsFgs6NOnDwDgzJkzcHFxweDBg6V+CoWibaokIiIiuotaFYgmTJgALy8vbN26FZ07dwZw62GNzz//PB577DEkJCS0aZFEREREd1Or5hCtXr0aK1eulMIQAHTu3BkrVqzgXWZERETkdFp1hshgMODq1avo37+/TXtRURHKy8vbpDByfmazEXq93qbN29sbAQEBdqqIiIiofq0KRE899RSef/55rF69Gg8//DAA4ODBg3j11VcxefLkNi2QnJPJVAK9/jzi4t6FSqWS2v38VEhL28BQREREDqVVgejDDz/E4sWLMW3aNJjN5lsv5OqKWbNmYdWqVW1aIDknq7UCFosSSuVC+Pj0BgBUVeWjpGQ1DAYDAxERETmUVgUiT09PrF+/HqtWrcK5c+cghECvXr2gVqvbuj5ycu7uXaFW95TWjUY7FkNERNSAO3owY0FBAQoKCtC7d2+o1WoIIdqqLiIiIqJ206pAVFJSghEjRqB3794YN24cCgoKAAAvvPACb7knIiIip9OqQLRw4UK4ubnh4sWL8PT0lNqffvpppKent1lxRERERO2hVXOIvvvuO3z77bfo2rWrTXtoaGid26yJiIiIHF2rzhBVVlbanBmqce3aNZtbrImIiIicQasC0eOPP45PP/1UWlcoFKiursaqVasQFRXVZsURERERtYdWXTJbtWoVIiMjcfjwYZhMJixZsgQnT57Er7/+in/9619tXSMRERHRXdWqM0T3338/jh07hoceegijRo1CZWUlJk+ejKNHj6Jnz55NvwARERGRA2nxGSKz2Yzo6Ghs3LgRb7311t2oiYiIiKhdtfgMkZubG06cOAGFQnE36iEiIiJqd626ZPbcc89h06ZNbV0LERERkV20KhCZTCZs2LAB4eHhmDNnDhYtWmSztKXLly9j2rRp8PPzg6enJx544AHk5ORI24UQSExMhE6ng4eHByIjI3Hy5Emb1zAajYiLi4O/vz/UajUmTpyIS5cutWmdRERE5LxaFIjOnz+P6upqnDhxAoMHD4a3tzfOnDmDo0ePSktubm6bFVdaWopHHnkEbm5u+Oabb/DLL79g9erV8PHxkfokJydjzZo1WLduHbKzs6HVajFq1CiUl5dLfeLj47Fz505s374d+/btQ0VFBWJiYmC1WtusViIiInJeLZpUHRoaioKCAmRmZgK49VUd77//PoKCgu5Kce+99x6Cg4OxefNmqa1Hjx7Sv4UQSElJwWuvvYbJkycDALZu3YqgoCCkpaVhzpw5KCsrw6ZNm/DZZ59h5MiRAIBt27YhODgYu3fvxujRo+9K7UREROQ8WnSGqPa32X/zzTeorKxs04Ju99VXX2HIkCGYMmUKAgMDMWjQIHz88cfS9ry8PBQWFiI6OlpqU6lUiIiIwP79+wEAOTk50p1xNXQ6HcLCwqQ+9TEajTAYDDYLERERdUytmkNUo3ZAamvnz5/Hhg0bEBoaim+//RYvvfQSFixYID0lu7CwEADqnKEKCgqSthUWFkKpVKJz584N9qnPypUrodFopCU4OLgth0ZEREQOpEWBSKFQ1Lnd/m7efl9dXY3BgwcjKSkJgwYNwpw5czB79mxs2LCh0RqEEE3W1VSf5cuXo6ysTFry8/NbPxAiIiJyaC2aQySEwMyZM6UvcL158yZeeuklqNVqm347duxok+K6dOmC+++/36atX79++N///V8AgFarBXDrLFCXLl2kPkVFRdJZI61WC5PJhNLSUpuzREVFRRg+fHiD761SqfhFtURERDLRojNEM2bMQGBgoHQZadq0adDpdDaXljQaTZsV98gjj+D06dM2bWfOnEH37t0BACEhIdBqtcjIyJC2m0wm7N27Vwo74eHhcHNzs+lTUFCAEydONBqIiIiISD5adIbo9ru92sPChQsxfPhwJCUlYerUqfjpp5/w0Ucf4aOPPgJw61JZfHw8kpKSEBoaitDQUCQlJcHT0xOxsbEAAI1Gg1mzZiEhIQF+fn7w9fXF4sWLMWDAAOmuM2o/ZrMRer1eWvf29kZAQIAdKyIiImrlt923lwcffBA7d+7E8uXL8fbbbyMkJAQpKSl49tlnpT5LlixBVVUV5s2bh9LSUgwdOhTfffcdvLy8pD5r166Fq6srpk6diqqqKowYMQJbtmyBi4uLPYYlWyZTCfT684iLe1e6HOnnp0Ja2gaGIiIisiuHDkQAEBMTg5iYmAa3KxQKJCYmIjExscE+7u7uSE1NRWpq6l2okJrLaq2AxaKEUrkQPj69UVWVj5KS1TAYDAxERERkVw4fiKjjcXfvCrW6JwDAaLRzMURERLjD5xARERERdQQMRERERCR7DEREREQkewxEREREJHsMRERERCR7DEREREQkewxEREREJHsMRERERCR7DEREREQkewxEREREJHsMRERERCR7DEREREQkewxEREREJHsMRERERCR7DEREREQkewxEREREJHsMRERERCR7DEREREQkewxEREREJHsMRERERCR7DEREREQkewxEREREJHsMRERERCR7rvYugOTNbDZCr9fbtHl7eyMgIMBOFRERkRwxEJHdmEwl0OvPIy7uXahUKqndywtYteoN+Pn5AWBAIiKiu4+BiOzGaq2AxaKEUrkQPj69AQAGw3EcPboYzz//uhSS/PxUSEvbwFBERER3DQMR2Z27e1eo1T0BAFVVepuQVFWVj5KS1TAYDAxERER01zAQkUO6PSQZjXYuhoiIOjzeZUZERESyx0BEREREssdARERERLLHQERERESyx0BEREREssdARERERLLHQERERESyx0BEREREssdARERERLLHQERERESy51SBaOXKlVAoFIiPj5fahBBITEyETqeDh4cHIiMjcfLkSZv9jEYj4uLi4O/vD7VajYkTJ+LSpUvtXD0RERE5KqcJRNnZ2fjoo4/wm9/8xqY9OTkZa9aswbp165CdnQ2tVotRo0ahvLxc6hMfH4+dO3di+/bt2LdvHyoqKhATEwOr1drewyAiIiIH5BSBqKKiAs8++yw+/vhjdO7cWWoXQiAlJQWvvfYaJk+ejLCwMGzduhU3btxAWloaAKCsrAybNm3C6tWrMXLkSAwaNAjbtm3D8ePHsXv3bnsNiYiIiByIUwSil19+GePHj8fIkSNt2vPy8lBYWIjo6GipTaVSISIiAvv37wcA5OTkwGw22/TR6XQICwuT+hAREZG8udq7gKZs374dR44cQXZ2dp1thYWFAICgoCCb9qCgIOj1eqmPUqm0ObNU06dm//oYjUYYjUZp3WAwtHoMRERE5Ngc+gxRfn4+XnnlFWzbtg3u7u4N9lMoFDbrQog6bbU11WflypXQaDTSEhwc3LLiiYiIyGk4dCDKyclBUVERwsPD4erqCldXV+zduxfvv/8+XF1dpTNDtc/0FBUVSdu0Wi1MJhNKS0sb7FOf5cuXo6ysTFry8/PbeHRERETkKBw6EI0YMQLHjx9Hbm6utAwZMgTPPvsscnNzcd9990Gr1SIjI0Pax2QyYe/evRg+fDgAIDw8HG5ubjZ9CgoKcOLECalPfVQqFby9vW0WIiIi6pgceg6Rl5cXwsLCbNrUajX8/Pyk9vj4eCQlJSE0NBShoaFISkqCp6cnYmNjAQAajQazZs1CQkIC/Pz84Ovri8WLF2PAgAF1JmmTYzKbjdKcsBre3t4ICAiwU0VERNTROHQgao4lS5agqqoK8+bNQ2lpKYYOHYrvvvsOXl5eUp+1a9fC1dUVU6dORVVVFUaMGIEtW7bAxcXFjpVTc5hMJdDrzyMu7l2oVCqp3c9PhbS0DQxFRETUJpwuEGVlZdmsKxQKJCYmIjExscF93N3dkZqaitTU1LtbHLU5q7UCFosSSuVC+Pj0BgBUVeWjpGQ1DAYDAxEREbUJpwtEJE/u7l2hVveU1m97IgIREdEdc+hJ1URERETtgYGIiIiIZI+BiIiIiGSPgYiIiIhkj4GIiIiIZI+BiIiIiGSPgYiIiIhkj4GIiIiIZI+BiIiIiGSPgYiIiIhkj4GIiIiIZI+BiIiIiGSPgYiIiIhkj4GIiIiIZI+BiIiIiGSPgYiIiIhkj4GIiIiIZI+BiIiIiGSPgYiIiIhkj4GIiIiIZI+BiIiIiGSPgYiIiIhkj4GIiIiIZI+BiIiIiGTP1d4FELWG2WyEXq+X1k0mE5RKpU0fb29vBAQEtHdpRETkhBiIyOmYTCXQ688jLu5dqFQqmM1GXLmSh3vv7QVX1//3kfbzUyEtbQNDERERNYmBiJyO1VoBi0UJpXIhfHx6o7T0IKqq3oGLywL4+PQGAFRV5aOkZDUMBgMDERERNYmBiJyWu3tXqNU9UVWlt1mvYTTaqzIiInI2nFRNREREssdARERERLLHQERERESyx0BEREREssdARERERLLHQERERESyx0BEREREssdARERERLLHQERERESyx0BEREREssdARERERLLn0IFo5cqVePDBB+Hl5YXAwEA8+eSTOH36tE0fIQQSExOh0+ng4eGByMhInDx50qaP0WhEXFwc/P39oVarMXHiRFy6dKk9h0JEREQOzKED0d69e/Hyyy/j4MGDyMjIgMViQXR0NCorK6U+ycnJWLNmDdatW4fs7GxotVqMGjUK5eXlUp/4+Hjs3LkT27dvx759+1BRUYGYmBhYrVZ7DIuIiIgcjEN/2316errN+ubNmxEYGIicnBw8/vjjEEIgJSUFr732GiZPngwA2Lp1K4KCgpCWloY5c+agrKwMmzZtwmeffYaRI0cCALZt24bg4GDs3r0bo0ePbvdxERERkWNx6DNEtZWVlQEAfH19AQB5eXkoLCxEdHS01EelUiEiIgL79+8HAOTk5MBsNtv00el0CAsLk/rUx2g0wmAw2CzkXMxmI/R6Pc6dOyctxcXF9i6LiIgckEOfIbqdEAKLFi3Co48+irCwMABAYWEhACAoKMimb1BQEPR6vdRHqVSic+fOdfrU7F+flStX4q233mrLIVA7MplKoNefR1zcu1CpVFK7n58KaWkbEBAQYMfqiIjI0TjNGaL58+fj2LFj+OKLL+psUygUNutCiDpttTXVZ/ny5SgrK5OW/Pz81hVOdmG1VsBiUUKpXAgfnxT4+KRApUpASYmRZ/uIiKgOpzhDFBcXh6+++go//PADunbtKrVrtVoAt84CdenSRWovKiqSzhpptVqYTCaUlpbanCUqKirC8OHDG3xPlUplc2aBnJO7e1eo1T2ldaPRjsUQEZHDcugzREIIzJ8/Hzt27MCePXsQEhJisz0kJARarRYZGRlSm8lkwt69e6WwEx4eDjc3N5s+BQUFOHHiRKOBiIiIiOTDoc8Qvfzyy0hLS8Pf//53eHl5SXN+NBoNPDw8oFAoEB8fj6SkJISGhiI0NBRJSUnw9PREbGys1HfWrFlISEiAn58ffH19sXjxYgwYMEC664yIiIjkzaED0YYNGwAAkZGRNu2bN2/GzJkzAQBLlixBVVUV5s2bh9LSUgwdOhTfffcdvLy8pP5r166Fq6srpk6diqqqKowYMQJbtmyBi4tLew2FiIiIHJhDByIhRJN9FAoFEhMTkZiY2GAfd3d3pKamIjU1tQ2rI2dUcyt+DW9vb95xRkREjh2IiNpSfbfi8zZ8IiICHHxSNVFbqn0rPm/DJyKiGjxDRLJz+634vA2fiIgAniEiIiIiYiAiIiIiYiAiIiIi2WMgIiIiItljICIiIiLZ411mJGu1H9QI8GGNRERyxEBEslXfgxoBPqyRiEiOGIhItmwf1NgbAFBVlY+SktUwGAwMREREMsJARLJ3+4MaAT6skYhIjjipmoiIiGSPgYiIiIhkj4GIiIiIZI+BiIiIiGSPgYiIiIhkj3eZETWhuLgYBoPBpo0PbyQi6lgYiIgaUVxcjNjYuSgpsb0Xnw9vJCLqWBiIiBphMBhQUmKESpUAD49gAHx4IxFRR8RARFTL7d9vptfrYbFY4OMTzIc3EhF1YAxERLep/f1mRmMl8vOvQqNhAiIi6sh4lxnRbWy/3ywFSuUsWCwCFovV3qUREdFdxEBEVI+a7zdzd+9i71KIiKgdMBARERGR7HEOEVEr3D7xGuBziYiInB0DEVEL1Z54DfC5REREzo6XzIhaqPbEa5UqASUlxjpPsyYiIufBM0RErVQz8Rqo/7lEtb/yg5fViIgcFwMR0V1Q31d+1HdZjaGJiMgxMBARtYHak6z1ej2uXq2EWr0UHh7B9X7dR3NDExER3X0MRER3qL5J1jVPuB44MLDBy2q1vyeN35FGRGQ/DEREd8h2knVvAEBp6UFYLO806wnXHh7Bjc5FIiKiu4+BiKiN3D7JuqpK30RvIiJyJAxERA6k9lwkgBOtiYjaAwMRUTupb+K1xWKR1uubiwQAXl7AqlVvwM/PDwADEhHR3cBARNQOGpt4rdHcmjhU31wkg+E4jh5djOeff90uT8Wu/VgAgIGMiDomBiKidtCSide15yLdvl973olW32MBAMd/NABDHBG1BgMRUTtq7cTrpp6KXVt9ocBkMkGpVErrTYWE2o8FuFVzPgoLk3D8+HF07969Wa/Tnpw1xBGR/ckqEK1fvx6rVq1CQUEB+vfvj5SUFDz22GP2Lovojt0egEpKSvDqqytQXi6k7WazEVeu5OHee3vB1fXWr31zQ8LtjwVw9C+2bSjEddTnO/FJ50RtRzaB6Msvv0R8fDzWr1+PRx55BBs3bsTYsWPxyy+/oFu3bvYuj6jZak/Orh2AauYm9emzFl5et4JMaelBVFW9AxeXBXd06a32pb+2DBtteanr9hAHdMznO/FJ50RtSzaBaM2aNZg1axZeeOEFAEBKSgq+/fZbbNiwAStXrrRzdUTN09jk7JoAVDM3ydW1S53Lcy299NaQ21+nouLOHxXQ3EtdTV0KrH3nXkfGJ50TtS1ZBCKTyYScnBwsW7bMpj06Ohr79++3U1VELdfY5OyaANSeD4Vs7qMCas9fAmxDU3MuddUXmmpfCqx95549NOcyVlte6uKTzhvGCfaOzdEu+coiEF27dg1WqxVBQUE27UFBQSgsLKx3H6PRCONtf13KysoAoM4v150qLy+H1WpGefm/YbGUAwAqK89BCCsqK8/Azc1aZ519Wt7H3u/f1n2s1hvS58VqvdHi16mqugyj8QZ++eUXlJeX1/vZzM/Ph9F4s9HPpsHwM8xmF1gsE+HhcS8A4MaN8zh79gM899wyqFRKmM0mFBbq0aXLfXB1dZFe38tLgbfffhW+vr7Se7m4VErvZbFU2tSYn5+PK1fK4OIyFUqlPwDAZPoFN27oYTSOh1J5LyyWX2A2f4qyslNQKEwA0KyxtpVff/0Vf/rTf6G8vLrecTa3T3PUPj7tOU5nUN/PGWjdz5raXn3Hx9dXhU2b1sLf379N36vm/7eFEI13FDJw+fJlAUDs37/fpn3FihWiT58+9e7z5ptvCgBcuHDhwoULlw6w5OfnN5oVZHGGyN/fHy4uLnXOBhUVFdU5a1Rj+fLlWLRokbReXV2NX3/9FX5+flAoFHdck8FgQHBwMPLz8+Ht7X3Hr+doOvL4OvLYgI49vo48NqBjj68jjw3o2OOz99iEECgvL4dOp2u0nywCkVKpRHh4ODIyMvDUU09J7RkZGZg0aVK9+6hUKps5EQDg4+PT5rV5e3t3uA//7Try+Dry2ICOPb6OPDagY4+vI48N6Njjs+fYNBpNk31kEYgAYNGiRZg+fTqGDBmCYcOG4aOPPsLFixfx0ksv2bs0IiIisjPZBKKnn34aJSUlePvtt1FQUICwsDD885//lJ62S0RERPIlm0AEAPPmzcO8efPsXQaAW5fk3nzzzTqX5TqKjjy+jjw2oGOPryOPDejY4+vIYwM69vicZWwKIZq6D42IiIioY+tk7wKIiIiI7I2BiIiIiGSPgYiIiIhkj4GIiIiIZI+ByE7Wr1+PkJAQuLu7Izw8HD/++KO9S2qxlStX4sEHH4SXlxcCAwPx5JNP4vTp0zZ9Zs6cCYVCYbM8/PDDdqq4+RITE+vUrdVqpe1CCCQmJkKn08HDwwORkZE4efKkHStumR49etQZn0KhwMsvvwzAuY7bDz/8gAkTJkCn00GhUGDXrl0225tzrIxGI+Li4uDv7w+1Wo2JEyfi0qVL7TiKhjU2PrPZjKVLl2LAgAFQq9XQ6XR47rnncOXKFZvXiIyMrHM8n3nmmXYeSf2aOn7N+Sw66vFramz1/Q4qFAqsWrVK6uOox645f/+d7XePgcgOvvzyS8THx+O1117D0aNH8dhjj2Hs2LG4ePGivUtrkb179+Lll1/GwYMHkZGRAYvFgujoaFRWVtr0GzNmDAoKCqTln//8p50qbpn+/fvb1H38+HFpW3JyMtasWYN169YhOzsbWq0Wo0aNcpov1czOzrYZW0ZGBgBgypQpUh9nOW6VlZUYOHAg1q1bV+/25hyr+Ph47Ny5E9u3b8e+fftQUVGBmJgYWK3W9hpGgxob340bN3DkyBG88cYbOHLkCHbs2IEzZ85g4sSJdfrOnj3b5nhu3LixPcpvUlPHD2j6s+iox6+psd0+poKCAnzyySdQKBT47W9/a9PPEY9dc/7+O93vXht8dyq10EMPPSReeuklm7a+ffuKZcuW2amitlFUVCQAiL1790ptM2bMEJMmTbJfUa305ptvioEDB9a7rbq6Wmi1WvHuu+9KbTdv3hQajUZ8+OGH7VRh23rllVdEz549RXV1tRDCeY8bALFz505pvTnH6vr168LNzU1s375d6nP58mXRqVMnkZ6e3m61N0ft8dXnp59+EgCEXq+X2iIiIsQrr7xyd4trA/WNr6nPorMcv+Ycu0mTJoknnnjCps1Zjl3tv//O+LvHM0TtzGQyIScnB9HR0Tbt0dHR2L9/v52qahtlZWUAAF9fX5v2rKwsBAYGonfv3pg9ezaKiorsUV6LnT17FjqdDiEhIXjmmWdw/vx5AEBeXh4KCwttjqFKpUJERIRTHkOTyYRt27bhD3/4g80XFzvrcbtdc45VTk4OzGazTR+dToewsDCnPJ5lZWVQKBR1vnvx888/h7+/P/r374/Fixc7zdlMoPHPYkc5flevXsXXX3+NWbNm1dnmDMeu9t9/Z/zdk9WTqh3BtWvXYLVaERQUZNMeFBSEwsJCO1V154QQWLRoER599FGEhYVJ7WPHjsWUKVPQvXt35OXl4Y033sATTzyBnJwch35q6dChQ/Hpp5+id+/euHr1KlasWIHhw4fj5MmT0nGq7xjq9Xp7lHtHdu3ahevXr2PmzJlSm7Met9qac6wKCwuhVCrRuXPnOn2c7Xfy5s2bWLZsGWJjY22+RPPZZ59FSEgItFotTpw4geXLl+Pnn3+WLpU6sqY+ix3l+G3duhVeXl6YPHmyTbszHLv6/v474+8eA5Gd3P5f4sCtD1TtNmcyf/58HDt2DPv27bNpf/rpp6V/h4WFYciQIejevTu+/vrrOr/4jmTs2LHSvwcMGIBhw4ahZ8+e2Lp1qzShs6Mcw02bNmHs2LHQ6XRSm7Met4a05lg52/E0m8145plnUF1djfXr19tsmz17tvTvsLAwhIaGYsiQIThy5AgGDx7c3qW2SGs/i852/D755BM8++yzcHd3t2l3hmPX0N9/wLl+93jJrJ35+/vDxcWlTvotKiqqk6SdRVxcHL766itkZmaia9eujfbt0qULunfvjrNnz7ZTdW1DrVZjwIABOHv2rHS3WUc4hnq9Hrt378YLL7zQaD9nPW7NOVZarRYmkwmlpaUN9nF0ZrMZU6dORV5eHjIyMmzODtVn8ODBcHNzc7rjCdT9LHaE4/fjjz/i9OnTTf4eAo537Br6+++Mv3sMRO1MqVQiPDy8zunOjIwMDB8+3E5VtY4QAvPnz8eOHTuwZ88ehISENLlPSUkJ8vPz0aVLl3aosO0YjUacOnUKXbp0kU5f334MTSYT9u7d63THcPPmzQgMDMT48eMb7eesx605xyo8PBxubm42fQoKCnDixAmnOJ41Yejs2bPYvXs3/Pz8mtzn5MmTMJvNTnc8gbqfRWc/fsCts7Th4eEYOHBgk30d5dg19fffKX/32n0aN4nt27cLNzc3sWnTJvHLL7+I+Ph4oVarxYULF+xdWovMnTtXaDQakZWVJQoKCqTlxo0bQgghysvLRUJCgti/f7/Iy8sTmZmZYtiwYeLee+8VBoPBztU3LiEhQWRlZYnz58+LgwcPipiYGOHl5SUdo3fffVdoNBqxY8cOcfz4cfH73/9edOnSxeHHdTur1Sq6desmli5datPubMetvLxcHD16VBw9elQAEGvWrBFHjx6V7rJqzrF66aWXRNeuXcXu3bvFkSNHxBNPPCEGDhwoLBaLvYYlaWx8ZrNZTJw4UXTt2lXk5uba/B4ajUYhhBD/+c9/xFtvvSWys7NFXl6e+Prrr0Xfvn3FoEGDHH58zf0sOurxa+qzKYQQZWVlwtPTU2zYsKHO/o587Jr6+y+E8/3uMRDZyQcffCC6d+8ulEqlGDx4sM2t6s4CQL3L5s2bhRBC3LhxQ0RHR4uAgADh5uYmunXrJmbMmCEuXrxo38Kb4emnnxZdunQRbm5uQqfTicmTJ4uTJ09K26urq8Wbb74ptFqtUKlU4vHHHxfHjx+3Y8Ut9+233woA4vTp0zbtznbcMjMz6/0czpgxQwjRvGNVVVUl5s+fL3x9fYWHh4eIiYlxmPE2Nr68vLwGfw8zMzOFEEJcvHhRPP7448LX11colUrRs2dPsWDBAlFSUmLfgf3/Ghtfcz+Ljnr8mvpsCiHExo0bhYeHh7h+/Xqd/R352DX1918I5/vdUwghxF06+URERETkFDiHiIiIiGSPgYiIiIhkj4GIiIiIZI+BiIiIiGSPgYiIiIhkj4GIiIiIZI+BiIiIiGSPgYiIZKFHjx5ISUmxdxlE5KAYiIjorvvwww/h5eUFi8UitVVUVMDNzQ2PPfaYTd8ff/wRCoUCZ86cae8yYTAY8Nprr6Fv375wd3eHVqvFyJEjsWPHDrT3M2wZ4Ijal6u9CyCiji8qKgoVFRU4fPgwHn74YQC3go9Wq0V2djZu3LgBT09PAEBWVhZ0Oh169+7d4vexWq1QKBTo1Knl/613/fp1PProoygrK8OKFSvw4IMPwtXVFXv37sWSJUvwxBNPwMfHp8WvS0TOgWeIiOiu69OnD3Q6HbKysqS2rKwsTJo0CT179sT+/ftt2qOiogAApaWleO6559C5c2d4enpi7NixOHv2rNR3y5Yt8PHxwT/+8Q/cf//9UKlU0Ov1KCoqwoQJE+Dh4YGQkBB8/vnnTdb4xz/+ERcuXMChQ4cwY8YM3H///ejduzdmz56N3Nxc3HPPPc2qKTExEQ888IDNa6ekpKBHjx7S+syZM/Hkk0/iv/7rv9ClSxf4+fnh5ZdfhtlsBgBERkZCr9dj4cKFUCgUUCgUzf5ZE1HrMBARUbuIjIxEZmamtJ6ZmYnIyEhERERI7SaTCQcOHJAC0cyZM3H48GF89dVXOHDgAIQQGDdunBQcAODGjRtYuXIl/ud//gcnT55EYGAgZs6ciQsXLmDPnj3429/+hvXr16OoqKjB2qqrq7F9+3Y8++yz0Ol0dbbfc889cHV1bXZNzZGZmYlz584hMzMTW7duxZYtW7BlyxYAwI4dO9C1a1e8/fbbKCgoQEFBQYtem4hajpfMiKhdREZGYuHChbBYLKiqqsLRo0fx+OOPw2q14v333wcAHDx4EFVVVYiKisLZs2fx1Vdf4V//+heGDx8OAPj8888RHByMXbt2YcqUKQAAs9mM9evXY+DAgQCAM2fO4JtvvsHBgwcxdOhQAMCmTZvQr1+/Bmu7du0aSktL0bdv30bH0NyamqNz585Yt24dXFxc0LdvX4wfPx7ff/89Zs+eDV9fX7i4uMDLywtarbbZr0lErcczRETULqKiolBZWYns7Gz8+OOP6N27NwIDAxEREYHs7GxUVlYiKysL3bp1w3333YdTp07B1dVVCjUA4Ofnhz59+uDUqVNSm1KpxG9+8xtpvWa/IUOGSG19+/ZtdP5PzYTppi5NNbem5ujfvz9cXFyk9S5dujR6FouI7i6eISKidtGrVy907doVmZmZKC0tRUREBABAq9UiJCQE//rXv5CZmYknnngCABq8q0sIYRNcPDw8bNabG25uFxAQgM6dOzcZappTU6dOner0q+9ympubm826QqFAdXV1s2smorbFM0RE1G6ioqKQlZWFrKwsREZGSu0RERH49ttvcfDgQWn+0P333w+LxYJDhw5J/UpKSnDmzJlGL3/169cPFosFhw8fltpOnz6N69evN7hPp06d8PTTT+Pzzz/HlStX6myvrKyExWJpVk0BAQEoLCy0CUW5ubkNvndDlEolrFZri/cjotZhICKidhMVFYV9+/YhNzdXOkME3ApEH3/8MW7evCkFotDQUEyaNAmzZ8/Gvn378PPPP2PatGm49957MWnSpAbfo0+fPhgzZgxmz56NQ4cOIScnBy+88AI8PDwarS0pKQnBwcEYOnQoPv30U/zyyy84e/YsPvnkEzzwwAOoqKhoVk2RkZEoLi5GcnIyzp07hw8++ADffPNNi39WPXr0wA8//IDLly/j2rVrLd6fiFqGgYiI2k1UVBSqqqrQq1cvBAUFSe0REREoLy9Hz549ERwcLLVv3rwZ4eHhiImJwbBhwyCEwD//+c86l5tq27x5M4KDgxEREYHJkyfjxRdfRGBgYKP7dO7cGQcPHsS0adOwYsUKDBo0CI899hi++OILrFq1ChqNplk19evXD+vXr8cHH3yAgQMH4qeffsLixYtb/LN6++23ceHCBfTs2RMBAQEt3p+IWkYh2vvxq0REREQOhmeIiIiISPYYiIiIiEj2GIiIiIhI9hiIiIiISPYYiIiIiEj2GIiIiIhI9hiIiIiISPYYiIiIiEj2GIiIiIhI9hiIiIiISPYYiIiIiEj2GIiIiIhI9v4/6vtA6j1VMfgAAAAASUVORK5CYII=",
      "text/plain": [
       "<Figure size 640x480 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Use for evaluating OPTION 1\n",
    "\n",
    "word_counts = all_line_data_filtered_df['new_line'].apply(lambda x: len(x.split()))-2\n",
    "plt.hist(word_counts, bins=100, alpha=0.7, color='blue', edgecolor='black')\n",
    "plt.xlabel('Word Count')\n",
    "plt.ylabel('Frequency')\n",
    "plt.title('Word Count Histogram')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# word_counts[word_counts == 98]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "# all_line_data_filtered_df['new_line'].iloc[3678]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# conda search -c conda-forge spacy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "CREATING DATA WITH SENTENCIZER"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def records_to_sentences(content_dict):\n",
    "    \"\"\"\n",
    "    Takes the content dictionary with the record and full text content for the record:\n",
    "    content_dict = {record: <all text in the record.txt file>}\n",
    "    \n",
    "    Returns sentences_dict with the record and the text broken by sentence number using the spacey sentencer.\n",
    "    \n",
    "    For example:\n",
    "    sentences_dict = {'record-106': {1:'Admission Date : 2015...after exercise .',\n",
    "                                     2: 'Pt had been in her usual state of health and performed here regular exercises .',\n",
    "                                     3: 'After returning to her room she w noted SSCP without radiation .',\n",
    "                                     ...}}\n",
    "    \"\"\"\n",
    "    nlp1 = spacy.load(\"en_ner_bc5cdr_md\",disable = ['parser'])\n",
    "    nlp1.add_pipe('sentencizer')\n",
    "\n",
    "    sentences_dict = {}\n",
    "    for key in content_dict:\n",
    "        sentences_dict[key] = {}\n",
    "        record_text = content_dict[key]\n",
    "        record_text = remove_adjacent_periods(record_text)\n",
    "        docs = nlp1(record_text)\n",
    "        i = 1\n",
    "        for s in docs.sents:\n",
    "            sentence = str.strip(str(s))\n",
    "            sentence = sentence.replace(\"\\n\",\" \")\n",
    "#             sentence = clean_text(sentence)\n",
    "\n",
    "            sentences_dict[key][i] = sentence\n",
    "            i += 1\n",
    "\n",
    "    return sentences_dict\n",
    "\n",
    "\n",
    "if data_to_test == \"Sentencizer Data\":\n",
    "    training_sentences_dict = records_to_sentences(training_content_dict)\n",
    "    test_sentences_dict = records_to_sentences(test_content_dict)\n",
    "    all_sentences_dict = records_to_sentences(all_content_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def update_data(data, sentences_dict): \n",
    "    \"\"\"\n",
    "    Takes the data dictionary and the sentences_dict dictionary\n",
    "    \n",
    "    Returns a sentencizer_dict that includes a processed line, which is the sentencizer sentence with the new_line that\n",
    "    contains the [entity] tokens inserted as well as a label key to classify the entity.\n",
    "    \n",
    "    For example:\n",
    "    sentencizer_dict = {'record-105': {...,\n",
    "                                       3: {'ast_line': 3,\n",
    "                                           'txt_line': 26,\n",
    "                                           'entity': 'htn',\n",
    "                                           'old_line': 'HTN elev chol right facial droop with metal plate secondary to GSW \n",
    "                                                        to face right nephrectomy',\n",
    "                                           'new_line': '[entity] HTN [entity] elev chol right facial droop with metal plate\n",
    "                                                        secondary to GSW to face right nephrectomy',\n",
    "                                           'label': 'present',\n",
    "                                           'processed': 'Past Medical History : ventral hernia [entity] HTN [entity] elev \n",
    "                                                         chol right facial droop with metal plate secondary to GSW to face \n",
    "                                                         right nephrectomy Social History : current smoker Family History \n",
    "                                                         : not given Physical Exam: Not done , taken to OR direct from \n",
    "                                                         ambulance transfer .'},\n",
    "                                        ...}}\n",
    "    \"\"\"\n",
    "    for record in data:\n",
    "        for ast_line in data[record]:\n",
    "            entity = data[record][ast_line]['entity']\n",
    "            old_line = data[record][ast_line]['old_line']\n",
    "            new_line = data[record][ast_line]['new_line']\n",
    "\n",
    "            # Look at every sentence in the record and store it if it matches the old_line\n",
    "            for sent in sentences_dict[record]:\n",
    "                full_sentence = sentences_dict[record][sent]\n",
    "                if old_line in full_sentence:\n",
    "                    proc_sent = full_sentence.replace(old_line, new_line)\n",
    "                    data[record][ast_line]['processed'] = proc_sent\n",
    "    return data\n",
    "\n",
    "if data_to_test == \"Sentencizer Data\":\n",
    "    sentencizer_training_data = update_data(training_line_data, training_sentences_dict)\n",
    "    sentencizer_test_data = update_data(test_line_data, test_sentences_dict)\n",
    "    sentencizer_all_data = update_data(all_line_data, all_sentences_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def count_missing_proc(data):\n",
    "    count = 0\n",
    "    for record in data:\n",
    "        for i in data[record]:\n",
    "            if 'processed' not in data[record][i]:\n",
    "#                 print(f\"Item {i} in {record} does not have the key 'processed'\")\n",
    "#                 print(\"ast_line:\", data[record][i]['ast_line'])\n",
    "#                 print(\"txt_line:\", data[record][i]['txt_line'])\n",
    "#                 print(\"old_line:\", data[record][i]['old_line'])\n",
    "#                 print(\"new_line:\", data[record][i]['new_line'])\n",
    "#                 print(\"entity:\", data[record][i]['entity'])\n",
    "#                 print(\"label:\", data[record][i]['label'])\n",
    "#                 print()\n",
    "#                 data[record][i]['processed'] = data[record][i]['new_line']\n",
    "                count += 1\n",
    "    return count\n",
    "\n",
    "if data_to_test == \"Sentencizer Data\":\n",
    "    count = count_missing_proc(sentencizer_all_data)\n",
    "    print(\"Number of examples NOT found (will be removed)\", count)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data_by_sentencizer(data):\n",
    "    record_dfs = []\n",
    "    for record, info in data.items():\n",
    "        df = pd.DataFrame.from_dict(info, orient='index')\n",
    "        df['record'] = record\n",
    "        record_dfs.append(df)\n",
    "    final_df = pd.concat(record_dfs).reset_index(drop=True)\n",
    "    final_df = final_df[['record', 'ast_line', 'txt_line', 'entity', 'processed', 'label']]\n",
    "\n",
    "    return final_df\n",
    "\n",
    "if data_to_test == \"Sentencizer Data\":\n",
    "    training_sentencizer_data_df = get_data_by_sentencizer(sentencizer_training_data)\n",
    "    test_sentencizer_data_df = get_data_by_sentencizer(sentencizer_test_data)\n",
    "    all_sentencizer_data_df = get_data_by_sentencizer(sentencizer_all_data)\n",
    "    training_sentencizer_data_filtered_df = filter_data(training_sentencizer_data_df)\n",
    "    test_sentencizer_data_filtered_df = filter_data(test_sentencizer_data_df)\n",
    "    all_sentencizer_data_filtered_df = filter_data(all_sentencizer_data_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# OPTION 2: Use sentencizer data\n",
    "\"\"\"\n",
    "Pros: larger sentences (gives context for assertion)\n",
    "Cons: more complex approach, sentencizer takes time to run, sentencizer requires removing adjacent periods, \n",
    "      some long sentences full of header of record, some small sentences due to periods on same line, some\n",
    "      data will need to be removed due to no exact match for line (line split by sentencizer)\n",
    "\"\"\"\n",
    "\n",
    "if data_to_test == \"Sentencizer Data\":\n",
    "    # all_sentencizer_data_filtered_df.to_csv(processed_path + '/ast_sentencizer_data.csv', index=False)\n",
    "    all_sentencizer_data_filtered_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use for evaluating OPTION 2\n",
    "\n",
    "if data_to_test == \"Sentencizer Data\":\n",
    "    word_counts = all_sentencizer_data_filtered_df['processed'].apply(lambda x: len(x.split()))-2\n",
    "    plt.hist(word_counts, bins=100, alpha=0.7, color='blue', edgecolor='black')\n",
    "    plt.xlabel('Word Count')\n",
    "    plt.ylabel('Frequency')\n",
    "    plt.title('Word Count Histogram')\n",
    "    plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nPros: Clean approach, uses all the data, no duplicates created\\nCons: Lines are extremely varaible in size, some entities on lines by themselves (no context for assertion task)\\n'"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# OPTION 3: Use n-tokens data - INCOMPLETE\n",
    "\"\"\"\n",
    "Pros: Clean approach, uses all the data, no duplicates created\n",
    "Cons: Lines are extremely varaible in size, some entities on lines by themselves (no context for assertion task)\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (3362177521.py, line 66)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  Cell \u001b[1;32mIn[27], line 66\u001b[1;36m\u001b[0m\n\u001b[1;33m    while len()\u001b[0m\n\u001b[1;37m               ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "def add_entity_and_assertion_n_tokens(new_labels, new_content, n):\n",
    "    \"\"\"\n",
    "    Takes the new_content dictionary and the new_label dictionary:\n",
    "    new_content_dict = {'record': {1: <all the text on line 1 of record.txt>,\n",
    "                               2: <all the text on line 2 of record.txt>,\n",
    "                               ...,\n",
    "                               n: <all the text on last line (n) of record.txt>}}\n",
    "\n",
    "    Returns a data dictionary that contains the record and each line of the record with infomration for the\n",
    "    ast_line number, txt_line number, entity, old_line text, new_line text, and ast label.\n",
    "    data = {'record': {1: {'ast_line': <line number the ast file (matches the key)>,\n",
    "                           'txt_line': <line number of the txt file where the target entity is located>,\n",
    "                           'entity': <the target entity (problem to be classified) from the line in the record.ast file>,\n",
    "                           'old_line': <all the text on the line in record.txt where the target entity is located>,\n",
    "                           'new_line': <all the text on the line in record.txt where the target entity is located \n",
    "                                       with [entity] tokens inserted around the target entity>,\n",
    "                           'label': <the label of the entity in the text (present, absent, possible, etc.)>},\n",
    "                           ...\n",
    "                           n: {...}}\n",
    "\n",
    "    For example:\n",
    "    data = {'record-105': {1: {'ast_line': 1,\n",
    "                               'txt_line': 55,\n",
    "                               'entity': 'left basilar atelectasis',\n",
    "                               'old_line': 'There has been interval improvement in left basilar atelectasis .',\n",
    "                               'new_line': 'There has been interval improvement in [entity] left basilar atelectasis [entity] .',\n",
    "                               'label': 'present'},\n",
    "                           ...\n",
    "                           n: {...}}\n",
    "    \"\"\"\n",
    "    data = {}\n",
    "    for record in new_labels:\n",
    "        i = 1\n",
    "        data[record] = {}\n",
    "        for key in new_labels[record]:\n",
    "            s_line = new_labels[record][key]['location'][0]\n",
    "            s_tok = new_labels[record][key]['location'][1]\n",
    "            e_line = new_labels[record][key]['location'][2]\n",
    "            e_tok = new_labels[record][key]['location'][3]\n",
    "            assertion = new_labels[record][key]['assertion']\n",
    "            entity = new_labels[record][key]['problem']\n",
    "            old_line = new_content[record][s_line]\n",
    "            \n",
    "#             entity = clean_text(entity)\n",
    "#             old_line = clean_text(old_line)\n",
    "            entity = remove_adjacent_periods(entity)\n",
    "            old_line = remove_adjacent_periods(old_line)\n",
    "            words = old_line.split()\n",
    "            words.insert(s_tok, '[entity]')\n",
    "            words.insert(e_tok + 2, '[entity]')\n",
    "            new_line = ' '.join(words)\n",
    "            \n",
    "#             print(new_content[record][s_line-1])\n",
    "            toks = words[s_tok:e_tok + 3]\n",
    "            front = words[0:s_tok]\n",
    "            \n",
    "            len_front_tokens_to_add = len(front)\n",
    "            add = front[-1:]\n",
    "\n",
    "            ntoks = front[-1:] + toks\n",
    "            \n",
    "            \n",
    "            if len(words) > n:\n",
    "                pass\n",
    "            else:\n",
    "                while len()\n",
    "            \n",
    "            print(\"n:\", n)\n",
    "            print(\"len:\", len(words))\n",
    "            print(\"len_front_tokens_to_add:\", len_front_tokens_to_add)\n",
    "            print(\"words:\", words)\n",
    "            print(\"toks:\", toks)\n",
    "            print(\"add:\", add)\n",
    "            print(\"front:\", words[0:s_tok])\n",
    "            print(\"ntoks:\", ntoks)\n",
    "            print()\n",
    "            \n",
    "            data[record][i] = {'ast_line': i, 'txt_line': s_line, \n",
    "                               'entity': entity,'old_line': old_line, \n",
    "                               'new_line': new_line, 'label': assertion}\n",
    "            i += 1\n",
    "        \n",
    "    return data\n",
    "\n",
    "training_ntok_data = add_entity_and_assertion_n_tokens(training_labels, training_content, 3)\n",
    "# test_ntok_data = add_entity_and_assertion_n_tokens(test_labels, test_content, 3)\n",
    "# all_ntok_data = add_entity_and_assertion_n_tokens(all_labels, all_content, 3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['kc', 'and', 'pb', 'the', 'party', 'was', 'for']\n"
     ]
    }
   ],
   "source": [
    "list1 = ['kc', 'and', 'pb']\n",
    "list2 = ['the', 'party', 'was', 'for']\n",
    "\n",
    "# Add elements from list2 to list1\n",
    "# list1.extend(list2)\n",
    "\n",
    "list1 = list1 + list2\n",
    "\n",
    "print(list1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Test the data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "X shape (184,) y shape : (184,)\n",
      "X_train shape (147,) y_train shape : (147,)\n",
      "X_val shape (19,) y_val shape : (19,)\n",
      "X_test shape (18,) y_test shape : (18,)\n"
     ]
    }
   ],
   "source": [
    "if data_to_test == \"Line Data\":\n",
    "    # LINE DATA TEST\n",
    "    # Test on 20% of data. Split to training, validation, and test sets\n",
    "    all_line_data_filtered_df_frac = all_line_data_filtered_df.sample(frac=0.01).copy()\n",
    "\n",
    "\n",
    "    X = all_line_data_filtered_df_frac['new_line']\n",
    "    y = all_line_data_filtered_df_frac['label']\n",
    "\n",
    "    X_train, X_test_valid, y_train, y_test_valid = train_test_split(X, y, test_size=0.2, stratify=y, random_state=seed)\n",
    "    X_test, X_valid, y_test, y_valid = train_test_split(X_test_valid, y_test_valid, test_size=0.5, random_state=seed, stratify=y_test_valid)\n",
    "\n",
    "    print(f\"X shape {X.shape} y shape : {y.shape}\")\n",
    "    print(f\"X_train shape {X_train.shape} y_train shape : {y_train.shape}\")\n",
    "    print(f\"X_val shape {X_valid.shape} y_val shape : {y_valid.shape}\")\n",
    "    print(f\"X_test shape {X_test.shape} y_test shape : {y_test.shape}\")\n",
    "        \n",
    "elif data_to_test == \"Sentencizer Data\":\n",
    "    # SENTENCIZER DATA TEST\n",
    "    # Test on 20% of data. Split to training, validation, and test sets\n",
    "    all_sentencizer_data_filtered_df_frac = all_sentencizer_data_filtered_df.sample(frac=0.2).copy()\n",
    "\n",
    "\n",
    "    X = all_sentencizer_data_filtered_df_frac['processed']\n",
    "    y = all_sentencizer_data_filtered_df_frac['label']\n",
    "\n",
    "    X_train, X_test_valid, y_train, y_test_valid = train_test_split(X, y, test_size=0.2, stratify=y, random_state=seed)\n",
    "    X_test, X_valid, y_test, y_valid = train_test_split(X_test_valid, y_test_valid, test_size=0.5, random_state=seed, stratify=y_test_valid)\n",
    "\n",
    "    print(f\"X shape {X.shape} y shape : {y.shape}\")\n",
    "    print(f\"X_train shape {X_train.shape} y_train shape : {y_train.shape}\")\n",
    "    print(f\"X_val shape {X_valid.shape} y_val shape : {y_valid.shape}\")\n",
    "    print(f\"X_test shape {X_test.shape} y_test shape : {y_test.shape}\")\n",
    "\n",
    "else:\n",
    "    print(data_to_test, \"not a valid choice\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Encoding Labels .....\n"
     ]
    }
   ],
   "source": [
    "print(\"Encoding Labels .....\")\n",
    "encoder = LabelEncoder()\n",
    "encoder.fit(y_train)\n",
    "y_train_encode = np.asarray(encoder.transform(y_train))\n",
    "y_valid_encode = np.asarray(encoder.transform(y_valid))\n",
    "y_test_encode = np.asarray(encoder.transform(y_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                               new_line  label\n",
      "217   Cardiovascular exam revealed a regular rate an...      0\n",
      "6298             Abdomen was [entity] gravid [entity] .      2\n",
      "5250  The electrocardiogram showed atrial fibrillati...      2\n",
      "1084  Moderate to severe 3+ mitral regurgitation and...      2\n",
      "8724  O2 95% on 3L NC mixed Quinn 82% genrl : in nad...      2\n"
     ]
    }
   ],
   "source": [
    "train_df = pd.DataFrame(X_train)\n",
    "valid_df = pd.DataFrame(X_valid)\n",
    "test_df = pd.DataFrame(X_test)\n",
    "\n",
    "train_df['label'] = y_train_encode.tolist()\n",
    "valid_df['label'] = y_valid_encode.tolist()\n",
    "test_df['label'] = y_test_encode.tolist()\n",
    "\n",
    "print(train_df.head())\n",
    "\n",
    "ds = DatasetDict ({\n",
    " 'train': Dataset.from_pandas(train_df),\n",
    " 'validation': Dataset.from_pandas(valid_df),\n",
    " 'test': Dataset.from_pandas(test_df)\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "device in use: cuda\n"
     ]
    }
   ],
   "source": [
    "# FOR MAC\n",
    "# device = torch.device('mps' if torch.backends.mps.is_available() else 'cpu')\n",
    "\n",
    "# FOR NVIDIA GPU\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
    "print(\"device in use:\", device)\n",
    "\n",
    "# If using GPU, set seed for CUDA operations\n",
    "if torch.cuda.is_available():\n",
    "    torch.cuda.manual_seed_all(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Could not locate the tokenizer configuration file, will try to use the model config instead.\n",
      "loading configuration file config.json from cache at C:\\Users\\kcaro/.cache\\huggingface\\hub\\models--emilyalsentzer--Bio_Discharge_Summary_BERT\\snapshots\\affde836a50e4d333f15dae9270f5a856d59540b\\config.json\n",
      "Model config BertConfig {\n",
      "  \"_name_or_path\": \"emilyalsentzer/Bio_Discharge_Summary_BERT\",\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"transformers_version\": \"4.35.2\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 28996\n",
      "}\n",
      "\n",
      "loading file vocab.txt from cache at C:\\Users\\kcaro/.cache\\huggingface\\hub\\models--emilyalsentzer--Bio_Discharge_Summary_BERT\\snapshots\\affde836a50e4d333f15dae9270f5a856d59540b\\vocab.txt\n",
      "loading file tokenizer.json from cache at None\n",
      "loading file added_tokens.json from cache at None\n",
      "loading file special_tokens_map.json from cache at None\n",
      "loading file tokenizer_config.json from cache at None\n",
      "loading configuration file config.json from cache at C:\\Users\\kcaro/.cache\\huggingface\\hub\\models--emilyalsentzer--Bio_Discharge_Summary_BERT\\snapshots\\affde836a50e4d333f15dae9270f5a856d59540b\\config.json\n",
      "Model config BertConfig {\n",
      "  \"_name_or_path\": \"emilyalsentzer/Bio_Discharge_Summary_BERT\",\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"transformers_version\": \"4.35.2\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 28996\n",
      "}\n",
      "\n",
      "loading configuration file config.json from cache at C:\\Users\\kcaro/.cache\\huggingface\\hub\\models--emilyalsentzer--Bio_Discharge_Summary_BERT\\snapshots\\affde836a50e4d333f15dae9270f5a856d59540b\\config.json\n",
      "Model config BertConfig {\n",
      "  \"_name_or_path\": \"emilyalsentzer/Bio_Discharge_Summary_BERT\",\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"transformers_version\": \"4.35.2\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 28996\n",
      "}\n",
      "\n",
      "loading configuration file config.json from cache at C:\\Users\\kcaro/.cache\\huggingface\\hub\\models--emilyalsentzer--Bio_Discharge_Summary_BERT\\snapshots\\affde836a50e4d333f15dae9270f5a856d59540b\\config.json\n",
      "Model config BertConfig {\n",
      "  \"_name_or_path\": \"emilyalsentzer/Bio_Discharge_Summary_BERT\",\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"classifier_dropout\": null,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"id2label\": {\n",
      "    \"0\": \"PRESENT\",\n",
      "    \"1\": \"ABSENT\",\n",
      "    \"2\": \"POSSIBLE\"\n",
      "  },\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"label2id\": {\n",
      "    \"LABEL_0\": 0,\n",
      "    \"LABEL_1\": 1,\n",
      "    \"LABEL_2\": 2\n",
      "  },\n",
      "  \"layer_norm_eps\": 1e-12,\n",
      "  \"max_position_embeddings\": 512,\n",
      "  \"model_type\": \"bert\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 0,\n",
      "  \"position_embedding_type\": \"absolute\",\n",
      "  \"transformers_version\": \"4.35.2\",\n",
      "  \"type_vocab_size\": 2,\n",
      "  \"use_cache\": true,\n",
      "  \"vocab_size\": 28996\n",
      "}\n",
      "\n",
      "loading weights file pytorch_model.bin from cache at C:\\Users\\kcaro/.cache\\huggingface\\hub\\models--emilyalsentzer--Bio_Discharge_Summary_BERT\\snapshots\\affde836a50e4d333f15dae9270f5a856d59540b\\pytorch_model.bin\n",
      "Some weights of the model checkpoint at emilyalsentzer/Bio_Discharge_Summary_BERT were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.seq_relationship.weight']\n",
      "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
      "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
      "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at emilyalsentzer/Bio_Discharge_Summary_BERT and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "from transformers import AutoModelForSequenceClassification\n",
    "from transformers import AutoTokenizer, AutoModel \n",
    "tokenizer_clinical_bio  = AutoTokenizer.from_pretrained(\"emilyalsentzer/Bio_Discharge_Summary_BERT\", model_max_length=150)\n",
    "model_clinical = AutoModelForSequenceClassification.from_pretrained(\"emilyalsentzer/Bio_Discharge_Summary_BERT\", \n",
    "                                                                    num_labels=3,\n",
    "                                                                    id2label={0: 'PRESENT', 1: 'ABSENT', 2:'POSSIBLE'})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_clinical = model_clinical.to(device)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "You are resizing the embedding layer without providing a `pad_to_multiple_of` parameter. This means that the new embedding dimension will be 28997. This might induce some performance reduction as *Tensor Cores* will not be available. For more details about this, or help on choosing the correct value for resizing, refer to this guide: https://docs.nvidia.com/deeplearning/performance/dl-performance-matrix-multiplication/index.html#requirements-tc\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We have added 1 tokens\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Embedding(28997, 768)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "special_tokens_dict = {\"additional_special_tokens\": [\"[entity]\"]}\n",
    "num_added_toks = tokenizer_clinical_bio.add_special_tokens(special_tokens_dict,False)\n",
    "\n",
    "print(\"We have added\", num_added_toks, \"tokens\")\n",
    "# Notice: resize_token_embeddings expect to receive the full size of the new vocabulary, i.e., the length of the tokenizer.\n",
    "model_clinical.resize_token_embeddings(len(tokenizer_clinical_bio))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3a07c4f26af6432fbce888662b094cab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/147 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b8cf9be33c2e4eb0aca40b1b34faff41",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/19 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0be6da423b9b44ce9b7cd379f3adf81a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/18 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "if data_to_test == \"Line Data\":\n",
    "    # LINE DATA TEST\n",
    "    def tokenize_function(example):\n",
    "        return tokenizer_clinical_bio(example[\"new_line\"],   padding=\"max_length\", truncation=True)\n",
    "    \n",
    "    # LINE DATA TEST\n",
    "    # Takes ds dataset and tokenizes\n",
    "    tokenized_ds = ds.map(tokenize_function, batched=True)\n",
    "    tokenized_ds = tokenized_ds.rename_column(\"label\", \"labels\")\n",
    "    tokenized_ds = tokenized_ds.remove_columns([\"new_line\"])\n",
    "    tokenized_ds = tokenized_ds.remove_columns([\"__index_level_0__\"])\n",
    "    tokenized_ds.set_format(\"torch\")\n",
    "    \n",
    "elif data_to_test == \"Sentencizer Data\":\n",
    "    # SENTENCIZER DATA TEST\n",
    "    def tokenize_function(example):\n",
    "        return tokenizer_clinical_bio(example[\"processed\"],   padding=\"max_length\", truncation=True)\n",
    "    \n",
    "    # SENTENCIZER DATA TEST\n",
    "    # Takes ds dataset and tokenizes\n",
    "    tokenized_ds = ds.map(tokenize_function, batched=True)\n",
    "    tokenized_ds = tokenized_ds.rename_column(\"label\", \"labels\")\n",
    "    tokenized_ds = tokenized_ds.remove_columns([\"processed\"])\n",
    "    tokenized_ds = tokenized_ds.remove_columns([\"__index_level_0__\"])\n",
    "    tokenized_ds.set_format(\"torch\")\n",
    "else:\n",
    "    print(data_to_test, \"not a valid choice\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_metrics(eval_pred):\n",
    "    logits, labels = eval_pred\n",
    "    predictions = np.argmax(logits, axis=-1)\n",
    "\n",
    "    accuracy = accuracy_score(labels, predictions)\n",
    "    precision = precision_score(labels, predictions, average='weighted')\n",
    "    recall = recall_score(labels, predictions, average='weighted')\n",
    "    f1 = f1_score(labels, predictions, average='weighted')\n",
    "\n",
    "    return {\n",
    "        'accuracy': accuracy,\n",
    "        'precision': precision,\n",
    "        'recall': recall,\n",
    "        'f1': f1,\n",
    "    }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Niharika\n",
    "\n",
    "# from transformers import AutoTokenizer\n",
    "\n",
    "# tokenizer = AutoTokenizer.from_pretrained(\"/content/my_awesome_wnut_model\")\n",
    "\n",
    "# from transformers import DataCollatorForTokenClassification\n",
    "\n",
    "# data_collator = DataCollatorForTokenClassification(tokenizer=tokenizer_clinical_bio)\n",
    "\n",
    "# training_args = TrainingArguments(\n",
    "#     output_dir=\"my_awesome_wnut_model\",\n",
    "#     learning_rate=2e-5,\n",
    "#     per_device_train_batch_size=16,\n",
    "#     per_device_eval_batch_size=16,\n",
    "#     num_train_epochs=20,\n",
    "#     weight_decay=0.01,\n",
    "#     evaluation_strategy=\"epoch\",\n",
    "#     save_strategy=\"epoch\",\n",
    "#     load_best_model_at_end=True,\n",
    "#     push_to_hub=True,\n",
    "# )\n",
    "\n",
    "# trainer = Trainer(\n",
    "#     model=model,\n",
    "#     args=training_args,\n",
    "#     train_dataset=tokenized_wnut[\"train\"],\n",
    "#     eval_dataset=tokenized_wnut[\"test\"],\n",
    "#     tokenizer=tokenizer,\n",
    "#     data_collator=data_collator,\n",
    "#     compute_metrics=compute_metrics,\n",
    "# )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # Kevin using Niharika's\n",
    "\n",
    "# training_args = TrainingArguments(\n",
    "#     output_dir=\"clinbert_trainer\", \n",
    "#     learning_rate=1e-5, \n",
    "#     per_device_train_batch_size=16,\n",
    "#     per_device_eval_batch_size=16,\n",
    "#     num_train_epochs=20,\n",
    "#     weight_decay=0.01,\n",
    "#     evaluation_strategy=\"epoch\",\n",
    "#     save_strategy=\"epoch\",\n",
    "#     load_best_model_at_end=True,\n",
    "#     push_to_hub=False,\n",
    "#     report_to=\"all\")\n",
    "\n",
    "# trainer = Trainer(\n",
    "#     model=model_clinical,\n",
    "#     args=training_args,\n",
    "#     train_dataset=tokenized_ds['train'],\n",
    "#     eval_dataset=tokenized_ds['validation'],\n",
    "#     tokenizer=tokenizer_clinical_bio,\n",
    "#     compute_metrics=compute_metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "PyTorch: setting up devices\n"
     ]
    }
   ],
   "source": [
    "# Kevin OLD\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=\"clinbert_trainer\", \n",
    "    evaluation_strategy=\"epoch\", \n",
    "    learning_rate=1e-5, \n",
    "    num_train_epochs=10,\n",
    "    per_device_train_batch_size=8,\n",
    "    per_device_eval_batch_size=8,\n",
    "    logging_dir=\"./logs\",\n",
    "    log_level=\"info\",\n",
    "    logging_steps=10,\n",
    "    report_to=\"all\")\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model_clinical,\n",
    "    args=training_args,\n",
    "    train_dataset=tokenized_ds['train'],\n",
    "    eval_dataset=tokenized_ds['validation'],\n",
    "    tokenizer=tokenizer_clinical_bio,\n",
    "    compute_metrics=compute_metrics)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running training *****\n",
      "  Num examples = 147\n",
      "  Num Epochs = 10\n",
      "  Instantaneous batch size per device = 8\n",
      "  Total train batch size (w. parallel, distributed & accumulation) = 8\n",
      "  Gradient Accumulation steps = 1\n",
      "  Total optimization steps = 190\n",
      "  Number of trainable parameters = 108,313,347\n",
      "You're using a BertTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='190' max='190' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [190/190 01:08, Epoch 10/10]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "      <th>Accuracy</th>\n",
       "      <th>Precision</th>\n",
       "      <th>Recall</th>\n",
       "      <th>F1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.888200</td>\n",
       "      <td>0.809424</td>\n",
       "      <td>0.684211</td>\n",
       "      <td>0.468144</td>\n",
       "      <td>0.684211</td>\n",
       "      <td>0.555921</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.804300</td>\n",
       "      <td>0.740175</td>\n",
       "      <td>0.684211</td>\n",
       "      <td>0.468144</td>\n",
       "      <td>0.684211</td>\n",
       "      <td>0.555921</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.637800</td>\n",
       "      <td>0.677273</td>\n",
       "      <td>0.684211</td>\n",
       "      <td>0.468144</td>\n",
       "      <td>0.684211</td>\n",
       "      <td>0.555921</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>0.655000</td>\n",
       "      <td>0.626592</td>\n",
       "      <td>0.736842</td>\n",
       "      <td>0.757310</td>\n",
       "      <td>0.736842</td>\n",
       "      <td>0.661573</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>0.554700</td>\n",
       "      <td>0.546181</td>\n",
       "      <td>0.842105</td>\n",
       "      <td>0.819079</td>\n",
       "      <td>0.842105</td>\n",
       "      <td>0.810799</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>0.389600</td>\n",
       "      <td>0.482178</td>\n",
       "      <td>0.842105</td>\n",
       "      <td>0.819079</td>\n",
       "      <td>0.842105</td>\n",
       "      <td>0.810799</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>0.420500</td>\n",
       "      <td>0.435677</td>\n",
       "      <td>0.842105</td>\n",
       "      <td>0.819079</td>\n",
       "      <td>0.842105</td>\n",
       "      <td>0.810799</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>0.360000</td>\n",
       "      <td>0.402120</td>\n",
       "      <td>0.894737</td>\n",
       "      <td>0.856140</td>\n",
       "      <td>0.894737</td>\n",
       "      <td>0.869256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>0.412800</td>\n",
       "      <td>0.380503</td>\n",
       "      <td>0.894737</td>\n",
       "      <td>0.856140</td>\n",
       "      <td>0.894737</td>\n",
       "      <td>0.869256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>0.415200</td>\n",
       "      <td>0.373001</td>\n",
       "      <td>0.894737</td>\n",
       "      <td>0.856140</td>\n",
       "      <td>0.894737</td>\n",
       "      <td>0.869256</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "***** Running Evaluation *****\n",
      "  Num examples = 19\n",
      "  Batch size = 8\n",
      "C:\\Users\\kcaro\\miniconda3\\envs\\clinical-adapter\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1469: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 19\n",
      "  Batch size = 8\n",
      "C:\\Users\\kcaro\\miniconda3\\envs\\clinical-adapter\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1469: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 19\n",
      "  Batch size = 8\n",
      "C:\\Users\\kcaro\\miniconda3\\envs\\clinical-adapter\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1469: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 19\n",
      "  Batch size = 8\n",
      "C:\\Users\\kcaro\\miniconda3\\envs\\clinical-adapter\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1469: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 19\n",
      "  Batch size = 8\n",
      "C:\\Users\\kcaro\\miniconda3\\envs\\clinical-adapter\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1469: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 19\n",
      "  Batch size = 8\n",
      "C:\\Users\\kcaro\\miniconda3\\envs\\clinical-adapter\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1469: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 19\n",
      "  Batch size = 8\n",
      "C:\\Users\\kcaro\\miniconda3\\envs\\clinical-adapter\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1469: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 19\n",
      "  Batch size = 8\n",
      "C:\\Users\\kcaro\\miniconda3\\envs\\clinical-adapter\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1469: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 19\n",
      "  Batch size = 8\n",
      "C:\\Users\\kcaro\\miniconda3\\envs\\clinical-adapter\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1469: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "***** Running Evaluation *****\n",
      "  Num examples = 19\n",
      "  Batch size = 8\n",
      "C:\\Users\\kcaro\\miniconda3\\envs\\clinical-adapter\\lib\\site-packages\\sklearn\\metrics\\_classification.py:1469: UndefinedMetricWarning: Precision is ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
      "  _warn_prf(average, modifier, msg_start, len(result))\n",
      "\n",
      "\n",
      "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
      "\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=190, training_loss=0.5254897054873015, metrics={'train_runtime': 70.6203, 'train_samples_per_second': 20.816, 'train_steps_per_second': 2.69, 'total_flos': 113313493377000.0, 'train_loss': 0.5254897054873015, 'epoch': 10.0})"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "clinical-adapter",
   "language": "python",
   "name": "clinical-adapter"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}