Final_Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import csv\n",
    "import gensim\n",
    "import numpy as np\n",
    "import nltk\n",
    "from nltk.tokenize import RegexpTokenizer\n",
    "from scipy import spatial\n",
    "import plotly.plotly as py\n",
    "import plotly.graph_objs as go\n",
    "import matplotlib.pyplot as plt\n",
    "from collections import Counter\n",
    "import gc\n",
    "import math\n",
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(324, 21)"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"FINAL_DATASET.csv\")\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Movie_Number</th>\n",
       "      <th>Movie_Name</th>\n",
       "      <th>Year_of_Release</th>\n",
       "      <th>IMDB_Rating</th>\n",
       "      <th>Number_of_Votes</th>\n",
       "      <th>Genres</th>\n",
       "      <th>rating</th>\n",
       "      <th>year</th>\n",
       "      <th>id_x</th>\n",
       "      <th>imdbid</th>\n",
       "      <th>...</th>\n",
       "      <th>crew</th>\n",
       "      <th>id_y</th>\n",
       "      <th>budget</th>\n",
       "      <th>genres</th>\n",
       "      <th>popularity</th>\n",
       "      <th>production_companies</th>\n",
       "      <th>revenue</th>\n",
       "      <th>title</th>\n",
       "      <th>vote_count</th>\n",
       "      <th>ROI</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>1999</td>\n",
       "      <td>6.9</td>\n",
       "      <td>62847</td>\n",
       "      <td>['comedy', 'romance']</td>\n",
       "      <td>3</td>\n",
       "      <td>1999</td>\n",
       "      <td>374</td>\n",
       "      <td>147800.0</td>\n",
       "      <td>...</td>\n",
       "      <td>[{'credit_id': '52fe43e6c3a36847f807731d', 'de...</td>\n",
       "      <td>4951</td>\n",
       "      <td>16000000</td>\n",
       "      <td>[{'id': 35, 'name': 'Comedy'}, {'id': 10749, '...</td>\n",
       "      <td>12.069966</td>\n",
       "      <td>[{'name': 'Mad Chance', 'id': 1757}, {'name': ...</td>\n",
       "      <td>53478166.0</td>\n",
       "      <td>10 Things I Hate About You</td>\n",
       "      <td>1768.0</td>\n",
       "      <td>3.342385</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>m3</td>\n",
       "      <td>2001: a space odyssey</td>\n",
       "      <td>1968</td>\n",
       "      <td>8.4</td>\n",
       "      <td>163227</td>\n",
       "      <td>['adventure', 'mystery', 'sci-fi']</td>\n",
       "      <td>0</td>\n",
       "      <td>1968</td>\n",
       "      <td>919</td>\n",
       "      <td>62622.0</td>\n",
       "      <td>...</td>\n",
       "      <td>[{'credit_id': '52fe4212c3a36847f8001a0b', 'de...</td>\n",
       "      <td>62</td>\n",
       "      <td>10500000</td>\n",
       "      <td>[{'id': 878, 'name': 'Science Fiction'}, {'id'...</td>\n",
       "      <td>22.494622</td>\n",
       "      <td>[{'name': 'Stanley Kubrick Productions', 'id':...</td>\n",
       "      <td>68700000.0</td>\n",
       "      <td>2001: A Space Odyssey</td>\n",
       "      <td>3075.0</td>\n",
       "      <td>6.542857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>m4</td>\n",
       "      <td>48 hrs.</td>\n",
       "      <td>1982</td>\n",
       "      <td>6.9</td>\n",
       "      <td>22289</td>\n",
       "      <td>['action', 'comedy', 'crime', 'drama', 'thrill...</td>\n",
       "      <td>2</td>\n",
       "      <td>1982</td>\n",
       "      <td>5355</td>\n",
       "      <td>83511.0</td>\n",
       "      <td>...</td>\n",
       "      <td>[{'credit_id': '52fe421fc3a36847f8005927', 'de...</td>\n",
       "      <td>150</td>\n",
       "      <td>12000000</td>\n",
       "      <td>[{'id': 53, 'name': 'Thriller'}, {'id': 28, 'n...</td>\n",
       "      <td>15.297121</td>\n",
       "      <td>[{'name': 'Paramount Pictures', 'id': 4}]</td>\n",
       "      <td>78868508.0</td>\n",
       "      <td>48 Hrs.</td>\n",
       "      <td>364.0</td>\n",
       "      <td>6.572376</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>m6</td>\n",
       "      <td>8mm</td>\n",
       "      <td>1999</td>\n",
       "      <td>6.3</td>\n",
       "      <td>48212</td>\n",
       "      <td>['crime', 'mystery', 'thriller']</td>\n",
       "      <td>1</td>\n",
       "      <td>1999</td>\n",
       "      <td>146</td>\n",
       "      <td>134273.0</td>\n",
       "      <td>...</td>\n",
       "      <td>[{'credit_id': '52fe4498c3a36847f809f18b', 'de...</td>\n",
       "      <td>8224</td>\n",
       "      <td>40000000</td>\n",
       "      <td>[{'id': 53, 'name': 'Thriller'}, {'id': 80, 'n...</td>\n",
       "      <td>7.473718</td>\n",
       "      <td>[{'name': 'Columbia Pictures Corporation', 'id...</td>\n",
       "      <td>96618699.0</td>\n",
       "      <td>8MM</td>\n",
       "      <td>479.0</td>\n",
       "      <td>2.415467</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>m7</td>\n",
       "      <td>a nightmare on elm street 4: the dream master</td>\n",
       "      <td>1988</td>\n",
       "      <td>5.2</td>\n",
       "      <td>13590</td>\n",
       "      <td>['fantasy', 'horror', 'thriller']</td>\n",
       "      <td>3</td>\n",
       "      <td>1988</td>\n",
       "      <td>6020</td>\n",
       "      <td>95742.0</td>\n",
       "      <td>...</td>\n",
       "      <td>[{'credit_id': '52fe43309251416c75006d1d', 'de...</td>\n",
       "      <td>10131</td>\n",
       "      <td>7000000</td>\n",
       "      <td>[{'id': 27, 'name': 'Horror'}, {'id': 53, 'nam...</td>\n",
       "      <td>8.120317</td>\n",
       "      <td>[{'name': 'New Line Cinema', 'id': 12}]</td>\n",
       "      <td>72500000.0</td>\n",
       "      <td>A Nightmare on Elm Street 4: The Dream Master</td>\n",
       "      <td>287.0</td>\n",
       "      <td>10.357143</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  Movie_Number                                     Movie_Name Year_of_Release  \\\n",
       "0           m0                     10 things i hate about you            1999   \n",
       "1           m3                          2001: a space odyssey            1968   \n",
       "2           m4                                        48 hrs.            1982   \n",
       "3           m6                                            8mm            1999   \n",
       "4           m7  a nightmare on elm street 4: the dream master            1988   \n",
       "\n",
       "   IMDB_Rating  Number_of_Votes  \\\n",
       "0          6.9            62847   \n",
       "1          8.4           163227   \n",
       "2          6.9            22289   \n",
       "3          6.3            48212   \n",
       "4          5.2            13590   \n",
       "\n",
       "                                              Genres  rating  year  id_x  \\\n",
       "0                              ['comedy', 'romance']       3  1999   374   \n",
       "1                 ['adventure', 'mystery', 'sci-fi']       0  1968   919   \n",
       "2  ['action', 'comedy', 'crime', 'drama', 'thrill...       2  1982  5355   \n",
       "3                   ['crime', 'mystery', 'thriller']       1  1999   146   \n",
       "4                  ['fantasy', 'horror', 'thriller']       3  1988  6020   \n",
       "\n",
       "     imdbid    ...                                                   crew  \\\n",
       "0  147800.0    ...      [{'credit_id': '52fe43e6c3a36847f807731d', 'de...   \n",
       "1   62622.0    ...      [{'credit_id': '52fe4212c3a36847f8001a0b', 'de...   \n",
       "2   83511.0    ...      [{'credit_id': '52fe421fc3a36847f8005927', 'de...   \n",
       "3  134273.0    ...      [{'credit_id': '52fe4498c3a36847f809f18b', 'de...   \n",
       "4   95742.0    ...      [{'credit_id': '52fe43309251416c75006d1d', 'de...   \n",
       "\n",
       "    id_y    budget                                             genres  \\\n",
       "0   4951  16000000  [{'id': 35, 'name': 'Comedy'}, {'id': 10749, '...   \n",
       "1     62  10500000  [{'id': 878, 'name': 'Science Fiction'}, {'id'...   \n",
       "2    150  12000000  [{'id': 53, 'name': 'Thriller'}, {'id': 28, 'n...   \n",
       "3   8224  40000000  [{'id': 53, 'name': 'Thriller'}, {'id': 80, 'n...   \n",
       "4  10131   7000000  [{'id': 27, 'name': 'Horror'}, {'id': 53, 'nam...   \n",
       "\n",
       "  popularity                               production_companies     revenue  \\\n",
       "0  12.069966  [{'name': 'Mad Chance', 'id': 1757}, {'name': ...  53478166.0   \n",
       "1  22.494622  [{'name': 'Stanley Kubrick Productions', 'id':...  68700000.0   \n",
       "2  15.297121          [{'name': 'Paramount Pictures', 'id': 4}]  78868508.0   \n",
       "3   7.473718  [{'name': 'Columbia Pictures Corporation', 'id...  96618699.0   \n",
       "4   8.120317            [{'name': 'New Line Cinema', 'id': 12}]  72500000.0   \n",
       "\n",
       "                                           title vote_count        ROI  \n",
       "0                     10 Things I Hate About You     1768.0   3.342385  \n",
       "1                          2001: A Space Odyssey     3075.0   6.542857  \n",
       "2                                        48 Hrs.      364.0   6.572376  \n",
       "3                                            8MM      479.0   2.415467  \n",
       "4  A Nightmare on Elm Street 4: The Dream Master      287.0  10.357143  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "analysis_df = pd.DataFrame(columns = ['Year',\n",
    "                                      'Movie_Title', \n",
    "                                      '#_of_Female_Cast', \n",
    "                                      '#_of_Male_Cast',\n",
    "                                      '#_of_Female_Dialogue',\n",
    "                                      '#_of_Male_Dialogue',\n",
    "                                      '#_of_Female_Crew',\n",
    "                                     '#_of_Male_Crew'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(324, 8)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "analysis_df['Movie_Title'] = df['Movie_Name']\n",
    "analysis_df['Year'] = df['Year_of_Release']\n",
    "analysis_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Movie_Title</th>\n",
       "      <th>#_of_Female_Cast</th>\n",
       "      <th>#_of_Male_Cast</th>\n",
       "      <th>#_of_Female_Dialogue</th>\n",
       "      <th>#_of_Male_Dialogue</th>\n",
       "      <th>#_of_Female_Crew</th>\n",
       "      <th>#_of_Male_Crew</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1999</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1968</td>\n",
       "      <td>2001: a space odyssey</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1982</td>\n",
       "      <td>48 hrs.</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1999</td>\n",
       "      <td>8mm</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1988</td>\n",
       "      <td>a nightmare on elm street 4: the dream master</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Year                                    Movie_Title #_of_Female_Cast  \\\n",
       "0  1999                     10 things i hate about you              NaN   \n",
       "1  1968                          2001: a space odyssey              NaN   \n",
       "2  1982                                        48 hrs.              NaN   \n",
       "3  1999                                            8mm              NaN   \n",
       "4  1988  a nightmare on elm street 4: the dream master              NaN   \n",
       "\n",
       "  #_of_Male_Cast #_of_Female_Dialogue #_of_Male_Dialogue #_of_Female_Crew  \\\n",
       "0            NaN                  NaN                NaN              NaN   \n",
       "1            NaN                  NaN                NaN              NaN   \n",
       "2            NaN                  NaN                NaN              NaN   \n",
       "3            NaN                  NaN                NaN              NaN   \n",
       "4            NaN                  NaN                NaN              NaN   \n",
       "\n",
       "  #_of_Male_Crew  \n",
       "0            NaN  \n",
       "1            NaN  \n",
       "2            NaN  \n",
       "3            NaN  \n",
       "4            NaN  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "analysis_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "mcm_df = pd.read_csv(\"mcm_csv.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Character_Id</th>\n",
       "      <th>Character_Name</th>\n",
       "      <th>Movie_Number</th>\n",
       "      <th>Movie_Title</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Position_in_Credits</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>u0</td>\n",
       "      <td>BIANCA</td>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>f</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>u1</td>\n",
       "      <td>BRUCE</td>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>m</td>\n",
       "      <td>?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>u2</td>\n",
       "      <td>CAMERON</td>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>m</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>u3</td>\n",
       "      <td>CHASTITY</td>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>f</td>\n",
       "      <td>?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>u4</td>\n",
       "      <td>JOEY</td>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>m</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>u5</td>\n",
       "      <td>KAT</td>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>f</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>u6</td>\n",
       "      <td>MANDELLA</td>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>f</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>u7</td>\n",
       "      <td>MICHAEL</td>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>m</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>u8</td>\n",
       "      <td>MISS PERKY</td>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>m</td>\n",
       "      <td>?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>u9</td>\n",
       "      <td>PATRICK</td>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>m</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>u10</td>\n",
       "      <td>SHARON</td>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>f</td>\n",
       "      <td>?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>u11</td>\n",
       "      <td>WALTER</td>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>m</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Character_Id Character_Name Movie_Number                 Movie_Title  \\\n",
       "0            u0         BIANCA           m0  10 things i hate about you   \n",
       "1            u1          BRUCE           m0  10 things i hate about you   \n",
       "2            u2        CAMERON           m0  10 things i hate about you   \n",
       "3            u3       CHASTITY           m0  10 things i hate about you   \n",
       "4            u4           JOEY           m0  10 things i hate about you   \n",
       "5            u5            KAT           m0  10 things i hate about you   \n",
       "6            u6       MANDELLA           m0  10 things i hate about you   \n",
       "7            u7        MICHAEL           m0  10 things i hate about you   \n",
       "8            u8     MISS PERKY           m0  10 things i hate about you   \n",
       "9            u9        PATRICK           m0  10 things i hate about you   \n",
       "10          u10         SHARON           m0  10 things i hate about you   \n",
       "11          u11         WALTER           m0  10 things i hate about you   \n",
       "\n",
       "   Gender Position_in_Credits  \n",
       "0       f                   4  \n",
       "1       m                   ?  \n",
       "2       m                   3  \n",
       "3       f                   ?  \n",
       "4       m                   6  \n",
       "5       f                   2  \n",
       "6       f                   7  \n",
       "7       m                   5  \n",
       "8       m                   ?  \n",
       "9       m                   1  \n",
       "10      f                   ?  \n",
       "11      m                   9  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mcm_df.loc[mcm_df[\"Movie_Number\"] == \"m0\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Getting the number of female and male characters\n",
    "for rows in analysis_df.itertuples():\n",
    "    f_count = 0\n",
    "    m_count = 0\n",
    "    for row2 in mcm_df.itertuples():\n",
    "        if row2[4] == rows[2]:\n",
    "            if row2[5] == 'f':\n",
    "                f_count += 1\n",
    "            else:\n",
    "                m_count += 1\n",
    "    analysis_df.at[rows.Index, \"#_of_Female_Cast\"] = f_count\n",
    "    analysis_df.at[rows.Index, \"#_of_Male_Cast\"] = m_count    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Movie_Title</th>\n",
       "      <th>#_of_Female_Cast</th>\n",
       "      <th>#_of_Male_Cast</th>\n",
       "      <th>#_of_Female_Dialogue</th>\n",
       "      <th>#_of_Male_Dialogue</th>\n",
       "      <th>#_of_Female_Crew</th>\n",
       "      <th>#_of_Male_Crew</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1999</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1968</td>\n",
       "      <td>2001: a space odyssey</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1982</td>\n",
       "      <td>48 hrs.</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1999</td>\n",
       "      <td>8mm</td>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1988</td>\n",
       "      <td>a nightmare on elm street 4: the dream master</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Year                                    Movie_Title #_of_Female_Cast  \\\n",
       "0  1999                     10 things i hate about you                5   \n",
       "1  1968                          2001: a space odyssey                2   \n",
       "2  1982                                        48 hrs.                7   \n",
       "3  1999                                            8mm                2   \n",
       "4  1988  a nightmare on elm street 4: the dream master                6   \n",
       "\n",
       "  #_of_Male_Cast #_of_Female_Dialogue #_of_Male_Dialogue #_of_Female_Crew  \\\n",
       "0              7                  NaN                NaN              NaN   \n",
       "1             11                  NaN                NaN              NaN   \n",
       "2              9                  NaN                NaN              NaN   \n",
       "3             13                  NaN                NaN              NaN   \n",
       "4              2                  NaN                NaN              NaN   \n",
       "\n",
       "  #_of_Male_Crew  \n",
       "0            NaN  \n",
       "1            NaN  \n",
       "2            NaN  \n",
       "3            NaN  \n",
       "4            NaN  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "analysis_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "del mcm_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "crew_df = pd.read_csv(\"Crew_Dataset.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Movie_Number</th>\n",
       "      <th>Movie_Name</th>\n",
       "      <th>Crew_Name</th>\n",
       "      <th>Job</th>\n",
       "      <th>Gender</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Charles Graffeo</td>\n",
       "      <td>Set Decoration</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>William Shakespeare</td>\n",
       "      <td>Theatre Play</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Mark Irwin</td>\n",
       "      <td>Director of Photography</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>O. Nicholas Brown</td>\n",
       "      <td>Editor</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Kimberly A. Tillman</td>\n",
       "      <td>Costume Design</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Richard Gibbs</td>\n",
       "      <td>Original Music Composer</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Andrew Lazar</td>\n",
       "      <td>Producer</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Carol Winstead Wood</td>\n",
       "      <td>Production Design</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Gil Junger</td>\n",
       "      <td>Director</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Karen McCullah Lutz</td>\n",
       "      <td>Writer</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Kirsten Smith</td>\n",
       "      <td>Writer</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Seth Jaret</td>\n",
       "      <td>Executive Producer</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Jeffrey Chernov</td>\n",
       "      <td>Executive Producer</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Greg Silverman</td>\n",
       "      <td>Associate Producer</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Gilbert Wong</td>\n",
       "      <td>Art Direction</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Marcia Ross</td>\n",
       "      <td>Casting</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Gail Goldberg</td>\n",
       "      <td>Casting</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Ralph Sall</td>\n",
       "      <td>Executive Music Producer</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Mel Metcalfe</td>\n",
       "      <td>Sound Re-Recording Mixer</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Terry Porter</td>\n",
       "      <td>Sound Re-Recording Mixer</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Albert Gasser</td>\n",
       "      <td>Dialogue Editor</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Dean A. Zupancic</td>\n",
       "      <td>Sound Re-Recording Mixer</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Ross Fanger</td>\n",
       "      <td>Unit Production Manager</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Donna Morong</td>\n",
       "      <td>Casting</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>24</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>K.C. Colwell</td>\n",
       "      <td>Assistant Director</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Robin Buerki</td>\n",
       "      <td>Camera Operator</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>26</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Robin Buerki</td>\n",
       "      <td>Steadycam</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>27</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Donald J. Malouf</td>\n",
       "      <td>Supervising Sound Effects Editor</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>28</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Nancy Karlin</td>\n",
       "      <td>Script Supervisor</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Jody Hedien</td>\n",
       "      <td>Co-Producer</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>\"Martin Vinnie Hagood\"</td>\n",
       "      <td>Makeup Artist</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>31</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Pauletta O. Lewis</td>\n",
       "      <td>Key Hair Stylist</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Alexandria Forster</td>\n",
       "      <td>Costume Supervisor</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Gary Katsuya Ushino</td>\n",
       "      <td>First Assistant Camera</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>34</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Marguerite Pomerhn Derricks</td>\n",
       "      <td>Choreographer</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>Ben Scott</td>\n",
       "      <td>Stunt Coordinator</td>\n",
       "      <td>f</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>David Schneider</td>\n",
       "      <td>Boom Operator</td>\n",
       "      <td>m</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Movie_Number                  Movie_Name                    Crew_Name  \\\n",
       "0            m0  10 things i hate about you              Charles Graffeo   \n",
       "1            m0  10 things i hate about you          William Shakespeare   \n",
       "2            m0  10 things i hate about you                   Mark Irwin   \n",
       "3            m0  10 things i hate about you            O. Nicholas Brown   \n",
       "4            m0  10 things i hate about you          Kimberly A. Tillman   \n",
       "5            m0  10 things i hate about you                Richard Gibbs   \n",
       "6            m0  10 things i hate about you                 Andrew Lazar   \n",
       "7            m0  10 things i hate about you          Carol Winstead Wood   \n",
       "8            m0  10 things i hate about you                   Gil Junger   \n",
       "9            m0  10 things i hate about you          Karen McCullah Lutz   \n",
       "10           m0  10 things i hate about you                Kirsten Smith   \n",
       "11           m0  10 things i hate about you                   Seth Jaret   \n",
       "12           m0  10 things i hate about you              Jeffrey Chernov   \n",
       "13           m0  10 things i hate about you               Greg Silverman   \n",
       "14           m0  10 things i hate about you                 Gilbert Wong   \n",
       "15           m0  10 things i hate about you                  Marcia Ross   \n",
       "16           m0  10 things i hate about you                Gail Goldberg   \n",
       "17           m0  10 things i hate about you                   Ralph Sall   \n",
       "18           m0  10 things i hate about you                 Mel Metcalfe   \n",
       "19           m0  10 things i hate about you                 Terry Porter   \n",
       "20           m0  10 things i hate about you                Albert Gasser   \n",
       "21           m0  10 things i hate about you             Dean A. Zupancic   \n",
       "22           m0  10 things i hate about you                  Ross Fanger   \n",
       "23           m0  10 things i hate about you                 Donna Morong   \n",
       "24           m0  10 things i hate about you                 K.C. Colwell   \n",
       "25           m0  10 things i hate about you                 Robin Buerki   \n",
       "26           m0  10 things i hate about you                 Robin Buerki   \n",
       "27           m0  10 things i hate about you             Donald J. Malouf   \n",
       "28           m0  10 things i hate about you                 Nancy Karlin   \n",
       "29           m0  10 things i hate about you                  Jody Hedien   \n",
       "30           m0  10 things i hate about you       \"Martin Vinnie Hagood\"   \n",
       "31           m0  10 things i hate about you            Pauletta O. Lewis   \n",
       "32           m0  10 things i hate about you           Alexandria Forster   \n",
       "33           m0  10 things i hate about you          Gary Katsuya Ushino   \n",
       "34           m0  10 things i hate about you  Marguerite Pomerhn Derricks   \n",
       "35           m0  10 things i hate about you                    Ben Scott   \n",
       "36           m0  10 things i hate about you              David Schneider   \n",
       "\n",
       "                                 Job Gender  \n",
       "0                     Set Decoration      m  \n",
       "1                       Theatre Play      m  \n",
       "2            Director of Photography      m  \n",
       "3                             Editor      m  \n",
       "4                     Costume Design      f  \n",
       "5            Original Music Composer      m  \n",
       "6                           Producer      m  \n",
       "7                  Production Design      f  \n",
       "8                           Director      m  \n",
       "9                             Writer      f  \n",
       "10                            Writer      f  \n",
       "11                Executive Producer      m  \n",
       "12                Executive Producer      m  \n",
       "13                Associate Producer      m  \n",
       "14                     Art Direction      f  \n",
       "15                           Casting      f  \n",
       "16                           Casting      f  \n",
       "17          Executive Music Producer      m  \n",
       "18          Sound Re-Recording Mixer      m  \n",
       "19          Sound Re-Recording Mixer      m  \n",
       "20                   Dialogue Editor      m  \n",
       "21          Sound Re-Recording Mixer      m  \n",
       "22           Unit Production Manager      m  \n",
       "23                           Casting      f  \n",
       "24                Assistant Director      m  \n",
       "25                   Camera Operator      m  \n",
       "26                         Steadycam      m  \n",
       "27  Supervising Sound Effects Editor      m  \n",
       "28                 Script Supervisor      f  \n",
       "29                       Co-Producer      m  \n",
       "30                     Makeup Artist      m  \n",
       "31                  Key Hair Stylist      f  \n",
       "32                Costume Supervisor      m  \n",
       "33            First Assistant Camera      m  \n",
       "34                     Choreographer      m  \n",
       "35                 Stunt Coordinator      f  \n",
       "36                     Boom Operator      m  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "crew_df.loc[crew_df[\"Movie_Number\"] == \"m0\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Getting the number of female and male characters\n",
    "for rows in analysis_df.itertuples():\n",
    "    f_count = 0\n",
    "    m_count = 0\n",
    "    for row2 in crew_df.itertuples():\n",
    "        if row2[2] == rows[2]:\n",
    "            if row2[5] == 'f':\n",
    "                f_count += 1\n",
    "            else:\n",
    "                m_count += 1\n",
    "    analysis_df.at[rows.Index, \"#_of_Female_Crew\"] = f_count\n",
    "    analysis_df.at[rows.Index, \"#_of_Male_Crew\"] = m_count   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Movie_Title</th>\n",
       "      <th>#_of_Female_Cast</th>\n",
       "      <th>#_of_Male_Cast</th>\n",
       "      <th>#_of_Female_Dialogue</th>\n",
       "      <th>#_of_Male_Dialogue</th>\n",
       "      <th>#_of_Female_Crew</th>\n",
       "      <th>#_of_Male_Crew</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1999</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>11</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1968</td>\n",
       "      <td>2001: a space odyssey</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1982</td>\n",
       "      <td>48 hrs.</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1999</td>\n",
       "      <td>8mm</td>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>15</td>\n",
       "      <td>62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1988</td>\n",
       "      <td>a nightmare on elm street 4: the dream master</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Year                                    Movie_Title #_of_Female_Cast  \\\n",
       "0  1999                     10 things i hate about you                5   \n",
       "1  1968                          2001: a space odyssey                2   \n",
       "2  1982                                        48 hrs.                7   \n",
       "3  1999                                            8mm                2   \n",
       "4  1988  a nightmare on elm street 4: the dream master                6   \n",
       "\n",
       "  #_of_Male_Cast #_of_Female_Dialogue #_of_Male_Dialogue #_of_Female_Crew  \\\n",
       "0              7                  NaN                NaN               11   \n",
       "1             11                  NaN                NaN                2   \n",
       "2              9                  NaN                NaN                1   \n",
       "3             13                  NaN                NaN               15   \n",
       "4              2                  NaN                NaN                3   \n",
       "\n",
       "  #_of_Male_Crew  \n",
       "0             26  \n",
       "1             17  \n",
       "2             14  \n",
       "3             62  \n",
       "4             12  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "del crew_df\n",
    "analysis_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "304713 (304713, 5)\n"
     ]
    }
   ],
   "source": [
    "## Getting the number of dialogues\n",
    "ml_df = pd.read_csv(\"ml_csv.csv\")\n",
    "mcm_df = pd.read_csv(\"mcm_csv.csv\")\n",
    "print(len(ml_df.Line_Id.unique()), ml_df.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Line_Id</th>\n",
       "      <th>Character_Id</th>\n",
       "      <th>Movie_Number</th>\n",
       "      <th>Character_Name</th>\n",
       "      <th>Dialogue</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>L1045</td>\n",
       "      <td>u0</td>\n",
       "      <td>m0</td>\n",
       "      <td>BIANCA</td>\n",
       "      <td>They do not!</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Line_Id Character_Id Movie_Number Character_Name      Dialogue\n",
       "0   L1045           u0           m0         BIANCA  They do not!"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ml_df.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Character_Id</th>\n",
       "      <th>Character_Name</th>\n",
       "      <th>Movie_Number</th>\n",
       "      <th>Movie_Title</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Position_in_Credits</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>u0</td>\n",
       "      <td>BIANCA</td>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>f</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Character_Id Character_Name Movie_Number                 Movie_Title Gender  \\\n",
       "0           u0         BIANCA           m0  10 things i hate about you      f   \n",
       "\n",
       "  Position_in_Credits  \n",
       "0                   4  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mcm_df.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(304713, 8)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df = pd.merge(ml_df, mcm_df, how='left')\n",
    "new_df.drop_duplicates(keep='first', inplace=True)\n",
    "new_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Line_Id</th>\n",
       "      <th>Character_Id</th>\n",
       "      <th>Movie_Number</th>\n",
       "      <th>Character_Name</th>\n",
       "      <th>Dialogue</th>\n",
       "      <th>Movie_Title</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Position_in_Credits</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>L1045</td>\n",
       "      <td>u0</td>\n",
       "      <td>m0</td>\n",
       "      <td>BIANCA</td>\n",
       "      <td>They do not!</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>f</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>L1044</td>\n",
       "      <td>u2</td>\n",
       "      <td>m0</td>\n",
       "      <td>CAMERON</td>\n",
       "      <td>They do to!</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>m</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>L985</td>\n",
       "      <td>u0</td>\n",
       "      <td>m0</td>\n",
       "      <td>BIANCA</td>\n",
       "      <td>I hope so.</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>f</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>L984</td>\n",
       "      <td>u2</td>\n",
       "      <td>m0</td>\n",
       "      <td>CAMERON</td>\n",
       "      <td>She okay?</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>m</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>L925</td>\n",
       "      <td>u0</td>\n",
       "      <td>m0</td>\n",
       "      <td>BIANCA</td>\n",
       "      <td>Let's go.</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>f</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Line_Id Character_Id Movie_Number Character_Name      Dialogue  \\\n",
       "0   L1045           u0           m0         BIANCA  They do not!   \n",
       "1   L1044           u2           m0        CAMERON   They do to!   \n",
       "2    L985           u0           m0         BIANCA    I hope so.   \n",
       "3    L984           u2           m0        CAMERON     She okay?   \n",
       "4    L925           u0           m0         BIANCA     Let's go.   \n",
       "\n",
       "                  Movie_Title Gender Position_in_Credits  \n",
       "0  10 things i hate about you      f                   4  \n",
       "1  10 things i hate about you      m                   3  \n",
       "2  10 things i hate about you      f                   4  \n",
       "3  10 things i hate about you      m                   3  \n",
       "4  10 things i hate about you      f                   4  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Getting the number of female and male characters\n",
    "for rows in analysis_df.itertuples():\n",
    "    f_count = 0\n",
    "    m_count = 0\n",
    "    for row2 in new_df.itertuples():\n",
    "        if row2[6] == rows[2]:\n",
    "            if row2[7] == 'f':\n",
    "                f_count += 1\n",
    "            else:\n",
    "                m_count += 1\n",
    "    analysis_df.at[rows.Index, \"#_of_Female_Dialogue\"] = f_count\n",
    "    analysis_df.at[rows.Index, \"#_of_Male_Dialogue\"] = m_count  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "del new_df\n",
    "del ml_df\n",
    "del mcm_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Movie_Title</th>\n",
       "      <th>#_of_Female_Cast</th>\n",
       "      <th>#_of_Male_Cast</th>\n",
       "      <th>#_of_Female_Dialogue</th>\n",
       "      <th>#_of_Male_Dialogue</th>\n",
       "      <th>#_of_Female_Crew</th>\n",
       "      <th>#_of_Male_Crew</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1999</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>295</td>\n",
       "      <td>374</td>\n",
       "      <td>11</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1968</td>\n",
       "      <td>2001: a space odyssey</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>17</td>\n",
       "      <td>255</td>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1982</td>\n",
       "      <td>48 hrs.</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>109</td>\n",
       "      <td>469</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1999</td>\n",
       "      <td>8mm</td>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>37</td>\n",
       "      <td>547</td>\n",
       "      <td>15</td>\n",
       "      <td>62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1988</td>\n",
       "      <td>a nightmare on elm street 4: the dream master</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>95</td>\n",
       "      <td>60</td>\n",
       "      <td>3</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Year                                    Movie_Title #_of_Female_Cast  \\\n",
       "0  1999                     10 things i hate about you                5   \n",
       "1  1968                          2001: a space odyssey                2   \n",
       "2  1982                                        48 hrs.                7   \n",
       "3  1999                                            8mm                2   \n",
       "4  1988  a nightmare on elm street 4: the dream master                6   \n",
       "\n",
       "  #_of_Male_Cast #_of_Female_Dialogue #_of_Male_Dialogue #_of_Female_Crew  \\\n",
       "0              7                  295                374               11   \n",
       "1             11                   17                255                2   \n",
       "2              9                  109                469                1   \n",
       "3             13                   37                547               15   \n",
       "4              2                   95                 60                3   \n",
       "\n",
       "  #_of_Male_Crew  \n",
       "0             26  \n",
       "1             17  \n",
       "2             14  \n",
       "3             62  \n",
       "4             12  "
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "analysis_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "analysis_df.to_csv(\"Analysis_csv.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting Sentiment scores in place"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(324, 8)\n"
     ]
    }
   ],
   "source": [
    "analy = pd.read_csv(\"Analysis_csv.csv\")\n",
    "print(analy.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Movie_Title</th>\n",
       "      <th>#_of_Female_Cast</th>\n",
       "      <th>#_of_Male_Cast</th>\n",
       "      <th>#_of_Female_Dialogue</th>\n",
       "      <th>#_of_Male_Dialogue</th>\n",
       "      <th>#_of_Female_Crew</th>\n",
       "      <th>#_of_Male_Crew</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1999</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>295</td>\n",
       "      <td>374</td>\n",
       "      <td>11</td>\n",
       "      <td>26</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1968</td>\n",
       "      <td>2001: a space odyssey</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>17</td>\n",
       "      <td>255</td>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1982</td>\n",
       "      <td>48 hrs.</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>109</td>\n",
       "      <td>469</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1999</td>\n",
       "      <td>8mm</td>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>37</td>\n",
       "      <td>547</td>\n",
       "      <td>15</td>\n",
       "      <td>62</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1988</td>\n",
       "      <td>a nightmare on elm street 4: the dream master</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>95</td>\n",
       "      <td>60</td>\n",
       "      <td>3</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Year                                    Movie_Title  #_of_Female_Cast  \\\n",
       "0  1999                     10 things i hate about you                 5   \n",
       "1  1968                          2001: a space odyssey                 2   \n",
       "2  1982                                        48 hrs.                 7   \n",
       "3  1999                                            8mm                 2   \n",
       "4  1988  a nightmare on elm street 4: the dream master                 6   \n",
       "\n",
       "   #_of_Male_Cast  #_of_Female_Dialogue  #_of_Male_Dialogue  #_of_Female_Crew  \\\n",
       "0               7                   295                 374                11   \n",
       "1              11                    17                 255                 2   \n",
       "2               9                   109                 469                 1   \n",
       "3              13                    37                 547                15   \n",
       "4               2                    95                  60                 3   \n",
       "\n",
       "   #_of_Male_Crew  \n",
       "0              26  \n",
       "1              17  \n",
       "2              14  \n",
       "3              62  \n",
       "4              12  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "analy.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(304713, 5)\n"
     ]
    }
   ],
   "source": [
    "ml = pd.read_csv(\"ml_csv.csv\")\n",
    "mcm = pd.read_csv(\"mcm_csv.csv\")\n",
    "print(ml.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(669, 5)"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ml.loc[ml[\"Movie_Number\"] == 'm0'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Line_Id</th>\n",
       "      <th>Character_Id</th>\n",
       "      <th>Movie_Number</th>\n",
       "      <th>Character_Name</th>\n",
       "      <th>Dialogue</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>L1045</td>\n",
       "      <td>u0</td>\n",
       "      <td>m0</td>\n",
       "      <td>BIANCA</td>\n",
       "      <td>They do not!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>L1044</td>\n",
       "      <td>u2</td>\n",
       "      <td>m0</td>\n",
       "      <td>CAMERON</td>\n",
       "      <td>They do to!</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>L985</td>\n",
       "      <td>u0</td>\n",
       "      <td>m0</td>\n",
       "      <td>BIANCA</td>\n",
       "      <td>I hope so.</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>L984</td>\n",
       "      <td>u2</td>\n",
       "      <td>m0</td>\n",
       "      <td>CAMERON</td>\n",
       "      <td>She okay?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>L925</td>\n",
       "      <td>u0</td>\n",
       "      <td>m0</td>\n",
       "      <td>BIANCA</td>\n",
       "      <td>Let's go.</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Line_Id Character_Id Movie_Number Character_Name      Dialogue\n",
       "0   L1045           u0           m0         BIANCA  They do not!\n",
       "1   L1044           u2           m0        CAMERON   They do to!\n",
       "2    L985           u0           m0         BIANCA    I hope so.\n",
       "3    L984           u2           m0        CAMERON     She okay?\n",
       "4    L925           u0           m0         BIANCA     Let's go."
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ml.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Character_Id</th>\n",
       "      <th>Character_Name</th>\n",
       "      <th>Movie_Number</th>\n",
       "      <th>Movie_Title</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Position_in_Credits</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>u0</td>\n",
       "      <td>BIANCA</td>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>f</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>u1</td>\n",
       "      <td>BRUCE</td>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>m</td>\n",
       "      <td>?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>u2</td>\n",
       "      <td>CAMERON</td>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>m</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>u3</td>\n",
       "      <td>CHASTITY</td>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>f</td>\n",
       "      <td>?</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>u4</td>\n",
       "      <td>JOEY</td>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>m</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Character_Id Character_Name Movie_Number                 Movie_Title Gender  \\\n",
       "0           u0         BIANCA           m0  10 things i hate about you      f   \n",
       "1           u1          BRUCE           m0  10 things i hate about you      m   \n",
       "2           u2        CAMERON           m0  10 things i hate about you      m   \n",
       "3           u3       CHASTITY           m0  10 things i hate about you      f   \n",
       "4           u4           JOEY           m0  10 things i hate about you      m   \n",
       "\n",
       "  Position_in_Credits  \n",
       "0                   4  \n",
       "1                   ?  \n",
       "2                   3  \n",
       "3                   ?  \n",
       "4                   6  "
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mcm.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(304713, 9)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ml_mcm = pd.merge(ml, mcm, on = [\"Movie_Number\", \"Character_Name\"], how = \"left\")\n",
    "ml_mcm.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Line_Id</th>\n",
       "      <th>Character_Id_x</th>\n",
       "      <th>Movie_Number</th>\n",
       "      <th>Character_Name</th>\n",
       "      <th>Dialogue</th>\n",
       "      <th>Character_Id_y</th>\n",
       "      <th>Movie_Title</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Position_in_Credits</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>L1045</td>\n",
       "      <td>u0</td>\n",
       "      <td>m0</td>\n",
       "      <td>BIANCA</td>\n",
       "      <td>They do not!</td>\n",
       "      <td>u0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>f</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>L1044</td>\n",
       "      <td>u2</td>\n",
       "      <td>m0</td>\n",
       "      <td>CAMERON</td>\n",
       "      <td>They do to!</td>\n",
       "      <td>u2</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>m</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>L985</td>\n",
       "      <td>u0</td>\n",
       "      <td>m0</td>\n",
       "      <td>BIANCA</td>\n",
       "      <td>I hope so.</td>\n",
       "      <td>u0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>f</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>L984</td>\n",
       "      <td>u2</td>\n",
       "      <td>m0</td>\n",
       "      <td>CAMERON</td>\n",
       "      <td>She okay?</td>\n",
       "      <td>u2</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>m</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>L925</td>\n",
       "      <td>u0</td>\n",
       "      <td>m0</td>\n",
       "      <td>BIANCA</td>\n",
       "      <td>Let's go.</td>\n",
       "      <td>u0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>f</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Line_Id Character_Id_x Movie_Number Character_Name      Dialogue  \\\n",
       "0   L1045             u0           m0         BIANCA  They do not!   \n",
       "1   L1044             u2           m0        CAMERON   They do to!   \n",
       "2    L985             u0           m0         BIANCA    I hope so.   \n",
       "3    L984             u2           m0        CAMERON     She okay?   \n",
       "4    L925             u0           m0         BIANCA     Let's go.   \n",
       "\n",
       "  Character_Id_y                 Movie_Title Gender Position_in_Credits  \n",
       "0             u0  10 things i hate about you      f                   4  \n",
       "1             u2  10 things i hate about you      m                   3  \n",
       "2             u0  10 things i hate about you      f                   4  \n",
       "3             u2  10 things i hate about you      m                   3  \n",
       "4             u0  10 things i hate about you      f                   4  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ml_mcm.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.sentiment.vader import SentimentIntensityAnalyzer\n",
    "import re\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "sid = SentimentIntensityAnalyzer()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.027764406779661036 ---- 0.02628743315508023\n",
      "0.24472352941176467 ---- 0.14193333333333336\n",
      "-0.026442201834862386 ---- 0.0028660980810234623\n",
      "-0.10827837837837839 ---- 0.011065265082266927\n",
      "-0.058179999999999996 ---- -0.052153333333333336\n",
      "0.021137864077669904 ---- 0.017488301886792467\n",
      "0.09267380952380952 ---- 0.06590328947368423\n",
      "0.11727142857142857 ---- 0.0332795847750865\n",
      "0.0014864406779661007 ---- 0.04003021978021977\n",
      "0.04253575757575758 ---- 0.1218619952494063\n",
      "0.01419607843137255 ---- -0.022654117647058825\n",
      "0.04387714285714286 ---- -0.0016661101836394\n",
      "0.07257290322580644 ---- 0.03607648183556403\n",
      "0.015314503816793898 ---- 0.07552151898734176\n",
      "0.1005069565217391 ---- 0.05452418478260873\n",
      "0.14561111111111108 ---- 0.08193978494623659\n",
      "0.17236323529411765 ---- 0.18250740740740728\n",
      "-0.02928759124087592 ---- -0.07126635859519406\n",
      "0.08284462809917355 ---- 0.08067237762237765\n",
      "0.010062222222222212 ---- 0.047314206128133716\n",
      "0.0554740524781341 ---- 0.10653218210361061\n",
      "0.08315777027027024 ---- 0.12668167938931307\n",
      "0.035391549295774656 ---- -0.05441666666666667\n",
      "0.05023402777777778 ---- 0.11611057971014467\n",
      "0.022361904761904767 ---- 0.019315189873417724\n",
      "0 ---- 0.005317361111111113\n",
      "0.0043006802721088444 ---- 0.010051267605633785\n",
      "0.09628204081632652 ---- 0.02936557377049182\n",
      "0.14988235294117644 ---- -0.0006792763157894681\n",
      "0.10373918918918913 ---- 0.09376593886462875\n",
      "0.02010779220779221 ---- 0.0836445945945946\n",
      "0.03416136363636364 ---- -0.02319253731343283\n",
      "0.012077586206896546 ---- 0.07135089820359283\n",
      "0.11939285714285709 ---- 0.05650599250936329\n",
      "0.05573181818181819 ---- -0.082035549132948\n",
      "0.05309024943310659 ---- 0.07350833333333337\n",
      "0.07480583333333336 ---- 0.028472398190045253\n",
      "0.016454081632653064 ---- 0.07017037037037036\n",
      "0.08877547169811319 ---- -0.018119746835443033\n",
      "0.024700711743772263 ---- 0.009079723502304158\n",
      "0.04100096153846155 ---- 0.05591940298507463\n",
      "0.010287155963302754 ---- -0.0010649462365591334\n",
      "0.02453988842398885 ---- 0.10216716417910449\n",
      "0.0717240384615385 ---- 0.09540786802030454\n",
      "0.08300219780219781 ---- 0.06023636363636369\n",
      "0.10532714285714288 ---- 0.0738371335504886\n",
      "0.08531350210970468 ---- 0.0774739495798319\n",
      "0.16922487804878047 ---- 0.0938852307692308\n",
      "0.04014473684210527 ---- 0.021090000000000005\n",
      "-0.05042941176470589 ---- -0.0067086274509804\n",
      "0.011812499999999998 ---- 0.014632692307692312\n",
      "0.07267075098814228 ---- 0.03459019264448334\n",
      "0.0017449760765550224 ---- 0.018107500000000006\n",
      "0.04848246073298429 ---- 0.06313794002607555\n",
      "0.036197619047619055 ---- 0.04708248587570623\n",
      "0.003549802371541508 ---- 0.04901222222222224\n",
      "0.12872392857142845 ---- 0.17225802919708028\n",
      "0.0728590909090909 ---- 0.07406415094339622\n",
      "-0.009250000000000001 ---- 0.0020120754716981145\n",
      "0.012997297297297301 ---- 0.06735748031496062\n",
      "0.0025821917808219142 ---- 0.017929816513761465\n",
      "0.09334115755627008 ---- 0.03574146341463416\n",
      "0.09391615384615379 ---- 0.11505741525423725\n",
      "-0.008787179487179485 ---- -0.06860256821829858\n",
      "0.028995909090909093 ---- -0.010751550387596892\n",
      "-0.0322735294117647 ---- -0.047660321100917453\n",
      "0.08533000000000002 ---- 0.0031906336088154486\n",
      "0.11981641468682488 ---- 0.1180128571428571\n",
      "0.05892727272727273 ---- 0.03971671826625388\n",
      "0.025475324675324686 ---- 0.0165059405940594\n",
      "0.08221820987654324 ---- 0.02260412371134022\n",
      "0.08269281045751635 ---- 0.051352403846153825\n",
      "0.11222604166666666 ---- 0.011619533527696804\n",
      "-0.033977343749999986 ---- 0.04583839779005529\n",
      "0.03405833333333333 ---- 0.09813865979381427\n",
      "0.04153260869565217 ---- 0.0264545706371191\n",
      "-0.031280291970802905 ---- 0.008356776556776562\n",
      "0.06318260869565218 ---- 0.02794686774941994\n",
      "0.058568699186991854 ---- 0.061634259259259236\n",
      "0.1158769696969697 ---- 0.0931515850144092\n",
      "0.012796875 ---- 0.013589952153110063\n",
      "0.11725000000000001 ---- 0.023165454545454534\n",
      "0.11807491289198607 ---- 0.11432916030534344\n",
      "0.10809452736318415 ---- 0.13548693467336678\n",
      "0.023549999999999998 ---- 0.02007229916897509\n",
      "-0.0097703125 ---- 0.014415647921760406\n",
      "0.11411294117647061 ---- 0.007489419795221858\n",
      "0.027564375000000002 ---- -0.03504368932038835\n",
      "0.140153033707865 ---- 0.16136894736842097\n",
      "-0.0001224719101123598 ---- 0.03192896281800391\n",
      "0.029150000000000006 ---- 0.042810344827586146\n",
      "0.10447672955974842 ---- 0.050526176470588234\n",
      "-0.01995 ---- -0.03659842105263157\n",
      "0.10065073170731711 ---- 0.10114316702819963\n",
      "0.10034363057324835 ---- 0.08543835920177377\n",
      "-0.04140799999999999 ---- -0.0052602857142857165\n",
      "0.03895507246376812 ---- 0.01768248730964467\n",
      "0.04961506849315067 ---- 0.029523311546840972\n",
      "-0.05307666666666666 ---- -0.10678620689655179\n",
      "0.06630000000000001 ---- 0.11903037974683547\n",
      "-0.07635057471264368 ---- 0.014938461538461545\n",
      "0.05845546391752581 ---- 0.015090585241730271\n",
      "0.057958415841584184 ---- 0.09531804878048783\n",
      "0.10492643678160918 ---- 0.09895485362095523\n",
      "0.024737142857142853 ---- 0.008241484716157207\n",
      "0.031019047619047613 ---- -0.012848888888888865\n",
      "-0.03736470588235294 ---- 0.009937500000000004\n",
      "-0.04090520231213872 ---- 0.06268865546218488\n",
      "0.07389166666666667 ---- 0.006730740740740741\n",
      "0.06598860759493672 ---- 0.06011333333333332\n",
      "0.05651857142857144 ---- 0.018651358024691346\n",
      "0.02561875 ---- -0.017551768488745983\n",
      "0.017595652173913044 ---- -0.000206808510638294\n",
      "0.031801694915254246 ---- 0.04431326923076924\n",
      "0.04997116279069771 ---- 0.0238158536585366\n",
      "0.008516969696969696 ---- 0.016590659340659338\n",
      "0.0840903846153846 ---- 0.04041556802244037\n",
      "0.07510256410256409 ---- 0.05747965616045843\n",
      "-0.012267415730337095 ---- 0.0055346733668341684\n",
      "0.051440157480315 ---- 0.017035502958579897\n",
      "-0.04572752293577983 ---- 0.012826111111111125\n",
      "0.07785967741935486 ---- -0.08312847682119212\n",
      "0.023261478599221785 ---- 0.023970118343195273\n",
      "0.0435993827160494 ---- 0.01431484375000001\n",
      "-0.002196756756756754 ---- 0.042336444444444406\n",
      "0.013886124401913861 ---- -0.05683333333333334\n",
      "0 ---- 0.008507221350078498\n",
      "0.10153785166240395 ---- 0.10060543209876537\n",
      "0.1620493506493506 ---- 0.07572058212058214\n",
      "-0.06318499999999999 ---- -0.062236363636363604\n",
      "0.0056555555555555526 ---- 0.07115430463576153\n",
      "0.13678183856502235 ---- 0.08570231404958678\n",
      "-0.0047884353741496815 ---- 0.09922080924855486\n",
      "-0.031636363636363636 ---- -0.01941836363636364\n",
      "0.04000330188679245 ---- 0.0908680555555555\n",
      "0.021230357142857148 ---- 0.020319302949061674\n",
      "0.056593877551020404 ---- 0.07296843750000002\n",
      "0.04331162790697674 ---- 0.05694270462633452\n",
      "0.07966000000000002 ---- 0.08897999999999996\n",
      "0.08270816326530614 ---- 0.07816724511930585\n",
      "0.02172343750000001 ---- -0.05087578947368419\n",
      "0.05327228915662654 ---- 0.08785105740181268\n",
      "0.010919607843137256 ---- 0.03716618357487922\n",
      "-0.0013723076923076874 ---- 0.020116849816849827\n",
      "-0.012439423076923075 ---- 0.05366682242990658\n",
      "0.12643125 ---- -0.03997971014492755\n",
      "0.20395353982300862 ---- 0.22469021113243715\n",
      "0.10853954372623573 ---- 0.1234818918918919\n",
      "0.06029565217391307 ---- 0.06435454545454546\n",
      "-0.08811739130434783 ---- -0.05454723926380368\n",
      "-0.0027142857142857094 ---- -0.009362576687116562\n",
      "0.08478778625954195 ---- 0.08856803571428556\n",
      "0.082432 ---- 0.04505046728971963\n",
      "0.015138499999999997 ---- -0.008210261194029839\n",
      "0.027506666666666683 ---- -0.06351187214611871\n",
      "0.05139343065693433 ---- 0.04704918032786889\n",
      "0.09505916666666671 ---- 0.08808612836438925\n",
      "0.15374018691588784 ---- 0.06277142857142853\n",
      "0.08623002680965142 ---- 0.038077659574468084\n",
      "0.062625 ---- 0.0263983305509182\n",
      "0.062071171171171176 ---- 0.062406611570247926\n",
      "0.014079746835443066 ---- 0.0014552521008403343\n",
      "0.06411612903225808 ---- 0.02963627684964201\n",
      "-0.011394573643410845 ---- -0.012222748815165881\n",
      "0.029744329896907222 ---- 0.03064005979073243\n",
      "0.054022766570605195 ---- 0.067151524390244\n",
      "0.03915608108108109 ---- 0.001820560747663548\n",
      "0.008432258064516121 ---- 0.07443036649214659\n",
      "-0.008615517241379309 ---- -0.023539038461538465\n",
      "-0.00806100917431192 ---- 0.0054681208053691255\n",
      "0.08058064516129036 ---- 0.04667642585551332\n",
      "-0.029656000000000012 ---- 0.06606458333333341\n",
      "-0.022749287749287753 ---- 0.0009502415458937222\n",
      "0.061620312499999996 ---- -0.05308999999999999\n",
      "0.056449767441860484 ---- 0.06463809523809523\n",
      "-0.012006024096385539 ---- 0.06985447761194033\n",
      "0.05771694915254237 ---- 0.0645759009009009\n",
      "-0.08184266666666667 ---- -0.09640681818181818\n",
      "0.11860612244897957 ---- 0.05706266968325791\n",
      "0.030570000000000003 ---- -0.010440282685512345\n",
      "-0.017874999999999995 ---- 0.017227016129032272\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.007757142857142853 ---- 0.009626978417266209\n",
      "-0.040465185185185173 ---- -0.04038037383177571\n",
      "-0.0036949494949494906 ---- -0.023687529976019188\n",
      "-0.00777708333333333 ---- 0.0472375\n",
      "0.09432749999999998 ---- 0.06388349673202612\n",
      "0.042046666666666684 ---- 0.063089485458613\n",
      "0.0875187919463087 ---- 0.15413799126637556\n",
      "0.08650886075949371 ---- -0.011226989619377178\n",
      "0.019527536231884076 ---- 0.009348578491965397\n",
      "0.022190389016018316 ---- 0.04484222972972972\n",
      "0.10978125 ---- 0.049861764705882355\n",
      "0.0025000000000000005 ---- -0.028413302752293576\n",
      "-0.0016075268817204369 ---- 0.021094881889763793\n",
      "0.11667500000000003 ---- -0.003003107344632764\n",
      "0.08229343434343435 ---- 0.060894901960784314\n",
      "-0.008597580645161293 ---- -0.01330610687022901\n",
      "-0.06495364238410593 ---- -0.07437692307692306\n",
      "-0.06641400000000001 ---- -0.04826516853932586\n",
      "-0.04989999999999999 ---- 0.0042691406250000025\n",
      "0.10477008928571431 ---- 0.05440457746478875\n",
      "0.06772298136645964 ---- 0.04141899641577062\n",
      "0.020293333333333344 ---- 0.017210576923076922\n",
      "0.05477615384615385 ---- -0.026495918367346934\n",
      "-0.07738265306122447 ---- -0.009183687943262409\n",
      "-0.0070176056338028215 ---- 0.045412195121951215\n",
      "0 ---- -0.007383638320775022\n",
      "-0.0031646153846153833 ---- 0.008494368600682598\n",
      "-0.04397384615384615 ---- -0.05845745614035086\n",
      "0.08075454545454547 ---- 0.04701622691292878\n",
      "0.05946818181818186 ---- 0.03432769230769232\n",
      "0.036397000000000006 ---- 0.014361875000000001\n",
      "-0.014003902439024375 ---- -0.01383157894736842\n",
      "0.11862182662538695 ---- 0.1050331797235023\n",
      "0.04485371179039301 ---- 0.08302785388127853\n",
      "0.018898947368421057 ---- -0.03611214285714286\n",
      "0.03076774193548387 ---- 0.19374761904761903\n",
      "0.065225 ---- 0.02614249084249085\n",
      "-0.05014850299401196 ---- -0.011334951456310681\n",
      "0.0887111111111111 ---- 0.20568412698412705\n",
      "0.04367096774193549 ---- -0.0023053872053872024\n",
      "0.0819968944099379 ---- 0.001489705882352951\n",
      "0.06533333333333335 ---- 0.013893301435406697\n",
      "0.06682466960352422 ---- -0.008038970588235295\n",
      "-0.021106250000000007 ---- -0.05562243902439025\n",
      "0.018054 ---- 0.0037553784860557774\n",
      "0.01065550239234451 ---- 0.03857958579881657\n",
      "0.02522307692307693 ---- 0.007736363636363644\n",
      "0.09962810810810813 ---- 0.08876469072164944\n",
      "0.16763648648648652 ---- 0.05798113207547169\n",
      "0.03077096774193548 ---- 0.05957031249999998\n",
      "-0.06771741573033706 ---- 0.004345652173913038\n",
      "0.006075 ---- 0.05442020202020202\n",
      "0.04873675675675678 ---- -0.0049332129963898875\n",
      "0.042998529411764734 ---- 0.06372549450549451\n",
      "0.07609636363636363 ---- -0.03346582733812948\n",
      "0.07 ---- 0.017810396039603944\n",
      "0.06296641791044773 ---- -0.0024936688311688257\n",
      "0.0984972789115646 ---- 0.09782099447513805\n",
      "-0.025936046511627904 ---- 0.02909402597402597\n",
      "0.010348333333333352 ---- 0.023783935018050543\n",
      "0.046562162162162156 ---- 0.044073899371069175\n",
      "0.12401501706484634 ---- 0.1058938826466914\n",
      "-0.003587951807228921 ---- -0.01430710526315789\n",
      "0.0797225806451613 ---- 0.005468636363636369\n",
      "-0.06027555555555557 ---- -0.0427669724770642\n",
      "0.1599547738693467 ---- 0.08907766497461929\n",
      "0.024349541284403683 ---- 0.019489625360230564\n",
      "0.0022531914893616995 ---- 0.02058144927536233\n",
      "-0.002367455621301771 ---- 0.043122522522522536\n",
      "-0.028109090909090917 ---- 0.049389855072463754\n",
      "0.030304629629629636 ---- 0.12836240601503765\n",
      "0.0700256544502618 ---- 0.10645111940298504\n",
      "0.05880517241379311 ---- 0.10599691011235957\n",
      "0.0753274193548387 ---- 0.012256571428571438\n",
      "0.10517739726027403 ---- 0.07632471264367818\n",
      "0.0264072625698324 ---- -0.011806874999999984\n",
      "0.08678977777777776 ---- 0.08959145496535796\n",
      "0.05106030534351145 ---- 0.1112549898167005\n",
      "0.023687066246056803 ---- 0.0307245925925926\n",
      "0.010381372549019597 ---- 0.024089565217391317\n",
      "0.0714411504424779 ---- 0.13300601503759402\n",
      "-0.007925757575757574 ---- 0.09219104166666667\n",
      "-0.057250295857988166 ---- 0.00949010989010989\n",
      "0.0503784090909091 ---- 0.0009704268292682942\n",
      "-0.12011818181818183 ---- -0.19657391304347827\n",
      "0.0011758620689655093 ---- 0.025784586466165424\n",
      "0 ---- -0.0076542857142857055\n",
      "0.10457122807017541 ---- 0.08086475644699144\n",
      "0.01692758620689655 ---- -0.007141599999999997\n",
      "0.04100999999999999 ---- 0.035085416666666695\n",
      "0.05173387096774194 ---- 0.06582620817843866\n",
      "0.022986206896551734 ---- 0.041007023411371246\n",
      "0.15724999999999997 ---- 0.06530055555555556\n",
      "-0.009024175824175825 ---- 0.009667602591792652\n",
      "0.1316837837837838 ---- -0.026234693877551007\n",
      "-0.03918813559322033 ---- 0.0028199248120300913\n",
      "0.09452793103448276 ---- 0.07214028985507248\n",
      "-0.0017656250000000057 ---- 0.005770175438596495\n",
      "0 ---- -0.011348240165631457\n",
      "0.09606580882352941 ---- 0.13175519801980193\n",
      "-0.050940293040293044 ---- -0.03009611650485437\n",
      "-0.07636858237547893 ---- -0.02507611111111111\n",
      "-0.0039662613981762875 ---- -0.058359480519480506\n",
      "0.12141948051948054 ---- 0.10564680851063822\n",
      "0.06951515151515154 ---- 0.04769411764705884\n",
      "0.0006651785714285681 ---- -0.0025147928994082897\n",
      "0.09779440789473685 ---- 0.0541057818659659\n",
      "-0.010707364341085273 ---- -0.00029373401534526535\n",
      "-0.1601142857142857 ---- 0.026318631178707242\n",
      "0.053261240310077534 ---- 0.06828651399491094\n",
      "0.08002000000000001 ---- 0.049550841750841734\n",
      "0.02007920000000002 ---- 0.04145795454545456\n",
      "-0.07370571428571428 ---- -0.008636720142602498\n",
      "0.06344440677966097 ---- 0.02239830508474576\n",
      "0.05196641221374045 ---- 0.06963371104815862\n",
      "0.07129134615384616 ---- 0.06710418604651162\n",
      "0.06986493506493506 ---- 0.043314024390243906\n",
      "0.07581974522292995 ---- 0.016168243243243254\n",
      "0.036332432432432435 ---- 0.09120349462365587\n",
      "0.026049484536082475 ---- -0.02163687943262411\n",
      "0.022211445783132534 ---- 0.1003571428571429\n",
      "0.07089666666666668 ---- 0.049909490740740754\n",
      "0.13036129032258065 ---- 0.02318086253369271\n",
      "0.049697297297297306 ---- 0.03817810650887575\n",
      "-0.0310554054054054 ---- 0.050774324324324306\n",
      "0.07093243243243244 ---- 0.003233435582822088\n",
      "0.09877924528301892 ---- 0.07385902255639099\n",
      "0.08680162162162161 ---- 0.04492688524590164\n",
      "0.10098235294117647 ---- 0.015613059033989284\n",
      "0.026790697674418613 ---- -0.04134464285714286\n",
      "-0.0018520833333333375 ---- -0.007508333333333328\n",
      "-0.04744117647058825 ---- -0.04245465838509318\n",
      "0.03468602484472051 ---- 0.06514209039548025\n",
      "0.025850229357798172 ---- 0.003194618834080714\n",
      "-0.11983333333333333 ---- 0.04416666666666669\n",
      "-0.003185306122448962 ---- -0.056313636363636344\n",
      "0.0931952380952381 ---- 0.08393853820598006\n",
      "-0.01943538461538461 ---- 0.035591416309012885\n",
      "0.05788739495798324 ---- -0.02233739495798319\n",
      "0.0020217391304347783 ---- -0.0376670634920635\n",
      "-0.002368493150684933 ---- -0.010368731563421817\n",
      "-0.017694871794871798 ---- 0.015544117647058823\n",
      "0.18254710743801664 ---- 0.08694747252747254\n"
     ]
    }
   ],
   "source": [
    "# Now doing it for the movie\n",
    "for rows in analy.itertuples():\n",
    "    sent_dict = {}\n",
    "    for rows2 in ml_mcm.itertuples():\n",
    "        if rows2[7] == rows[2]:\n",
    "            l = rows2[5]\n",
    "            line = re.sub(r'\\.\\.+', '', str(l))\n",
    "            s_score = sid.polarity_scores(line)\n",
    "            sent_dict[rows2.Index] = {rows2[8] : s_score.get('compound')}\n",
    "#     print(sent_dict, len(sent_dict))\n",
    "    g_listm = []\n",
    "    genre_listm = []\n",
    "    g_listf = []\n",
    "    genre_listf = []\n",
    "    sum_f = 0; sum_m = 0\n",
    "    for key, vals in sent_dict.items():\n",
    "        gender = str(list(sent_dict.get(key, {}).keys())).replace('[','').replace(']', '').replace('\\'', '')\n",
    "        genre = str(list(sent_dict.get(key, {}).values())).replace('[','').replace(']', '').replace('\\'', '')\n",
    "        if gender == 'm' or gender == 'M':\n",
    "            g_listm.append(gender)\n",
    "            genre_listm.append(genre)\n",
    "        else:\n",
    "            g_listf.append(gender)\n",
    "            genre_listf.append(genre)\n",
    "    genre_listf = np.array(genre_listf, dtype = float)\n",
    "    genre_listm = np.array(genre_listm, dtype = float)\n",
    "#     print(genre_listf,  \"\\n----\\n\", genre_listm)\n",
    "    try:\n",
    "        sum_f = sum(genre_listf)/len(genre_listf)\n",
    "    except:\n",
    "        sum_f = 0\n",
    "    try:\n",
    "        sum_m = sum(genre_listm)/len(genre_listm)\n",
    "    except:\n",
    "        sum_m = 0\n",
    "    print(sum_f, \"----\", sum_m)\n",
    "    analy.at[rows.Index, \"Sentiment_Female\"] = sum_f\n",
    "    analy.at[rows.Index, \"Sentiment_Male\"] = sum_m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Movie_Title</th>\n",
       "      <th>#_of_Female_Cast</th>\n",
       "      <th>#_of_Male_Cast</th>\n",
       "      <th>#_of_Female_Dialogue</th>\n",
       "      <th>#_of_Male_Dialogue</th>\n",
       "      <th>#_of_Female_Crew</th>\n",
       "      <th>#_of_Male_Crew</th>\n",
       "      <th>Sentiment_Female</th>\n",
       "      <th>Sentiment_Male</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>319</th>\n",
       "      <td>1985</td>\n",
       "      <td>witness</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>119</td>\n",
       "      <td>238</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0.057887</td>\n",
       "      <td>-0.022337</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>320</th>\n",
       "      <td>2009</td>\n",
       "      <td>watchmen</td>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>115</td>\n",
       "      <td>252</td>\n",
       "      <td>6</td>\n",
       "      <td>18</td>\n",
       "      <td>0.002022</td>\n",
       "      <td>-0.037667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>321</th>\n",
       "      <td>2002</td>\n",
       "      <td>xxx</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "      <td>407</td>\n",
       "      <td>37</td>\n",
       "      <td>102</td>\n",
       "      <td>-0.002368</td>\n",
       "      <td>-0.010369</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>322</th>\n",
       "      <td>2000</td>\n",
       "      <td>x-men</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>39</td>\n",
       "      <td>136</td>\n",
       "      <td>3</td>\n",
       "      <td>20</td>\n",
       "      <td>-0.017695</td>\n",
       "      <td>0.015544</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323</th>\n",
       "      <td>1974</td>\n",
       "      <td>young frankenstein</td>\n",
       "      <td>3</td>\n",
       "      <td>11</td>\n",
       "      <td>121</td>\n",
       "      <td>455</td>\n",
       "      <td>9</td>\n",
       "      <td>35</td>\n",
       "      <td>0.182547</td>\n",
       "      <td>0.086947</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Year         Movie_Title  #_of_Female_Cast  #_of_Male_Cast  \\\n",
       "319  1985             witness                 2               8   \n",
       "320  2009            watchmen                 4              12   \n",
       "321  2002                 xxx                 1              10   \n",
       "322  2000               x-men                 2               7   \n",
       "323  1974  young frankenstein                 3              11   \n",
       "\n",
       "     #_of_Female_Dialogue  #_of_Male_Dialogue  #_of_Female_Crew  \\\n",
       "319                   119                 238                 0   \n",
       "320                   115                 252                 6   \n",
       "321                     5                 407                37   \n",
       "322                    39                 136                 3   \n",
       "323                   121                 455                 9   \n",
       "\n",
       "     #_of_Male_Crew  Sentiment_Female  Sentiment_Male  \n",
       "319               7          0.057887       -0.022337  \n",
       "320              18          0.002022       -0.037667  \n",
       "321             102         -0.002368       -0.010369  \n",
       "322              20         -0.017695        0.015544  \n",
       "323              35          0.182547        0.086947  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "analy.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "analy.to_csv(\"Analysis_csv.csv\", index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Code for the Genre tagging"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim\n",
    "try:\n",
    "    model = gensim.models.KeyedVectors.load_word2vec_format('../Google_Billion_Word_Corpus/GoogleNews-vectors-negative300.bin', binary=True)\n",
    "except Exception as e:\n",
    "    print(e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.corpus import stopwords\n",
    "stop_words = set(stopwords.words('english'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(304713, 5)\n"
     ]
    }
   ],
   "source": [
    "ml = pd.read_csv(\"ml_csv.csv\")\n",
    "mcm = pd.read_csv(\"mcm_csv.csv\")\n",
    "print(ml.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(304713, 9)"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ml_mcm = pd.merge(ml, mcm, on = [\"Movie_Number\", \"Character_Name\"], how = \"left\")\n",
    "ml_mcm.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Line_Id</th>\n",
       "      <th>Character_Id_x</th>\n",
       "      <th>Movie_Number</th>\n",
       "      <th>Character_Name</th>\n",
       "      <th>Dialogue</th>\n",
       "      <th>Character_Id_y</th>\n",
       "      <th>Movie_Title</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Position_in_Credits</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>L1045</td>\n",
       "      <td>u0</td>\n",
       "      <td>m0</td>\n",
       "      <td>BIANCA</td>\n",
       "      <td>They do not!</td>\n",
       "      <td>u0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>f</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>L1044</td>\n",
       "      <td>u2</td>\n",
       "      <td>m0</td>\n",
       "      <td>CAMERON</td>\n",
       "      <td>They do to!</td>\n",
       "      <td>u2</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>m</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>L985</td>\n",
       "      <td>u0</td>\n",
       "      <td>m0</td>\n",
       "      <td>BIANCA</td>\n",
       "      <td>I hope so.</td>\n",
       "      <td>u0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>f</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>L984</td>\n",
       "      <td>u2</td>\n",
       "      <td>m0</td>\n",
       "      <td>CAMERON</td>\n",
       "      <td>She okay?</td>\n",
       "      <td>u2</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>m</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>L925</td>\n",
       "      <td>u0</td>\n",
       "      <td>m0</td>\n",
       "      <td>BIANCA</td>\n",
       "      <td>Let's go.</td>\n",
       "      <td>u0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>f</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Line_Id Character_Id_x Movie_Number Character_Name      Dialogue  \\\n",
       "0   L1045             u0           m0         BIANCA  They do not!   \n",
       "1   L1044             u2           m0        CAMERON   They do to!   \n",
       "2    L985             u0           m0         BIANCA    I hope so.   \n",
       "3    L984             u2           m0        CAMERON     She okay?   \n",
       "4    L925             u0           m0         BIANCA     Let's go.   \n",
       "\n",
       "  Character_Id_y                 Movie_Title Gender Position_in_Credits  \n",
       "0             u0  10 things i hate about you      f                   4  \n",
       "1             u2  10 things i hate about you      m                   3  \n",
       "2             u0  10 things i hate about you      f                   4  \n",
       "3             u2  10 things i hate about you      m                   3  \n",
       "4             u0  10 things i hate about you      f                   4  "
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ml_mcm.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(324, 10)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Movie_Number</th>\n",
       "      <th>Movie_Title</th>\n",
       "      <th>Year_of_Release</th>\n",
       "      <th>IMDB_Rating</th>\n",
       "      <th>Number_of_Votes</th>\n",
       "      <th>Genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>m0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>1999</td>\n",
       "      <td>6.9</td>\n",
       "      <td>62847</td>\n",
       "      <td>['comedy', 'romance']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>m1</td>\n",
       "      <td>1492: conquest of paradise</td>\n",
       "      <td>1992</td>\n",
       "      <td>6.2</td>\n",
       "      <td>10421</td>\n",
       "      <td>['adventure', 'biography', 'drama', 'history']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>m2</td>\n",
       "      <td>15 minutes</td>\n",
       "      <td>2001</td>\n",
       "      <td>6.1</td>\n",
       "      <td>25854</td>\n",
       "      <td>['action', 'crime', 'drama', 'thriller']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>m3</td>\n",
       "      <td>2001: a space odyssey</td>\n",
       "      <td>1968</td>\n",
       "      <td>8.4</td>\n",
       "      <td>163227</td>\n",
       "      <td>['adventure', 'mystery', 'sci-fi']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>m4</td>\n",
       "      <td>48 hrs.</td>\n",
       "      <td>1982</td>\n",
       "      <td>6.9</td>\n",
       "      <td>22289</td>\n",
       "      <td>['action', 'comedy', 'crime', 'drama', 'thrill...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Movie_Number                 Movie_Title Year_of_Release  IMDB_Rating  \\\n",
       "0           m0  10 things i hate about you            1999          6.9   \n",
       "1           m1  1492: conquest of paradise            1992          6.2   \n",
       "2           m2                  15 minutes            2001          6.1   \n",
       "3           m3       2001: a space odyssey            1968          8.4   \n",
       "4           m4                     48 hrs.            1982          6.9   \n",
       "\n",
       "   Number_of_Votes                                             Genres  \n",
       "0            62847                              ['comedy', 'romance']  \n",
       "1            10421     ['adventure', 'biography', 'drama', 'history']  \n",
       "2            25854           ['action', 'crime', 'drama', 'thriller']  \n",
       "3           163227                 ['adventure', 'mystery', 'sci-fi']  \n",
       "4            22289  ['action', 'comedy', 'crime', 'drama', 'thrill...  "
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mtm_df = pd.read_csv(\"mtm_csv.csv\")\n",
    "analy = pd.read_csv(\"Analysis_csv.csv\")\n",
    "print(analy.shape)\n",
    "mtm_df.rename(columns = {'Movie_Name': 'Movie_Title'}, inplace = True)\n",
    "mtm_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Movie_Title</th>\n",
       "      <th>#_of_Female_Cast</th>\n",
       "      <th>#_of_Male_Cast</th>\n",
       "      <th>#_of_Female_Dialogue</th>\n",
       "      <th>#_of_Male_Dialogue</th>\n",
       "      <th>#_of_Female_Crew</th>\n",
       "      <th>#_of_Male_Crew</th>\n",
       "      <th>Sentiment_Female</th>\n",
       "      <th>Sentiment_Male</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1999</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>295</td>\n",
       "      <td>374</td>\n",
       "      <td>11</td>\n",
       "      <td>26</td>\n",
       "      <td>0.027764</td>\n",
       "      <td>0.026287</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1968</td>\n",
       "      <td>2001: a space odyssey</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>17</td>\n",
       "      <td>255</td>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "      <td>0.244724</td>\n",
       "      <td>0.141933</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Year                 Movie_Title  #_of_Female_Cast  #_of_Male_Cast  \\\n",
       "0  1999  10 things i hate about you                 5               7   \n",
       "1  1968       2001: a space odyssey                 2              11   \n",
       "\n",
       "   #_of_Female_Dialogue  #_of_Male_Dialogue  #_of_Female_Crew  #_of_Male_Crew  \\\n",
       "0                   295                 374                11              26   \n",
       "1                    17                 255                 2              17   \n",
       "\n",
       "   Sentiment_Female  Sentiment_Male  \n",
       "0          0.027764        0.026287  \n",
       "1          0.244724        0.141933  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "analy.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "324\n"
     ]
    }
   ],
   "source": [
    "movie_list = analy.Movie_Title.unique()\n",
    "print(len(movie_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_Df = ml_mcm.loc[ml_mcm['Movie_Title'].isin(movie_list)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(669, 9)"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_Df.loc[new_Df['Movie_Number'] == 'm0'].shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(163017, 9)"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_Df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:1: SettingWithCopyWarning:\n",
      "\n",
      "\n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "\n"
     ]
    }
   ],
   "source": [
    "new_Df['Dialogue'] = new_Df['Dialogue'].str.lower()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Line_Id</th>\n",
       "      <th>Character_Id_x</th>\n",
       "      <th>Movie_Number</th>\n",
       "      <th>Character_Name</th>\n",
       "      <th>Dialogue</th>\n",
       "      <th>Character_Id_y</th>\n",
       "      <th>Movie_Title</th>\n",
       "      <th>Gender</th>\n",
       "      <th>Position_in_Credits</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>L1045</td>\n",
       "      <td>u0</td>\n",
       "      <td>m0</td>\n",
       "      <td>BIANCA</td>\n",
       "      <td>they do not!</td>\n",
       "      <td>u0</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>f</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Line_Id Character_Id_x Movie_Number Character_Name      Dialogue  \\\n",
       "0   L1045             u0           m0         BIANCA  they do not!   \n",
       "\n",
       "  Character_Id_y                 Movie_Title Gender Position_in_Credits  \n",
       "0             u0  10 things i hate about you      f                   4  "
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_Df.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(324, 11)\n"
     ]
    }
   ],
   "source": [
    "analy = pd.merge(analy, mtm_df, on = 'Movie_Title', how = 'left')\n",
    "analy.drop(['Movie_Number', 'Year_of_Release', 'IMDB_Rating', 'Number_of_Votes'], axis = 1, inplace = True)\n",
    "print(analy.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Movie_Title</th>\n",
       "      <th>#_of_Female_Cast</th>\n",
       "      <th>#_of_Male_Cast</th>\n",
       "      <th>#_of_Female_Dialogue</th>\n",
       "      <th>#_of_Male_Dialogue</th>\n",
       "      <th>#_of_Female_Crew</th>\n",
       "      <th>#_of_Male_Crew</th>\n",
       "      <th>Sentiment_Female</th>\n",
       "      <th>Sentiment_Male</th>\n",
       "      <th>Genres</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1999</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>295</td>\n",
       "      <td>374</td>\n",
       "      <td>11</td>\n",
       "      <td>26</td>\n",
       "      <td>0.027764</td>\n",
       "      <td>0.026287</td>\n",
       "      <td>['comedy', 'romance']</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1968</td>\n",
       "      <td>2001: a space odyssey</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>17</td>\n",
       "      <td>255</td>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "      <td>0.244724</td>\n",
       "      <td>0.141933</td>\n",
       "      <td>['adventure', 'mystery', 'sci-fi']</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Year                 Movie_Title  #_of_Female_Cast  #_of_Male_Cast  \\\n",
       "0  1999  10 things i hate about you                 5               7   \n",
       "1  1968       2001: a space odyssey                 2              11   \n",
       "\n",
       "   #_of_Female_Dialogue  #_of_Male_Dialogue  #_of_Female_Crew  #_of_Male_Crew  \\\n",
       "0                   295                 374                11              26   \n",
       "1                    17                 255                 2              17   \n",
       "\n",
       "   Sentiment_Female  Sentiment_Male                              Genres  \n",
       "0          0.027764        0.026287               ['comedy', 'romance']  \n",
       "1          0.244724        0.141933  ['adventure', 'mystery', 'sci-fi']  "
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "analy.head(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/ipykernel_launcher.py:64: RuntimeWarning:\n",
      "\n",
      "invalid value encountered in true_divide\n",
      "\n",
      "/Library/Frameworks/Python.framework/Versions/3.7/lib/python3.7/site-packages/scipy/spatial/distance.py:698: RuntimeWarning:\n",
      "\n",
      "invalid value encountered in double_scalars\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['comedy', 'romance']\n",
      "comedy romance\n",
      "['adventure', 'mystery', 'science']\n",
      "adventure mystery science\n",
      "['action', 'comedy', 'crime', 'drama', 'thriller']\n",
      "action comedy crime\n",
      "['crime', 'mystery', 'thriller']\n",
      "crime mystery thriller\n",
      "['fantasy', 'horror', 'thriller']\n",
      "fantasy horror thriller\n",
      "['action', 'drama', 'thriller']\n",
      "action drama thriller\n",
      "['comedy', 'romance']\n",
      "comedy romance\n",
      "['crime', 'drama', 'science', 'thriller']\n",
      "crime drama science\n",
      "['action', 'science', 'thriller']\n",
      "action science thriller\n",
      "['biography', 'drama', 'music']\n",
      "biography drama music\n",
      "['horror', 'romance']\n",
      "horror romance\n",
      "['action', 'western']\n",
      "action western\n",
      "['drama', 'thriller']\n",
      "drama thriller\n",
      "['drama', 'thriller']\n",
      "drama thriller\n",
      "['action', 'adventure', 'comedy', 'crime']\n",
      "action adventure comedy\n",
      "['comedy', 'romance']\n",
      "comedy romance\n",
      "['drama', 'romance', 'war']\n",
      "drama romance war\n",
      "['crime', 'drama', 'mystery', 'thriller']\n",
      "crime drama mystery\n",
      "['adventure', 'drama', 'fantasy']\n",
      "adventure drama fantasy\n",
      "['drama', 'science', 'thriller']\n",
      "drama science thriller\n",
      "['comedy', 'drama', 'romance']\n",
      "comedy drama romance\n",
      "['crime', 'mystery', 'thriller']\n",
      "crime mystery thriller\n",
      "['action', 'biography', 'drama', 'history', 'war']\n",
      "action biography drama\n",
      "['drama', 'romance', 'war']\n",
      "drama romance war\n",
      "['adventure', 'drama']\n",
      "adventure drama\n",
      "['comedy', 'science', 'thriller']\n",
      "comedy science thriller\n",
      "['drama', 'mystery', 'science']\n",
      "drama mystery science\n",
      "['comedy', 'adult']\n",
      "comedy adult\n",
      "['comedy', 'musical']\n",
      "comedy musical\n",
      "['drama', 'romance', 'science']\n",
      "drama romance science\n",
      "['comedy', 'drama', 'romance', 'western']\n",
      "comedy drama romance\n",
      "['horror', 'mystery', 'mystery', 'science', 'science']\n",
      "horror mystery\n",
      "['action', 'adventure', 'fantasy', 'science', 'fantasy']\n",
      "action adventure fantasy\n",
      "['comedy', 'drama', 'romance']\n",
      "comedy drama romance\n",
      "['drama', 'fantasy']\n",
      "drama fantasy\n",
      "['biography', 'drama']\n",
      "biography drama\n",
      "['drama', 'horror', 'science']\n",
      "drama horror science\n",
      "['horror']\n",
      "horror\n",
      "['action', 'crime', 'horror', 'thriller']\n",
      "action crime horror\n",
      "['action', 'drama']\n",
      "action drama\n",
      "['action', 'horror', 'science', 'thriller']\n",
      "action horror science\n",
      "['action', 'adventure', 'comedy', 'science']\n",
      "action adventure comedy\n",
      "['comedy', 'drama']\n",
      "comedy drama\n",
      "['action', 'adventure', 'comedy', 'fantasy', 'science', 'action', 'adventure', 'comedy', 'science']\n",
      "action adventure comedy\n",
      "['animation', 'comedy', 'fantasy', 'science', 'horror']\n",
      "animation comedy fantasy\n",
      "['action', 'adventure', 'drama']\n",
      "action adventure drama\n",
      "['drama', 'romance']\n",
      "drama romance\n",
      "['comedy', 'romance']\n",
      "comedy romance\n",
      "['drama', 'horror', 'thriller']\n",
      "drama horror thriller\n",
      "['action', 'adventure', 'fantasy']\n",
      "action adventure fantasy\n",
      "['horror']\n",
      "horror\n",
      "['comedy', 'drama', 'music', 'romance']\n",
      "comedy drama music\n",
      "['action', 'fantasy']\n",
      "action fantasy\n",
      "['comedy', 'drama', 'romance']\n",
      "comedy drama romance\n",
      "['drama', 'war']\n",
      "drama war\n",
      "['horror']\n",
      "horror\n",
      "['comedy', 'drama']\n",
      "comedy drama\n",
      "['drama', 'science', 'thriller']\n",
      "drama science thriller\n",
      "['action', 'adventure', 'science', 'thriller']\n",
      "action adventure science\n",
      "['action', 'adventure', 'thriller', 'action', 'adventure', 'fantasy']\n",
      "action adventure thriller\n",
      "['action', 'adventure']\n",
      "action adventure\n",
      "['comedy', 'romance', 'crime']\n",
      "comedy romance crime\n",
      "['comedy', 'romance']\n",
      "comedy romance\n",
      "['biography', 'drama', 'history', 'mystery', 'thriller']\n",
      "biography drama history\n",
      "['crime', 'drama', 'thriller']\n",
      "crime drama thriller\n",
      "['horror', 'science']\n",
      "horror science\n",
      "['thriller']\n",
      "thriller\n",
      "['comedy', 'drama']\n",
      "comedy drama\n",
      "['crime', 'drama']\n",
      "crime drama\n",
      "['fantasy', 'action', 'adventure', 'science']\n",
      "fantasy action adventure\n",
      "['comedy']\n",
      "comedy\n",
      "['fantasy', 'adventure', 'romance']\n",
      "fantasy adventure romance\n",
      "['fantasy', 'comedy', 'romance', 'animation', 'comedy', 'fantasy', 'horror']\n",
      "fantasy comedy romance\n",
      "['drama', 'horror', 'mystery', 'thriller']\n",
      "drama horror mystery\n",
      "['adventure', 'drama', 'fantasy', 'mystery']\n",
      "adventure drama fantasy\n",
      "['action', 'adventure', 'comedy', 'science']\n",
      "action adventure comedy\n",
      "['action', 'crime', 'mystery', 'science', 'thriller', 'action', 'animation', 'drama', 'mystery', 'science', 'thriller']\n",
      "action crime mystery\n",
      "['biography', 'drama', 'history']\n",
      "biography drama history\n",
      "['drama', 'romance']\n",
      "drama romance\n",
      "['comedy', 'drama', 'romance']\n",
      "comedy drama romance\n",
      "['adventure', 'drama', 'science']\n",
      "adventure drama science\n",
      "['animation', 'comedy', 'fantasy', 'romance']\n",
      "animation comedy fantasy\n",
      "['drama']\n",
      "drama\n",
      "['comedy', 'romance']\n",
      "comedy romance\n",
      "['action', 'comedy', 'fantasy']\n",
      "action comedy fantasy\n",
      "['comedy']\n",
      "comedy\n",
      "['crime', 'drama', 'thriller']\n",
      "crime drama thriller\n",
      "['horror']\n",
      "horror\n",
      "['comedy', 'romance']\n",
      "comedy romance\n",
      "['crime', 'drama', 'mystery', 'thriller', 'western']\n",
      "crime drama mystery\n",
      "['comedy', 'adventure', 'crime', 'music']\n",
      "comedy adventure crime\n",
      "['drama', 'romance']\n",
      "drama romance\n",
      "['thriller']\n",
      "thriller\n",
      "['comedy', 'drama', 'fantasy']\n",
      "comedy drama fantasy\n",
      "['comedy', 'drama', 'romance']\n",
      "comedy drama romance\n",
      "['biography', 'drama', 'sport']\n",
      "biography drama sport\n",
      "['crime', 'mystery', 'romance', 'thriller']\n",
      "crime mystery romance\n",
      "['drama', 'romance']\n",
      "drama romance\n",
      "['crime', 'mystery', 'thriller']\n",
      "crime mystery thriller\n",
      "['comedy']\n",
      "comedy\n",
      "['comedy', 'thriller']\n",
      "comedy thriller\n",
      "['drama']\n",
      "drama\n",
      "['comedy', 'musical']\n",
      "comedy musical\n",
      "['comedy', 'drama', 'romance']\n",
      "comedy drama romance\n",
      "['drama', 'mystery', 'science', 'thriller']\n",
      "drama mystery science\n",
      "['comedy', 'horror', 'science']\n",
      "comedy horror science\n",
      "['drama', 'mystery', 'romance', 'science']\n",
      "drama mystery romance\n",
      "['action', 'adventure', 'fantasy', 'science', 'action', 'adventure', 'science', 'thriller']\n",
      "action adventure fantasy\n",
      "['drama', 'war']\n",
      "drama war\n",
      "['action', 'adventure', 'science', 'thriller']\n",
      "action adventure science\n",
      "['action', 'adventure', 'science', 'thriller']\n",
      "action adventure science\n",
      "['action', 'adventure', 'science', 'thriller']\n",
      "action adventure science\n",
      "['action', 'mystery', 'science', 'thriller']\n",
      "action mystery science\n",
      "['action', 'science', 'thriller']\n",
      "action science thriller\n",
      "['adventure', 'drama', 'romance', 'science']\n",
      "adventure drama romance\n",
      "['action', 'crime', 'drama', 'fantasy', 'music', 'mystery', 'science', 'thriller']\n",
      "action crime drama\n",
      "['comedy', 'drama']\n",
      "comedy drama\n",
      "['drama', 'thriller']\n",
      "drama thriller\n",
      "['drama', 'history', 'thriller']\n",
      "drama history thriller\n",
      "['action', 'drama', 'romance']\n",
      "action drama romance\n",
      "['action', 'adventure', 'science', 'thriller']\n",
      "action adventure science\n",
      "['action', 'comedy', 'horror', 'thriller']\n",
      "action comedy horror\n",
      "['drama', 'horror', 'mystery', 'thriller']\n",
      "drama horror mystery\n",
      "['fantasy', 'horror', 'mystery', 'thriller']\n",
      "fantasy horror mystery\n",
      "['crime', 'romance', 'thriller']\n",
      "crime romance thriller\n",
      "['drama', 'mystery', 'thriller']\n",
      "drama mystery thriller\n",
      "['adventure', 'horror', 'science', 'thriller']\n",
      "adventure horror science\n",
      "['drama']\n",
      "drama\n",
      "['comedy', 'romance']\n",
      "comedy romance\n",
      "['horror', 'romance', 'thriller', 'comedy']\n",
      "horror romance thriller\n",
      "['animation', 'adventure', 'drama', 'family', 'musical', 'animation', 'family']\n",
      "animation adventure drama\n",
      "['comedy', 'drama', 'romance']\n",
      "comedy drama romance\n",
      "['animation', 'adventure', 'comedy', 'family']\n",
      "animation adventure comedy\n",
      "['drama', 'war']\n",
      "drama war\n",
      "['comedy', 'drama', 'romance']\n",
      "comedy drama romance\n",
      "['action', 'thriller', 'crime']\n",
      "action thriller crime\n",
      "['drama', 'romance']\n",
      "drama romance\n",
      "['adventure', 'family', 'science']\n",
      "adventure family science\n",
      "['crime', 'drama', 'romance', 'thriller']\n",
      "crime drama romance\n",
      "['drama', 'mystery', 'thriller']\n",
      "drama mystery thriller\n",
      "['mystery', 'romance', 'thriller']\n",
      "mystery romance thriller\n",
      "['biography', 'drama']\n",
      "biography drama\n",
      "['action', 'crime', 'fantasy', 'thriller', 'animation', 'crime', 'fantasy', 'action', 'thriller']\n",
      "action crime fantasy\n",
      "['action', 'crime', 'fantasy', 'thriller']\n",
      "action crime fantasy\n",
      "['crime', 'drama', 'thriller', 'action', 'adventure', 'crime']\n",
      "crime drama thriller\n",
      "['animation', 'adventure', 'comedy', 'crime']\n",
      "animation adventure comedy\n",
      "['drama', 'comedy']\n",
      "drama comedy\n",
      "['comedy', 'drama', 'fantasy', 'romance']\n",
      "comedy drama fantasy\n",
      "['drama', 'mystery']\n",
      "drama mystery\n",
      "['action', 'fantasy', 'horror', 'thriller', 'action', 'adventure', 'horror']\n",
      "action fantasy horror\n",
      "['action', 'adventure', 'fantasy', 'horror', 'thriller']\n",
      "action adventure fantasy\n",
      "['biography', 'crime', 'drama']\n",
      "biography crime drama\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['drama', 'history', 'war']\n",
      "drama history war\n",
      "['drama', 'romance', 'thriller']\n",
      "drama romance thriller\n",
      "['crime', 'drama', 'thriller']\n",
      "crime drama thriller\n",
      "['drama', 'fantasy', 'science']\n",
      "drama fantasy science\n",
      "['comedy', 'drama', 'romance']\n",
      "comedy drama romance\n",
      "['comedy', 'horror']\n",
      "comedy horror\n",
      "['horror', 'comedy', 'action']\n",
      "horror comedy action\n",
      "['biography', 'crime', 'drama']\n",
      "biography crime drama\n",
      "['action', 'crime', 'fantasy']\n",
      "action crime fantasy\n",
      "['action', 'crime', 'thriller']\n",
      "action crime thriller\n",
      "['comedy', 'mystery', 'romance', 'thriller']\n",
      "comedy mystery romance\n",
      "['horror', 'mystery', 'thriller']\n",
      "horror mystery thriller\n",
      "['crime', 'drama', 'mystery', 'thriller']\n",
      "crime drama mystery\n",
      "['drama', 'mystery']\n",
      "drama mystery\n",
      "['action', 'adventure', 'thriller']\n",
      "action adventure thriller\n",
      "['crime', 'drama', 'science']\n",
      "crime drama science\n",
      "['crime', 'drama', 'thriller']\n",
      "crime drama thriller\n",
      "['action', 'crime', 'mystery', 'romance', 'thriller']\n",
      "action crime mystery\n",
      "['drama', 'mystery', 'science', 'thriller']\n",
      "drama mystery science\n",
      "['crime', 'drama']\n",
      "crime drama\n",
      "['crime', 'mystery', 'thriller']\n",
      "crime mystery thriller\n",
      "['crime', 'drama']\n",
      "crime drama\n",
      "['drama', 'romance', 'thriller']\n",
      "drama romance thriller\n",
      "['mystery', 'science', 'thriller']\n",
      "mystery science thriller\n",
      "['comedy', 'romance']\n",
      "comedy romance\n",
      "['horror', 'science']\n",
      "horror science\n",
      "['drama']\n",
      "drama\n",
      "['action', 'crime', 'science']\n",
      "action crime science\n",
      "['action', 'crime', 'thriller']\n",
      "action crime thriller\n",
      "['crime', 'drama']\n",
      "crime drama\n",
      "['action', 'crime', 'drama', 'thriller']\n",
      "action crime drama\n",
      "['drama']\n",
      "drama\n",
      "['action', 'adventure', 'science']\n",
      "action adventure science\n",
      "['biography', 'comedy', 'drama']\n",
      "biography comedy drama\n",
      "['comedy']\n",
      "comedy\n",
      "['comedy', 'drama']\n",
      "comedy drama\n",
      "['crime', 'drama', 'mystery', 'thriller']\n",
      "crime drama mystery\n",
      "['action', 'drama', 'thriller']\n",
      "action drama thriller\n",
      "['biography', 'drama', 'romance']\n",
      "biography drama romance\n",
      "['science', 'thriller']\n",
      "science thriller\n",
      "['action', 'adventure', 'science', 'thriller']\n",
      "action adventure science\n",
      "['action', 'crime', 'drama', 'science', 'thriller']\n",
      "action crime drama\n",
      "['adventure', 'science']\n",
      "adventure science\n",
      "['crime', 'drama', 'thriller']\n",
      "crime drama thriller\n",
      "['crime', 'drama', 'mystery', 'thriller']\n",
      "crime drama mystery\n",
      "['horror', 'thriller']\n",
      "horror thriller\n",
      "['horror', 'thriller']\n",
      "horror thriller\n",
      "['action', 'adventure', 'thriller']\n",
      "action adventure thriller\n",
      "['drama']\n",
      "drama\n",
      "['comedy', 'crime', 'mystery']\n",
      "comedy crime mystery\n",
      "['horror']\n",
      "horror\n",
      "['horror']\n",
      "horror\n",
      "['drama', 'romance', 'science', 'thriller']\n",
      "drama romance science\n",
      "['action', 'crime', 'drama', 'thriller']\n",
      "action crime drama\n",
      "['drama']\n",
      "drama\n",
      "['action', 'crime', 'thriller']\n",
      "action crime thriller\n",
      "['crime', 'drama', 'thriller']\n",
      "crime drama thriller\n",
      "['drama']\n",
      "drama\n",
      "['comedy', 'crime', 'romance', 'thriller']\n",
      "comedy crime romance\n",
      "['action', 'crime', 'drama', 'thriller']\n",
      "action crime drama\n",
      "['horror', 'thriller']\n",
      "horror thriller\n",
      "['comedy', 'drama', 'romance']\n",
      "comedy drama romance\n",
      "['crime', 'thriller']\n",
      "crime thriller\n",
      "['comedy', 'crime']\n",
      "comedy crime\n",
      "['crime', 'drama', 'fantasy', 'thriller']\n",
      "crime drama fantasy\n",
      "['comedy', 'drama']\n",
      "comedy drama\n",
      "['horror', 'mystery', 'thriller']\n",
      "horror mystery thriller\n",
      "['horror']\n",
      "horror\n",
      "['action', 'adventure', 'comedy', 'action', 'adventure', 'comedy', 'crime']\n",
      "action adventure comedy\n",
      "['horror']\n",
      "horror\n",
      "['crime', 'drama', 'mystery', 'thriller']\n",
      "crime drama mystery\n",
      "['horror', 'mystery', 'thriller']\n",
      "horror mystery thriller\n",
      "['drama', 'horror', 'mystery', 'thriller']\n",
      "drama horror mystery\n",
      "['thriller']\n",
      "thriller\n",
      "['thriller']\n",
      "thriller\n",
      "['thriller']\n",
      "thriller\n",
      "['comedy', 'drama', 'romance', 'sport']\n",
      "comedy drama romance\n",
      "['action', 'adventure', 'science', 'thriller']\n",
      "action adventure science\n",
      "['action', 'adventure', 'family', 'science', 'action', 'adventure', 'fantasy', 'science']\n",
      "action adventure family\n",
      "['horror', 'thriller']\n",
      "horror thriller\n",
      "['adventure', 'drama', 'romance']\n",
      "adventure drama romance\n",
      "['mystery', 'romance', 'thriller']\n",
      "mystery romance thriller\n",
      "['drama']\n",
      "drama\n",
      "['adventure', 'horror', 'mystery', 'science', 'thriller']\n",
      "adventure horror mystery\n",
      "['crime', 'thriller']\n",
      "crime thriller\n",
      "['drama', 'mystery', 'romance', 'western']\n",
      "drama mystery romance\n",
      "['drama', 'romance']\n",
      "drama romance\n",
      "['drama']\n",
      "drama\n",
      "['crime', 'thriller']\n",
      "crime thriller\n",
      "['biography', 'comedy', 'drama']\n",
      "biography comedy drama\n",
      "['drama', 'fantasy', 'mystery', 'romance']\n",
      "drama fantasy mystery\n",
      "['crime', 'drama', 'mystery', 'thriller']\n",
      "crime drama mystery\n",
      "['drama']\n",
      "drama\n",
      "['biography', 'crime', 'drama', 'thriller']\n",
      "biography crime drama\n",
      "['thriller']\n",
      "thriller\n",
      "['action', 'adventure', 'thriller']\n",
      "action adventure thriller\n",
      "['action', 'adventure', 'thriller']\n",
      "action adventure thriller\n",
      "['comedy', 'romance', 'drama']\n",
      "comedy romance drama\n",
      "['adventure', 'comedy']\n",
      "adventure comedy\n",
      "['drama', 'mystery', 'thriller']\n",
      "drama mystery thriller\n",
      "['comedy', 'drama', 'family', 'romance']\n",
      "comedy drama family\n",
      "['drama', 'music']\n",
      "drama music\n",
      "['action', 'crime', 'drama', 'romance', 'thriller']\n",
      "action crime drama\n",
      "['comedy', 'drama', 'romance']\n",
      "comedy drama romance\n",
      "['fantasy', 'horror', 'thriller']\n",
      "fantasy horror thriller\n",
      "['comedy', 'romance']\n",
      "comedy romance\n",
      "['crime', 'drama', 'romance']\n",
      "crime drama romance\n",
      "['comedy', 'crime', 'romance', 'thriller']\n",
      "comedy crime romance\n",
      "['action', 'drama', 'romance', 'war']\n",
      "action drama romance\n",
      "['comedy', 'drama', 'fantasy', 'romance']\n",
      "comedy drama fantasy\n",
      "['drama']\n",
      "drama\n",
      "['action', 'science', 'thriller']\n",
      "action science thriller\n",
      "['action', 'adventure', 'science', 'thriller', 'animation', 'action', 'adventure', 'science']\n",
      "action adventure science\n",
      "['action', 'drama', 'war']\n",
      "action drama war\n",
      "['action', 'adventure', 'crime', 'drama', 'sport', 'thriller']\n",
      "action adventure crime\n",
      "['action', 'adventure', 'science', 'thriller']\n",
      "action adventure science\n",
      "['comedy', 'romance']\n",
      "comedy romance\n",
      "['drama', 'horror', 'mystery', 'thriller', 'thriller']\n",
      "drama horror mystery\n",
      "['drama', 'horror', 'thriller']\n",
      "drama horror thriller\n",
      "['drama', 'romance', 'sport']\n",
      "drama romance sport\n",
      "['animation', 'comedy', 'crime', 'family', 'fantasy', 'mystery', 'animation', 'adventure', 'action', 'fantasy']\n",
      "animation comedy crime\n",
      "['drama', 'romance']\n",
      "drama romance\n",
      "['action', 'crime', 'thriller']\n",
      "action crime thriller\n",
      "['action', 'comedy', 'crime', 'thriller']\n",
      "action comedy crime\n",
      "['action', 'comedy', 'thriller', 'crime']\n",
      "action comedy thriller\n",
      "['comedy', 'romance', 'drama']\n",
      "comedy romance drama\n",
      "['drama', 'history']\n",
      "drama history\n",
      "['action', 'drama', 'history', 'war']\n",
      "action drama history\n",
      "['comedy', 'drama', 'romance']\n",
      "comedy drama romance\n",
      "['horror', 'mystery', 'thriller']\n",
      "horror mystery thriller\n",
      "['horror', 'mystery', 'thriller']\n",
      "horror mystery thriller\n",
      "['crime', 'horror', 'mystery', 'thriller']\n",
      "crime horror mystery\n",
      "['comedy', 'drama', 'romance']\n",
      "comedy drama romance\n",
      "['comedy', 'crime', 'music']\n",
      "comedy crime music\n",
      "['fantasy', 'mystery', 'thriller']\n",
      "fantasy mystery thriller\n",
      "['comedy']\n",
      "comedy\n",
      "['drama', 'horror', 'mystery', 'science', 'thriller']\n",
      "drama horror mystery\n",
      "['action', 'adventure', 'fantasy', 'science']\n",
      "action adventure fantasy\n",
      "['action', 'adventure', 'science', 'thriller']\n",
      "action adventure science\n",
      "['adventure', 'fantasy', 'mystery', 'science']\n",
      "adventure fantasy mystery\n",
      "['comedy', 'drama']\n",
      "comedy drama\n",
      "['comedy', 'drama']\n",
      "comedy drama\n",
      "['action', 'adventure', 'science']\n",
      "action adventure science\n",
      "['action', 'adventure', 'comedy', 'fantasy', 'science']\n",
      "action adventure comedy\n",
      "['action', 'adventure', 'fantasy', 'romance', 'science']\n",
      "action adventure fantasy\n",
      "['action', 'adventure', 'family', 'fantasy', 'science']\n",
      "action adventure family\n",
      "['action', 'family', 'science']\n",
      "action family science\n",
      "['drama', 'film-noir']\n",
      "drama film-noir\n",
      "['drama', 'mystery', 'science', 'thriller']\n",
      "drama mystery science\n",
      "['drama', 'history', 'romance']\n",
      "drama history romance\n",
      "['action', 'drama', 'history', 'romance', 'western']\n",
      "action drama history\n",
      "['animation', 'adventure', 'comedy', 'family', 'fantasy']\n",
      "animation adventure comedy\n",
      "['crime', 'drama', 'thriller']\n",
      "crime drama thriller\n",
      "['crime', 'drama']\n",
      "crime drama\n",
      "['action', 'adventure', 'science', 'thriller']\n",
      "action adventure science\n",
      "['action', 'adventure', 'science', 'thriller', 'science']\n",
      "action adventure science\n",
      "['action', 'thriller']\n",
      "action thriller\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['crime', 'romance', 'thriller']\n",
      "crime romance thriller\n",
      "['mystery', 'science', 'thriller']\n",
      "mystery science thriller\n",
      "['drama', 'fantasy', 'mystery', 'thriller']\n",
      "drama fantasy mystery\n",
      "['drama', 'western']\n",
      "drama western\n",
      "['crime', 'mystery', 'romance', 'thriller']\n",
      "crime mystery romance\n",
      "['drama', 'horror', 'mystery', 'thriller']\n",
      "drama horror mystery\n",
      "['comedy', 'fantasy', 'romance']\n",
      "comedy fantasy romance\n",
      "['crime', 'mystery', 'thriller']\n",
      "crime mystery thriller\n",
      "['action', 'western', 'comedy', 'science']\n",
      "action western comedy\n",
      "['action', 'adventure', 'drama', 'fantasy', 'romance']\n",
      "action adventure drama\n",
      "['drama', 'romance', 'thriller']\n",
      "drama romance thriller\n",
      "['action', 'crime', 'fantasy', 'mystery', 'science', 'thriller']\n",
      "action crime fantasy\n",
      "['action', 'adventure', 'crime']\n",
      "action adventure crime\n",
      "['action', 'science']\n",
      "action science\n",
      "['comedy', 'science']\n",
      "comedy science\n"
     ]
    }
   ],
   "source": [
    "tokenizer = RegexpTokenizer(r'\\w+')\n",
    "\n",
    "for rows in analy.itertuples():\n",
    "    gdict = {}\n",
    "    # Genre information\n",
    "    j = rows[11]\n",
    "    j=j.replace('[',\"\")\n",
    "    j=j.replace(']',\"\")\n",
    "    j=j.replace(\"'\",\"\")\n",
    "    j=j.split(\", \")\n",
    "    j = ['science' if x == 'sci-fi' else x for x in j]\n",
    "    print(j)\n",
    "    try:\n",
    "        j = [j[0], j[1], j[2]]\n",
    "    except:\n",
    "        if len(j) == 2:\n",
    "            j = [j[0], j[1]]\n",
    "        else:\n",
    "            j = [j[0]]\n",
    "    \n",
    "    j = pd.Series(j).drop_duplicates().tolist()    \n",
    "    \n",
    "    try:\n",
    "        print(j[0], j[1], j[2])\n",
    "    except:\n",
    "        if len(j) == 2:\n",
    "            print(j[0], j[1])\n",
    "        else:\n",
    "            print(j[0])\n",
    "        \n",
    "    gdict[j[0]] = model[j[0]]\n",
    "    try:\n",
    "        gdict[j[1]] = model[j[1]]\n",
    "    except:\n",
    "        gdict[\"Nan\"] = np.zeros(300)\n",
    "    try:\n",
    "        gdict[j[2]] = model[j[2]]\n",
    "    except:\n",
    "        gdict[\"NaN\"] = np.zeros(300)\n",
    "        \n",
    "    women_vals = [0, 0, 0]\n",
    "    men_vals = [0, 0, 0]\n",
    "    fc = 0\n",
    "    mc = 0\n",
    "    final_vals = []\n",
    "    \n",
    "    for rows2 in new_Df.itertuples():\n",
    "        if rows2[7] == rows[2]:\n",
    "            try:\n",
    "                line = tokenizer.tokenize(rows2[5])\n",
    "            except:\n",
    "                line=''\n",
    "            av = np.zeros(300)\n",
    "            val2={}\n",
    "            \n",
    "            line=[w for w in line if not w in stop_words]\n",
    "            l1=len(line)\n",
    "            for tokens in line:\n",
    "                try:\n",
    "                    av+=model[tokens]\n",
    "                except:\n",
    "                    l1-=1\n",
    "                    pass\n",
    "            av=av/l1\n",
    "            # Cosine similarities\n",
    "            for key, vals in gdict.items():\n",
    "                try:\n",
    "                    val2[key] = 1 - spatial.distance.cosine(av, vals)\n",
    "                except:\n",
    "                    pass\n",
    "            final_vals = [val2[k] for k in val2]\n",
    "            final_vals = [0 if math.isnan(x) else x for x in final_vals]\n",
    "#             print(final_vals, rows2[8])\n",
    "            \n",
    "            if rows2[8] == 'f' or rows2[8] == 'F':\n",
    "                women_vals[0] += final_vals[0]\n",
    "                women_vals[1] += final_vals[1]\n",
    "                women_vals[2] += final_vals[2]\n",
    "                fc += 1\n",
    "            else:\n",
    "                men_vals[0] += final_vals[0]\n",
    "                men_vals[1] += final_vals[1]\n",
    "                men_vals[2] += final_vals[2]\n",
    "                mc += 1\n",
    "    \n",
    "#     print(women_vals[0]/fc, men_vals[0]/mc, women_vals[1]/fc, men_vals[1]/mc)\n",
    "    try:\n",
    "        analy.at[rows.Index, \"Genre1_Tag_Female\"] = float(women_vals[0]/fc)\n",
    "    except:\n",
    "        analy.at[rows.Index, \"Genre1_Tag_Female\"] = float(0)\n",
    "    try:\n",
    "        analy.at[rows.Index, \"Genre1_Tag_Male\"] = float(men_vals[0]/mc)\n",
    "    except:\n",
    "        analy.at[rows.Index, \"Genre1_Tag_Male\"] = float(0)\n",
    "    try:\n",
    "        analy.at[rows.Index, \"Genre2_Tag_Female\"] = float(women_vals[1]/fc)\n",
    "    except:\n",
    "        analy.at[rows.Index, \"Genre2_Tag_Female\"] = float(0)\n",
    "    try:\n",
    "        analy.at[rows.Index, \"Genre2_Tag_Male\"] = float(men_vals[1]/mc)\n",
    "    except:\n",
    "        analy.at[rows.Index, \"Genre2_Tag_Male\"] = float(0)\n",
    "    try:\n",
    "        analy.at[rows.Index, \"Genre3_Tag_Female\"] = float(women_vals[2]/fc)\n",
    "    except:\n",
    "        analy.at[rows.Index, \"Genre3_Tag_Female\"] = float(0)\n",
    "    try:\n",
    "        analy.at[rows.Index, \"Genre3_Tag_Male\"] = float(men_vals[2]/mc)\n",
    "    except:\n",
    "        analy.at[rows.Index, \"Genre3_Tag_Male\"] = float(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Movie_Title</th>\n",
       "      <th>#_of_Female_Cast</th>\n",
       "      <th>#_of_Male_Cast</th>\n",
       "      <th>#_of_Female_Dialogue</th>\n",
       "      <th>#_of_Male_Dialogue</th>\n",
       "      <th>#_of_Female_Crew</th>\n",
       "      <th>#_of_Male_Crew</th>\n",
       "      <th>Sentiment_Female</th>\n",
       "      <th>Sentiment_Male</th>\n",
       "      <th>Genres</th>\n",
       "      <th>Genre1_Tag_Female</th>\n",
       "      <th>Genre1_Tag_Male</th>\n",
       "      <th>Genre2_Tag_Female</th>\n",
       "      <th>Genre2_Tag_Male</th>\n",
       "      <th>Genre3_Tag_Female</th>\n",
       "      <th>Genre3_Tag_Male</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>314</th>\n",
       "      <td>2000</td>\n",
       "      <td>what lies beneath</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>436</td>\n",
       "      <td>223</td>\n",
       "      <td>22</td>\n",
       "      <td>89</td>\n",
       "      <td>0.025850</td>\n",
       "      <td>0.003195</td>\n",
       "      <td>['drama', 'horror', 'mystery', 'thriller']</td>\n",
       "      <td>0.083183</td>\n",
       "      <td>0.087377</td>\n",
       "      <td>0.122338</td>\n",
       "      <td>0.118682</td>\n",
       "      <td>0.120105</td>\n",
       "      <td>0.126256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>315</th>\n",
       "      <td>2000</td>\n",
       "      <td>what women want</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "      <td>24</td>\n",
       "      <td>71</td>\n",
       "      <td>-0.119833</td>\n",
       "      <td>0.044167</td>\n",
       "      <td>['comedy', 'fantasy', 'romance']</td>\n",
       "      <td>0.079808</td>\n",
       "      <td>0.151906</td>\n",
       "      <td>0.094969</td>\n",
       "      <td>0.181789</td>\n",
       "      <td>0.087453</td>\n",
       "      <td>0.145327</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>316</th>\n",
       "      <td>1998</td>\n",
       "      <td>wild things</td>\n",
       "      <td>6</td>\n",
       "      <td>7</td>\n",
       "      <td>245</td>\n",
       "      <td>264</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>-0.003185</td>\n",
       "      <td>-0.056314</td>\n",
       "      <td>['crime', 'mystery', 'thriller']</td>\n",
       "      <td>0.125601</td>\n",
       "      <td>0.142793</td>\n",
       "      <td>0.135009</td>\n",
       "      <td>0.138269</td>\n",
       "      <td>0.070724</td>\n",
       "      <td>0.075105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>317</th>\n",
       "      <td>1999</td>\n",
       "      <td>wild wild west</td>\n",
       "      <td>1</td>\n",
       "      <td>8</td>\n",
       "      <td>21</td>\n",
       "      <td>301</td>\n",
       "      <td>1</td>\n",
       "      <td>11</td>\n",
       "      <td>0.093195</td>\n",
       "      <td>0.083939</td>\n",
       "      <td>['action', 'western', 'comedy', 'sci-fi']</td>\n",
       "      <td>0.127692</td>\n",
       "      <td>0.112407</td>\n",
       "      <td>0.117143</td>\n",
       "      <td>0.109587</td>\n",
       "      <td>0.106009</td>\n",
       "      <td>0.107918</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>318</th>\n",
       "      <td>1988</td>\n",
       "      <td>willow</td>\n",
       "      <td>3</td>\n",
       "      <td>7</td>\n",
       "      <td>65</td>\n",
       "      <td>233</td>\n",
       "      <td>11</td>\n",
       "      <td>45</td>\n",
       "      <td>-0.019435</td>\n",
       "      <td>0.035591</td>\n",
       "      <td>['action', 'adventure', 'drama', 'fantasy', 'r...</td>\n",
       "      <td>0.080072</td>\n",
       "      <td>0.089630</td>\n",
       "      <td>0.117216</td>\n",
       "      <td>0.125480</td>\n",
       "      <td>0.099258</td>\n",
       "      <td>0.096071</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>319</th>\n",
       "      <td>1985</td>\n",
       "      <td>witness</td>\n",
       "      <td>2</td>\n",
       "      <td>8</td>\n",
       "      <td>119</td>\n",
       "      <td>238</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>0.057887</td>\n",
       "      <td>-0.022337</td>\n",
       "      <td>['drama', 'romance', 'thriller']</td>\n",
       "      <td>0.084969</td>\n",
       "      <td>0.086732</td>\n",
       "      <td>0.112467</td>\n",
       "      <td>0.110036</td>\n",
       "      <td>0.050484</td>\n",
       "      <td>0.061333</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>320</th>\n",
       "      <td>2009</td>\n",
       "      <td>watchmen</td>\n",
       "      <td>4</td>\n",
       "      <td>12</td>\n",
       "      <td>115</td>\n",
       "      <td>252</td>\n",
       "      <td>6</td>\n",
       "      <td>18</td>\n",
       "      <td>0.002022</td>\n",
       "      <td>-0.037667</td>\n",
       "      <td>['action', 'crime', 'fantasy', 'mystery', 'sci...</td>\n",
       "      <td>0.093158</td>\n",
       "      <td>0.107956</td>\n",
       "      <td>0.123250</td>\n",
       "      <td>0.141138</td>\n",
       "      <td>0.162767</td>\n",
       "      <td>0.165617</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>321</th>\n",
       "      <td>2002</td>\n",
       "      <td>xxx</td>\n",
       "      <td>1</td>\n",
       "      <td>10</td>\n",
       "      <td>5</td>\n",
       "      <td>407</td>\n",
       "      <td>37</td>\n",
       "      <td>102</td>\n",
       "      <td>-0.002368</td>\n",
       "      <td>-0.010369</td>\n",
       "      <td>['action', 'adventure', 'crime']</td>\n",
       "      <td>0.126709</td>\n",
       "      <td>0.129786</td>\n",
       "      <td>0.146824</td>\n",
       "      <td>0.147584</td>\n",
       "      <td>0.138890</td>\n",
       "      <td>0.152288</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>322</th>\n",
       "      <td>2000</td>\n",
       "      <td>x-men</td>\n",
       "      <td>2</td>\n",
       "      <td>7</td>\n",
       "      <td>39</td>\n",
       "      <td>136</td>\n",
       "      <td>3</td>\n",
       "      <td>20</td>\n",
       "      <td>-0.017695</td>\n",
       "      <td>0.015544</td>\n",
       "      <td>['action', 'sci-fi']</td>\n",
       "      <td>0.102302</td>\n",
       "      <td>0.108213</td>\n",
       "      <td>0.134645</td>\n",
       "      <td>0.115685</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323</th>\n",
       "      <td>1974</td>\n",
       "      <td>young frankenstein</td>\n",
       "      <td>3</td>\n",
       "      <td>11</td>\n",
       "      <td>121</td>\n",
       "      <td>455</td>\n",
       "      <td>9</td>\n",
       "      <td>35</td>\n",
       "      <td>0.182547</td>\n",
       "      <td>0.086947</td>\n",
       "      <td>['comedy', 'sci-fi']</td>\n",
       "      <td>0.107888</td>\n",
       "      <td>0.094795</td>\n",
       "      <td>0.096118</td>\n",
       "      <td>0.100548</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Year         Movie_Title  #_of_Female_Cast  #_of_Male_Cast  \\\n",
       "314  2000   what lies beneath                 5               5   \n",
       "315  2000     what women want                 1               1   \n",
       "316  1998         wild things                 6               7   \n",
       "317  1999      wild wild west                 1               8   \n",
       "318  1988              willow                 3               7   \n",
       "319  1985             witness                 2               8   \n",
       "320  2009            watchmen                 4              12   \n",
       "321  2002                 xxx                 1              10   \n",
       "322  2000               x-men                 2               7   \n",
       "323  1974  young frankenstein                 3              11   \n",
       "\n",
       "     #_of_Female_Dialogue  #_of_Male_Dialogue  #_of_Female_Crew  \\\n",
       "314                   436                 223                22   \n",
       "315                     3                   3                24   \n",
       "316                   245                 264                 1   \n",
       "317                    21                 301                 1   \n",
       "318                    65                 233                11   \n",
       "319                   119                 238                 0   \n",
       "320                   115                 252                 6   \n",
       "321                     5                 407                37   \n",
       "322                    39                 136                 3   \n",
       "323                   121                 455                 9   \n",
       "\n",
       "     #_of_Male_Crew  Sentiment_Female  Sentiment_Male  \\\n",
       "314              89          0.025850        0.003195   \n",
       "315              71         -0.119833        0.044167   \n",
       "316               8         -0.003185       -0.056314   \n",
       "317              11          0.093195        0.083939   \n",
       "318              45         -0.019435        0.035591   \n",
       "319               7          0.057887       -0.022337   \n",
       "320              18          0.002022       -0.037667   \n",
       "321             102         -0.002368       -0.010369   \n",
       "322              20         -0.017695        0.015544   \n",
       "323              35          0.182547        0.086947   \n",
       "\n",
       "                                                Genres  Genre1_Tag_Female  \\\n",
       "314         ['drama', 'horror', 'mystery', 'thriller']           0.083183   \n",
       "315                   ['comedy', 'fantasy', 'romance']           0.079808   \n",
       "316                   ['crime', 'mystery', 'thriller']           0.125601   \n",
       "317          ['action', 'western', 'comedy', 'sci-fi']           0.127692   \n",
       "318  ['action', 'adventure', 'drama', 'fantasy', 'r...           0.080072   \n",
       "319                   ['drama', 'romance', 'thriller']           0.084969   \n",
       "320  ['action', 'crime', 'fantasy', 'mystery', 'sci...           0.093158   \n",
       "321                   ['action', 'adventure', 'crime']           0.126709   \n",
       "322                               ['action', 'sci-fi']           0.102302   \n",
       "323                               ['comedy', 'sci-fi']           0.107888   \n",
       "\n",
       "     Genre1_Tag_Male  Genre2_Tag_Female  Genre2_Tag_Male  Genre3_Tag_Female  \\\n",
       "314         0.087377           0.122338         0.118682           0.120105   \n",
       "315         0.151906           0.094969         0.181789           0.087453   \n",
       "316         0.142793           0.135009         0.138269           0.070724   \n",
       "317         0.112407           0.117143         0.109587           0.106009   \n",
       "318         0.089630           0.117216         0.125480           0.099258   \n",
       "319         0.086732           0.112467         0.110036           0.050484   \n",
       "320         0.107956           0.123250         0.141138           0.162767   \n",
       "321         0.129786           0.146824         0.147584           0.138890   \n",
       "322         0.108213           0.134645         0.115685           0.000000   \n",
       "323         0.094795           0.096118         0.100548           0.000000   \n",
       "\n",
       "     Genre3_Tag_Male  \n",
       "314         0.126256  \n",
       "315         0.145327  \n",
       "316         0.075105  \n",
       "317         0.107918  \n",
       "318         0.096071  \n",
       "319         0.061333  \n",
       "320         0.165617  \n",
       "321         0.152288  \n",
       "322         0.000000  \n",
       "323         0.000000  "
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "analy.tail(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "529"
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gc.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "analy.drop(['Genres'], axis = 1, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Year</th>\n",
       "      <th>Movie_Title</th>\n",
       "      <th>#_of_Female_Cast</th>\n",
       "      <th>#_of_Male_Cast</th>\n",
       "      <th>#_of_Female_Dialogue</th>\n",
       "      <th>#_of_Male_Dialogue</th>\n",
       "      <th>#_of_Female_Crew</th>\n",
       "      <th>#_of_Male_Crew</th>\n",
       "      <th>Sentiment_Female</th>\n",
       "      <th>Sentiment_Male</th>\n",
       "      <th>Genre1_Tag_Female</th>\n",
       "      <th>Genre1_Tag_Male</th>\n",
       "      <th>Genre2_Tag_Female</th>\n",
       "      <th>Genre2_Tag_Male</th>\n",
       "      <th>Genre3_Tag_Female</th>\n",
       "      <th>Genre3_Tag_Male</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1999</td>\n",
       "      <td>10 things i hate about you</td>\n",
       "      <td>5</td>\n",
       "      <td>7</td>\n",
       "      <td>295</td>\n",
       "      <td>374</td>\n",
       "      <td>11</td>\n",
       "      <td>26</td>\n",
       "      <td>0.027764</td>\n",
       "      <td>0.026287</td>\n",
       "      <td>0.099730</td>\n",
       "      <td>0.114050</td>\n",
       "      <td>0.127356</td>\n",
       "      <td>0.130247</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1968</td>\n",
       "      <td>2001: a space odyssey</td>\n",
       "      <td>2</td>\n",
       "      <td>11</td>\n",
       "      <td>17</td>\n",
       "      <td>255</td>\n",
       "      <td>2</td>\n",
       "      <td>17</td>\n",
       "      <td>0.244724</td>\n",
       "      <td>0.141933</td>\n",
       "      <td>0.140718</td>\n",
       "      <td>0.107436</td>\n",
       "      <td>0.126419</td>\n",
       "      <td>0.143354</td>\n",
       "      <td>0.086336</td>\n",
       "      <td>0.109146</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1982</td>\n",
       "      <td>48 hrs.</td>\n",
       "      <td>7</td>\n",
       "      <td>9</td>\n",
       "      <td>109</td>\n",
       "      <td>469</td>\n",
       "      <td>1</td>\n",
       "      <td>14</td>\n",
       "      <td>-0.026442</td>\n",
       "      <td>0.002866</td>\n",
       "      <td>0.109943</td>\n",
       "      <td>0.106253</td>\n",
       "      <td>0.121081</td>\n",
       "      <td>0.130147</td>\n",
       "      <td>0.137317</td>\n",
       "      <td>0.138767</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1999</td>\n",
       "      <td>8mm</td>\n",
       "      <td>2</td>\n",
       "      <td>13</td>\n",
       "      <td>37</td>\n",
       "      <td>547</td>\n",
       "      <td>15</td>\n",
       "      <td>62</td>\n",
       "      <td>-0.108278</td>\n",
       "      <td>0.011065</td>\n",
       "      <td>0.142303</td>\n",
       "      <td>0.135797</td>\n",
       "      <td>0.129424</td>\n",
       "      <td>0.144191</td>\n",
       "      <td>0.059458</td>\n",
       "      <td>0.074539</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1988</td>\n",
       "      <td>a nightmare on elm street 4: the dream master</td>\n",
       "      <td>6</td>\n",
       "      <td>2</td>\n",
       "      <td>95</td>\n",
       "      <td>60</td>\n",
       "      <td>3</td>\n",
       "      <td>12</td>\n",
       "      <td>-0.058180</td>\n",
       "      <td>-0.052153</td>\n",
       "      <td>0.178471</td>\n",
       "      <td>0.189962</td>\n",
       "      <td>0.173313</td>\n",
       "      <td>0.171728</td>\n",
       "      <td>0.087794</td>\n",
       "      <td>0.091161</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Year                                    Movie_Title  #_of_Female_Cast  \\\n",
       "0  1999                     10 things i hate about you                 5   \n",
       "1  1968                          2001: a space odyssey                 2   \n",
       "2  1982                                        48 hrs.                 7   \n",
       "3  1999                                            8mm                 2   \n",
       "4  1988  a nightmare on elm street 4: the dream master                 6   \n",
       "\n",
       "   #_of_Male_Cast  #_of_Female_Dialogue  #_of_Male_Dialogue  #_of_Female_Crew  \\\n",
       "0               7                   295                 374                11   \n",
       "1              11                    17                 255                 2   \n",
       "2               9                   109                 469                 1   \n",
       "3              13                    37                 547                15   \n",
       "4               2                    95                  60                 3   \n",
       "\n",
       "   #_of_Male_Crew  Sentiment_Female  Sentiment_Male  Genre1_Tag_Female  \\\n",
       "0              26          0.027764        0.026287           0.099730   \n",
       "1              17          0.244724        0.141933           0.140718   \n",
       "2              14         -0.026442        0.002866           0.109943   \n",
       "3              62         -0.108278        0.011065           0.142303   \n",
       "4              12         -0.058180       -0.052153           0.178471   \n",
       "\n",
       "   Genre1_Tag_Male  Genre2_Tag_Female  Genre2_Tag_Male  Genre3_Tag_Female  \\\n",
       "0         0.114050           0.127356         0.130247           0.000000   \n",
       "1         0.107436           0.126419         0.143354           0.086336   \n",
       "2         0.106253           0.121081         0.130147           0.137317   \n",
       "3         0.135797           0.129424         0.144191           0.059458   \n",
       "4         0.189962           0.173313         0.171728           0.087794   \n",
       "\n",
       "   Genre3_Tag_Male  \n",
       "0         0.000000  \n",
       "1         0.109146  \n",
       "2         0.138767  \n",
       "3         0.074539  \n",
       "4         0.091161  "
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "analy.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "analy.to_csv(\"Analysis_csv.csv\", index = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}