Web Scraping Cinema.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Web Scraping Cinema\n",
    "\n",
    "Extracting movies schedule on Natal movie theaters and getting information about these movies in reference sites."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import time\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "url_imdb = 'http://www.imdb.com/title/tt0451279/?ref_=nv_sr_1'\n",
    "url_tomates = 'https://www.rottentomatoes.com/m/wonder_woman_2017'\n",
    "url_filmow = 'https://filmow.com/mulher-maravilha-t48706/'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# IMDb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Get the page content and set up a new parser\n",
    "response = requests.get(url_imdb)\n",
    "content = response.content\n",
    "soup = BeautifulSoup(content, 'html.parser')\n",
    "\n",
    "# Get title\n",
    "title_imd = soup.find_all(\"h1\", itemprop=\"name\")[0].text\n",
    "\n",
    "# Get rating\n",
    "rating_imdb = soup.find_all(\"span\", itemprop=\"ratingValue\")[0].text\n",
    "userRating_imdb = soup.find_all(\"span\", itemprop=\"ratingCount\")[0].text\n",
    "\n",
    "# Get time\n",
    "time_imdb = soup.find_all(\"time\", itemprop=\"duration\")[0].text\n",
    "\n",
    "# Get genres\n",
    "genres = soup.find_all(\"span\", itemprop=\"genre\", class_=\"itemprop\")\n",
    "g_imdb = \"\"\n",
    "for genre in genres:\n",
    "    g_imdb = g_imdb + genre.text + \", \""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Rotten tomates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Get the page content and set up a new parser\n",
    "response = requests.get(url_tomates)\n",
    "content = response.content\n",
    "soup = BeautifulSoup(content, 'html.parser')\n",
    "\n",
    "### TOMATOMETER\n",
    "# Get rating\n",
    "rating_tmt = soup.find_all(\"span\", class_=\"meter-value superPageFontColor\")[0].text\n",
    "divs_tmt = soup.find_all(\"div\", id=\"scoreStats\")[0].find_all(\"div\", class_=\"superPageFontColor\")\n",
    "critic_tmt = soup.find_all(\"p\", class_=\"critic_consensus superPageFontColor\")[0]\n",
    "\n",
    "### AUDIENCE SCORE\n",
    "# Get audience\n",
    "audience_tmt = soup.find_all(\"span\", class_=\"superPageFontColor\")[0].text\n",
    "divs_tmt_audience = soup.find_all(\"div\", class_=\"audience-info hidden-xs superPageFontColor\")[0].find_all(\"div\")\n",
    "\n",
    "# Synopsis\n",
    "movie_synopsis_tmt = soup.find_all(\"div\", id=\"movieSynopsis\")[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Filmow"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Get the page content and set up a new parser\n",
    "response = requests.get(url_filmow)\n",
    "content = response.content\n",
    "soup = BeautifulSoup(content, 'html.parser')\n",
    "\n",
    "# Get cast\n",
    "rating_filmow = soup.find_all(\"span\", itemprop=\"ratingValue\")[0].text\n",
    "ratingCount_filmow = soup.find_all(\"span\", itemprop=\"ratingCount\")[0].text\n",
    "\n",
    "# Get cast\n",
    "lis_filmow = soup.find_all(\"ul\", id=\"casting\")[0].find_all(\"li\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# IMDB\n",
    "print(\"Title: \" + title_imd.strip())\n",
    "print(\"Rating (IMDB): \" + rating_imdb + \"/10 (\" + userRating_imdb + \" users rating)\")\n",
    "print(\"Time: \" + time_imdb.strip())\n",
    "print(\"Genres: \" + g_imdb[0:-2])\n",
    "\n",
    "# Rotten Tomates\n",
    "print(\"\\n----- TOMATOMETER -----\")\n",
    "print(\"Rating: \" + rating_tmt)\n",
    "for div in divs_tmt:\n",
    "    if(len(div.find_all(\"span\")) == 2):\n",
    "        print(div.find_all(\"span\")[0].text + div.find_all(\"span\")[1].text)\n",
    "    else:\n",
    "        print(div.text.strip())\n",
    "print(critic_tmt.text_tmt)\n",
    "print(\"\\n----- AUDIENCE SCORE -----\")\n",
    "print(\"Audience rating: \" + audience_tmt)\n",
    "for div in divs_tmt_audience:\n",
    "    print(div.text.strip())\n",
    "print(\"\\n----- Movie Info -----\")\n",
    "print(\"Synopsis: \" + movie_synopsis_tmt.text)\n",
    "\n",
    "# Filmow\n",
    "print(\"Rating (Filmow): \" + rating_filmow + \" (Based on \" + ratingCount_filmow + \" votes)\")\n",
    "print(\"\\n----- Casting -----\")\n",
    "for li in lis_filmow:\n",
    "    actor = li.find_all(\"span\", itemprop=\"name\")[0].text\n",
    "    paper = li.find_all(\"em\")\n",
    "    if paper:\n",
    "        print(actor + \" - \" + paper[0].text)\n",
    "    else:\n",
    "        print(actor)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Getting movie theater schedule data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Add the lib package to the system path, so that we can include modules from there.\n",
    "\n",
    "import os\n",
    "import sys\n",
    "\n",
    "module_path = os.path.abspath(os.path.join('./'))\n",
    "\n",
    "if module_path not in sys.path:\n",
    "    sys.path.append(module_path)\n",
    "    \n",
    "from lib.movie_theater_schedule.cinepolis_schedule import CinepolisSchedule\n",
    "from lib.movie_theater_schedule.cinemark_schedule import CinemarkSchedule"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cinépolis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def generate_cinepolis_soup(cod_cinema, cod_claquete):\n",
    "    body = {\n",
    "        'cod_cinema'   : cod_cinema,\n",
    "        'cod_claquete' : cod_claquete,\n",
    "        'cod_horario'  : time.strftime(\"%Y-%m-%d\"),\n",
    "        'cod_filme'    : '0'\n",
    "    }\n",
    "    page = requests.post('http://www.cinepolis.com.br/programacao/ajax/ajax.conteudo_horarios.php', data = body)\n",
    "    return BeautifulSoup(page.content, 'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_cinepolis_schedule_data(soup, movie_theater):\n",
    "    table = soup.find(\"table\", { \"class\" : \"tabelahorarios\" })\n",
    "    rows = table.find_all('tr')\n",
    "    \n",
    "    has_movie = lambda row : row.find(attrs={'data-order': re.compile('\\w+')})\n",
    "\n",
    "    entries = [row for row in rows if has_movie(row)]\n",
    "\n",
    "    rows_list = []\n",
    "\n",
    "    for entry in entries:\n",
    "        title = entry.find_all(href=re.compile('http://www.cinepolis.com.br/filmes/filme.php'))[0].text\n",
    "\n",
    "        room = entry.find_all('td')[0].text\n",
    "\n",
    "        tags = []\n",
    "        if(entry.find(\"a\", { \"class\" : \"icovip\" })):\n",
    "            tags.append('VIP')\n",
    "        if(entry.find(\"a\", { \"class\" : \"icomacroxe\" })):\n",
    "            tags.append('MacroXE')\n",
    "        if(entry.find(\"a\", { \"class\" : \"ico3d\" })):\n",
    "            tags.append('3D')\n",
    "        tags = ' / '.join(tags)\n",
    "\n",
    "        content_rating = entry.find_all('td')[2].find('img').get('alt')\n",
    "\n",
    "        category = entry.find(\"td\", { \"class\" : \"horarios\" }).find('span').get('aria-label')\n",
    "\n",
    "        schedules_tags = entry.find(\"td\", { \"class\" : \"horarios\" }).select(\"span + span, a\")\n",
    "        schedules = ' / '.join([s.text for s in schedules_tags])\n",
    "\n",
    "        row_dict = {\n",
    "            'room': room,\n",
    "            'title': title,\n",
    "            'tags': tags,\n",
    "            'content_rating': content_rating,\n",
    "            'category': category,\n",
    "            'schedules': schedules,\n",
    "            'movie_theater': movie_theater\n",
    "        }\n",
    "        rows_list.append(row_dict)\n",
    "\n",
    "    return pd.DataFrame(rows_list)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cinépolis Natal Shopping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cinepolis_natal_soup = generate_cinepolis_soup('31', '769')\n",
    "cinepolis_natal_schedule = CinepolisSchedule(cinepolis_natal_soup, 'Cinépolis Natal Shopping')\n",
    "cinepolis_natal_schedule_df = cinepolis_natal_schedule.get_dataframe()\n",
    "cinepolis_natal_schedule_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cinépolis Partage Norte Shopping Natal"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "cinepolis_norte_soup = generate_cinepolis_soup('33', '770')\n",
    "cinepolis_norte_schedule = CinepolisSchedule(cinepolis_norte_soup, 'Cinépolis Partage Norte Shopping Natal')\n",
    "cinepolis_norte_schedule_df = cinepolis_norte_schedule.get_dataframe()\n",
    "cinepolis_norte_schedule_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Cinemark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "response = requests.get('https://www.cinemark.com.br/natal/cinemas')\n",
    "cinemark_soup = BeautifulSoup(response.content, 'html.parser')\n",
    "cinemark_schedule = CinemarkSchedule(cinemark_soup, 'Cinemark Midway Mall Natal')\n",
    "cinemark_schedule_df = cinemark_schedule.get_dataframe()\n",
    "cinemark_schedule_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Movie theaters schedule data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "frames = [cinepolis_natal_schedule_df, cinepolis_norte_schedule_df, cinemark_schedule_df]\n",
    "movie_theaters_schedule_data = pd.concat(frames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "movie_theaters_schedule_data.sample()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Movies data on IMDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "movies_data = movie_theaters_schedule_data.groupby('title')['movie_url'].unique().reset_index()\n",
    "\n",
    "imdb_url_series = pd.Series(None, index = movies_data.index, name = 'imdb_url')\n",
    "movies_data = movies_data.join(imdb_url_series)\n",
    "movies_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Obtain movie cast on original theater site"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def get_cinemark_movie_cast(soup):\n",
    "    detail_items = soup.find_all('div', { 'class' : 'detail-title' })\n",
    "\n",
    "    has_label_data = lambda tag : 'Elenco' in tag.select('strong')[0].text\n",
    "\n",
    "    detail_tag = [item for item in detail_items if has_label_data(item)][0]\n",
    "    actors = detail_tag.get_text().split('Elenco: ')[1].strip().split(', ')\n",
    "    return [' '.join(actor.split(' ')[:2]) for actor in actors]\n",
    "\n",
    "def get_cinepolis_movie_cast(soup):\n",
    "    content = soup.select('.conteudo > .direita')[0].get_text()\n",
    "    content = re.sub('[\\s][ ]+', '', content)\n",
    "    content = re.sub('\\r\\n', '', content)\n",
    "    content = re.sub('\\n', '', content)\n",
    "    \n",
    "    m = re.search('Elenco(.*)Roteiro', content)\n",
    "    if m:\n",
    "        found = m.group(1)\n",
    "        items = found.split(',')[:3]\n",
    "        return [item.strip() for item in items]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find movie IMDB url using title and cast as reference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for index, row in movies_data.iterrows():\n",
    "    url = row.movie_url[0]\n",
    "    title = row.title\n",
    "\n",
    "    movie_response = requests.get(url)\n",
    "    movie_soup = BeautifulSoup(movie_response.content, 'html.parser')\n",
    "    print('Searching IMDB url for: ' + title)\n",
    "    \n",
    "    if('cinepolis' in url):\n",
    "        cast = get_cinepolis_movie_cast(movie_soup)\n",
    "    else:\n",
    "        cast = get_cinemark_movie_cast(movie_soup)\n",
    "\n",
    "    imdb_url = ''\n",
    "\n",
    "    if(cast is not None):\n",
    "        query_token = '+'.join(title.split(' '))\n",
    "        search_url = 'http://www.imdb.com/find?q={}&s=tt'.format(query_token)\n",
    "        page = requests.get(search_url)\n",
    "        soup = BeautifulSoup(page.content, 'html.parser')\n",
    "\n",
    "        search_results = soup.select('.findResult .result_text > a')\n",
    "\n",
    "        for link in search_results:\n",
    "            if(imdb_url != ''):\n",
    "                break\n",
    "\n",
    "            movie_url = 'http://www.imdb.com' + link.get('href')\n",
    "            movie_page = requests.get(movie_url)\n",
    "            movie_soup = BeautifulSoup(movie_page.content, 'html.parser')\n",
    "\n",
    "            actors_tags = movie_soup.find_all('span', { 'itemprop': 'name' })\n",
    "            cast_imdb = [actor.text for actor in actors_tags]\n",
    "            if(all(actor in cast_imdb for actor in cast)):\n",
    "                imdb_url = movie_url\n",
    "\n",
    "    movies_data.loc[index, 'imdb_url'] = imdb_url"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "movies_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Add movie data using IMDB"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for serie_name in ['original_title', 'rating', 'time', 'genres']:\n",
    "    series = pd.Series(None, index = movies_data.index, name = serie_name)\n",
    "    movies_data = movies_data.join(series)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "for index, row in movies_data.iterrows():\n",
    "    url = row.imdb_url\n",
    "    \n",
    "    if('imdb' in url):   \n",
    "        movie_response = requests.get(url)\n",
    "        movie_soup = BeautifulSoup(movie_response.content, 'html.parser')\n",
    "\n",
    "        title = ''\n",
    "        rating = ''\n",
    "        time = ''\n",
    "        genres_str = ''\n",
    "\n",
    "        title = movie_soup.find_all(\"h1\", itemprop=\"name\")[0].text\n",
    "\n",
    "        rating = movie_soup.find_all(\"span\", itemprop=\"ratingValue\")[0].text\n",
    "\n",
    "        time_tags = movie_soup.find_all(\"time\", itemprop=\"duration\")\n",
    "        if(len(time_tags) > 0):\n",
    "            time = movie_soup.find_all(\"time\", itemprop=\"duration\")[0].text\n",
    "\n",
    "        genres = movie_soup.find_all(\"span\", itemprop=\"genre\", class_=\"itemprop\")\n",
    "        genres_str = \"\"\n",
    "        for genre in genres:\n",
    "            genres_str = genres_str + genre.text + \", \"\n",
    "\n",
    "        movies_data.loc[index, 'original_title'] = title\n",
    "        movies_data.loc[index, 'rating'] = rating\n",
    "        movies_data.loc[index, 'time'] = time\n",
    "        movies_data.loc[index, 'genres'] = genres_str"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "movies_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Developers\n",
    "\n",
    "- Álvaro Ferreira - [github.com/alvarofpp](https://github.com/alvarofpp)\n",
    "- Gabriel Ribeiro - [github.com/Bib7/](https://github.com/Bib7/)\n",
    "- Kaio Max - [github.com/kaiomax](https://github.com/kaiomax)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# References\n",
    "\n",
    "\n",
    "- [https://www.crummy.com/software/BeautifulSoup/bs4/doc/][1]\n",
    "- [https://imasters.com.br/desenvolvimento/aprendendo-sobre-web-scraping-em-python-utilizando-beautifulsoup/?trace=1519021197&source=single][2]\n",
    "- [http://docs.python-requests.org/en/master/user/quickstart/#more-complicated-post-requests][3]\n",
    "- [http://akul.me/blog/2016/beautifulsoup-cheatsheet/][4]\n",
    "- [https://pymotw.com/2/abc/][5]\n",
    "\n",
    "[1]: https://www.crummy.com/software/BeautifulSoup/bs4/doc/\n",
    "[2]: https://imasters.com.br/desenvolvimento/aprendendo-sobre-web-scraping-em-python-utilizando-beautifulsoup/?trace=1519021197&source=single\n",
    "[3]: http://docs.python-requests.org/en/master/user/quickstart/#more-complicated-post-requests\n",
    "[4]: http://akul.me/blog/2016/beautifulsoup-cheatsheet/\n",
    "[5]: https://pymotw.com/2/abc/"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}