All Scrapping process.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "from bs4 import BeautifulSoup\n",
    "import requests\n",
    "import time\n",
    "from datetime import datetime\n",
    "import json\n",
    "from zipfile import ZipFile\n",
    "import pandas as pd\n",
    "import shutil\n",
    "import zipfile\n",
    "import random"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define PATH constant"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "BASE_PATH = \"data\"\n",
    "HTML_PATH = BASE_PATH + \"/html\"\n",
    "USER_PATH = BASE_PATH + \"/users\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Get Clubs IDs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_number(string):\n",
    "    return int(string.strip().replace(\",\", \"\"))\n",
    "\n",
    "\n",
    "clubs_id = set()\n",
    "possibles_users = 0\n",
    "page = 1\n",
    "\n",
    "while True:\n",
    "    print(f\"\\r{page}\", end=\"\")\n",
    "\n",
    "    time.sleep(3)  # Wait 3 seconds per page\n",
    "    data = requests.get(f\"https://myanimelist.net/clubs.php?p={page}\")\n",
    "    soup = BeautifulSoup(data.text, \"html.parser\")\n",
    "    rows = soup.find_all(\"tr\", {\"class\": \"table-data\"})\n",
    "    for row in rows:\n",
    "        members = get_number(row.find(\"td\", {\"class\": \"ac\"}).text)\n",
    "        club_id = get_number(\n",
    "            row.find(\"a\", {\"class\": \"fw-b\"}).get(\"href\").split(\"=\")[-1]\n",
    "        )\n",
    "        if (\n",
    "            club_id not in clubs_id and members > 30\n",
    "        ):  # Only save groups with more than 30 members\n",
    "            possibles_users += members\n",
    "            clubs_id.add(club_id)\n",
    "\n",
    "    page += 1\n",
    "    if possibles_users > 1000000:  # Threshold to stop\n",
    "        break\n",
    "\n",
    "with open(f\"{BASE_PATH}/clubs.txt\", \"w\") as file:\n",
    "    for club in clubs_id:\n",
    "        file.write(f\"{club}\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2. Get usernames in every clubs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists(f\"{BASE_PATH}/users_list.txt\"):\n",
    "    with open(f\"{BASE_PATH}/users_list.txt\", \"w\", encoding=\"UTF-8\") as file:\n",
    "        pass\n",
    "    \n",
    "if not os.path.exists(f\"{BASE_PATH}/_revised_clubs.txt\"):\n",
    "    with open(f\"{BASE_PATH}/_revised_clubs.txt\", \"w\", encoding=\"UTF-8\") as file:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f\"{BASE_PATH}/clubs.txt\") as file:\n",
    "    clubs_id = [x.strip() for x in file.readlines()]\n",
    "\n",
    "with open(f\"{BASE_PATH}/users_list.txt\", encoding=\"UTF-8\") as file:\n",
    "    users = set([x.strip() for x in file.readlines()])\n",
    "\n",
    "with open(f\"{BASE_PATH}/_revised_clubs.txt\", encoding=\"UTF-8\") as file:\n",
    "    revised_clubs = set([int(x.strip()) for x in file.readlines()])\n",
    "\n",
    "len(users), len(revised_clubs), len(clubs_id)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext jupyternotify"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%notify -m \"get username finish\"\n",
    "\n",
    "for i, club_id in enumerate(clubs_id):\n",
    "    if club_id in revised_clubs:\n",
    "        continue\n",
    "\n",
    "    page = 1\n",
    "    while True:\n",
    "        print(f\"\\r{i+1}/{len(clubs_id)} --> {str(page).zfill(2)}\", end=\"\")\n",
    "        link = f\"https://api.jikan.moe/v3/club/{club_id}/members/{page}\"\n",
    "\n",
    "        try:\n",
    "            time.sleep(4.2)\n",
    "            data = requests.get(link)\n",
    "        except KeyboardInterrupt:\n",
    "            raise KeyboardInterrupt()\n",
    "        except:  # Other exception wait 2 min and try again\n",
    "            time.sleep(120)\n",
    "            continue\n",
    "\n",
    "        if data.status_code != 200:\n",
    "            break\n",
    "\n",
    "        with open(f\"{BASE_PATH}/users_list.txt\", \"a\", encoding=\"UTF-8\") as file:\n",
    "            for user in map(lambda x: x[\"username\"], json.loads(data.text)[\"members\"]):\n",
    "                if user not in users and user != \"\":\n",
    "                    file.write(f\"{user}\\n\")\n",
    "                    users.add(user)\n",
    "        page += 1\n",
    "\n",
    "    revised_clubs.add(club_id)\n",
    "    with open(f\"{BASE_PATH}/_revised_clubs.txt\", \"a\", encoding=\"UTF-8\") as file:\n",
    "        file.write(f\"{club_id}\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f\"{BASE_PATH}/users_list.txt\", encoding=\"UTF-8\") as file:\n",
    "    users = list(set([x.strip() for x in file.readlines()]))[1:]\n",
    "    random.shuffle(users)\n",
    "\n",
    "with open(f\"{BASE_PATH}/users.csv\", \"w\", encoding=\"UTF-8\") as file:\n",
    "    file.write(\"user_id,username\\n\")\n",
    "    for i, user in enumerate(users):\n",
    "        file.write(f\"{i},{user}\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3. Get animelist per user"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(f\"{BASE_PATH}/users.csv\", \"r\", encoding=\"UTF-8\") as file:\n",
    "    file.readline()\n",
    "    users = [x.strip().split(\",\") for x in file.readlines()]\n",
    "    users = [(int(x[0]), x[1]) for x in users]\n",
    "\n",
    "last_revised_users = -1\n",
    "if os.path.exists(f\"{BASE_PATH}/_last_revised_users.txt\"):\n",
    "    with open(f\"{BASE_PATH}/_last_revised_users.txt\", \"r\", encoding=\"UTF-8\") as file:\n",
    "        last_revised_users = int(file.readline())\n",
    "\n",
    "len(users), last_revised_users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext jupyternotify"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%notify -m \"animelist finish\"\n",
    "for i, (user_id, username) in enumerate(users):\n",
    "    if user_id <= last_revised_users:\n",
    "        continue\n",
    "\n",
    "    now = datetime.now()\n",
    "    print(f'\\r{str(now).split(\".\")[0]} --> {i+1}/{len(users)}', end=\"\")\n",
    "    page = 1\n",
    "    all_animes = []\n",
    "\n",
    "    while True:\n",
    "        link = f\"https://api.jikan.moe/v3/user/{username}/animelist/all?page={page}\"\n",
    "        try:\n",
    "            time.sleep(4.2)\n",
    "            data = requests.get(link, timeout=15)\n",
    "        except KeyboardInterrupt:\n",
    "            raise KeyboardInterrupt()\n",
    "        except:  # Other exception wait 2 min and try again\n",
    "            time.sleep(120)\n",
    "            continue\n",
    "\n",
    "        if data.status_code != 200:\n",
    "            break\n",
    "\n",
    "        data = json.loads(data.text)\n",
    "        for anime in data[\"anime\"]:\n",
    "            all_animes.append((anime[\"mal_id\"], anime[\"score\"], anime[\"watching_status\"], anime[\"watched_episodes\"]))\n",
    "\n",
    "        page += 1\n",
    "        if len(data[\"anime\"]) < 300:\n",
    "            break\n",
    "\n",
    "    if len(all_animes) != 0:\n",
    "        with open(f\"{USER_PATH}/{user_id}.csv\", \"w\") as f1:\n",
    "            f1.write(f\"anime_id,score,watching_status,watched_episodes\\n\")\n",
    "            for anime_id, anime_score, watching_status, watched_episodes in all_animes:\n",
    "                f1.write(\n",
    "                    f\"{anime_id},{anime_score},{watching_status},{watched_episodes}\\n\"\n",
    "                )\n",
    "\n",
    "    revised_users = user_id\n",
    "    with open(f\"{BASE_PATH}/_last_revised_users.txt\", \"w\", encoding=\"UTF-8\") as file:\n",
    "        file.write(f\"{user_id}\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4. Download Anime HTML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_anime = set()\n",
    "folder = os.listdir(USER_PATH)\n",
    "for i, user_file in enumerate(folder):\n",
    "    if \".csv\" not in user_file:\n",
    "        continue\n",
    "\n",
    "    print(f\"\\r{i + 1}/{len(folder)}\", end=\"\")\n",
    "    with open(f\"{USER_PATH}/{user_file}\", \"r\") as file:\n",
    "        file.readline()\n",
    "        for line in file:\n",
    "            anime = line.strip().split(\",\")[0]\n",
    "            unique_anime.add(anime)\n",
    "\n",
    "print(\"         \")\n",
    "print(len(unique_anime))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "MAX = 7  # MAX SECOND TO WAIT PER REQUEST\n",
    "MIN = 4  # MIN SECONDS TO WAIT PER REQUEST\n",
    "\n",
    "\n",
    "def sleep():\n",
    "    time_to_sleep = random.random() * (MAX - MIN) + MIN\n",
    "    time.sleep(time_to_sleep)\n",
    "\n",
    "\n",
    "def get_link_by_text(soup, anime_id, text):\n",
    "    links = list(filter(lambda x: anime_id in x[\"href\"], soup.find_all(\"a\", text=text)))\n",
    "    return links[0][\"href\"]\n",
    "\n",
    "\n",
    "def save(path, data):\n",
    "    with open(path, \"w\", encoding=\"UTF-8\") as file:\n",
    "        file.write(data)\n",
    "\n",
    "\n",
    "def save_link(link, anime_id, name):\n",
    "    sleep()\n",
    "    path = f\"{HTML_PATH}/{anime_id}/{name}.html\"\n",
    "    data = requests.get(link)\n",
    "    soup = BeautifulSoup(data.text, \"html.parser\")\n",
    "    soup.script.decompose()\n",
    "    save(path, soup.prettify())\n",
    "    return soup\n",
    "\n",
    "\n",
    "def save_reviews(link, anime_id):\n",
    "    page = 1\n",
    "    while True:\n",
    "        sleep()\n",
    "        actual_link = f\"{link}?p={page}\"\n",
    "        data = requests.get(actual_link)\n",
    "        soup = BeautifulSoup(data.text, \"html.parser\")\n",
    "        reviews = soup.find_all(\"a\", text=\"Overall Rating\")\n",
    "        if len(reviews) == 0:\n",
    "            break\n",
    "\n",
    "        path = f\"{HTML_PATH}/{anime_id}/reviews_{page}.html\"\n",
    "        soup.script.decompose()\n",
    "        save(path, soup.prettify())\n",
    "        page += 1\n",
    "\n",
    "\n",
    "def scrap_anime(anime_id):\n",
    "    path = f\"{HTML_PATH}/{anime_id}\"\n",
    "    os.makedirs(path, exist_ok=True)\n",
    "    sleep()\n",
    "    data = requests.get(f\"https://myanimelist.net/anime/{anime_id}\")\n",
    "\n",
    "    anime_info = data.text\n",
    "    soup = BeautifulSoup(anime_info, \"html.parser\")\n",
    "    soup.script.decompose()\n",
    "    save(f\"{HTML_PATH}/{anime_id}/details.html\", soup.prettify())\n",
    "\n",
    "    link_review = get_link_by_text(soup, anime_id, \"Reviews\")\n",
    "    link_recomendations = get_link_by_text(soup, anime_id, \"Recommendations\")\n",
    "    link_stats = get_link_by_text(soup, anime_id, \"Stats\")\n",
    "    link_staff = get_link_by_text(soup, anime_id, \"Characters & Staff\")\n",
    "    link_pictures = get_link_by_text(soup, anime_id, \"Pictures\")\n",
    "\n",
    "    save_link(link_pictures, anime_id, \"pictures\")\n",
    "    save_link(link_staff, anime_id, \"staff\")\n",
    "    save_link(link_stats, anime_id, \"stats\")\n",
    "    save_link(link_recomendations, anime_id, \"recomendations\")\n",
    "    save_reviews(link_review, anime_id)\n",
    "\n",
    "\n",
    "def zipdir(path, ziph):\n",
    "    # ziph is zipfile handle\n",
    "    for root, dirs, files in os.walk(path):\n",
    "        for file in files:\n",
    "            ziph.write(\n",
    "                os.path.join(root, file),\n",
    "                os.path.relpath(os.path.join(root, file), path),\n",
    "            )\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext jupyternotify"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%%notify -m \"Anime scrapping finish\"\n",
    "\n",
    "for i, anime_id in enumerate(unique_anime):\n",
    "    if os.path.isfile(f\"{HTML_PATH}/{anime_id}.zip\"):\n",
    "        continue\n",
    "\n",
    "    print(f\"\\r{i+1}/{len(unique_anime)}\", end=\"\")\n",
    "\n",
    "    try:\n",
    "        scrap_anime(anime_id)\n",
    "    except KeyboardInterrupt:\n",
    "        break\n",
    "    except:  # Other exception wait 2 min and try again\n",
    "        time.sleep(120)\n",
    "        continue\n",
    "\n",
    "    path = f\"{HTML_PATH}/{anime_id}\"\n",
    "    zipf = zipfile.ZipFile(f\"{path}.zip\", \"w\", zipfile.ZIP_DEFLATED)\n",
    "    zipdir(path, zipf)\n",
    "    zipf.close()\n",
    "\n",
    "    shutil.rmtree(path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5. Scrapping Anime info from HTML"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_zip(input_zip):\n",
    "    input_zip = ZipFile(input_zip)\n",
    "    return {name: input_zip.read(name) for name in input_zip.namelist()}\n",
    "\n",
    "KEYS = ['MAL_ID', 'Name', 'Score', 'Genders', 'English name', 'Japanese name', 'Type', 'Episodes',\n",
    "        'Aired', 'Premiered', 'Producers', 'Licensors', 'Studios', 'Source', 'Duration', 'Rating',\n",
    "        'Ranked', 'Popularity', 'Members', 'Favorites', 'Watching', 'Completed', 'On-Hold', 'Dropped',\n",
    "        'Plan to Watch', 'Score-10', 'Score-9', 'Score-8', 'Score-7', 'Score-6', 'Score-5', 'Score-4',\n",
    "        'Score-3', 'Score-2', 'Score-1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_name(info):\n",
    "    return info.find(\"h1\", {\"class\": \"title-name h1_bold_none\"}).text.strip()\n",
    "\n",
    "\n",
    "def get_english_name(info):\n",
    "    span = info.findAll(\"span\", {\"class\": \"dark_text\"})\n",
    "    return span.parent.text.strip()\n",
    "\n",
    "\n",
    "def get_table(a_soup):\n",
    "    return a_soup.find(\"div\", {\"class\": \"po-r js-statistics-info di-ib\"})\n",
    "\n",
    "\n",
    "def get_score(stats):\n",
    "    score = stats.find(\"span\", {\"itemprop\": \"ratingValue\"})\n",
    "    if score is None:\n",
    "        return \"Unknown\"\n",
    "    return score.text.strip()\n",
    "\n",
    "\n",
    "def get_gender(sum_info):\n",
    "    text = \", \".join(\n",
    "        [x.text.strip() for x in sum_info.findAll(\"span\", {\"itemprop\": \"genre\"})]\n",
    "    )\n",
    "    return text\n",
    "\n",
    "\n",
    "def get_description(sum_info):\n",
    "    return sum_info.find(\"td\", {\"class\": \"borderClass\", \"width\": \"225\"})\n",
    "\n",
    "\n",
    "def get_all_stats(soup):\n",
    "    return soup.find(\"div\", {\"id\": \"horiznav_nav\"}).parent.findAll(\n",
    "        \"div\", {\"class\": \"spaceit_pad\"}\n",
    "    )\n",
    "\n",
    "\n",
    "def get_info_anime(anime_id):\n",
    "    data = extract_zip(f\"data/html/{anime_id}.zip\")\n",
    "    anime_info = data[\"stats.html\"].decode()\n",
    "    soup = BeautifulSoup(anime_info, \"html.parser\")\n",
    "\n",
    "    stats = get_table(soup)\n",
    "    description = get_description(soup)\n",
    "    anime_info = {key: \"Unknown\" for key in KEYS}\n",
    "\n",
    "    anime_info[\"MAL_ID\"] = anime_id\n",
    "    anime_info[\"Name\"] = get_name(soup)\n",
    "    anime_info[\"Score\"] = get_score(stats)\n",
    "    anime_info[\"Genders\"] = get_gender(description)\n",
    "\n",
    "    for d in description.findAll(\"span\", {\"class\": \"dark_text\"}):\n",
    "        information = [x.strip().replace(\" \", \" \") for x in d.parent.text.split(\":\")]\n",
    "        category, value = information[0], \":\".join(information[1:])\n",
    "        value.replace(\"\\t\", \"\")\n",
    "\n",
    "        if category in [\"Broadcast\", \"Synonyms\", \"Genres\", \"Score\", \"Status\"]:\n",
    "            continue\n",
    "\n",
    "        if category in [\"Ranked\"]:\n",
    "            value = value.split(\"\\n\")[0]\n",
    "        if category in [\"Producers\", \"Licensors\", \"Studios\"]:\n",
    "            value = \", \".join([x.strip() for x in value.split(\",\")])\n",
    "        if category in [\"Ranked\", \"Popularity\"]:\n",
    "            value = value.replace(\"#\", \"\")\n",
    "        if category in [\"Members\", \"Favorites\"]:\n",
    "            value = value.replace(\",\", \"\")\n",
    "        if category in [\"English\", \"Japanese\"]:\n",
    "            category += \" name\"\n",
    "\n",
    "        anime_info[category] = value\n",
    "\n",
    "    # Stats (Watching, Completed, On-Hold, Dropped, Plan to Watch)\n",
    "    for d in get_all_stats(soup)[:5]:\n",
    "        category, value = [x.strip().replace(\" \", \" \") for x in d.text.split(\":\")]\n",
    "        value = value.replace(\",\", \"\")\n",
    "        anime_info[category] = value\n",
    "\n",
    "    # Stast votes per score\n",
    "    for d in get_all_stats(soup)[6:]:\n",
    "        score = d.parent.parent.find(\"td\", {\"class\": \"score-label\"}).text.strip()\n",
    "        value = [x.strip().replace(\" \", \" \") for x in d.text.split(\"%\")][1].strip(\n",
    "            \"(votes)\"\n",
    "        )\n",
    "        label = f\"Score-{score}\"\n",
    "        anime_info[label] = value.strip()\n",
    "\n",
    "    for key, value in anime_info.items():\n",
    "        if str(value) in [\"?\", \"None found, add some\", \"None\", \"N/A\", \"Not available\"]:\n",
    "            anime_info[key] = \"Unknown\"\n",
    "    return anime_info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "get_info_anime(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate anime.tsv\n",
    "anime_revised = set()\n",
    "exist_file = os.path.exists(f\"{BASE_PATH}/anime.tsv\")\n",
    "\n",
    "if exist_file:\n",
    "    # If the file exist, include new data.\n",
    "    actual_data = pd.read_csv(f\"{BASE_PATH}/anime.tsv\", sep=\"\\t\")\n",
    "    anime_revised = list(actual_data.MAL_ID.unique())\n",
    "\n",
    "actual_data.head()\n",
    "total_data = []\n",
    "zips = os.listdir(HTML_PATH)\n",
    "for i, anime in enumerate(zips):\n",
    "    if not \".zip\" in anime:\n",
    "        continue\n",
    "\n",
    "    anime_id = int(anime.strip(\".zip\"))\n",
    "\n",
    "    if int(anime_id) in anime_revised:\n",
    "        continue\n",
    "\n",
    "    print(f\"\\r{i+1}/{len(zips)} ({anime_id})\", end=\"\")\n",
    "\n",
    "    anime_id = anime.strip(\".zip\")\n",
    "    total_data.append(get_info_anime(anime_id))\n",
    "\n",
    "if len(total_data):\n",
    "    df = pd.DataFrame.from_dict(total_data)\n",
    "    df[\"MAL_ID\"] = pd.to_numeric(df[\"MAL_ID\"])\n",
    "    df = df.sort_values(by=\"MAL_ID\").reset_index(drop=True)\n",
    "\n",
    "    if exist_file:\n",
    "        df = (\n",
    "            pd.concat([actual_data, df]).sort_values(by=\"MAL_ID\").reset_index(drop=True)\n",
    "        )\n",
    "\n",
    "else:\n",
    "    df = actual_data\n",
    "\n",
    "pd.set_option(\"display.max_columns\", None)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv(f\"{BASE_PATH}/anime.tsv\", index=False, sep=\"\\t\", encoding=\"UTF-8\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6. Create rating_complete.csv\n",
    "\n",
    "This file only contain animes with `watching_status==2`(complete) and have been rated (`score!=0`)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists(f\"{BASE_PATH}/rating_complete.csv\"):\n",
    "    with open(f\"{BASE_PATH}/rating_complete.csv\", \"w\", encoding=\"UTF-8\") as file:\n",
    "        file.write(\"user_id,anime_id,rating\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_anime = set()\n",
    "all_users = sorted(os.listdir(USER_PATH), key=lambda x:int(x.split(\".\")[0]))\n",
    "\n",
    "with open(f\"{BASE_PATH}/rating_complete.csv\", \"a\") as f1:\n",
    "\n",
    "    for i, user_file in enumerate(all_users):\n",
    "        if not user_file.endswith(\".csv\"):\n",
    "            continue\n",
    "\n",
    "        print(f\"\\r{i+1}/{len(all_users)}\", end=\"\")\n",
    "\n",
    "        user_id = user_file.split(\".\")[0]\n",
    "        with open(f\"{USER_PATH}/{user_file}\", \"r\") as file:\n",
    "            file.readline()\n",
    "            for line in file:\n",
    "                anime_id, score, watching_status, _ = line.strip().split(\",\")\n",
    "                if int(watching_status) == 2 and int(score) != 0:\n",
    "                    f1.write(f\"{user_id},{anime_id},{score}\\n\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 7. Unified animelist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not os.path.exists(f\"{BASE_PATH}/animelist.csv\"):\n",
    "    with open(f\"{BASE_PATH}/animelist.csv\", \"w\", encoding=\"UTF-8\") as file:\n",
    "        file.write(\"user_id,anime_id,rating,watching_status,watched_episodes\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_anime = set()\n",
    "all_users = sorted(os.listdir(USER_PATH), key=lambda x:int(x.split(\".\")[0]))\n",
    "\n",
    "with open(f\"{BASE_PATH}/animelist.csv\", \"a\") as f1:\n",
    "\n",
    "    for i, user_file in enumerate(all_users):\n",
    "        if not user_file.endswith(\".csv\"):\n",
    "            continue\n",
    "\n",
    "        print(f\"\\r{i+1}/{len(all_users)}\", end=\"\")\n",
    "\n",
    "        user_id = user_file.split(\".\")[0]\n",
    "        with open(f\"{USER_PATH}/{user_file}\", \"r\") as file:\n",
    "            file.readline()\n",
    "            for line in file:\n",
    "                anime_id, score, watching_status, watched_episodes = line.strip().split(\",\")\n",
    "                f1.write(f\"{user_id},{anime_id},{score},{watching_status},{watched_episodes}\\n\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}