Workshop 3 - Paginacja.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generatory, Yield"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mylist = [0, 1, 4]\n",
    "for i in mylist:\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mylist = [x*x for x in range(3)]\n",
    "for i in mylist:\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mylist = (x*x for x in range(3))\n",
    "for i in mylist:\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def createGenerator():\n",
    "    mylist = range(3)\n",
    "    for i in mylist:\n",
    "        yield i*i\n",
    "        \n",
    "for i in createGenerator():\n",
    "    print(i)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fail2Ban"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As you can see in my example, I have set up 300 maxretry and 300 for findtime, so, we need to have 300 GETs from the same IP in a time window of 300 seconds to have the originating IP blocked."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Scraper"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Scrapy doesn’t wait a fixed amount of time between requests, but uses a random interval between 0.5 * DOWNLOAD_DELAY and 1.5 * DOWNLOAD_DELAY."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scrapy\n",
    "import scrapy.crawler as crawler\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "from scrapy.crawler import CrawlerProcess\n",
    "\n",
    "class MySpider(scrapy.Spider):\n",
    "    name = 'myspider'\n",
    "    start_urls = [\n",
    "        'https://www.gumtree.pl/s-mieszkania-i-domy-sprzedam-i-kupie/mazowieckie/v1c9073l3200001p1'\n",
    "        ]\n",
    "   \n",
    "    custom_settings = {\n",
    "        'DOWNLOAD_DELAY': '4.0',\n",
    "    }\n",
    "\n",
    "    top_url = 'https://www.gumtree.pl'\n",
    "    def parse(self, response):\n",
    "        self.logger.info('Got successful response from {}'.format(response.url))\n",
    "        soup = BeautifulSoup(response.body, 'lxml')\n",
    "        link_tabs = soup.findAll(\"div\", {\"class\": \"result-link\"})\n",
    "        item_urls = []\n",
    "        for tab in link_tabs:\n",
    "            hrefs = tab.findAll(\"a\", {\"class\": \"href-link\"})\n",
    "            for h in hrefs:\n",
    "                item_urls.append(self.top_url + h[\"href\"])\n",
    "            \n",
    "        \n",
    "        for item_url in item_urls:\n",
    "            yield scrapy.Request(item_url, self.parse_item)\n",
    "\n",
    "    def parse_item(self, response):\n",
    "        self.logger.info('Got successful response from {}'.format(response.url))\n",
    "        # <processing code not shown>\n",
    "        #item = MyItem()\n",
    "        # populate `item` fields\n",
    "        # and extract item_details_url\n",
    "        #yield scrapy.Request(item_details_url, self.parse_details, meta={'item': item})\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "process = CrawlerProcess({\n",
    "    'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'\n",
    "})\n",
    "process.crawl(MySpider)\n",
    "process.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scrapy\n",
    "import scrapy.crawler as crawler\n",
    "from bs4 import BeautifulSoup\n",
    "\n",
    "from scrapy.crawler import CrawlerProcess\n",
    "\n",
    "class MyPaginatingSpider(scrapy.Spider):\n",
    "    name = 'mypaginatingspider'\n",
    "    start_urls = [\n",
    "        'https://www.gumtree.pl/s-mieszkania-i-domy-sprzedam-i-kupie/mazowieckie/v1c9073l3200001p1'\n",
    "        ]\n",
    "   \n",
    "    custom_settings = {\n",
    "        'DOWNLOAD_DELAY': '6.0',\n",
    "    }\n",
    "\n",
    "    top_url = 'https://www.gumtree.pl'\n",
    "    def parse(self, response):\n",
    "        self.logger.info('Got successful response from {}'.format(response.url))\n",
    "        soup = BeautifulSoup(response.body, 'lxml')\n",
    "        link_tabs = soup.findAll(\"div\", {\"class\": \"result-link\"})\n",
    "        item_urls = []\n",
    "        next_urls = []\n",
    "        for tab in link_tabs:\n",
    "            hrefs = tab.findAll(\"a\", {\"class\": \"href-link\"})\n",
    "            for h in hrefs:\n",
    "                item_urls.append(self.top_url + h[\"href\"])\n",
    "            \n",
    "        nexts = soup.findAll(\"a\", {\"class\": \"next\"})\n",
    "        \n",
    "        for h in nexts:\n",
    "            for h in hrefs:\n",
    "                next_urls.append(self.top_url + h[\"href\"])\n",
    "        \n",
    "        print (next_urls)\n",
    "            \n",
    "        for item_url in item_urls:\n",
    "            yield scrapy.Request(item_url, self.parse_item)\n",
    "\n",
    "        #for next_url in next_urls:\n",
    "        #    yield scrapy.Request(item_url, self.parse)\n",
    "    \n",
    "            \n",
    "    def parse_item(self, response):\n",
    "        self.logger.info('Got successful response from {}'.format(response.url))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "process = CrawlerProcess({\n",
    "    'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'\n",
    "})\n",
    "process.crawl(MyPaginatingSpider)\n",
    "process.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}