Added documentation.

Signed-off-by: RobMcH <[email protected]>

mindfactory/pipelines/database_pipeline.py

@@ -5,7 +5,8 @@ class DatabasePipeline(object):
     def __init__(self):
         self.connection = sqlite3.connect('./scrapedata.db')
         self.cursor = self.connection.cursor()
-        # If this returns None, the table does not exist. In this case INSERT instead of UPDATE is used.
+        # If this returns None, the table does not exist. In this case INSERT instead of UPDATE is used when processing
+        # items.
         self.mode = self.cursor.execute("""
                                         SELECT name FROM sqlite_master
                                         WHERE type='table' AND name='productdata'

mindfactory/settings.py

@@ -60,9 +60,9 @@
 
 # Enable or disable extensions
 # See https://doc.scrapy.org/en/latest/topics/extensions.html
-#EXTENSIONS = {
-#    'scrapy.extensions.telnet.TelnetConsole': None,
-#}
+EXTENSIONS = {
+    'scrapy.extensions.telnet.TelnetConsole': None,
+}
 
 # Configure item pipelines
 # See https://doc.scrapy.org/en/latest/topics/item-pipeline.html

mindfactory/spiders/product_spider.py

@@ -56,12 +56,11 @@ def parse_product(self, response):
         # Extract information on product page and process reviews.
         item = MindfactoryItem()
         item["url"] = response.url
-        item["category"] = response.xpath(self.product_category).extract_first()
-        item["name"] = response.xpath(self.product_name_xpath).extract_first()
-        item["brand"] = response.xpath(self.product_brand_xpath).extract_first()
+        item["category"] = response.xpath(self.product_category).extract_first(default=None)
+        item["name"] = response.xpath(self.product_name_xpath).extract_first(default=None)
+        item["brand"] = response.xpath(self.product_brand_xpath).extract_first(default=None)
         item["ean"] = response.xpath(self.product_ean_xpath).extract_first(default=None)
         item["sku"] = response.xpath(self.product_sku_xpath).extract_first(default=None)
-        # There are prices and special prices for some reason.
         price = response.xpath(self.product_price_xpath).extract_first(default=None)
         if price is not None:
             text = price.rstrip()[1:-1]
@@ -71,6 +70,7 @@ def parse_product(self, response):
             item["price"] = None
         count_and_people = response.xpath(self.product_count_xpath).extract()
         sold_or_people = response.xpath(self.product_sold_or_people).extract_first(default=None)
+        # Assign the amount of sold products and watching people depending on the present information.
         if len(count_and_people) == 2:
             item["count_sold"] = int(count_and_people[0].replace(".", ""))
             item["people_watching"] = int(count_and_people[1].replace(".", ""))