WIP crawlers

Vagrantfile

@@ -12,7 +12,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
     web.vm.network "forwarded_port", guest: 8000, host: 8080
 
     web.vm.provider "virtualbox" do |vb|
-      vb.customize ["modifyvm", :id, "--memory", "512"]
+      vb.customize ["modifyvm", :id, "--memory", "1024"]
     end
 
     web.vm.provision "ansible" do |ansible|

amiibofindr/apps/amiibo/management/commands/fetch_price.py

@@ -0,0 +1,12 @@
+# coding: utf-8
+
+# python3
+from __future__ import unicode_literals
+
+# django
+from django.core.management.base import BaseCommand
+
+
+class Command(BaseCommand):
+    def handle(self, *args, **kwargs):
+        print(args, kwargs)

amiibofindr/apps/amiibo/models.py

@@ -6,6 +6,9 @@ import os
 # django
 from django.db import models
 
+# project
+from amiibofindr.apps.shop.crawlers import Crawler
+
 
 #
 # Upload_to helpers
@@ -124,6 +127,11 @@ class AmiiboPrice(models.Model):
             self.price, self.currency
         )
 
+    def fetch(self):
+        crawler = Crawler(self.amiibo_shop.shop.slug)
+        price = crawler.fetch_from_id(self.amiibo_shop.item_id)
+        return price
+
     def save_history(self, old_price, new_price):
         history = AmiiboPriceHistory(
             amiibo=self.amiibo,

amiibofindr/apps/shop/crawlers/__init__.py

@@ -0,0 +1,31 @@
+# coding: utf-8
+
+# py3
+from __future__ import unicode_literals
+
+from .amazon import (
+    AmazonUSCrawler,
+    AmazonESCrawler,
+    AmazonUKCrawler,
+    AmazonITCrawler,
+    AmazonDECrawler,
+    AmazonJPCrawler,
+    AmazonFRCrawler,
+)
+
+
+class Crawler(object):
+    crawler_classes = {
+        'amazon-us': AmazonUSCrawler,
+        'amazon-uk': AmazonUKCrawler,
+        'amazon-fr': AmazonFRCrawler,
+        'amazon-es': AmazonESCrawler,
+        'amazon-de': AmazonDECrawler,
+        'amazon-it': AmazonITCrawler,
+        'amazon-jp': AmazonJPCrawler,
+    }
+
+    def __new__(self, shop_slug):
+        if shop_slug in self.crawler_classes:
+            return self.crawler_classes[shop_slug]()
+        raise Exception('Shop slug {} not found!'.format(shop_slug))

amiibofindr/apps/shop/crawlers/amazon.py

@@ -0,0 +1,61 @@
+# coding: utf-8
+
+# py3
+from __future__ import unicode_literals
+
+# third party
+from amazon.api import AmazonAPI
+
+# django
+from django.conf import settings
+
+
+class AmazonBaseCrawler(object):
+    region = 'US'
+
+    def __init__(self):
+        self.amazon = AmazonAPI(
+            settings.AMAZON_ACCESS_KEY,
+            settings.AMAZON_SECRET_KEY,
+            settings.AMAZON_ASSOC_TAG,
+            region=self.region
+        )
+
+    def fetch_by_id(self, product_id):
+        product = self.amazon.lookup(ItemId=product_id)
+        price_and_currency = product.price_and_currency
+        amiibo_price = {
+            'shop_product_id': product_id,
+            'price': price_and_currency[0],
+            'currency': price_and_currency[1],
+        }
+
+        return amiibo_price
+
+
+class AmazonUSCrawler(AmazonBaseCrawler):
+    pass
+
+
+class AmazonESCrawler(AmazonBaseCrawler):
+    region = 'ES'
+
+
+class AmazonFRCrawler(AmazonBaseCrawler):
+    region = 'FR'
+
+
+class AmazonUKCrawler(AmazonBaseCrawler):
+    region = 'UK'
+
+
+class AmazonDECrawler(AmazonBaseCrawler):
+    region = 'DE'
+
+
+class AmazonITCrawler(AmazonBaseCrawler):
+    region = 'IT'
+
+
+class AmazonJPCrawler(AmazonBaseCrawler):
+    region = 'JP'

provision/hosts

@@ -1,8 +1,8 @@
-default ansible_ssh_host=127.0.0.1 ansible_ssh_user=vagrant ansible_ssh_port=2222
+vagrant ansible_ssh_host=127.0.0.1 ansible_ssh_user=vagrant ansible_ssh_port=2222
 web-1 ansible_ssh_host=46.101.172.158 ansible_ssh_user=root ansible_ssh_port=22
 
 [localdev]
-default
+vagrant
 
 [development]
 

requirements/base.txt

@@ -27,3 +27,6 @@ requests==2.7.0
 # Import / Export
 openpyxl==2.2.3
 django-import-export==0.2.7
+
+# Amazon
+python-amazon-simple-product-api==1.5.0