WIP crawlers
This commit is contained in:
parent
643755ae66
commit
ddedb0b199
|
@ -12,7 +12,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
|
||||||
web.vm.network "forwarded_port", guest: 8000, host: 8080
|
web.vm.network "forwarded_port", guest: 8000, host: 8080
|
||||||
|
|
||||||
web.vm.provider "virtualbox" do |vb|
|
web.vm.provider "virtualbox" do |vb|
|
||||||
vb.customize ["modifyvm", :id, "--memory", "512"]
|
vb.customize ["modifyvm", :id, "--memory", "1024"]
|
||||||
end
|
end
|
||||||
|
|
||||||
web.vm.provision "ansible" do |ansible|
|
web.vm.provision "ansible" do |ansible|
|
||||||
|
|
|
@ -0,0 +1,12 @@
|
||||||
|
# coding: utf-8
|
||||||
|
|
||||||
|
# python3
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
# django
|
||||||
|
from django.core.management.base import BaseCommand
|
||||||
|
|
||||||
|
|
||||||
|
class Command(BaseCommand):
|
||||||
|
def handle(self, *args, **kwargs):
|
||||||
|
print(args, kwargs)
|
|
@ -6,6 +6,9 @@ import os
|
||||||
# django
|
# django
|
||||||
from django.db import models
|
from django.db import models
|
||||||
|
|
||||||
|
# project
|
||||||
|
from amiibofindr.apps.shop.crawlers import Crawler
|
||||||
|
|
||||||
|
|
||||||
#
|
#
|
||||||
# Upload_to helpers
|
# Upload_to helpers
|
||||||
|
@ -124,6 +127,11 @@ class AmiiboPrice(models.Model):
|
||||||
self.price, self.currency
|
self.price, self.currency
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def fetch(self):
|
||||||
|
crawler = Crawler(self.amiibo_shop.shop.slug)
|
||||||
|
price = crawler.fetch_from_id(self.amiibo_shop.item_id)
|
||||||
|
return price
|
||||||
|
|
||||||
def save_history(self, old_price, new_price):
|
def save_history(self, old_price, new_price):
|
||||||
history = AmiiboPriceHistory(
|
history = AmiiboPriceHistory(
|
||||||
amiibo=self.amiibo,
|
amiibo=self.amiibo,
|
||||||
|
|
|
@ -0,0 +1,31 @@
|
||||||
|
# coding: utf-8
|
||||||
|
|
||||||
|
# py3
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
from .amazon import (
|
||||||
|
AmazonUSCrawler,
|
||||||
|
AmazonESCrawler,
|
||||||
|
AmazonUKCrawler,
|
||||||
|
AmazonITCrawler,
|
||||||
|
AmazonDECrawler,
|
||||||
|
AmazonJPCrawler,
|
||||||
|
AmazonFRCrawler,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
class Crawler(object):
|
||||||
|
crawler_classes = {
|
||||||
|
'amazon-us': AmazonUSCrawler,
|
||||||
|
'amazon-uk': AmazonUKCrawler,
|
||||||
|
'amazon-fr': AmazonFRCrawler,
|
||||||
|
'amazon-es': AmazonESCrawler,
|
||||||
|
'amazon-de': AmazonDECrawler,
|
||||||
|
'amazon-it': AmazonITCrawler,
|
||||||
|
'amazon-jp': AmazonJPCrawler,
|
||||||
|
}
|
||||||
|
|
||||||
|
def __new__(self, shop_slug):
|
||||||
|
if shop_slug in self.crawler_classes:
|
||||||
|
return self.crawler_classes[shop_slug]()
|
||||||
|
raise Exception('Shop slug {} not found!'.format(shop_slug))
|
|
@ -0,0 +1,61 @@
|
||||||
|
# coding: utf-8
|
||||||
|
|
||||||
|
# py3
|
||||||
|
from __future__ import unicode_literals
|
||||||
|
|
||||||
|
# third party
|
||||||
|
from amazon.api import AmazonAPI
|
||||||
|
|
||||||
|
# django
|
||||||
|
from django.conf import settings
|
||||||
|
|
||||||
|
|
||||||
|
class AmazonBaseCrawler(object):
|
||||||
|
region = 'US'
|
||||||
|
|
||||||
|
def __init__(self):
|
||||||
|
self.amazon = AmazonAPI(
|
||||||
|
settings.AMAZON_ACCESS_KEY,
|
||||||
|
settings.AMAZON_SECRET_KEY,
|
||||||
|
settings.AMAZON_ASSOC_TAG,
|
||||||
|
region=self.region
|
||||||
|
)
|
||||||
|
|
||||||
|
def fetch_by_id(self, product_id):
|
||||||
|
product = self.amazon.lookup(ItemId=product_id)
|
||||||
|
price_and_currency = product.price_and_currency
|
||||||
|
amiibo_price = {
|
||||||
|
'shop_product_id': product_id,
|
||||||
|
'price': price_and_currency[0],
|
||||||
|
'currency': price_and_currency[1],
|
||||||
|
}
|
||||||
|
|
||||||
|
return amiibo_price
|
||||||
|
|
||||||
|
|
||||||
|
class AmazonUSCrawler(AmazonBaseCrawler):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
class AmazonESCrawler(AmazonBaseCrawler):
|
||||||
|
region = 'ES'
|
||||||
|
|
||||||
|
|
||||||
|
class AmazonFRCrawler(AmazonBaseCrawler):
|
||||||
|
region = 'FR'
|
||||||
|
|
||||||
|
|
||||||
|
class AmazonUKCrawler(AmazonBaseCrawler):
|
||||||
|
region = 'UK'
|
||||||
|
|
||||||
|
|
||||||
|
class AmazonDECrawler(AmazonBaseCrawler):
|
||||||
|
region = 'DE'
|
||||||
|
|
||||||
|
|
||||||
|
class AmazonITCrawler(AmazonBaseCrawler):
|
||||||
|
region = 'IT'
|
||||||
|
|
||||||
|
|
||||||
|
class AmazonJPCrawler(AmazonBaseCrawler):
|
||||||
|
region = 'JP'
|
|
@ -1,8 +1,8 @@
|
||||||
default ansible_ssh_host=127.0.0.1 ansible_ssh_user=vagrant ansible_ssh_port=2222
|
vagrant ansible_ssh_host=127.0.0.1 ansible_ssh_user=vagrant ansible_ssh_port=2222
|
||||||
web-1 ansible_ssh_host=46.101.172.158 ansible_ssh_user=root ansible_ssh_port=22
|
web-1 ansible_ssh_host=46.101.172.158 ansible_ssh_user=root ansible_ssh_port=22
|
||||||
|
|
||||||
[localdev]
|
[localdev]
|
||||||
default
|
vagrant
|
||||||
|
|
||||||
[development]
|
[development]
|
||||||
|
|
||||||
|
|
|
@ -27,3 +27,6 @@ requests==2.7.0
|
||||||
# Import / Export
|
# Import / Export
|
||||||
openpyxl==2.2.3
|
openpyxl==2.2.3
|
||||||
django-import-export==0.2.7
|
django-import-export==0.2.7
|
||||||
|
|
||||||
|
# Amazon
|
||||||
|
python-amazon-simple-product-api==1.5.0
|
||||||
|
|
Loading…
Reference in New Issue