Amazon crawlers working
This commit is contained in:
Felipe Martín
parent
295ffcfe7a
commit
4254284ef8
|
|
@ -5,6 +5,7 @@ from __future__ import unicode_literals
|
|||
|
||||
# python
|
||||
import copy
|
||||
from decimal import Decimal
|
||||
|
||||
# amiibo
|
||||
from .signals import amiibo_price_changed
|
||||
|
|
@ -12,7 +13,11 @@ from .signals import amiibo_price_changed
|
|||
|
||||
def save_historical_price(sender, instance, **kwargs):
|
||||
old_price = kwargs.get('old_price')
|
||||
if old_price:
|
||||
old_price = Decimal(old_price)
|
||||
new_price = kwargs.get('new_price')
|
||||
if new_price:
|
||||
new_price = Decimal(new_price)
|
||||
instance.save_history(old_price, new_price)
|
||||
|
||||
|
||||
|
|
@ -25,7 +30,7 @@ def post_check_price_change(sender, instance, created, **kwargs):
|
|||
amiibo_price_changed.send(
|
||||
sender=instance.__class__,
|
||||
instance=instance,
|
||||
amiibo=instance.amiibo,
|
||||
amiibo=instance.amiibo_shop.amiibo,
|
||||
old_price=instance.old_price,
|
||||
new_price=instance.price
|
||||
)
|
||||
|
|
|
|||
|
|
@ -8,9 +8,21 @@ from django.core.management.base import BaseCommand
|
|||
|
||||
# amiibo
|
||||
from amiibofindr.apps.amiibo.models import AmiiboShop
|
||||
from amiibofindr.apps.shop.models import Shop
|
||||
from amiibofindr.apps.shop.crawlers import Crawler
|
||||
|
||||
|
||||
class Command(BaseCommand):
|
||||
def handle(self, *args, **kwargs):
|
||||
for amiibo_shop in AmiiboShop.objects.order_by('shop__flag_code'):
|
||||
print(amiibo_shop.shop.flag_code, amiibo_shop.item_id, amiibo_shop.url)
|
||||
regions = Shop.objects.all().order_by('flag_code').distinct('flag_code').values_list('flag_code', 'slug')
|
||||
|
||||
for region in regions:
|
||||
item_codes = AmiiboShop.objects.filter(shop__flag_code=region[0]).values_list('item_id', flat=True)
|
||||
amazon = Crawler(region[1])
|
||||
products = amazon.fetch_batch(item_codes)
|
||||
for product in products:
|
||||
amiibo_shop = AmiiboShop.objects.get(
|
||||
item_id=product['shop_product_id'],
|
||||
shop__flag_code=region[0]
|
||||
)
|
||||
amiibo_shop.update_price(product['price'], product['currency'])
|
||||
|
|
|
|||
|
|
@ -0,0 +1,19 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from django.db import models, migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('amiibo', '0013_auto_20150623_0006'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='amiiboprice',
|
||||
name='price',
|
||||
field=models.DecimalField(null=True, max_digits=6, decimal_places=2),
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from django.db import models, migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('amiibo', '0014_auto_20150624_0049'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='amiibopricehistory',
|
||||
name='price',
|
||||
field=models.DecimalField(null=True, max_digits=6, decimal_places=2),
|
||||
),
|
||||
]
|
||||
|
|
@ -0,0 +1,19 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from django.db import models, migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('amiibo', '0015_auto_20150624_0050'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterField(
|
||||
model_name='amiiboprice',
|
||||
name='date',
|
||||
field=models.DateTimeField(auto_now=True),
|
||||
),
|
||||
]
|
||||
|
|
@ -4,8 +4,12 @@
|
|||
import os
|
||||
|
||||
# django
|
||||
from django.conf import settings
|
||||
from django.db import models
|
||||
|
||||
# 3rd party
|
||||
from amazonify import amazonify
|
||||
|
||||
# project
|
||||
from amiibofindr.apps.shop.crawlers import Crawler
|
||||
|
||||
|
|
@ -101,6 +105,21 @@ class AmiiboShop(models.Model):
|
|||
class Meta:
|
||||
ordering = ('shop__name', )
|
||||
|
||||
def get_url(self):
|
||||
return amazonify(self.url, settings.AMAZON_ASSOC_TAG)
|
||||
|
||||
def update_price(self, price, currency):
|
||||
price_obj, is_new = AmiiboPrice.objects.get_or_create(
|
||||
amiibo_shop_id=self.pk)
|
||||
|
||||
price_obj.price = price
|
||||
price_obj.stock = price is not None
|
||||
|
||||
if is_new and currency:
|
||||
price_obj.currency = currency
|
||||
|
||||
price_obj.save()
|
||||
|
||||
@property
|
||||
def last_price(self):
|
||||
return self.price_set.first()
|
||||
|
|
@ -111,10 +130,10 @@ class AmiiboShop(models.Model):
|
|||
|
||||
class AmiiboPrice(models.Model):
|
||||
amiibo_shop = models.ForeignKey(AmiiboShop, related_name='price_set')
|
||||
price = models.DecimalField(max_digits=6, decimal_places=2)
|
||||
price = models.DecimalField(max_digits=6, decimal_places=2, null=True)
|
||||
stock = models.BooleanField(default=False)
|
||||
currency = models.CharField(default='EUR', max_length=3)
|
||||
date = models.DateTimeField(auto_now_add=True)
|
||||
date = models.DateTimeField(auto_now=True)
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(AmiiboPrice, self).__init__(*args, **kwargs)
|
||||
|
|
@ -133,12 +152,16 @@ class AmiiboPrice(models.Model):
|
|||
return price
|
||||
|
||||
def save_history(self, old_price, new_price):
|
||||
if new_price is None or old_price is None:
|
||||
diff = 0
|
||||
else:
|
||||
diff = new_price-old_price
|
||||
|
||||
history = AmiiboPriceHistory(
|
||||
amiibo=self.amiibo,
|
||||
shop_id=self.shop_id,
|
||||
amiibo_shop_id=self.amiibo_shop_id,
|
||||
price=self.price,
|
||||
currency=self.currency,
|
||||
diff=new_price-old_price
|
||||
diff=diff
|
||||
)
|
||||
return history.save()
|
||||
|
||||
|
|
@ -147,7 +170,7 @@ class AmiiboPriceHistory(models.Model):
|
|||
amiibo_shop = models.ForeignKey(AmiiboShop,
|
||||
related_name='price_history_set')
|
||||
stock = models.BooleanField(default=False)
|
||||
price = models.DecimalField(max_digits=6, decimal_places=2)
|
||||
price = models.DecimalField(max_digits=6, decimal_places=2, null=True)
|
||||
currency = models.CharField(default='EUR', max_length=3)
|
||||
date = models.DateTimeField(auto_now_add=True)
|
||||
diff = models.DecimalField(max_digits=6, decimal_places=2)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,13 @@
|
|||
# coding: utf-8
|
||||
|
||||
# py3
|
||||
from __future__ import unicode_literals
|
||||
|
||||
|
||||
def chunks(l, n):
|
||||
"""
|
||||
Yield successive n-sized chunks from l.
|
||||
http://stackoverflow.com/a/312464
|
||||
"""
|
||||
for i in xrange(0, len(l), n):
|
||||
yield l[i:i+n]
|
||||
|
|
@ -1,7 +1,8 @@
|
|||
# coding: utf-8
|
||||
|
||||
# py3
|
||||
# py
|
||||
from __future__ import unicode_literals
|
||||
from time import sleep
|
||||
|
||||
# third party
|
||||
from amazon.api import AmazonAPI
|
||||
|
|
@ -9,9 +10,13 @@ from amazon.api import AmazonAPI
|
|||
# django
|
||||
from django.conf import settings
|
||||
|
||||
# amiibofindr
|
||||
from amiibofindr.apps.core.utils import chunks
|
||||
|
||||
|
||||
class AmazonBaseCrawler(object):
|
||||
region = 'US'
|
||||
max_batch_lookup = 10
|
||||
|
||||
def __init__(self):
|
||||
self.amazon = AmazonAPI(
|
||||
|
|
@ -21,7 +26,22 @@ class AmazonBaseCrawler(object):
|
|||
region=self.region
|
||||
)
|
||||
|
||||
def fetch_by_id(self, product_id):
|
||||
def fetch_batch(self, product_ids):
|
||||
result = []
|
||||
for chunk_product_ids in chunks(product_ids, self.max_batch_lookup):
|
||||
products = self.amazon.lookup(ItemId=','.join(chunk_product_ids))
|
||||
for product in products:
|
||||
price_and_currency = product.price_and_currency
|
||||
result.append({
|
||||
'shop_product_id': product.asin,
|
||||
'price': price_and_currency[0],
|
||||
'currency': price_and_currency[1],
|
||||
})
|
||||
sleep(1)
|
||||
|
||||
return result
|
||||
|
||||
def fetch_from_id(self, product_id):
|
||||
product = self.amazon.lookup(ItemId=product_id)
|
||||
price_and_currency = product.price_and_currency
|
||||
amiibo_price = {
|
||||
|
|
|
|||
|
|
@ -0,0 +1,18 @@
|
|||
# -*- coding: utf-8 -*-
|
||||
from __future__ import unicode_literals
|
||||
|
||||
from django.db import models, migrations
|
||||
|
||||
|
||||
class Migration(migrations.Migration):
|
||||
|
||||
dependencies = [
|
||||
('shop', '0002_shop_flag_code'),
|
||||
]
|
||||
|
||||
operations = [
|
||||
migrations.AlterModelOptions(
|
||||
name='shop',
|
||||
options={'ordering': ('name',)},
|
||||
),
|
||||
]
|
||||
|
|
@ -56,13 +56,21 @@
|
|||
<tbody>
|
||||
{% for relation in amiibo.shops_set.all %}
|
||||
<tr>
|
||||
<td><a href="{{ relation.url }}">{{ relation.shop.name }}</a></td>
|
||||
<td class="center aligned">{% if relation.shop.flag_code %}<i class="{{ relation.shop.flag_code }} flag"></i>{% endif %}</td>
|
||||
<td class="center aligned {{ relation.stock|yesno:'positive,negative' }}">
|
||||
<i class="icon {{ relation.stock|yesno:'checkmark,close' }}"></i>
|
||||
{{ relation.stock|yesno|capfirst }}
|
||||
{% with price=relation.last_price.price %}
|
||||
<td><a href="{{ relation.get_url }}">{{ relation.shop.name }}</a> <i class="info icon" title="{{ relation.last_price.date }}"></i></td>
|
||||
<td class="center aligned">{% if relation.shop.flag_code %}<i class="{% if relation.shop.flag_code == 'uk' %}gb{% else %}{{ relation.shop.flag_code }}{% endif %} flag"></i>{% endif %}</td>
|
||||
<td class="center aligned {{ price|yesno:'positive,negative' }}">
|
||||
<i class="icon {{ price|yesno:'checkmark,close' }}"></i>
|
||||
{{ price|yesno|capfirst }}
|
||||
</td>
|
||||
<td class="center aligned">{{ relation.last_price.price }}</td>
|
||||
<td class="center aligned">
|
||||
{% if price %}
|
||||
{{ price }} {{ relation.last_price.currency }}
|
||||
{% else %}
|
||||
--
|
||||
{% endif %}
|
||||
</td>
|
||||
{% endwith %}
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
|
|
|
|||
|
|
@ -29,4 +29,5 @@ openpyxl==2.2.3
|
|||
django-import-export==0.2.7
|
||||
|
||||
# Amazon
|
||||
amazonify==0.1 # Links with aff tag
|
||||
python-amazon-simple-product-api==1.5.0
|
||||
|
|
|
|||
Loading…
Reference in New Issue