fmartingr
/
amiibofindr
mirror of https://github.com/fmartingr/amiibofindr.git
1
0
Fork 0
Code Issues Releases Wiki Activity

Crawl parsing error when is only one element

This commit is contained in:
Felipe Martín 2015-09-22 20:18:01 +02:00
parent 3e3d12c0c5
commit 0254532e31
1 changed files with 23 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
amiibofindr/apps/shop/crawlers/_amazon.py
View File

@ -26,11 +26,7 @@ class AmazonBaseCrawler(object):
region=self.region
)
def fetch_batch(self, product_ids):
result = []
for chunk_product_ids in chunks(product_ids, self.max_batch_lookup):
products = self.amazon.lookup(ItemId=','.join(chunk_product_ids))
for product in products:
def parse_product(self, product):
price_and_currency = product.price_and_currency
price = price_and_currency[0]
@ -39,14 +35,24 @@ class AmazonBaseCrawler(object):
if currency == 'JPY':
price = float(price)*100
print(self.region, product.asin, price, currency)
print(self.region, product.asin, price, currency, product.title)
result.append({
return {
'shop_product_id': product.asin,
'price': price,
'currency': currency,
'title': product.title,
})
}
def fetch_batch(self, product_ids):
result = []
for chunk_product_ids in chunks(product_ids, self.max_batch_lookup):
products = self.amazon.lookup(ItemId=','.join(chunk_product_ids))
try:
for product in products:
result.append(self.parse_product(product))
except TypeError:
result.append(self.parse_product(products))
sleep(1)
return result