Crawl parsing error when is only one element
This commit is contained in:
parent
3e3d12c0c5
commit
0254532e31
|
@ -26,27 +26,33 @@ class AmazonBaseCrawler(object):
|
||||||
region=self.region
|
region=self.region
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def parse_product(self, product):
|
||||||
|
price_and_currency = product.price_and_currency
|
||||||
|
|
||||||
|
price = price_and_currency[0]
|
||||||
|
currency = price_and_currency[1]
|
||||||
|
|
||||||
|
if currency == 'JPY':
|
||||||
|
price = float(price)*100
|
||||||
|
|
||||||
|
print(self.region, product.asin, price, currency, product.title)
|
||||||
|
|
||||||
|
return {
|
||||||
|
'shop_product_id': product.asin,
|
||||||
|
'price': price,
|
||||||
|
'currency': currency,
|
||||||
|
'title': product.title,
|
||||||
|
}
|
||||||
|
|
||||||
def fetch_batch(self, product_ids):
|
def fetch_batch(self, product_ids):
|
||||||
result = []
|
result = []
|
||||||
for chunk_product_ids in chunks(product_ids, self.max_batch_lookup):
|
for chunk_product_ids in chunks(product_ids, self.max_batch_lookup):
|
||||||
products = self.amazon.lookup(ItemId=','.join(chunk_product_ids))
|
products = self.amazon.lookup(ItemId=','.join(chunk_product_ids))
|
||||||
for product in products:
|
try:
|
||||||
price_and_currency = product.price_and_currency
|
for product in products:
|
||||||
|
result.append(self.parse_product(product))
|
||||||
price = price_and_currency[0]
|
except TypeError:
|
||||||
currency = price_and_currency[1]
|
result.append(self.parse_product(products))
|
||||||
|
|
||||||
if currency == 'JPY':
|
|
||||||
price = float(price)*100
|
|
||||||
|
|
||||||
print(self.region, product.asin, price, currency)
|
|
||||||
|
|
||||||
result.append({
|
|
||||||
'shop_product_id': product.asin,
|
|
||||||
'price': price,
|
|
||||||
'currency': currency,
|
|
||||||
'title': product.title,
|
|
||||||
})
|
|
||||||
sleep(1)
|
sleep(1)
|
||||||
|
|
||||||
return result
|
return result
|
||||||
|
|
Loading…
Reference in New Issue