Jocaru/wallamanta
1
0
Fork 0
Code Issues 15 Packages Projects Releases Wiki Activity

Reduced steps for normal searches, kept 15 for initial one

Browse Source
This commit is contained in:
Joan
2023-08-09 12:03:39 +02:00
parent 4d0a329996
commit c05d58907c
1 changed files with 27 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
wallamanta/worker.py
View File

@@ -17,7 +17,7 @@ class Worker:
_stop = False _stop = False
def request(self, product_name, n_articles, latitude=constants.LATITUDE, longitude=constants.LONGITUDE, distance='0', condition='all', min_price=0, max_price=10000000, category="", subcategories=[]): def request(self, product_name, steps=15, latitude=constants.LATITUDE, longitude=constants.LONGITUDE, distance='0', condition='all', min_price=0, max_price=10000000, category="", subcategories=[]):
distance = str(int(distance) * 1000) distance = str(int(distance) * 1000)
url = (f"https://api.wallapop.com/api/v3/general/search?keywords={product_name}" url = (f"https://api.wallapop.com/api/v3/general/search?keywords={product_name}"
f"&order_by=newest&latitude={latitude}" f"&order_by=newest&latitude={latitude}"
@@ -41,18 +41,19 @@ class Worker:
search_objects = list() search_objects = list()
for step in range(15): for step in range(steps):
while True: tries = 5
for _ in range(tries):
helpers.random_wait() helpers.random_wait()
response = requests.get(url+f"&step={step+1}") response = requests.get(url+f"&step={step}")
search_objects = search_objects + response.json()['search_objects']
try: try:
if response.status_code == 200: if response.status_code == 200:
search_objects = search_objects + response.json()['search_objects']
break break
else: else:
logging.info(f"\'{product_name}\' -> Wallapop returned status {response.status_code}. Illegal parameters or Wallapop service is down. Retrying...") logging.info(f"\'{product_name}\' -> Wallapop returned status {response.status_code}. Illegal parameters or Wallapop service is down. Retrying...")
except Exception as e: except Exception as e:
logging.info("Exception: " + e) logging.info("Error while querying Wallapop, try #{_}: " + e)
time.sleep(3) time.sleep(3)
return search_objects return search_objects
@@ -61,60 +62,63 @@ class Worker:
logging.info(f"First run for {product['product_name']} for {walladb.get_user(product['telegram_user_id'])} ({walladb.get_user(product['telegram_user_id'])})") logging.info(f"First run for {product['product_name']} for {walladb.get_user(product['telegram_user_id'])} ({walladb.get_user(product['telegram_user_id'])})")
for i in range(5): for i in range(5):
helpers.random_wait() helpers.random_wait()
list = {} list = []
if not helpers.is_valid_request(product): if not helpers.is_valid_request(product):
return list return list
if product['category'] == '': if product['category'] == '':
articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], product['category']) articles = self.request(product['product_name'], 15, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], product['category'])
for article in articles: for article in articles:
list[article['id']] = 1 #list[article['id']] = 1
list.insert(0, article['id'])
else: else:
if '0' in product['category'].split(','): if '0' in product['category'].split(','):
articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price']) articles = self.request(product['product_name'], 15, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'])
for article in articles: for article in articles:
list[article['id']] = 1 #list[article['id']] = 1
list.insert(0, article['id'])
else: else:
for category in product['category'].split(','): for category in product['category'].split(','):
if product['subcategory'] == '' or not helpers.has_subcategory(category): if product['subcategory'] == '' or not helpers.has_subcategory(category):
articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category) articles = self.request(product['product_name'], 15, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category)
for article in articles: for article in articles:
list[article['id']] = 1 #list[article['id']] = 1
list.insert(0, article['id'])
else: else:
subcategories = [] subcategories = []
for subcategory in product['subcategory'].split(','): for subcategory in product['subcategory'].split(','):
if helpers.is_subcategory(category, subcategory): if helpers.is_subcategory(category, subcategory):
subcategories.append(subcategory) subcategories.append(subcategory)
articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category, subcategories) articles = self.request(product['product_name'], 15, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category, subcategories)
for article in articles: for article in articles:
list[article['id']] = 1 #list[article['id']] = 1
list.insert(0, article['id'])
return list return list
async def work(self, product, list): async def work(self, product, list):
helpers.random_wait() # Random wait to make requests separated in time in order to prevent API rate limit helpers.random_wait() # Random wait to make requests separated in time in order to prevent API rate limit
exec_times = [] exec_times = []
while True: while True:
#logging.info(f"List for {product['product_name']} length is: {len(list)}")
if not helpers.is_valid_request(product) or self._stop == True: if not helpers.is_valid_request(product) or self._stop == True:
logging.info(f"{product['product_name']} not valid anymore, exiting worker") logging.info(f"{product['product_name']} not valid anymore, exiting worker")
break # Exits and ends worker thread break # Exits and ends worker thread
start_time = time.time() start_time = time.time()
articles_list = [] articles_list = []
if product['category'] == '': if product['category'] == '':
articles_list.append(self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], product['category'])) articles_list.append(self.request(product['product_name'], 1, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price']))
else: else:
if '0' in product['category'].split(','): if '0' in product['category'].split(','):
articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price']) articles_list.append(self.request(product['product_name'], 1, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price']))
for article in articles:
list[article['id']] = 1
else: else:
for category in product['category'].split(','): for category in product['category'].split(','):
if product['subcategory'] == '' or not helpers.has_subcategory(category): if product['subcategory'] == '' or not helpers.has_subcategory(category):
articles_list.append(self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category)) articles_list.append(self.request(product['product_name'], 1, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category))
else: else:
subcategories = [] subcategories = []
for subcategory in product['subcategory'].split(','): for subcategory in product['subcategory'].split(','):
if helpers.is_subcategory(category, subcategory): if helpers.is_subcategory(category, subcategory):
subcategories.append(subcategory) subcategories.append(subcategory)
articles_list.append(self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category, subcategories)) articles_list.append(self.request(product['product_name'], 1, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category, subcategories))
for articles in articles_list: for articles in articles_list:
for article in articles: for article in articles:
if not article['id'] in list: if not article['id'] in list:
@@ -127,7 +131,8 @@ class Worker:
await helpers.send_article(article, product) await helpers.send_article(article, product)
await asyncio.sleep(1) await asyncio.sleep(1)
#time.sleep(1) # Avoid Telegram flood restriction #time.sleep(1) # Avoid Telegram flood restriction
list[article['id']] = 1 #list[article['id']] = 1
list.insert(0, article['id'])
except Exception as e: except Exception as e:
logging.info("---------- EXCEPTION -----------") logging.info("---------- EXCEPTION -----------")
logging.info(f"{product['product_name']} worker crashed. {e}") logging.info(f"{product['product_name']} worker crashed. {e}")