Jocaru/wallamanta
1
0
Fork 0
Code Issues 15 Packages Projects Releases Wiki Activity

Reduced steps for normal searches, kept 15 for initial one

Browse Source
This commit is contained in:
Joan
2023-08-09 12:03:39 +02:00
parent 4d0a329996
commit c05d58907c
1 changed files with 27 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

49
wallamanta/worker.py
View File

@@ -17,7 +17,7 @@ class Worker:
_stop = False
def request(self, product_name, n_articles, latitude=constants.LATITUDE, longitude=constants.LONGITUDE, distance='0', condition='all', min_price=0, max_price=10000000, category="", subcategories=[]):
def request(self, product_name, steps=15, latitude=constants.LATITUDE, longitude=constants.LONGITUDE, distance='0', condition='all', min_price=0, max_price=10000000, category="", subcategories=[]):
distance = str(int(distance) * 1000)
url = (f"https://api.wallapop.com/api/v3/general/search?keywords={product_name}"
f"&order_by=newest&latitude={latitude}"
@@ -41,18 +41,19 @@ class Worker:
search_objects = list()
for step in range(15):
while True:
for step in range(steps):
tries = 5
for _ in range(tries):
helpers.random_wait()
response = requests.get(url+f"&step={step+1}")
search_objects = search_objects + response.json()['search_objects']
response = requests.get(url+f"&step={step}")
try:
if response.status_code == 200:
search_objects = search_objects + response.json()['search_objects']
break
else:
logging.info(f"\'{product_name}\' -> Wallapop returned status {response.status_code}. Illegal parameters or Wallapop service is down. Retrying...")
except Exception as e:
logging.info("Exception: " + e)
logging.info("Error while querying Wallapop, try #{_}: " + e)
time.sleep(3)
return search_objects
@@ -61,60 +62,63 @@ class Worker:
logging.info(f"First run for {product['product_name']} for {walladb.get_user(product['telegram_user_id'])} ({walladb.get_user(product['telegram_user_id'])})")
for i in range(5):
helpers.random_wait()
list = {}
list = []
if not helpers.is_valid_request(product):
return list
if product['category'] == '':
articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], product['category'])
articles = self.request(product['product_name'], 15, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], product['category'])
for article in articles:
list[article['id']] = 1
#list[article['id']] = 1
list.insert(0, article['id'])
else:
if '0' in product['category'].split(','):
articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'])
articles = self.request(product['product_name'], 15, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'])
for article in articles:
list[article['id']] = 1
#list[article['id']] = 1
list.insert(0, article['id'])
else:
for category in product['category'].split(','):
if product['subcategory'] == '' or not helpers.has_subcategory(category):
articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category)
articles = self.request(product['product_name'], 15, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category)
for article in articles:
list[article['id']] = 1
#list[article['id']] = 1
list.insert(0, article['id'])
else:
subcategories = []
for subcategory in product['subcategory'].split(','):
if helpers.is_subcategory(category, subcategory):
subcategories.append(subcategory)
articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category, subcategories)
articles = self.request(product['product_name'], 15, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category, subcategories)
for article in articles:
list[article['id']] = 1
#list[article['id']] = 1
list.insert(0, article['id'])
return list
async def work(self, product, list):
helpers.random_wait() # Random wait to make requests separated in time in order to prevent API rate limit
exec_times = []
while True:
#logging.info(f"List for {product['product_name']} length is: {len(list)}")
if not helpers.is_valid_request(product) or self._stop == True:
logging.info(f"{product['product_name']} not valid anymore, exiting worker")
break # Exits and ends worker thread
start_time = time.time()
articles_list = []
if product['category'] == '':
articles_list.append(self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], product['category']))
articles_list.append(self.request(product['product_name'], 1, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price']))
else:
if '0' in product['category'].split(','):
articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'])
for article in articles:
list[article['id']] = 1
articles_list.append(self.request(product['product_name'], 1, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price']))
else:
for category in product['category'].split(','):
if product['subcategory'] == '' or not helpers.has_subcategory(category):
articles_list.append(self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category))
articles_list.append(self.request(product['product_name'], 1, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category))
else:
subcategories = []
for subcategory in product['subcategory'].split(','):
if helpers.is_subcategory(category, subcategory):
subcategories.append(subcategory)
articles_list.append(self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category, subcategories))
articles_list.append(self.request(product['product_name'], 1, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category, subcategories))
for articles in articles_list:
for article in articles:
if not article['id'] in list:
@@ -127,7 +131,8 @@ class Worker:
await helpers.send_article(article, product)
await asyncio.sleep(1)
#time.sleep(1) # Avoid Telegram flood restriction
list[article['id']] = 1
#list[article['id']] = 1
list.insert(0, article['id'])
except Exception as e:
logging.info("---------- EXCEPTION -----------")
logging.info(f"{product['product_name']} worker crashed. {e}")