From c05d58907ce27ea7c56912e8a3e02e1a51a37c98 Mon Sep 17 00:00:00 2001 From: Joan Date: Wed, 9 Aug 2023 12:03:39 +0200 Subject: [PATCH] Reduced steps for normal searches, kept 15 for initial one --- wallamanta/worker.py | 49 ++++++++++++++++++++++++-------------------- 1 file changed, 27 insertions(+), 22 deletions(-) diff --git a/wallamanta/worker.py b/wallamanta/worker.py index 00798ea..e1479bf 100644 --- a/wallamanta/worker.py +++ b/wallamanta/worker.py @@ -17,7 +17,7 @@ class Worker: _stop = False - def request(self, product_name, n_articles, latitude=constants.LATITUDE, longitude=constants.LONGITUDE, distance='0', condition='all', min_price=0, max_price=10000000, category="", subcategories=[]): + def request(self, product_name, steps=15, latitude=constants.LATITUDE, longitude=constants.LONGITUDE, distance='0', condition='all', min_price=0, max_price=10000000, category="", subcategories=[]): distance = str(int(distance) * 1000) url = (f"https://api.wallapop.com/api/v3/general/search?keywords={product_name}" f"&order_by=newest&latitude={latitude}" @@ -41,18 +41,19 @@ class Worker: search_objects = list() - for step in range(15): - while True: + for step in range(steps): + tries = 5 + for _ in range(tries): helpers.random_wait() - response = requests.get(url+f"&step={step+1}") - search_objects = search_objects + response.json()['search_objects'] + response = requests.get(url+f"&step={step}") try: if response.status_code == 200: + search_objects = search_objects + response.json()['search_objects'] break else: logging.info(f"\'{product_name}\' -> Wallapop returned status {response.status_code}. Illegal parameters or Wallapop service is down. Retrying...") except Exception as e: - logging.info("Exception: " + e) + logging.info("Error while querying Wallapop, try #{_}: " + e) time.sleep(3) return search_objects @@ -61,60 +62,63 @@ class Worker: logging.info(f"First run for {product['product_name']} for {walladb.get_user(product['telegram_user_id'])} ({walladb.get_user(product['telegram_user_id'])})") for i in range(5): helpers.random_wait() - list = {} + list = [] if not helpers.is_valid_request(product): return list if product['category'] == '': - articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], product['category']) + articles = self.request(product['product_name'], 15, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], product['category']) for article in articles: - list[article['id']] = 1 + #list[article['id']] = 1 + list.insert(0, article['id']) else: if '0' in product['category'].split(','): - articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price']) + articles = self.request(product['product_name'], 15, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price']) for article in articles: - list[article['id']] = 1 + #list[article['id']] = 1 + list.insert(0, article['id']) else: for category in product['category'].split(','): if product['subcategory'] == '' or not helpers.has_subcategory(category): - articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category) + articles = self.request(product['product_name'], 15, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category) for article in articles: - list[article['id']] = 1 + #list[article['id']] = 1 + list.insert(0, article['id']) else: subcategories = [] for subcategory in product['subcategory'].split(','): if helpers.is_subcategory(category, subcategory): subcategories.append(subcategory) - articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category, subcategories) + articles = self.request(product['product_name'], 15, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category, subcategories) for article in articles: - list[article['id']] = 1 + #list[article['id']] = 1 + list.insert(0, article['id']) return list async def work(self, product, list): helpers.random_wait() # Random wait to make requests separated in time in order to prevent API rate limit exec_times = [] while True: + #logging.info(f"List for {product['product_name']} length is: {len(list)}") if not helpers.is_valid_request(product) or self._stop == True: logging.info(f"{product['product_name']} not valid anymore, exiting worker") break # Exits and ends worker thread start_time = time.time() articles_list = [] if product['category'] == '': - articles_list.append(self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], product['category'])) + articles_list.append(self.request(product['product_name'], 1, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'])) else: if '0' in product['category'].split(','): - articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price']) - for article in articles: - list[article['id']] = 1 + articles_list.append(self.request(product['product_name'], 1, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'])) else: for category in product['category'].split(','): if product['subcategory'] == '' or not helpers.has_subcategory(category): - articles_list.append(self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category)) + articles_list.append(self.request(product['product_name'], 1, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category)) else: subcategories = [] for subcategory in product['subcategory'].split(','): if helpers.is_subcategory(category, subcategory): subcategories.append(subcategory) - articles_list.append(self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category, subcategories)) + articles_list.append(self.request(product['product_name'], 1, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category, subcategories)) for articles in articles_list: for article in articles: if not article['id'] in list: @@ -127,7 +131,8 @@ class Worker: await helpers.send_article(article, product) await asyncio.sleep(1) #time.sleep(1) # Avoid Telegram flood restriction - list[article['id']] = 1 + #list[article['id']] = 1 + list.insert(0, article['id']) except Exception as e: logging.info("---------- EXCEPTION -----------") logging.info(f"{product['product_name']} worker crashed. {e}")