diff --git a/wallamanta/worker.py b/wallamanta/worker.py index b2ab673..92ca4ac 100644 --- a/wallamanta/worker.py +++ b/wallamanta/worker.py @@ -4,6 +4,7 @@ import logging import helpers import walladb import constants +import sys # Enable logging logging.basicConfig( @@ -134,9 +135,13 @@ class Worker: logging.info("---------- EXCEPTION -----------") logging.info(f"{product['product_name']} worker crashed. {e}") logging.info(f"{product['product_name']}: Trying to parse {article['id']}: {article['title']} .\n") + if len(list) > 600: + del list[600:] + if len(exec_times) > 50: + del exec_times[50:] time.sleep(constants.SLEEP_TIME) - exec_times.append(time.time() - start_time) - logging.info(f"\'{product['product_name']}\' for {walladb.get_user(product['telegram_user_id'])} ({product['telegram_user_id']}) node-> last: {exec_times[-1]:.2f} max: {self.get_max_time(exec_times):.2f} avg: {self.get_average_time(exec_times):.2f}") + exec_times.insert(0, time.time() - start_time) + logging.info(f"\'{product['product_name']}\' for {walladb.get_user(product['telegram_user_id'])} ({product['telegram_user_id']}) node-> last: {exec_times[0]:.2f} max: {self.get_max_time(exec_times):.2f} avg: {self.get_average_time(exec_times):.2f} - Size of articles_list: {round(sys.getsizeof(articles_list)/1024, 2)}Kb. Size of list: {round(sys.getsizeof(list)/1024, 2)}Kb. Size of exec_times: {round(sys.getsizeof(exec_times)/1024, 2)}Kb") def has_excluded_words(self, title, description, excluded_words): if len(excluded_words) > 0: