From 17e9cd05ec14e58a30b1c82a3b6e3f61ea35c60a Mon Sep 17 00:00:00 2001 From: Joan Cano Date: Mon, 6 Mar 2023 00:22:07 +0100 Subject: [PATCH] Added database and deleted json. Renamed main file. Modified functions to work with database. And more things... --- .gitignore | 3 +- data/products.json | 1 - wallamanta/Dockerfile | 2 +- wallamanta/walladb.py | 102 ++++++++++++++++++++----- wallamanta/{alert.py => wallamanta.py} | 77 ++++++++++++------- wallamanta/worker.py | 57 +++++++------- 6 files changed, 163 insertions(+), 79 deletions(-) delete mode 100644 data/products.json rename wallamanta/{alert.py => wallamanta.py} (67%) diff --git a/.gitignore b/.gitignore index 2eea525..1d2dcc0 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -.env \ No newline at end of file +.env +/data/wallamanta.db \ No newline at end of file diff --git a/data/products.json b/data/products.json deleted file mode 100644 index 0637a08..0000000 --- a/data/products.json +++ /dev/null @@ -1 +0,0 @@ -[] \ No newline at end of file diff --git a/wallamanta/Dockerfile b/wallamanta/Dockerfile index 2304443..f3ea6dd 100644 --- a/wallamanta/Dockerfile +++ b/wallamanta/Dockerfile @@ -6,4 +6,4 @@ RUN pip install -r /app/requirements.txt WORKDIR /app -CMD [ "python", "/app/alert.py" ] \ No newline at end of file +CMD [ "python", "/app/wallamanta.py" ] \ No newline at end of file diff --git a/wallamanta/walladb.py b/wallamanta/walladb.py index c80cf91..e536104 100644 --- a/wallamanta/walladb.py +++ b/wallamanta/walladb.py @@ -1,12 +1,32 @@ import sqlite3 +import os +import logging + +LATITUDE = os.getenv("LATITUDE") +LONGITUDE = os.getenv("LONGITUDE") DB = "/app/data/wallamanta.db" +# Enable logging +logging.basicConfig( + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO +) + +logger = logging.getLogger(__name__) + +def dict_factory(cursor, row): + d = {} + for idx, col in enumerate(cursor.description): + d[col[0]] = row[idx] + return d + def setup_db(): con = sqlite3.connect(DB) cur = con.cursor() cur.execute("CREATE TABLE IF NOT EXISTS users(telegram_user_id, active)") - cur.execute("CREATE TABLE IF NOT EXISTS products(product_name, distance, latitude, longitude, condition, min_price, max_price, title_keyword_exclude, exclude, telegram_user_id)") + cur.execute("CREATE TABLE IF NOT EXISTS products(product_name, distance, \ + latitude, longitude, condition, min_price, max_price, \ + title_exclude, title_description_exclude, telegram_user_id)") con.close() def is_user_valid(telegram_user_id): @@ -18,6 +38,7 @@ def is_user_valid(telegram_user_id): return ret def add_valid_user(telegram_user_id): + found = False con = sqlite3.connect(DB) cur = con.cursor() res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id={telegram_user_id}") @@ -27,7 +48,9 @@ def add_valid_user(telegram_user_id): else: cur.execute(f"UPDATE users SET active = True WHERE telegram_user_id={telegram_user_id}") con.commit() + found = True con.close() + return found def remove_valid_user(telegram_user_id): con = sqlite3.connect(DB) @@ -46,28 +69,69 @@ def get_user_list(): con.close() return ret -def get_product(args, telegram_user_id): - product_name = args[0] +def get_product(product): + product_name = product.get('product_name') + telegram_user_id = product.get('telegram_user_id') con = sqlite3.connect(DB) - con.row_factory = sqlite3.Row + con.row_factory = dict_factory cur = con.cursor() - res = cur.execute(f"SELECT * FROM products WHERE telegram_user_id={telegram_user_id} AND product_name='{product_name}'") + res = cur.execute(f"SELECT * FROM products WHERE telegram_user_id={telegram_user_id} \ + AND product_name='{product_name}'") ret = res.fetchone() con.close() return ret -def add_product(args, telegram_user_id): - title_exclude, title_description_exclude, latitude, longitude, distance = [], [], LATITUDE, LONGITUDE, "0" - product_name, min_price, max_price = args[0:3] - if len(args) > 3 and args[3]: - title_exclude = args[3].split(",") - if len(args) > 4 and args[4]: - title_description_exclude = args[4].split(",") - if len(args) > 5 and args[5]: - latitude = args[5] - if len(args) > 6 and args[6]: - longitude = args[6] - if len(args) > 7 and args[7]: - distance = args[7] +def get_products_from_user(telegram_user_id): + con = sqlite3.connect(DB) + con.row_factory = dict_factory + cur = con.cursor() + res = cur.execute(f"SELECT * FROM products WHERE telegram_user_id={telegram_user_id}") + ret = res.fetchall() + con.close() + return ret - +def get_all_products(): + con = sqlite3.connect(DB) + con.row_factory = dict_factory + cur = con.cursor() + res = cur.execute(f"SELECT * FROM products") + ret = res.fetchall() + con.close() + return ret + +def add_product(product): + condition = 'all' + product_name = product.get('product_name') + distance = product.get('distance', 0) + latitude = product.get('latitude', LATITUDE) + longitude = product.get('longitude', LONGITUDE) + min_price = product.get('min_price') + max_price = product.get('max_price') + title_exclude = product.get('title_exclude', '') + title_description_exclude = product.get('title_description_exclude', '') + telegram_user_id = product.get('telegram_user_id') + params = (product_name, \ + distance, latitude, longitude, condition, min_price, \ + max_price, title_exclude, title_description_exclude, telegram_user_id) + logging.info(f"Trying to add: {product_name}, {title_exclude}, {title_description_exclude}, {telegram_user_id}") + + con = sqlite3.connect(DB) + cur = con.cursor() + res = cur.execute(f"SELECT * FROM products WHERE telegram_user_id={product.get('telegram_user_id')} \ + AND product_name='{product.get('product_name')}'") + if res.fetchone() is None: + cur.execute("INSERT INTO products VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", params) + con.commit() + con.close() + +def remove_product(product): + removed = False + if get_product(product): + con = sqlite3.connect(DB) + cur = con.cursor() + res = cur.execute(f"DELETE FROM products WHERE telegram_user_id={product.get('telegram_user_id')} \ + AND product_name='{product.get('product_name')}'") + con.commit() + con.close() + removed = True + return removed diff --git a/wallamanta/alert.py b/wallamanta/wallamanta.py similarity index 67% rename from wallamanta/alert.py rename to wallamanta/wallamanta.py index f87a0c4..e612403 100644 --- a/wallamanta/alert.py +++ b/wallamanta/wallamanta.py @@ -42,7 +42,8 @@ async def help_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> No await update.message.reply_markdown_v2(helpers.telegram_escape_characters(message)) async def add_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: - if walladb.is_user_valid(helpers.get_telegram_user_id(update)): + telegram_user_id = helpers.get_telegram_user_id(update) + if walladb.is_user_valid(telegram_user_id): message = """Tienes que pasar el número correcto de parámetros: `/add producto;precio_mínimo;precio_máximo,excluir_título(opcional, separado por comas);excluir_descripción_y_título(opciona, separado por comas);latitud(opcional);longitud(opcional),distancia(opcional)`\n Ejemplo: `/add placa base itx;0;150`\n Ejemplo 2: `/add cpu;10;30;;intel,core 2 duo,celeron;;;100`\n @@ -52,45 +53,57 @@ async def add_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> Non if len(args) == 1: pass elif len(args[1].split(";")) > 2: + product = dict() + product['telegram_user_id'] = telegram_user_id args = args[1].split(";") - logging.info(f'Adding: {args}') - if not walladb.get_product(args, helpers.get_telegram_user_id(update)): - walladb.add_product(args, helpers.get_telegram_user_id(update)) - p = threading.Thread(target=Worker.run, args=(walladb.get_product(args, helpers.get_telegram_user_id(update)), )) + product['product_name'], product['min_price'], product['max_price'] = args[0:3] + if len(args) > 3 and args[3]: + product['title_exclude'] = args[3] + if len(args) > 4 and args[4]: + product['title_description_exclude'] = args[4] + if len(args) > 5 and args[5]: + product['latitude'] = args[5] + if len(args) > 6 and args[6]: + product['longitude'] = args[6] + if len(args) > 7 and args[7]: + product['distance'] = args[7] + + logging.info(f'Adding: {product}') + if not walladb.get_product(product): + walladb.add_product(product) + p = threading.Thread(target=Worker.run, args=(walladb.get_product(product), )) p.start() - message = f"Añadido {walladb.get_product(args, helpers.get_telegram_user_id(update))['product_name']} a seguimiento" + message = f"Añadido {walladb.get_product(product)['product_name']} a seguimiento" else: - message = f"{walladb.get_product(args, helpers.get_telegram_user_id(update))['product_name']} ya está en seguimiento!" + message = f"{walladb.get_product(product)['product_name']} ya está en seguimiento!" await update.message.reply_markdown_v2(helpers.telegram_escape_characters(message)) async def remove_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: - if walladb.is_user_valid(helpers.get_telegram_user_id(update)): + telegram_user_id = helpers.get_telegram_user_id(update) + if walladb.is_user_valid(telegram_user_id): product_to_remove = update.message.text[len('/remove '):] message = f"{product_to_remove} no está en seguimiento!" - products = parse_json_file() - for product in products: - if product['product_name'] == product_to_remove: - products.remove(product) - message = f"{product_to_remove} borrado!" - save_json_file(products) + if walladb.remove_product({'product_name' : product_to_remove, \ + 'telegram_user_id' : telegram_user_id}): + message = f"{product_to_remove} borrado!" await update.message.reply_text(message) async def list_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: - if walladb.is_user_valid(helpers.get_telegram_user_id(update)): - products = parse_json_file() + telegram_user_id = helpers.get_telegram_user_id(update) + if walladb.is_user_valid(telegram_user_id): + products = walladb.get_products_from_user(telegram_user_id) args = update.message.text.split("/list ") found = False if len(args) > 1: - table = prettytable.PrettyTable(['Campo', 'Valor']) - table.align['Campo'] = 'l' - table.align['Valor'] = 'r' - for product in products: - if product['product_name'] == args[1]: - for key in product: - table.add_row([key, product[key]]) - found = True - break + product = walladb.get_product({'product_name':args[1],'telegram_user_id':telegram_user_id}) + if product: + table = prettytable.PrettyTable(['Campo', 'Valor']) + table.align['Campo'] = 'l' + table.align['Valor'] = 'r' + for key in product: + table.add_row([key, product[key]]) + found = True if not found: table = prettytable.PrettyTable(['Producto', 'Mín', 'Máx']) table.align['Producto'] = 'l' @@ -107,8 +120,16 @@ async def admin_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> N async def add_user_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: if helpers.is_user_admin(update.message.chat_id): telegram_user_id = update.message.text.split('/add_user ')[1] - walladb.add_valid_user(telegram_user_id) - await update.message.reply_markdown_v2(helpers.telegram_escape_characters(f"{telegram_user_id} añadido.")) + if walladb.add_valid_user(telegram_user_id): + products = walladb.get_products_from_user(telegram_user_id) + + for product in products: + logging.info(product) + p = threading.Thread(target=Worker.run, args=(product, )) + p.start() + await update.message.reply_markdown_v2(helpers.telegram_escape_characters(f"{telegram_user_id} re-activado. Re-activando productos.")) + else: + await update.message.reply_markdown_v2(helpers.telegram_escape_characters(f"{telegram_user_id} añadido.")) async def remove_user_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: if helpers.is_user_admin(update.message.chat_id): @@ -118,7 +139,7 @@ async def remove_user_command(update: Update, context: ContextTypes.DEFAULT_TYPE def main()->None: walladb.setup_db() - products = parse_json_file() + products = walladb.get_all_products() for product in products: logging.info(product) diff --git a/wallamanta/worker.py b/wallamanta/worker.py index aa4d50f..3257c96 100644 --- a/wallamanta/worker.py +++ b/wallamanta/worker.py @@ -5,6 +5,7 @@ import os import logging import json import helpers +import walladb TELEGRAM_CHANNEL_ID = os.getenv("TELEGRAM_CHANNEL_ID") TELEGRAM_TOKEN = os.getenv("TELEGRAM_TOKEN") @@ -34,16 +35,17 @@ class Worker: if condition != "all": url = url + f"&condition={condition}" # new, as_good_as_new, good, fair, has_given_it_all - while True: - response = requests.get(url) - try: - if response.status_code == 200: - break - else: - logging.info(f"\'{product_name}\' -> Wallapop returned status {response.status_code}. Illegal parameters or Wallapop service is down. Retrying...") - except Exception as e: - logging.info("Exception: " + e) - time.sleep(3) + for step in range(15): + while True: + response = requests.get(url+ f"&step={step+1}") + try: + if response.status_code == 200: + break + else: + logging.info(f"\'{product_name}\' -> Wallapop returned status {response.status_code}. Illegal parameters or Wallapop service is down. Retrying...") + except Exception as e: + logging.info("Exception: " + e) + time.sleep(3) json_data = response.json() return json_data['search_objects'] @@ -59,22 +61,17 @@ class Worker: exec_times = [] bot = telegram.Bot(token = TELEGRAM_TOKEN) while True: - f = open("/app/data/products.json") - products = json.load(f) - found = False - for fproduct in products: - if fproduct['product_name'] == product['product_name']: - found = True - break - if not found: - break # Exits worker if product not in list anymore + if not walladb.get_product(product): + break # Exits worker if product not in DB anymore + if not walladb.is_user_valid(product['telegram_user_id']): + break # Exits worker if user is not active anymore start_time = time.time() articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price']) for article in articles: if not article['id'] in list: logging.info(f"Found article {article['title']}") try: - if not self.has_excluded_words(article['title'].lower(), article['description'].lower(), product['exclude']) and not self.is_title_key_word_excluded(article['title'].lower(), product['title_keyword_exclude']): + if not self.has_excluded_words(article['title'].lower(), article['description'].lower(), product['title_description_exclude']) and not self.is_title_key_word_excluded(article['title'].lower(), product['title_exclude']): try: text = f"*Artículo*: {helpers.telegram_escape_characters(article['title'])}\n*Descripción*: {helpers.telegram_escape_characters(article['description'])}\n*Precio*: {helpers.telegram_escape_characters(str(article['price']))} {helpers.telegram_escape_characters(article['currency'])}\n[Ir al anuncio](https://es\.wallapop\.com/item/{helpers.telegram_escape_characters(article['web_slug'])})" url = f"https://api.telegram.org/bot{TELEGRAM_TOKEN}/sendMessage?chat_id={TELEGRAM_CHANNEL_ID}&text={text}&parse_mode=MarkdownV2" @@ -97,18 +94,20 @@ class Worker: logging.info(f"\'{product['product_name']}\' node-> last: {exec_times[-1]} max: {self.get_max_time(exec_times)} avg: {self.get_average_time(exec_times)}") def has_excluded_words(self, title, description, excluded_words): - for word in excluded_words: - logging.info("EXCLUDER: Checking '" + word + "' for title: '" + title) - if word in title or word in description: - logging.info("EXCLUDE!") - return True + if len(excluded_words) > 0: + for word in excluded_words.split(","): + logging.info("EXCLUDER: Checking '" + word + "' for title: '" + title) + if word in title or word in description: + logging.info("EXCLUDE!") + return True return False def is_title_key_word_excluded(self, title, excluded_words): - for word in excluded_words: - logging.info("Checking '" + word + "' for title: '" + title) - if word in title: - return True + if len(excluded_words) > 0: + for word in excluded_words.split(","): + logging.info("Checking '" + word + "' for title: '" + title) + if word in title: + return True return False def get_average_time(self, exec_times):