Added database and deleted json. Renamed main file. Modified functions to work with database. And more things...

.gitignore

@@ -1 +1,2 @@
 .env
+/data/wallamanta.db

data/products.json

@@ -1 +0,0 @@
-[]

wallamanta/Dockerfile

@@ -6,4 +6,4 @@ RUN pip install -r /app/requirements.txt
 
 WORKDIR /app
 
-CMD [ "python", "/app/alert.py" ]
+CMD [ "python", "/app/wallamanta.py" ]

wallamanta/walladb.py

@@ -1,12 +1,32 @@
 import sqlite3
+import os
+import logging
+
+LATITUDE = os.getenv("LATITUDE")
+LONGITUDE = os.getenv("LONGITUDE")
 
 DB = "/app/data/wallamanta.db"
 
+# Enable logging
+logging.basicConfig(
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO
+)
+
+logger = logging.getLogger(__name__)
+
+def dict_factory(cursor, row):
+    d = {}
+    for idx, col in enumerate(cursor.description):
+        d[col[0]] = row[idx]
+    return d
+
 def setup_db():
     con = sqlite3.connect(DB)
     cur = con.cursor()
     cur.execute("CREATE TABLE IF NOT EXISTS users(telegram_user_id, active)")
-    cur.execute("CREATE TABLE IF NOT EXISTS products(product_name, distance, latitude, longitude, condition, min_price, max_price, title_keyword_exclude, exclude, telegram_user_id)")
+    cur.execute("CREATE TABLE IF NOT EXISTS products(product_name, distance, \
+                latitude, longitude, condition, min_price, max_price, \
+                title_exclude, title_description_exclude, telegram_user_id)")
     con.close()
 
 def is_user_valid(telegram_user_id):
@@ -18,6 +38,7 @@ def is_user_valid(telegram_user_id):
     return ret
 
 def add_valid_user(telegram_user_id):
+    found = False
     con = sqlite3.connect(DB)
     cur = con.cursor()
     res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id={telegram_user_id}")
@@ -27,7 +48,9 @@ def add_valid_user(telegram_user_id):
     else:
         cur.execute(f"UPDATE users SET active = True WHERE telegram_user_id={telegram_user_id}")
         con.commit()
+        found = True
     con.close()
+    return found
 
 def remove_valid_user(telegram_user_id):
     con = sqlite3.connect(DB)
@@ -46,28 +69,69 @@ def get_user_list():
     con.close()
     return ret
 
-def get_product(args, telegram_user_id):
+def get_product(product):
-    product_name = args[0]
+    product_name = product.get('product_name')
+    telegram_user_id = product.get('telegram_user_id')
     con = sqlite3.connect(DB)
-    con.row_factory = sqlite3.Row
+    con.row_factory = dict_factory
     cur = con.cursor()
-    res = cur.execute(f"SELECT * FROM products WHERE telegram_user_id={telegram_user_id} AND product_name='{product_name}'")
+    res = cur.execute(f"SELECT * FROM products WHERE telegram_user_id={telegram_user_id} \
+                      AND product_name='{product_name}'")
     ret = res.fetchone()
     con.close()
     return ret
 
-def add_product(args, telegram_user_id):
+def get_products_from_user(telegram_user_id):
-    title_exclude, title_description_exclude, latitude, longitude, distance = [], [], LATITUDE, LONGITUDE, "0"
+    con = sqlite3.connect(DB)
-    product_name, min_price, max_price = args[0:3]
+    con.row_factory = dict_factory
-    if len(args) > 3 and args[3]:
+    cur = con.cursor()
-        title_exclude = args[3].split(",")
+    res = cur.execute(f"SELECT * FROM products WHERE telegram_user_id={telegram_user_id}")
-    if len(args) > 4 and args[4]:
+    ret = res.fetchall()
-        title_description_exclude = args[4].split(",")
+    con.close()
-    if len(args) > 5 and args[5]:
+    return ret
-        latitude = args[5]
-    if len(args) > 6 and args[6]:
-        longitude = args[6]
-    if len(args) > 7 and args[7]:
-        distance = args[7]
 
+def get_all_products():
+    con = sqlite3.connect(DB)
+    con.row_factory = dict_factory
+    cur = con.cursor()
+    res = cur.execute(f"SELECT * FROM products")
+    ret = res.fetchall()
+    con.close()
+    return ret
 
+def add_product(product):
+    condition = 'all'
+    product_name = product.get('product_name')
+    distance = product.get('distance', 0)
+    latitude = product.get('latitude', LATITUDE)
+    longitude = product.get('longitude', LONGITUDE)
+    min_price = product.get('min_price')
+    max_price = product.get('max_price')
+    title_exclude = product.get('title_exclude', '')
+    title_description_exclude = product.get('title_description_exclude', '')
+    telegram_user_id = product.get('telegram_user_id')
+    params = (product_name, \
+                    distance, latitude, longitude, condition, min_price, \
+                    max_price, title_exclude, title_description_exclude, telegram_user_id)
+    logging.info(f"Trying to add: {product_name}, {title_exclude}, {title_description_exclude}, {telegram_user_id}")
+
+    con = sqlite3.connect(DB)
+    cur = con.cursor()
+    res = cur.execute(f"SELECT * FROM products WHERE telegram_user_id={product.get('telegram_user_id')} \
+                      AND product_name='{product.get('product_name')}'")
+    if res.fetchone() is None:
+        cur.execute("INSERT INTO products VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", params)
+        con.commit()
+    con.close()
+
+def remove_product(product):
+    removed = False
+    if get_product(product):
+        con = sqlite3.connect(DB)
+        cur = con.cursor()
+        res = cur.execute(f"DELETE FROM products WHERE telegram_user_id={product.get('telegram_user_id')} \
+                        AND product_name='{product.get('product_name')}'")
+        con.commit()
+        con.close()
+        removed = True
+    return removed

wallamanta/wallamanta.py

@@ -42,7 +42,8 @@ async def help_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> No
         await update.message.reply_markdown_v2(helpers.telegram_escape_characters(message))
 
 async def add_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
-    if walladb.is_user_valid(helpers.get_telegram_user_id(update)):
+    telegram_user_id = helpers.get_telegram_user_id(update)
+    if walladb.is_user_valid(telegram_user_id):
         message = """Tienes que pasar el número correcto de parámetros: `/add producto;precio_mínimo;precio_máximo,excluir_título(opcional, separado por comas);excluir_descripción_y_título(opciona, separado por comas);latitud(opcional);longitud(opcional),distancia(opcional)`\n
     Ejemplo: `/add placa base itx;0;150`\n
     Ejemplo 2: `/add cpu;10;30;;intel,core 2 duo,celeron;;;100`\n
@@ -52,45 +53,57 @@ async def add_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> Non
         if len(args) == 1:
             pass
         elif len(args[1].split(";")) > 2:
+            product = dict()
+            product['telegram_user_id'] = telegram_user_id
             args = args[1].split(";")
-            logging.info(f'Adding: {args}')
+            product['product_name'], product['min_price'], product['max_price'] = args[0:3]
-            if not walladb.get_product(args, helpers.get_telegram_user_id(update)):
+            if len(args) > 3 and args[3]:
-                walladb.add_product(args, helpers.get_telegram_user_id(update))
+                product['title_exclude'] = args[3]
-                p = threading.Thread(target=Worker.run, args=(walladb.get_product(args, helpers.get_telegram_user_id(update)), ))
+            if len(args) > 4 and args[4]:
+                product['title_description_exclude'] = args[4]
+            if len(args) > 5 and args[5]:
+                product['latitude'] = args[5]
+            if len(args) > 6 and args[6]:
+                product['longitude'] = args[6]
+            if len(args) > 7 and args[7]:
+                product['distance'] = args[7]
+            
+            logging.info(f'Adding: {product}')
+            if not walladb.get_product(product):
+                walladb.add_product(product)
+                p = threading.Thread(target=Worker.run, args=(walladb.get_product(product), ))
                 p.start()
-                message = f"Añadido {walladb.get_product(args, helpers.get_telegram_user_id(update))['product_name']} a seguimiento"
+                message = f"Añadido {walladb.get_product(product)['product_name']} a seguimiento"
             else:
-                message = f"{walladb.get_product(args, helpers.get_telegram_user_id(update))['product_name']} ya está en seguimiento!"
+                message = f"{walladb.get_product(product)['product_name']} ya está en seguimiento!"
         await update.message.reply_markdown_v2(helpers.telegram_escape_characters(message))
 
 async def remove_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
-    if walladb.is_user_valid(helpers.get_telegram_user_id(update)):
+    telegram_user_id = helpers.get_telegram_user_id(update)
+    if walladb.is_user_valid(telegram_user_id):
         product_to_remove = update.message.text[len('/remove '):]
         message = f"{product_to_remove} no está en seguimiento!"
-        products = parse_json_file()
+        if walladb.remove_product({'product_name' : product_to_remove, \
-        for product in products:
+                                'telegram_user_id' : telegram_user_id}):
-            if product['product_name'] == product_to_remove:
-                products.remove(product)
             message = f"{product_to_remove} borrado!"
-        save_json_file(products)
         await update.message.reply_text(message)
 
 async def list_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
-    if walladb.is_user_valid(helpers.get_telegram_user_id(update)):
+    telegram_user_id = helpers.get_telegram_user_id(update)
-        products = parse_json_file()
+    if walladb.is_user_valid(telegram_user_id):
+        products = walladb.get_products_from_user(telegram_user_id)
 
         args = update.message.text.split("/list ")
         found = False
         if len(args) > 1:
+            product = walladb.get_product({'product_name':args[1],'telegram_user_id':telegram_user_id})
+            if product:
                 table = prettytable.PrettyTable(['Campo', 'Valor'])
                 table.align['Campo'] = 'l'
                 table.align['Valor'] = 'r'
-            for product in products:
-                if product['product_name'] == args[1]:
                 for key in product:
                     table.add_row([key, product[key]])
             found = True
-                    break
         if not found:
             table = prettytable.PrettyTable(['Producto', 'Mín', 'Máx'])
             table.align['Producto'] = 'l'
@@ -107,7 +120,15 @@ async def admin_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> N
 async def add_user_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
     if helpers.is_user_admin(update.message.chat_id):
         telegram_user_id = update.message.text.split('/add_user ')[1]
-        walladb.add_valid_user(telegram_user_id)
+        if walladb.add_valid_user(telegram_user_id):
+            products = walladb.get_products_from_user(telegram_user_id)
+
+            for product in products:
+                logging.info(product)
+                p = threading.Thread(target=Worker.run, args=(product, ))
+                p.start()
+            await update.message.reply_markdown_v2(helpers.telegram_escape_characters(f"{telegram_user_id} re-activado. Re-activando productos."))
+        else:
             await update.message.reply_markdown_v2(helpers.telegram_escape_characters(f"{telegram_user_id} añadido."))
 
 async def remove_user_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
@@ -118,7 +139,7 @@ async def remove_user_command(update: Update, context: ContextTypes.DEFAULT_TYPE
 
 def main()->None:
     walladb.setup_db()
-    products = parse_json_file()
+    products = walladb.get_all_products()
 
     for product in products:
         logging.info(product)

wallamanta/worker.py

@@ -5,6 +5,7 @@ import os
 import logging
 import json
 import helpers
+import walladb
 
 TELEGRAM_CHANNEL_ID = os.getenv("TELEGRAM_CHANNEL_ID") 
 TELEGRAM_TOKEN = os.getenv("TELEGRAM_TOKEN")
@@ -34,8 +35,9 @@ class Worker:
         if condition != "all":
             url = url + f"&condition={condition}" # new, as_good_as_new, good, fair, has_given_it_all
 
+        for step in range(15):
             while True:
-            response = requests.get(url)
+                response = requests.get(url+ f"&step={step+1}")
                 try:
                     if response.status_code == 200:
                         break
@@ -59,22 +61,17 @@ class Worker:
         exec_times = []
         bot = telegram.Bot(token = TELEGRAM_TOKEN)
         while True:
-            f = open("/app/data/products.json")
+            if not walladb.get_product(product):
-            products = json.load(f)
+                break # Exits worker if product not in DB anymore
-            found = False
+            if not walladb.is_user_valid(product['telegram_user_id']):
-            for fproduct in products:
+                break # Exits worker if user is not active anymore
-                if fproduct['product_name'] == product['product_name']:
-                    found = True
-                    break
-            if not found:
-                break # Exits worker if product not in list anymore
             start_time = time.time()
             articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'])
             for article in articles:
                 if not article['id'] in list:
                     logging.info(f"Found article {article['title']}")
                     try:
-                        if not self.has_excluded_words(article['title'].lower(), article['description'].lower(), product['exclude']) and not self.is_title_key_word_excluded(article['title'].lower(), product['title_keyword_exclude']):
+                        if not self.has_excluded_words(article['title'].lower(), article['description'].lower(), product['title_description_exclude']) and not self.is_title_key_word_excluded(article['title'].lower(), product['title_exclude']):
                             try:
                                 text =  f"*Artículo*: {helpers.telegram_escape_characters(article['title'])}\n*Descripción*: {helpers.telegram_escape_characters(article['description'])}\n*Precio*: {helpers.telegram_escape_characters(str(article['price']))} {helpers.telegram_escape_characters(article['currency'])}\n[Ir al anuncio](https://es\.wallapop\.com/item/{helpers.telegram_escape_characters(article['web_slug'])})"
                                 url = f"https://api.telegram.org/bot{TELEGRAM_TOKEN}/sendMessage?chat_id={TELEGRAM_CHANNEL_ID}&text={text}&parse_mode=MarkdownV2"
@@ -97,7 +94,8 @@ class Worker:
             logging.info(f"\'{product['product_name']}\' node-> last: {exec_times[-1]} max: {self.get_max_time(exec_times)} avg: {self.get_average_time(exec_times)}")
 
     def has_excluded_words(self, title, description, excluded_words):
-        for word in excluded_words:
+        if len(excluded_words) > 0:
+            for word in excluded_words.split(","):
                 logging.info("EXCLUDER: Checking '" + word + "' for title: '" + title)
                 if word in title or word in description:
                     logging.info("EXCLUDE!")
@@ -105,7 +103,8 @@ class Worker:
         return False
     
     def is_title_key_word_excluded(self, title, excluded_words):
-        for word in excluded_words:
+        if len(excluded_words) > 0:
+            for word in excluded_words.split(","):
                 logging.info("Checking '" + word + "' for title: '" + title)
                 if word in title:
                     return True