Modified to be used in homelabs vip

docker-compose.yml

@@ -1,10 +1,10 @@
 version: "3"
 
 services:
-  wallamanta-bot-multiuser:
+  wallamanta_homelabs:
     build: wallamanta
-    image: wallamanta:multiuser
+    image: wallamanta:homelabs
-    container_name: wallamanta-bot-multiuser
+    container_name: wallamanta_homelabs
     volumes:
       - ./data:/app/data
     restart: unless-stopped
@@ -20,10 +20,12 @@ services:
       - DB_PASSWORD=${DB_PASSWORD}
       - NEW_RELIC_INSERT_KEY=${NEW_RELIC_INSERT_KEY}
       - NR_ENV=${NR_ENV}
+    networks:
+      - wallamanta_homelabs
 
-  wallamanta_db:
+  wallamanta_homelabs_db:
     image: mysql:8
-    container_name: wallamanta_db
+    container_name: wallamanta_homelabs_db
     volumes:
       - ./db:/var/lib/mysql
     environment:
@@ -32,10 +34,18 @@ services:
       - MYSQL_USER=${DB_USER}
       - MYSQL_PASSWORD=${DB_PASSWORD}
     restart: unless-stopped
+    networks:
+      - wallamanta_homelabs
 
-  wallamanta_adminer:
+  wallamanta_homelabs_adminer:
     image: adminer
-    container_name: wallamanta_adminer
+    container_name: wallamanta_homelabs_adminer
     restart: unless-stopped
     ports:
-      - 8555:8080
+      - 8655:8080
+    networks:
+      - wallamanta_homelabs
+
+networks:
+  wallamanta_homelabs:
+    name: wallamanta_homelabs

wallamanta/account_checker.py

@@ -12,16 +12,16 @@ logging.basicConfig(
 def work(sleep_time):
     time.sleep(30)
     while True:
-        user_list = walladb.get_user_list()
+        user_list = walladb.get_active_user_list()
         for user in user_list:
-            if walladb.is_user_valid(user['telegram_user_id']):
+            if walladb.is_user_active(user['telegram_user_id']):
                 if walladb.is_user_expired(user['telegram_user_id']):
-                    message = "¡Hola!\n\nTu cuenta ha caducado. Si quieres seguir usando @wallamanta_bot ponte en contacto con @jocarduck\n\n¡Gracias!"
+                    message = "¡Hola!\n\nTu cuenta ha caducado. Si quieres seguir usando @wallamanta_homelabs_bot renueva tu membresía a Homelabs VIP\n\n¡Gracias!"
                     try:
                         time.sleep(1)
                         helpers.send_message(user['telegram_user_id'], message)
                     except Exception as e:
-                        logging.error(f"Couldn't sent message to {user['telegram_user_id']}. Reason: {e}")
+                        logging.error(f"Couldn't send message to {user['telegram_user_id']}. Reason: {e}")
                     walladb.deactivate_user(user['telegram_user_id'])
         time.sleep(sleep_time)
 

wallamanta/constants.py

@@ -14,7 +14,11 @@ LATITUDE = os.getenv("LATITUDE")
 LONGITUDE = os.getenv("LONGITUDE")
 SLEEP_TIME = int(os.getenv("SLEEP_TIME"))
 NEW_RELIC_INSERT_KEY = os.getenv("NEW_RELIC_INSERT_KEY")
+NEW_RELIC_METRICS_KEY = os.getenv("NEW_RELIC_METRICS_KEY")
 NR_ENV = os.getenv("NR_ENV")
+NR_HOST_INSIGHTS = os.getenv("NR_HOST_INSIGHTS")
+NR_HOST_METRICS = os.getenv("NR_HOST_METRICS")
+PROXY_SOCKS = os.getenv("PROXY_SOCKS")
 MAX_WORKERS = 12
 
 

wallamanta/helpers.py

@@ -6,7 +6,8 @@ import constants
 import pytz
 import walladb
 import json
-from newrelic_telemetry_sdk import Event, EventClient
+import string
+from newrelic_telemetry_sdk import Event, EventClient, GaugeMetric, MetricClient
 
 from PIL import Image, ImageDraw, ImageFont
 from datetime import datetime, timedelta, timezone, date
@@ -59,15 +60,14 @@ def is_date_expired(until):
 def random_wait():
     time.sleep(random.random())
 
-def download_image(article):
+def download_image(article, product_id):
     r = requests.get(article['images'][0]['original'])
     if r.status_code == 200:
-        image = open(f"/app/data/images/products/{article['id']}.jpg", "wb")
+        with open(f"/app/data/images/products/{article['id']}_{product_id}.jpg", "wb") as image:
-        image.write(r.content)
+            image.write(r.content)
-        image.close()
 
-def create_image(article):
+def create_image(article, product_id):
-    download_image(article)
+    download_image(article, product_id)
     currency = '?'
     if article['currency'] == 'EUR':
         currency = '€'
@@ -84,64 +84,69 @@ def create_image(article):
     font = ImageFont.truetype("/app/data/fonts/Roboto-Bold.ttf", 90)
     wallamanta_text_font = ImageFont.truetype("/app/data/fonts/Roboto-Bold.ttf", 40)
     # inicializamos canvas
-    image = Image.new('RGBA', (width, height), (255, 255, 255))
+    with Image.new('RGBA', (width, height), (255, 255, 255)) as image:
-    # logo homelabers, redimensionamos y ponemos en la parte derecha arriba
+        draw = ImageDraw.Draw(image)
-    #logo_image = Image.open("/app/data/images/logo.png").convert("RGBA")
+        # escribimos @wallamanta_bot
-    #hlogo = int((float(logo_image.size[1]) * float(lpercent)))
+        wtext, htext = draw.textsize(wallamanta_text, font=wallamanta_text_font)
-    #lpercent = wlogo / float(logo_image.size[0])
+        draw.text(((width / 6) * 5 - wtext / 2, int(height - height * 0.2)), wallamanta_text, "#13C1AC", font=wallamanta_text_font)
-    #logo_image = logo_image.resize((wlogo, hlogo), Image.Resampling.LANCZOS)
+        # escribimos el precio
-    #image.paste(logo_image, (int((width / 6) * 5 - logo_image.size[0] / 2), int(height * 0.1)), logo_image)
+        wtext, htext = draw.textsize(price, font=font)
-    # logo wallamanta, redimensionamos y ponemos en la parte derecha abajo
+        draw.text(((width / 6) * 5 - wtext / 2, height / 2 - htext / 2), price, (0, 0, 0), font=font)
-    #wallamanta_logo = Image.open("/app/data/images/wallamanta_logo.png").convert("RGBA")
+        # dibujamos rectángulo verde externo, con un margen externo y ancho determinado
-    #lpercent = wlogo / float(wallamanta_logo.size[0])
+        draw.rectangle([15, 15, width - 15, height - 15], width = 15, outline="#13C1AC")
-    #hlogo = int((float(wallamanta_logo.size[1]) * float(lpercent)))
+        # ponemos la imagen del producto en la parte izquierda y se redimensiona dependiendo de lo ancho
-    #wallamanta_logo = wallamanta_logo.resize((wlogo, hlogo), Image.Resampling.LANCZOS)
+        with Image.open(f"/app/data/images/products/{article['id']}_{product_id}.jpg") as product_image:
-    #image.paste(wallamanta_logo, (int((width / 6) * 5 - wallamanta_logo.size[0] / 2), int(height - height * 0.2)), wallamanta_logo)
+            hpercent = (baseheight / float(product_image.size[1]))
-    draw = ImageDraw.Draw(image)
+            wsize = int((float(product_image.size[0]) * float(hpercent)))
-    # escribimos @wallamanta_bot
+            if wsize < wlimit:
-    wtext, htext = draw.textsize(wallamanta_text, font=wallamanta_text_font)
+                resized_product_image = product_image.resize((wsize, baseheight), Image.Resampling.LANCZOS)
-    draw.text(((width / 6) * 5 - wtext / 2, int(height - height * 0.2)), wallamanta_text, "#13C1AC", font=wallamanta_text_font)
+            else:
-    # escribimos el precio
+                wpercent = wlimit / float(product_image.size[0])
-    wtext, htext = draw.textsize(price, font=font)
+                hsize = int((float(product_image.size[1]) * float(wpercent)))
-    draw.text(((width / 6) * 5 - wtext / 2, height / 2 - htext / 2), price, (0, 0, 0), font=font)
+                resized_product_image = product_image.resize((wlimit, hsize), Image.Resampling.LANCZOS)
-    # dibujamos rectángulo verde externo, con un margen externo y ancho determinado
+            image.paste(resized_product_image, (int((width/3)-(resized_product_image.size[0]/2)), int((height/2) - (resized_product_image.size[1]/2))))
-    draw.rectangle([15, 15, width - 15, height - 15], width = 15, outline="#13C1AC")
+            resized_product_image.close()
-    # ponemos la imagen del producto en la parte izquierda y se redimensiona dependiendo de lo ancho
+            # guardamos la imagen con otro nombre
-    product_image = Image.open(f"/app/data/images/products/{article['id']}.jpg")
+            image.save(f"/app/data/images/products/{article['id']}_{product_id}_composed.png", quality=95)
-    hpercent = (baseheight / float(product_image.size[1]))
-    wsize = int((float(product_image.size[0]) * float(hpercent)))
-    if wsize < wlimit:
-        product_image = product_image.resize((wsize, baseheight), Image.Resampling.LANCZOS)
-    else:
-        wpercent = wlimit / float(product_image.size[0])
-        hsize = int((float(product_image.size[1]) * float(wpercent)))
-        product_image = product_image.resize((wlimit, hsize), Image.Resampling.LANCZOS)
-    image.paste(product_image, (int((width/3)-(product_image.size[0]/2)), int((height/2) - (product_image.size[1]/2))))
-    # guardamos la imagen con otro nombre
-    image.save(f"/app/data/images/products/{article['id']}_composed.png", quality=95)
-    image.close()
-    product_image.close()
 
 def get_publish_date(article):
     article_date = article['creation_date']
-    return datetime.fromtimestamp(int(int(article_date)/1000)).astimezone(pytz.timezone("Europe/Madrid")).strftime("%d/%m/%Y - %H:%M:%S")
+    formato_fecha = "%Y-%m-%dT%H:%M:%S.%f%z"
+    return datetime.strptime(article_date, formato_fecha).astimezone(pytz.timezone("Europe/Madrid")).strftime("%d/%m/%Y - %H:%M:%S")
 
 def get_modified_date(article):
     article_date = article['modification_date']
-    return datetime.fromtimestamp(int(int(article_date)/1000)).astimezone(pytz.timezone("Europe/Madrid")).strftime("%d/%m/%Y - %H:%M:%S")
+    formato_fecha = "%Y-%m-%dT%H:%M:%S.%f%z"
+    return datetime.strptime(article_date, formato_fecha).astimezone(pytz.timezone("Europe/Madrid")).strftime("%d/%m/%Y - %H:%M:%S")
+
+def get_random_string(length):
+    result_str = ''.join(random.choice(string.ascii_letters) for i in range(length))
+    return result_str
 
 def send_message(telegram_user_id, message):
     files = {
         'chat_id': (None, telegram_user_id),
         'text': (None, message),
     }
-    requests.post(f'https://api.telegram.org/bot{constants.TELEGRAM_TOKEN}/sendMessage', files=files)
+    try:
-    #application = Application.builder().get_updates_http_version('1.1').http_version('1.1').token(constants.TELEGRAM_TOKEN).build()
+        requests.post(f'https://api.telegram.org/bot{constants.TELEGRAM_TOKEN}/sendMessage', files=files)
-    #await application.bot.send_message(chat_id=telegram_user_id, text=message)
+    except:
+        time.sleep(1)
+        requests.post(f'https://api.telegram.org/bot{constants.TELEGRAM_TOKEN}/sendMessage', files=files)
+
+def send_message_to_all(message):
+    for user in walladb.get_user_list:
+        try:
+            send_message(user['telegram_user_id'], message)
+        except Exception as e:
+            logging.info(f"Error sending message to {user['telegram_user_id']}: {e}")
+
+def check_code(code):
+    walladb.get_code(code)
 
 def send_article(article, product):
     #application = Application.builder().get_updates_http_version('1.1').http_version('1.1').token(constants.TELEGRAM_TOKEN).build()
-    create_image(article)
+    create_image(article, product['id'])
     title = f"*{telegram_escape_characters(article['title'])}*"
     description = f"*📝 Descripción*: {telegram_escape_characters(article['description'])}"
     found_by = f"*🔍 Encontrado por la búsqueda de:* {telegram_escape_characters(product['product_name'])}"
@@ -165,27 +170,31 @@ def send_article(article, product):
     #markup = InlineKeyboardMarkup(keyboard)
     keyboard = {'inline_keyboard':[[{'text':'Ir al anuncio','url':f'https://es.wallapop.com/item/{article["web_slug"]}'}]]}
 
-    image = open(f"/app/data/images/products/{article['id']}_composed.png", 'rb')
+    with open(f"/app/data/images/products/{article['id']}_{product['id']}_composed.png", 'rb') as image:
-    files = {
+        files = {
-        'chat_id': (None, product['telegram_user_id']),
+            'chat_id': (None, product['telegram_user_id']),
-        'photo': image,
+            'photo': image,
-        'caption': (None, text),
+            'caption': (None, text),
-        'parse_mode': (None, ParseMode.MARKDOWN_V2),
+            'parse_mode': (None, ParseMode.MARKDOWN_V2),
-        'reply_markup': (None, json.dumps(keyboard)),
+            'reply_markup': (None, json.dumps(keyboard)),
-    }
+        }
 
-    response = requests.post(f'https://api.telegram.org/bot{constants.TELEGRAM_TOKEN}/sendPhoto', files=files)
-    while response.status_code != 200:
-        logging.info(f"Error sending to Telegram, probably flood restricted. {product['product_name']} ({product['id']}) - ({walladb.get_user(product['telegram_user_id'])}")
-        random_wait()
         response = requests.post(f'https://api.telegram.org/bot{constants.TELEGRAM_TOKEN}/sendPhoto', files=files)
-    image.close()
+        tries = 0
+        while response.status_code != 200:
+            logging.info(f"Error sending to Telegram, probably flood restricted. {product['product_name']} ({product['id']}) - ({walladb.get_user(product['telegram_user_id'])}) - {response.status_code}")
+            random_wait()
+            response = requests.post(f'https://api.telegram.org/bot{constants.TELEGRAM_TOKEN}/sendPhoto', files=files)
+            tries = tries + 1
+            if tries > 5:
+                logging.info(f"Gave up after 5 retries. Error sending to Telegram, probably flood restricted. {product['product_name']} ({product['id']}) - ({walladb.get_user(product['telegram_user_id'])}) - {response.status_code}")
+                break
 
     #response = await application.bot.send_photo(chat_id=product['telegram_user_id'], photo=open(f"/app/data/images/products/{article['id']}_composed.png", 'rb'), caption=text, parse_mode=ParseMode.MARKDOWN_V2, reply_markup=markup)
     #logging.info(requests.post(url, files=files).content)
     send_to_nr(article, product)
     #logging.info(response.content)
-    logging.info(f"'{title}' (https://es.wallapop.com/item/{article['web_slug']}) sent to {walladb.get_user(product['telegram_user_id'])}")
+    logging.info(f"'{title}' (https://es.wallapop.com/item/{article['web_slug']}) found by {product['product_name']} ({product['id']}) - ({walladb.get_user(product['telegram_user_id'])})")
 
 def get_category_name(category):
     category = int(category)
@@ -334,18 +343,36 @@ def send_to_nr(article, product):
             "environment": constants.NR_ENV
         }
     )
-    event_client = EventClient(insert_key=constants.NEW_RELIC_INSERT_KEY, host="insights-collector.eu01.nr-data.net")
+    event_client = EventClient(insert_key=constants.NEW_RELIC_INSERT_KEY, host=constants.NR_HOST_INSIGHTS)
     try:
-        response = event_client.send(event)
+        logging.info(f"=== Sending product {article['title']} info to NR ===")
+        response = event_client.send(event, timeout=30)
+        response.raise_for_status()
+        logging.info(f"=== Sent product {article['title']} info to NR ===")
+    except Exception as e:
+        logging.error(f"Error sending to NR: {e}")
+
+def send_statistics_to_nr(query_time, products_found, number_of_searches):
+    metric_client = MetricClient(insert_key=constants.NEW_RELIC_INSERT_KEY, host=constants.NR_HOST_METRICS)
+
+    query_time = GaugeMetric("query_time", query_time, {"units": "Seconds"})
+    products_found = GaugeMetric("products_found", products_found)
+    number_of_searches = GaugeMetric("active_searches", number_of_searches)
+
+    batch = [query_time, products_found, number_of_searches]
+    try:
+        logging.info("=== Sending stats to NR ===")
+        response = metric_client.send_batch(batch, timeout=30)
+        response.raise_for_status()
+        logging.info("=== Stats sent to NR ===")
     except Exception as e:
         logging.error(f"Error sending to NR: {e}")
-    response.raise_for_status()
 
 def is_valid_request(product):
     is_valid = False
     if walladb.get_product(product):
         if not walladb.is_user_expired(product['telegram_user_id']):
-            if walladb.is_user_valid(product['telegram_user_id']):
+            if walladb.is_user_active(product['telegram_user_id']):
                 if walladb.is_user_premium(product['telegram_user_id']) or \
                 walladb.is_user_testing(product['telegram_user_id']):
                     is_valid = True

wallamanta/requirements.txt

@@ -1,6 +1,7 @@
 python-telegram-bot==20.4
 python-telegram-bot[job-queue]==20.4
 requests==2.31.0
+requests[socks]==2.31.0
 prettytable==3.6.0
 Pillow==9.4.0
 mysql-connector-python==8.0.32

wallamanta/search_manager.py

@@ -2,6 +2,7 @@ import time
 import concurrent.futures
 import logging
 
+import helpers
 import worker
 import walladb
 import constants
@@ -17,6 +18,8 @@ def work():
     searches = {}
     
     while True:
+        start_time = time.time()
+        logging.info(f"=== Beginning Searches ===")
         products = walladb.get_all_valid_products()
         first_run_products = []
 
@@ -24,7 +27,7 @@ def work():
             try:
                 searches[product['id']]
             except:
-                logging.info(f"New product added: {product['product_name']} ({product['id']}) - ({walladb.get_user(product['telegram_user_id'])}")
+                logging.info(f"New product added: {product['product_name']} ({product['id']}) - ({walladb.get_user(product['telegram_user_id'])})")
                 first_run_products.append(product)
 
         if len(first_run_products) > 0:
@@ -39,14 +42,23 @@ def work():
                     else:
                         logging.info(f"First run for {product['product_name']} ({product['id']}) - ({walladb.get_user(product['telegram_user_id'])}) got: {len(searches[product['id']])}")
 
+        products_found = 0
         with concurrent.futures.ThreadPoolExecutor(max_workers=constants.MAX_WORKERS) as executor:
             future_to_search = {executor.submit(worker.work, product, searches[product['id']]): product for product in products}
             for future in concurrent.futures.as_completed(future_to_search):
                 product = future_to_search[future]
                 try:
-                    searches[product['id']] = future.result()
+                    new_searches = future.result()
+                    search_products_found = len(set(new_searches) - set(searches[product['id']]))
+                    searches[product['id']] = new_searches
                 except Exception as e:
                     logging.info(f"Error occurred in search for {product['product_name']} ({product['id']}) - ({walladb.get_user(product['telegram_user_id'])}): {e}")
                 else:
-                    logging.info(f"Search for {product['product_name']} ({product['id']}) - ({walladb.get_user(product['telegram_user_id'])})")
+                    products_found = products_found + search_products_found
-        time.sleep(constants.SLEEP_TIME)
+                    logging.info(f"Search for {product['product_name']} ({product['id']}) - ({walladb.get_user(product['telegram_user_id'])}) completed ({len(searches[product['id']])}) (+{search_products_found})")
+        #gc.collect()
+        completed_time = round(time.time() - start_time, 2)
+        helpers.send_statistics_to_nr(completed_time, products_found, len(products))
+        wait_time = max(constants.SLEEP_TIME - completed_time, 1)
+        logging.info(f"=== Searches finished in {completed_time} seconds. Found a total of {products_found} products from {len(products)} searches. Sleeping for {wait_time} seconds ===")
+        time.sleep(wait_time)

wallamanta/walladb.py

@@ -74,7 +74,7 @@ def setup_db():
                 else:
                     logging.info("OK")
 
-def is_user_valid(telegram_user_id):
+def is_user_active(telegram_user_id):
     if telegram_user_id < 0:
         ret = False
     else:
@@ -114,6 +114,17 @@ def is_user_premium(telegram_user_id):
                 ret = False
     return ret
 
+def is_user_homelabs(telegram_user_id):
+    with connect_db() as con:
+        with con.cursor(prepared=True) as cur:
+            params = (telegram_user_id,)
+            cur.execute(f"SELECT * FROM users WHERE telegram_user_id=%s AND active=True AND type='homelabs'", params)
+            try:
+                ret = cur.fetchone() != None
+            except:
+                ret = False
+    return ret
+
 def is_user_testing(telegram_user_id):
     with connect_db() as con:
         with con.cursor(prepared=True) as cur:
@@ -125,12 +136,31 @@ def is_user_testing(telegram_user_id):
                 ret = False
     return ret
 
+def enable_user(telegram_user_id, telegram_name):
+    found = False
+    with connect_db() as con:
+        with con.cursor(prepared=True) as cur:
+            params = (telegram_user_id,)
+            res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id=%s", params)
+            try:
+                if res == None:
+                    params = (telegram_user_id, False, None, None, telegram_name.first_name)
+                    cur.execute("INSERT INTO users VALUES (%s, %s, %s, %s, %s)", params)
+                    con.commit()
+                    logging.info(f"Enabled user {telegram_user_id} - {telegram_name.first_name}")
+                else:
+                    found = True
+                    logging.info(f"User {telegram_user_id} - {telegram_name.first_name} was already enabled")
+            except Exception as e:
+                logging.error(f"Couldn't find username with id {telegram_user_id}: {e}")
+    return found
+
 def add_premium_user(telegram_user_id, until):
     found = False
     with connect_db() as con:
         with con.cursor(prepared=True) as cur:
-            res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id=%s", params)
+            res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id={telegram_user_id}")
-            if res.fetchone() != None:
+            if res != None:
                 params = (until, telegram_user_id)
                 cur.execute(f"UPDATE users SET active = True, type = 'premium', until = %s WHERE telegram_user_id=%s", params)
                 con.commit()
@@ -138,33 +168,46 @@ def add_premium_user(telegram_user_id, until):
             logging.info(f"Added premium user {telegram_user_id} until {until}")
     return found
 
-def add_test_user(telegram_user_id, telegram_name, until):
+def add_test_user(telegram_user_id, until):
     found = False
     with connect_db() as con:
         with con.cursor(prepared=True) as cur:
-            params = (telegram_user_id,)
+            res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id={telegram_user_id}")
-            cur.execute(f"SELECT * FROM users WHERE telegram_user_id=%s", params)
+            if res != None:
-            try:
+                params = (until, telegram_user_id)
-                cur.fetchone()
+                cur.execute(f"UPDATE users SET active = True, type = 'testing', until = %s WHERE telegram_user_id=%s", params)
-                params = (telegram_user_id, True, 'testing', until, telegram_name.first_name)
-                cur.execute("INSERT INTO users VALUES (%s, %s, %s, %s, %s)", params)
                 con.commit()
                 found = True
-            except Exception as e:
+            logging.info(f"Added testing user {telegram_user_id} until {until}")
-                logging.error(f"Couldn't find username with id {telegram_user_id}: {e}")
+    return found
-            logging.info(f"Added test user {telegram_user_id} until {until}")
-    return not found
 
 def remove_valid_user(telegram_user_id):
     with connect_db() as con:
         with con.cursor(prepared=True) as cur:
             params = (telegram_user_id,)
-            res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id=%s", params)
+            res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id={telegram_user_id}")
-            if res.fetchone() != None:
+            if res != None:
                 cur.execute(f"UPDATE users SET active = False WHERE telegram_user_id=%s", params)
                 con.commit()
     logging.info(f"De-activated user {telegram_user_id}")
 
+def get_code(code):
+    with connect_db() as con:
+        with con.cursor(dictionary=True, prepared=True) as cur:
+            params = (code,)
+            res = cur.execute(f"SELECT * FROM codes WHERE code_id=%s", params)
+            ret = cur.fetchone()
+    return ret
+
+def use_code(code):
+    with connect_db() as con:
+        with con.cursor(prepared=True) as cur:
+            params = (code,)
+            res = cur.execute(f"SELECT * FROM codes WHERE code_id=%s", params)
+            if res.fetchone() != None:
+                cur.execute(f"UPDATE codes SET used = True WHERE code_id=%s", params)
+                con.commit()
+
 def get_user_list():
     with connect_db() as con:
         with con.cursor(dictionary=True, prepared=True) as cur:
@@ -172,6 +215,13 @@ def get_user_list():
             ret = cur.fetchall()
     return ret
 
+def get_active_user_list():
+    with connect_db() as con:
+        with con.cursor(dictionary=True, prepared=True) as cur:
+            res = cur.execute(f"SELECT * FROM users WHERE active=True")
+            ret = cur.fetchall()
+    return ret
+
 def get_user(telegram_user_id):
     with connect_db() as con:
         with con.cursor(dictionary=True, prepared=True) as cur:

wallamanta/wallamanta.py

@@ -26,6 +26,10 @@ ACTION, ADD_PRODUCT_NAME, ADD_PRODUCT_MIN_PRICE, ADD_PRODUCT_MAX_PRICE, \
     ADD_PRODUCT_CATEGORY, ADD_PRODUCT_TITLE_EXCLUDE, ADD_PRODUCT_DESCRIPTION_EXCLUDE, \
     ADD_PRODUCT_COORDS, ADD_PRODUCT_DISTANCE, REMOVE_PRODUCT, LIST, FINISH, CONTINUE_OR_FINISH = range(13)
 
+MAX_PRODUCTS_TESTING = 5
+MAX_PRODUCTS_PREMIUM = 30
+MAX_PRODUCTS_HOMELABS = 5
+
 # Enable logging
 logging.basicConfig(
     format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO
@@ -37,10 +41,10 @@ httpx_logger = logging.getLogger('httpx')
 httpx_logger.setLevel(logging.WARNING)
 
 async def help_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
-    if walladb.is_user_valid(helpers.get_telegram_user_id(update)):
+    if walladb.is_user_active(helpers.get_telegram_user_id(update)):
         message = "Utiliza el menú de la conversación para añadir un producto y sigue los pasos indicados. Si tienes cualquier duda contacta con @jocarduck para más información."
     else:
-        message = "Activa tu periodo de prueba de 7 días con `/test` o contacta con @jocarduck para más información."   
+        message = "No tienes permitido usar este bot."   
     await update.message.reply_markdown_v2(helpers.telegram_escape_characters(message))
 
 async def main_menu(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
@@ -58,21 +62,27 @@ async def main_menu(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
 async def menu_click_handler(update: Update, context: CallbackContext):
     await update.callback_query.edit_message_reply_markup(None)
     telegram_user_id = helpers.get_telegram_user_id(update)
-    if walladb.is_user_valid(telegram_user_id):
+    if walladb.is_user_active(telegram_user_id):
         context.user_data['telegram_user_id'] = telegram_user_id
         query = update.callback_query
+        number_of_products = walladb.count_user_products(telegram_user_id)
         if query.data == 'add':
             valid = False
             number_of_products = walladb.count_user_products(telegram_user_id)
             if walladb.is_user_testing(telegram_user_id):
                 valid = True
-                if number_of_products >= 5:
+                if number_of_products >= MAX_PRODUCTS_TESTING:
-                    message = "Ya tienes 5 productos en seguimiento. Con premium puedes tener hasta 20."
+                    message = f"Ya tienes {MAX_PRODUCTS_TESTING} productos en seguimiento. Borra algunos para añadir más."
                     valid = False
             elif walladb.is_user_premium(telegram_user_id):
                 valid = True
-                if number_of_products >= 20:
+                if number_of_products >= MAX_PRODUCTS_PREMIUM:
-                    message = "Ya tienes 20 productos en seguimiento. Borra algunos para añadir más."
+                    message = f"Ya tienes {MAX_PRODUCTS_PREMIUM} productos en seguimiento. Borra algunos para añadir más."
+                    valid = False
+            elif walladb.is_user_homelabs(telegram_user_id):
+                valid = True
+                if number_of_products >= MAX_PRODUCTS_HOMELABS:
+                    message = f"Ya tienes {MAX_PRODUCTS_HOMELABS} productos en seguimiento. Borra algunos para añadir más."
                     valid = False
             if valid:
                 await context.bot.send_message(chat_id=update.effective_chat.id,
@@ -82,14 +92,22 @@ async def menu_click_handler(update: Update, context: CallbackContext):
                 await context.bot.send_message(chat_id=update.effective_chat.id, text=f'{message}')
                 return ConversationHandler.END
         if query.data == 'remove':
-            await send_list(update, context)
+            if number_of_products != 0:
-            return REMOVE_PRODUCT
+                await send_list(update, context)
+                return REMOVE_PRODUCT
+            message = "No tienes ninguna búsqueda activa"
+            await context.bot.send_message(chat_id=update.effective_chat.id, text=f'{message}')
+            return ConversationHandler.END
         if query.data == 'list':
-            await send_list(update, context)
+            if number_of_products != 0:
-            return LIST
+                await send_list(update, context)
+                return LIST
+            message = "No tienes ninguna búsqueda activa"
+            await context.bot.send_message(chat_id=update.effective_chat.id, text=f'{message}')
+            return ConversationHandler.END
     else:
         await context.bot.send_message(chat_id=update.effective_chat.id,
-                                    text=helpers.telegram_escape_characters('Activa tu periodo de prueba de 7 días con `/test` o contacta con @jocarduck para más información.'), parse_mode=ParseMode.MARKDOWN_V2)
+                                    text=helpers.telegram_escape_characters('No tienes permiso para usar este bot.'), parse_mode=ParseMode.MARKDOWN_V2)
         return ConversationHandler.END
     
 async def add_product_name(update: Update, context: CallbackContext):
@@ -340,7 +358,7 @@ async def product_details(update: Update, context: ContextTypes.DEFAULT_TYPE) ->
 
 async def send_list(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
     telegram_user_id = helpers.get_telegram_user_id(update)
-    if walladb.is_user_valid(telegram_user_id):
+    if walladb.is_user_active(telegram_user_id):
         if walladb.is_user_testing(telegram_user_id) or walladb.is_user_premium(telegram_user_id):
             query = update.callback_query
             if query.data == 'remove':
@@ -373,28 +391,21 @@ async def remove_user_command(update: Update, context: ContextTypes.DEFAULT_TYPE
 
 async def status_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
     telegram_user_id = helpers.get_telegram_user_id(update)
-    if walladb.is_user_valid(telegram_user_id):
+    if walladb.is_user_active(telegram_user_id):
         type = walladb.get_user_type(telegram_user_id)
         until = walladb.get_user_until(telegram_user_id)
         message = f"Tu cuenta es tipo: {type} y caduca el {helpers.get_spanish_date(until)}."
         await update.message.reply_markdown_v2(helpers.telegram_escape_characters(message))
 
-async def test_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
+async def start_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
     telegram_user_id = helpers.get_telegram_user_id(update)
     telegram_user_name = helpers.get_telegram_user_name(update)
     #if not walladb.is_user_valid(telegram_user_id):
     if telegram_user_id < 0:
         message = "Este bot no se puede usar en grupos."
     elif walladb.get_user(telegram_user_id) == "NoName":
-        until = helpers.get_date_ahead(7)
+        walladb.enable_user(telegram_user_id, telegram_user_name)
-        walladb.add_test_user(telegram_user_id, telegram_user_name, until)
+        message = f"Bienvenido a @wallamanta, usa el comando /help para más información."
-        message = f"Periodo de prueba activado hasta el {helpers.get_spanish_date(until)}."
-    else:
-        message = "Ya has utilizado el periodo de prueba."
-        if walladb.is_user_testing(telegram_user_id):
-            message = "Ya estás en el periodo de prueba."
-        elif walladb.is_user_premium(telegram_user_id):
-            message = "Ya eres premium. No puedes volver al periodo de prueba."
     await update.message.reply_markdown_v2(helpers.telegram_escape_characters(message))
 
 async def add_to_db_and_send(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
@@ -428,6 +439,36 @@ async def list_threads(update: Update, context: ContextTypes.DEFAULT_TYPE) -> No
         else: 
             await update.message.reply_markdown_v2(helpers.telegram_escape_characters(f"{threads_string}"))
 
+async def message_to_all(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
+    if helpers.is_user_admin(update.message.chat_id):
+        message = update.message.text.split('/message_to_all ')[1] 
+        helpers.send_message_to_all(message)
+        await update.message.reply_markdown_v2(helpers.telegram_escape_characters(f"Messages sent to all users"))
+
+async def code(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
+    code = update.message.text.split('/code ')[1]
+    telegram_user_id = update.message.chat_id
+    if len(code) != 12:
+        await update.message.reply_markdown_v2(helpers.telegram_escape_characters(f"El código no tiene el formato correcto"))
+    else:
+        code = walladb.get_code(code)
+        if code:
+            until = helpers.get_date_ahead(code['days'])
+            try:
+                if code['type'] == 'premium':
+                    walladb.add_premium_user(telegram_user_id, until)
+                    await update.message.reply_markdown_v2(helpers.telegram_escape_characters(f"Activado periodo premium hasta el {until}"))
+                elif code['type'] == 'testing':
+                    walladb.add_test_user(telegram_user_id, until)
+                    await update.message.reply_markdown_v2(helpers.telegram_escape_characters(f"Activado periodo testing hasta el {until}"))
+                walladb.use_code(code)
+            except Exception as e:
+                error_code = helpers.get_random_string(8)
+                logging.info(f"Error trying to checkout code {code}: {e}. {error_code}")
+                await update.message.reply_markdown_v2(helpers.telegram_escape_characters(f"Ha habido un error, contacta con @jocarduck y dale el siguiente código de error: `{error_code}`"))
+        else:
+            await update.message.reply_markdown_v2(helpers.telegram_escape_characters(f"El código no es válido o ya ha sido usado"))
+
 def count_threads():
     time.sleep(10)
     while True:
@@ -447,17 +488,19 @@ def main()->None:
     p = threading.Thread(target=account_checker.work, args=(3600, ))
     p.start()
 
-    p = threading.Thread(target=count_threads)
+    #p = threading.Thread(target=count_threads)
-    p.start()
+    #p.start()
 
     # on different commands - answer in Telegram
+    application.add_handler(CommandHandler("start", start_command))
     application.add_handler(CommandHandler("help", help_command))
     application.add_handler(CommandHandler("admin", admin_command))
     application.add_handler(CommandHandler("add_premium_user", add_premium_user_command))
     application.add_handler(CommandHandler("remove_user", remove_user_command))
     application.add_handler(CommandHandler("status", status_command))
-    application.add_handler(CommandHandler("test", test_command))
     application.add_handler(CommandHandler("list_threads", list_threads))
+    application.add_handler(CommandHandler("message_to_all", message_to_all))
+    application.add_handler(CommandHandler("code", code))
     #application.add_handler(CallbackQueryHandler("list", send_list()))
     #application.add_handler(CallbackQueryHandler(pattern="list", callback=send_list()))
 

wallamanta/worker.py

@@ -1,5 +1,5 @@
-import time
 import requests
+from random import randint
 import logging
 import helpers
 import walladb
@@ -13,6 +13,7 @@ logging.basicConfig(
 logger = logging.getLogger(__name__)
 
 def request(product_name, steps=15, latitude=constants.LATITUDE, longitude=constants.LONGITUDE, distance='0', condition='all', min_price=0, max_price=10000000, category="", subcategories=[]):
+    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'}
     distance = str(int(distance) * 1000)
     url = (f"https://api.wallapop.com/api/v3/general/search?keywords={product_name}"
                 f"&order_by=newest&latitude={latitude}"
@@ -22,8 +23,8 @@ def request(product_name, steps=15, latitude=constants.LATITUDE, longitude=const
                 f"&max_sale_price={max_price}" 
                 f"&filters_source=quick_filters&language=es_ES")
 
-    if condition != "all":
+    #if condition != "all":
-        url = url + f"&condition={condition}" # new, as_good_as_new, good, fair, has_given_it_all
+    #    url = url + f"&condition={condition}" # new, as_good_as_new, good, fair, has_given_it_all
 
     if category != "":
         url = url + f"&category_ids={category}"
@@ -37,39 +38,45 @@ def request(product_name, steps=15, latitude=constants.LATITUDE, longitude=const
     search_objects = list()
 
     for step in range(steps):
+        #helpers.random_wait()
         tries = 5
         for _ in range(tries):
-            response = requests.get(url+f"&step={step}")
             try:
+                if randint(0, 1) == 0:
+                    response = requests.get(url+f"&step={step}", timeout=1, headers=headers)
+                else:
+                    response = requests.get(url+f"&step={step}", timeout=1, headers=headers, proxies=dict(https=f'socks5://{constants.PROXY_SOCKS}'))
                 if response.status_code == 200:
                     search_objects = search_objects + response.json()['search_objects']
                     break
                 else:
                     logging.info(f"\'{product_name}\' -> Wallapop returned status {response.status_code}. Illegal parameters or Wallapop service is down. Retrying...")
             except Exception as e:
-                logging.info("Error while querying Wallapop, try #{_}: " + e)
+                logging.info(f"Error while querying Wallapop, try #{_}: {e}")
-                time.sleep(3)
+                for __ in range(_):
+                    helpers.random_wait()
 
     return search_objects
 
 def first_run(product):
-    logging.info(f"First run for {product['product_name']} for {walladb.get_user(product['telegram_user_id'])} ({walladb.get_user(product['telegram_user_id'])})")
+    steps = 1
+    #logging.info(f"First run for {product['product_name']} for {walladb.get_user(product['telegram_user_id'])} ({walladb.get_user(product['telegram_user_id'])})")
     list = []
     if not helpers.is_valid_request(product):
         return list
     if product['category'] == '':
-        articles = request(product['product_name'], 15, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], product['category'])
+        articles = request(product['product_name'], steps, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], product['category'])
         for article in articles: 
             list.insert(0, article['id'])
     else:
         if '0' in product['category'].split(','):
-            articles = request(product['product_name'], 15, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'])
+            articles = request(product['product_name'], steps, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'])
             for article in articles: 
                 list.insert(0, article['id'])
         else:
             for category in product['category'].split(','):
                 if product['subcategory'] == '' or not helpers.has_subcategory(category):
-                    articles = request(product['product_name'], 15, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category)
+                    articles = request(product['product_name'], steps, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category)
                     for article in articles: 
                         list.insert(0, article['id'])
                 else:
@@ -77,33 +84,35 @@ def first_run(product):
                     for subcategory in product['subcategory'].split(','):
                         if helpers.is_subcategory(category, subcategory):
                             subcategories.append(subcategory)
-                    articles = request(product['product_name'], 15, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category, subcategories)
+                    articles = request(product['product_name'], steps, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category, subcategories)
                     for article in articles: 
                         list.insert(0, article['id'])
     return list
 
-def work(product, list):
+def work(product, searches_list):
-    logging.info(f"Searching {product['product_name']}")
+    list = searches_list.copy()
+    steps = 1
+    logging.info(f"Searching {product['product_name']} ({product['id']}) - ({walladb.get_user(product['telegram_user_id'])})")
     articles_list = []
     if product['category'] == '':
-        articles_list.append(request(product['product_name'], 1, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price']))
+        articles_list.append(request(product['product_name'], steps, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price']))
     else:
         if '0' in product['category'].split(','):
-            articles_list.append(request(product['product_name'], 1, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price']))
+            articles_list.append(request(product['product_name'], steps, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price']))
         else:
             for category in product['category'].split(','):
                 if product['subcategory'] == '' or not helpers.has_subcategory(category):
-                    articles_list.append(request(product['product_name'], 1, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category))
+                    articles_list.append(request(product['product_name'], steps, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category))
                 else:
                     subcategories = []
                     for subcategory in product['subcategory'].split(','):
                         if helpers.is_subcategory(category, subcategory):
                             subcategories.append(subcategory)
-                    articles_list.append(request(product['product_name'], 1, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category, subcategories))
+                    articles_list.append(request(product['product_name'], steps, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category, subcategories))
     for articles in articles_list:
         for article in articles:
             if not article['id'] in list:
-                logging.info(f"Found article {article['title']} for {walladb.get_user(product['telegram_user_id'])} ({product['telegram_user_id']})")
+                logging.info(f"Found article {article['title']} for {product['product_name']} ({product['id']}) - ({walladb.get_user(product['telegram_user_id'])})")
                 try:
                     if not has_excluded_words(article['title'].lower(), article['description'].lower(), product['title_description_exclude']) and not is_title_key_word_excluded(article['title'].lower(), product['title_exclude']):
                         helpers.send_article(article, product)
@@ -112,7 +121,7 @@ def work(product, list):
                     logging.info("---------- EXCEPTION -----------")
                     logging.info(f"{product['product_name']} worker crashed. {e}")
                     logging.info(f"{product['product_name']}: Trying to parse {article['id']}: {article['title']} .\n")
-    return list[:600]
+    return list[:100]
    
 def has_excluded_words(title, description, excluded_words):
     if len(excluded_words) > 0:
@@ -129,4 +138,4 @@ def is_title_key_word_excluded(title, excluded_words):
             logging.info("Checking '" + word + "' for title: '" + title)
             if word.lower().lstrip().rstrip() in title.lower():
                 return True
-    return False
+    return False