diff --git a/docker-compose.yml b/docker-compose.yml index 61cce1a..84b9aed 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,10 +1,10 @@ version: "3" services: - wallamanta-bot-multiuser: + wallamanta_homelabs: build: wallamanta - image: wallamanta:multiuser - container_name: wallamanta-bot-multiuser + image: wallamanta:homelabs + container_name: wallamanta_homelabs volumes: - ./data:/app/data restart: unless-stopped @@ -20,10 +20,12 @@ services: - DB_PASSWORD=${DB_PASSWORD} - NEW_RELIC_INSERT_KEY=${NEW_RELIC_INSERT_KEY} - NR_ENV=${NR_ENV} + networks: + - wallamanta_homelabs - wallamanta_db: + wallamanta_homelabs_db: image: mysql:8 - container_name: wallamanta_db + container_name: wallamanta_homelabs_db volumes: - ./db:/var/lib/mysql environment: @@ -32,10 +34,18 @@ services: - MYSQL_USER=${DB_USER} - MYSQL_PASSWORD=${DB_PASSWORD} restart: unless-stopped + networks: + - wallamanta_homelabs - wallamanta_adminer: + wallamanta_homelabs_adminer: image: adminer - container_name: wallamanta_adminer + container_name: wallamanta_homelabs_adminer restart: unless-stopped ports: - - 8555:8080 \ No newline at end of file + - 8655:8080 + networks: + - wallamanta_homelabs + +networks: + wallamanta_homelabs: + name: wallamanta_homelabs \ No newline at end of file diff --git a/wallamanta/account_checker.py b/wallamanta/account_checker.py index b103ef0..9ba9efd 100644 --- a/wallamanta/account_checker.py +++ b/wallamanta/account_checker.py @@ -12,16 +12,16 @@ logging.basicConfig( def work(sleep_time): time.sleep(30) while True: - user_list = walladb.get_user_list() + user_list = walladb.get_active_user_list() for user in user_list: - if walladb.is_user_valid(user['telegram_user_id']): + if walladb.is_user_active(user['telegram_user_id']): if walladb.is_user_expired(user['telegram_user_id']): - message = "¡Hola!\n\nTu cuenta ha caducado. Si quieres seguir usando @wallamanta_bot ponte en contacto con @jocarduck\n\n¡Gracias!" + message = "¡Hola!\n\nTu cuenta ha caducado. Si quieres seguir usando @wallamanta_homelabs_bot renueva tu membresía a Homelabs VIP\n\n¡Gracias!" try: time.sleep(1) helpers.send_message(user['telegram_user_id'], message) except Exception as e: - logging.error(f"Couldn't sent message to {user['telegram_user_id']}. Reason: {e}") + logging.error(f"Couldn't send message to {user['telegram_user_id']}. Reason: {e}") walladb.deactivate_user(user['telegram_user_id']) time.sleep(sleep_time) diff --git a/wallamanta/constants.py b/wallamanta/constants.py index 9b3c9d6..bb64508 100644 --- a/wallamanta/constants.py +++ b/wallamanta/constants.py @@ -14,7 +14,11 @@ LATITUDE = os.getenv("LATITUDE") LONGITUDE = os.getenv("LONGITUDE") SLEEP_TIME = int(os.getenv("SLEEP_TIME")) NEW_RELIC_INSERT_KEY = os.getenv("NEW_RELIC_INSERT_KEY") +NEW_RELIC_METRICS_KEY = os.getenv("NEW_RELIC_METRICS_KEY") NR_ENV = os.getenv("NR_ENV") +NR_HOST_INSIGHTS = os.getenv("NR_HOST_INSIGHTS") +NR_HOST_METRICS = os.getenv("NR_HOST_METRICS") +PROXY_SOCKS = os.getenv("PROXY_SOCKS") MAX_WORKERS = 12 diff --git a/wallamanta/helpers.py b/wallamanta/helpers.py index e92810f..271ad8d 100644 --- a/wallamanta/helpers.py +++ b/wallamanta/helpers.py @@ -6,7 +6,8 @@ import constants import pytz import walladb import json -from newrelic_telemetry_sdk import Event, EventClient +import string +from newrelic_telemetry_sdk import Event, EventClient, GaugeMetric, MetricClient from PIL import Image, ImageDraw, ImageFont from datetime import datetime, timedelta, timezone, date @@ -59,15 +60,14 @@ def is_date_expired(until): def random_wait(): time.sleep(random.random()) -def download_image(article): +def download_image(article, product_id): r = requests.get(article['images'][0]['original']) if r.status_code == 200: - image = open(f"/app/data/images/products/{article['id']}.jpg", "wb") - image.write(r.content) - image.close() + with open(f"/app/data/images/products/{article['id']}_{product_id}.jpg", "wb") as image: + image.write(r.content) -def create_image(article): - download_image(article) +def create_image(article, product_id): + download_image(article, product_id) currency = '?' if article['currency'] == 'EUR': currency = '€' @@ -84,64 +84,69 @@ def create_image(article): font = ImageFont.truetype("/app/data/fonts/Roboto-Bold.ttf", 90) wallamanta_text_font = ImageFont.truetype("/app/data/fonts/Roboto-Bold.ttf", 40) # inicializamos canvas - image = Image.new('RGBA', (width, height), (255, 255, 255)) - # logo homelabers, redimensionamos y ponemos en la parte derecha arriba - #logo_image = Image.open("/app/data/images/logo.png").convert("RGBA") - #hlogo = int((float(logo_image.size[1]) * float(lpercent))) - #lpercent = wlogo / float(logo_image.size[0]) - #logo_image = logo_image.resize((wlogo, hlogo), Image.Resampling.LANCZOS) - #image.paste(logo_image, (int((width / 6) * 5 - logo_image.size[0] / 2), int(height * 0.1)), logo_image) - # logo wallamanta, redimensionamos y ponemos en la parte derecha abajo - #wallamanta_logo = Image.open("/app/data/images/wallamanta_logo.png").convert("RGBA") - #lpercent = wlogo / float(wallamanta_logo.size[0]) - #hlogo = int((float(wallamanta_logo.size[1]) * float(lpercent))) - #wallamanta_logo = wallamanta_logo.resize((wlogo, hlogo), Image.Resampling.LANCZOS) - #image.paste(wallamanta_logo, (int((width / 6) * 5 - wallamanta_logo.size[0] / 2), int(height - height * 0.2)), wallamanta_logo) - draw = ImageDraw.Draw(image) - # escribimos @wallamanta_bot - wtext, htext = draw.textsize(wallamanta_text, font=wallamanta_text_font) - draw.text(((width / 6) * 5 - wtext / 2, int(height - height * 0.2)), wallamanta_text, "#13C1AC", font=wallamanta_text_font) - # escribimos el precio - wtext, htext = draw.textsize(price, font=font) - draw.text(((width / 6) * 5 - wtext / 2, height / 2 - htext / 2), price, (0, 0, 0), font=font) - # dibujamos rectángulo verde externo, con un margen externo y ancho determinado - draw.rectangle([15, 15, width - 15, height - 15], width = 15, outline="#13C1AC") - # ponemos la imagen del producto en la parte izquierda y se redimensiona dependiendo de lo ancho - product_image = Image.open(f"/app/data/images/products/{article['id']}.jpg") - hpercent = (baseheight / float(product_image.size[1])) - wsize = int((float(product_image.size[0]) * float(hpercent))) - if wsize < wlimit: - product_image = product_image.resize((wsize, baseheight), Image.Resampling.LANCZOS) - else: - wpercent = wlimit / float(product_image.size[0]) - hsize = int((float(product_image.size[1]) * float(wpercent))) - product_image = product_image.resize((wlimit, hsize), Image.Resampling.LANCZOS) - image.paste(product_image, (int((width/3)-(product_image.size[0]/2)), int((height/2) - (product_image.size[1]/2)))) - # guardamos la imagen con otro nombre - image.save(f"/app/data/images/products/{article['id']}_composed.png", quality=95) - image.close() - product_image.close() + with Image.new('RGBA', (width, height), (255, 255, 255)) as image: + draw = ImageDraw.Draw(image) + # escribimos @wallamanta_bot + wtext, htext = draw.textsize(wallamanta_text, font=wallamanta_text_font) + draw.text(((width / 6) * 5 - wtext / 2, int(height - height * 0.2)), wallamanta_text, "#13C1AC", font=wallamanta_text_font) + # escribimos el precio + wtext, htext = draw.textsize(price, font=font) + draw.text(((width / 6) * 5 - wtext / 2, height / 2 - htext / 2), price, (0, 0, 0), font=font) + # dibujamos rectángulo verde externo, con un margen externo y ancho determinado + draw.rectangle([15, 15, width - 15, height - 15], width = 15, outline="#13C1AC") + # ponemos la imagen del producto en la parte izquierda y se redimensiona dependiendo de lo ancho + with Image.open(f"/app/data/images/products/{article['id']}_{product_id}.jpg") as product_image: + hpercent = (baseheight / float(product_image.size[1])) + wsize = int((float(product_image.size[0]) * float(hpercent))) + if wsize < wlimit: + resized_product_image = product_image.resize((wsize, baseheight), Image.Resampling.LANCZOS) + else: + wpercent = wlimit / float(product_image.size[0]) + hsize = int((float(product_image.size[1]) * float(wpercent))) + resized_product_image = product_image.resize((wlimit, hsize), Image.Resampling.LANCZOS) + image.paste(resized_product_image, (int((width/3)-(resized_product_image.size[0]/2)), int((height/2) - (resized_product_image.size[1]/2)))) + resized_product_image.close() + # guardamos la imagen con otro nombre + image.save(f"/app/data/images/products/{article['id']}_{product_id}_composed.png", quality=95) def get_publish_date(article): article_date = article['creation_date'] - return datetime.fromtimestamp(int(int(article_date)/1000)).astimezone(pytz.timezone("Europe/Madrid")).strftime("%d/%m/%Y - %H:%M:%S") + formato_fecha = "%Y-%m-%dT%H:%M:%S.%f%z" + return datetime.strptime(article_date, formato_fecha).astimezone(pytz.timezone("Europe/Madrid")).strftime("%d/%m/%Y - %H:%M:%S") def get_modified_date(article): article_date = article['modification_date'] - return datetime.fromtimestamp(int(int(article_date)/1000)).astimezone(pytz.timezone("Europe/Madrid")).strftime("%d/%m/%Y - %H:%M:%S") + formato_fecha = "%Y-%m-%dT%H:%M:%S.%f%z" + return datetime.strptime(article_date, formato_fecha).astimezone(pytz.timezone("Europe/Madrid")).strftime("%d/%m/%Y - %H:%M:%S") + +def get_random_string(length): + result_str = ''.join(random.choice(string.ascii_letters) for i in range(length)) + return result_str def send_message(telegram_user_id, message): files = { 'chat_id': (None, telegram_user_id), 'text': (None, message), } - requests.post(f'https://api.telegram.org/bot{constants.TELEGRAM_TOKEN}/sendMessage', files=files) - #application = Application.builder().get_updates_http_version('1.1').http_version('1.1').token(constants.TELEGRAM_TOKEN).build() - #await application.bot.send_message(chat_id=telegram_user_id, text=message) + try: + requests.post(f'https://api.telegram.org/bot{constants.TELEGRAM_TOKEN}/sendMessage', files=files) + except: + time.sleep(1) + requests.post(f'https://api.telegram.org/bot{constants.TELEGRAM_TOKEN}/sendMessage', files=files) + +def send_message_to_all(message): + for user in walladb.get_user_list: + try: + send_message(user['telegram_user_id'], message) + except Exception as e: + logging.info(f"Error sending message to {user['telegram_user_id']}: {e}") + +def check_code(code): + walladb.get_code(code) def send_article(article, product): #application = Application.builder().get_updates_http_version('1.1').http_version('1.1').token(constants.TELEGRAM_TOKEN).build() - create_image(article) + create_image(article, product['id']) title = f"*{telegram_escape_characters(article['title'])}*" description = f"*📝 Descripción*: {telegram_escape_characters(article['description'])}" found_by = f"*🔍 Encontrado por la búsqueda de:* {telegram_escape_characters(product['product_name'])}" @@ -165,27 +170,31 @@ def send_article(article, product): #markup = InlineKeyboardMarkup(keyboard) keyboard = {'inline_keyboard':[[{'text':'Ir al anuncio','url':f'https://es.wallapop.com/item/{article["web_slug"]}'}]]} - image = open(f"/app/data/images/products/{article['id']}_composed.png", 'rb') - files = { - 'chat_id': (None, product['telegram_user_id']), - 'photo': image, - 'caption': (None, text), - 'parse_mode': (None, ParseMode.MARKDOWN_V2), - 'reply_markup': (None, json.dumps(keyboard)), - } + with open(f"/app/data/images/products/{article['id']}_{product['id']}_composed.png", 'rb') as image: + files = { + 'chat_id': (None, product['telegram_user_id']), + 'photo': image, + 'caption': (None, text), + 'parse_mode': (None, ParseMode.MARKDOWN_V2), + 'reply_markup': (None, json.dumps(keyboard)), + } - response = requests.post(f'https://api.telegram.org/bot{constants.TELEGRAM_TOKEN}/sendPhoto', files=files) - while response.status_code != 200: - logging.info(f"Error sending to Telegram, probably flood restricted. {product['product_name']} ({product['id']}) - ({walladb.get_user(product['telegram_user_id'])}") - random_wait() response = requests.post(f'https://api.telegram.org/bot{constants.TELEGRAM_TOKEN}/sendPhoto', files=files) - image.close() + tries = 0 + while response.status_code != 200: + logging.info(f"Error sending to Telegram, probably flood restricted. {product['product_name']} ({product['id']}) - ({walladb.get_user(product['telegram_user_id'])}) - {response.status_code}") + random_wait() + response = requests.post(f'https://api.telegram.org/bot{constants.TELEGRAM_TOKEN}/sendPhoto', files=files) + tries = tries + 1 + if tries > 5: + logging.info(f"Gave up after 5 retries. Error sending to Telegram, probably flood restricted. {product['product_name']} ({product['id']}) - ({walladb.get_user(product['telegram_user_id'])}) - {response.status_code}") + break #response = await application.bot.send_photo(chat_id=product['telegram_user_id'], photo=open(f"/app/data/images/products/{article['id']}_composed.png", 'rb'), caption=text, parse_mode=ParseMode.MARKDOWN_V2, reply_markup=markup) #logging.info(requests.post(url, files=files).content) send_to_nr(article, product) #logging.info(response.content) - logging.info(f"'{title}' (https://es.wallapop.com/item/{article['web_slug']}) sent to {walladb.get_user(product['telegram_user_id'])}") + logging.info(f"'{title}' (https://es.wallapop.com/item/{article['web_slug']}) found by {product['product_name']} ({product['id']}) - ({walladb.get_user(product['telegram_user_id'])})") def get_category_name(category): category = int(category) @@ -334,18 +343,36 @@ def send_to_nr(article, product): "environment": constants.NR_ENV } ) - event_client = EventClient(insert_key=constants.NEW_RELIC_INSERT_KEY, host="insights-collector.eu01.nr-data.net") + event_client = EventClient(insert_key=constants.NEW_RELIC_INSERT_KEY, host=constants.NR_HOST_INSIGHTS) try: - response = event_client.send(event) + logging.info(f"=== Sending product {article['title']} info to NR ===") + response = event_client.send(event, timeout=30) + response.raise_for_status() + logging.info(f"=== Sent product {article['title']} info to NR ===") + except Exception as e: + logging.error(f"Error sending to NR: {e}") + +def send_statistics_to_nr(query_time, products_found, number_of_searches): + metric_client = MetricClient(insert_key=constants.NEW_RELIC_INSERT_KEY, host=constants.NR_HOST_METRICS) + + query_time = GaugeMetric("query_time", query_time, {"units": "Seconds"}) + products_found = GaugeMetric("products_found", products_found) + number_of_searches = GaugeMetric("active_searches", number_of_searches) + + batch = [query_time, products_found, number_of_searches] + try: + logging.info("=== Sending stats to NR ===") + response = metric_client.send_batch(batch, timeout=30) + response.raise_for_status() + logging.info("=== Stats sent to NR ===") except Exception as e: logging.error(f"Error sending to NR: {e}") - response.raise_for_status() def is_valid_request(product): is_valid = False if walladb.get_product(product): if not walladb.is_user_expired(product['telegram_user_id']): - if walladb.is_user_valid(product['telegram_user_id']): + if walladb.is_user_active(product['telegram_user_id']): if walladb.is_user_premium(product['telegram_user_id']) or \ walladb.is_user_testing(product['telegram_user_id']): is_valid = True diff --git a/wallamanta/requirements.txt b/wallamanta/requirements.txt index f9eff1c..9527898 100644 --- a/wallamanta/requirements.txt +++ b/wallamanta/requirements.txt @@ -1,6 +1,7 @@ python-telegram-bot==20.4 python-telegram-bot[job-queue]==20.4 requests==2.31.0 +requests[socks]==2.31.0 prettytable==3.6.0 Pillow==9.4.0 mysql-connector-python==8.0.32 diff --git a/wallamanta/search_manager.py b/wallamanta/search_manager.py index a70660f..68038d2 100644 --- a/wallamanta/search_manager.py +++ b/wallamanta/search_manager.py @@ -2,6 +2,7 @@ import time import concurrent.futures import logging +import helpers import worker import walladb import constants @@ -17,6 +18,8 @@ def work(): searches = {} while True: + start_time = time.time() + logging.info(f"=== Beginning Searches ===") products = walladb.get_all_valid_products() first_run_products = [] @@ -24,7 +27,7 @@ def work(): try: searches[product['id']] except: - logging.info(f"New product added: {product['product_name']} ({product['id']}) - ({walladb.get_user(product['telegram_user_id'])}") + logging.info(f"New product added: {product['product_name']} ({product['id']}) - ({walladb.get_user(product['telegram_user_id'])})") first_run_products.append(product) if len(first_run_products) > 0: @@ -39,14 +42,23 @@ def work(): else: logging.info(f"First run for {product['product_name']} ({product['id']}) - ({walladb.get_user(product['telegram_user_id'])}) got: {len(searches[product['id']])}") + products_found = 0 with concurrent.futures.ThreadPoolExecutor(max_workers=constants.MAX_WORKERS) as executor: future_to_search = {executor.submit(worker.work, product, searches[product['id']]): product for product in products} for future in concurrent.futures.as_completed(future_to_search): product = future_to_search[future] try: - searches[product['id']] = future.result() + new_searches = future.result() + search_products_found = len(set(new_searches) - set(searches[product['id']])) + searches[product['id']] = new_searches except Exception as e: logging.info(f"Error occurred in search for {product['product_name']} ({product['id']}) - ({walladb.get_user(product['telegram_user_id'])}): {e}") else: - logging.info(f"Search for {product['product_name']} ({product['id']}) - ({walladb.get_user(product['telegram_user_id'])})") - time.sleep(constants.SLEEP_TIME) \ No newline at end of file + products_found = products_found + search_products_found + logging.info(f"Search for {product['product_name']} ({product['id']}) - ({walladb.get_user(product['telegram_user_id'])}) completed ({len(searches[product['id']])}) (+{search_products_found})") + #gc.collect() + completed_time = round(time.time() - start_time, 2) + helpers.send_statistics_to_nr(completed_time, products_found, len(products)) + wait_time = max(constants.SLEEP_TIME - completed_time, 1) + logging.info(f"=== Searches finished in {completed_time} seconds. Found a total of {products_found} products from {len(products)} searches. Sleeping for {wait_time} seconds ===") + time.sleep(wait_time) \ No newline at end of file diff --git a/wallamanta/walladb.py b/wallamanta/walladb.py index b526600..ae1bf7f 100644 --- a/wallamanta/walladb.py +++ b/wallamanta/walladb.py @@ -74,7 +74,7 @@ def setup_db(): else: logging.info("OK") -def is_user_valid(telegram_user_id): +def is_user_active(telegram_user_id): if telegram_user_id < 0: ret = False else: @@ -114,6 +114,17 @@ def is_user_premium(telegram_user_id): ret = False return ret +def is_user_homelabs(telegram_user_id): + with connect_db() as con: + with con.cursor(prepared=True) as cur: + params = (telegram_user_id,) + cur.execute(f"SELECT * FROM users WHERE telegram_user_id=%s AND active=True AND type='homelabs'", params) + try: + ret = cur.fetchone() != None + except: + ret = False + return ret + def is_user_testing(telegram_user_id): with connect_db() as con: with con.cursor(prepared=True) as cur: @@ -125,12 +136,31 @@ def is_user_testing(telegram_user_id): ret = False return ret +def enable_user(telegram_user_id, telegram_name): + found = False + with connect_db() as con: + with con.cursor(prepared=True) as cur: + params = (telegram_user_id,) + res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id=%s", params) + try: + if res == None: + params = (telegram_user_id, False, None, None, telegram_name.first_name) + cur.execute("INSERT INTO users VALUES (%s, %s, %s, %s, %s)", params) + con.commit() + logging.info(f"Enabled user {telegram_user_id} - {telegram_name.first_name}") + else: + found = True + logging.info(f"User {telegram_user_id} - {telegram_name.first_name} was already enabled") + except Exception as e: + logging.error(f"Couldn't find username with id {telegram_user_id}: {e}") + return found + def add_premium_user(telegram_user_id, until): found = False with connect_db() as con: with con.cursor(prepared=True) as cur: - res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id=%s", params) - if res.fetchone() != None: + res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id={telegram_user_id}") + if res != None: params = (until, telegram_user_id) cur.execute(f"UPDATE users SET active = True, type = 'premium', until = %s WHERE telegram_user_id=%s", params) con.commit() @@ -138,33 +168,46 @@ def add_premium_user(telegram_user_id, until): logging.info(f"Added premium user {telegram_user_id} until {until}") return found -def add_test_user(telegram_user_id, telegram_name, until): +def add_test_user(telegram_user_id, until): found = False with connect_db() as con: with con.cursor(prepared=True) as cur: - params = (telegram_user_id,) - cur.execute(f"SELECT * FROM users WHERE telegram_user_id=%s", params) - try: - cur.fetchone() - params = (telegram_user_id, True, 'testing', until, telegram_name.first_name) - cur.execute("INSERT INTO users VALUES (%s, %s, %s, %s, %s)", params) + res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id={telegram_user_id}") + if res != None: + params = (until, telegram_user_id) + cur.execute(f"UPDATE users SET active = True, type = 'testing', until = %s WHERE telegram_user_id=%s", params) con.commit() found = True - except Exception as e: - logging.error(f"Couldn't find username with id {telegram_user_id}: {e}") - logging.info(f"Added test user {telegram_user_id} until {until}") - return not found + logging.info(f"Added testing user {telegram_user_id} until {until}") + return found def remove_valid_user(telegram_user_id): with connect_db() as con: with con.cursor(prepared=True) as cur: params = (telegram_user_id,) - res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id=%s", params) - if res.fetchone() != None: + res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id={telegram_user_id}") + if res != None: cur.execute(f"UPDATE users SET active = False WHERE telegram_user_id=%s", params) con.commit() logging.info(f"De-activated user {telegram_user_id}") +def get_code(code): + with connect_db() as con: + with con.cursor(dictionary=True, prepared=True) as cur: + params = (code,) + res = cur.execute(f"SELECT * FROM codes WHERE code_id=%s", params) + ret = cur.fetchone() + return ret + +def use_code(code): + with connect_db() as con: + with con.cursor(prepared=True) as cur: + params = (code,) + res = cur.execute(f"SELECT * FROM codes WHERE code_id=%s", params) + if res.fetchone() != None: + cur.execute(f"UPDATE codes SET used = True WHERE code_id=%s", params) + con.commit() + def get_user_list(): with connect_db() as con: with con.cursor(dictionary=True, prepared=True) as cur: @@ -172,6 +215,13 @@ def get_user_list(): ret = cur.fetchall() return ret +def get_active_user_list(): + with connect_db() as con: + with con.cursor(dictionary=True, prepared=True) as cur: + res = cur.execute(f"SELECT * FROM users WHERE active=True") + ret = cur.fetchall() + return ret + def get_user(telegram_user_id): with connect_db() as con: with con.cursor(dictionary=True, prepared=True) as cur: diff --git a/wallamanta/wallamanta.py b/wallamanta/wallamanta.py index 0c9264c..cc3ec86 100644 --- a/wallamanta/wallamanta.py +++ b/wallamanta/wallamanta.py @@ -26,6 +26,10 @@ ACTION, ADD_PRODUCT_NAME, ADD_PRODUCT_MIN_PRICE, ADD_PRODUCT_MAX_PRICE, \ ADD_PRODUCT_CATEGORY, ADD_PRODUCT_TITLE_EXCLUDE, ADD_PRODUCT_DESCRIPTION_EXCLUDE, \ ADD_PRODUCT_COORDS, ADD_PRODUCT_DISTANCE, REMOVE_PRODUCT, LIST, FINISH, CONTINUE_OR_FINISH = range(13) +MAX_PRODUCTS_TESTING = 5 +MAX_PRODUCTS_PREMIUM = 30 +MAX_PRODUCTS_HOMELABS = 5 + # Enable logging logging.basicConfig( format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO @@ -37,10 +41,10 @@ httpx_logger = logging.getLogger('httpx') httpx_logger.setLevel(logging.WARNING) async def help_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: - if walladb.is_user_valid(helpers.get_telegram_user_id(update)): + if walladb.is_user_active(helpers.get_telegram_user_id(update)): message = "Utiliza el menú de la conversación para añadir un producto y sigue los pasos indicados. Si tienes cualquier duda contacta con @jocarduck para más información." else: - message = "Activa tu periodo de prueba de 7 días con `/test` o contacta con @jocarduck para más información." + message = "No tienes permitido usar este bot." await update.message.reply_markdown_v2(helpers.telegram_escape_characters(message)) async def main_menu(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: @@ -58,21 +62,27 @@ async def main_menu(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: async def menu_click_handler(update: Update, context: CallbackContext): await update.callback_query.edit_message_reply_markup(None) telegram_user_id = helpers.get_telegram_user_id(update) - if walladb.is_user_valid(telegram_user_id): + if walladb.is_user_active(telegram_user_id): context.user_data['telegram_user_id'] = telegram_user_id query = update.callback_query + number_of_products = walladb.count_user_products(telegram_user_id) if query.data == 'add': valid = False number_of_products = walladb.count_user_products(telegram_user_id) if walladb.is_user_testing(telegram_user_id): valid = True - if number_of_products >= 5: - message = "Ya tienes 5 productos en seguimiento. Con premium puedes tener hasta 20." + if number_of_products >= MAX_PRODUCTS_TESTING: + message = f"Ya tienes {MAX_PRODUCTS_TESTING} productos en seguimiento. Borra algunos para añadir más." valid = False elif walladb.is_user_premium(telegram_user_id): valid = True - if number_of_products >= 20: - message = "Ya tienes 20 productos en seguimiento. Borra algunos para añadir más." + if number_of_products >= MAX_PRODUCTS_PREMIUM: + message = f"Ya tienes {MAX_PRODUCTS_PREMIUM} productos en seguimiento. Borra algunos para añadir más." + valid = False + elif walladb.is_user_homelabs(telegram_user_id): + valid = True + if number_of_products >= MAX_PRODUCTS_HOMELABS: + message = f"Ya tienes {MAX_PRODUCTS_HOMELABS} productos en seguimiento. Borra algunos para añadir más." valid = False if valid: await context.bot.send_message(chat_id=update.effective_chat.id, @@ -82,14 +92,22 @@ async def menu_click_handler(update: Update, context: CallbackContext): await context.bot.send_message(chat_id=update.effective_chat.id, text=f'{message}') return ConversationHandler.END if query.data == 'remove': - await send_list(update, context) - return REMOVE_PRODUCT + if number_of_products != 0: + await send_list(update, context) + return REMOVE_PRODUCT + message = "No tienes ninguna búsqueda activa" + await context.bot.send_message(chat_id=update.effective_chat.id, text=f'{message}') + return ConversationHandler.END if query.data == 'list': - await send_list(update, context) - return LIST + if number_of_products != 0: + await send_list(update, context) + return LIST + message = "No tienes ninguna búsqueda activa" + await context.bot.send_message(chat_id=update.effective_chat.id, text=f'{message}') + return ConversationHandler.END else: await context.bot.send_message(chat_id=update.effective_chat.id, - text=helpers.telegram_escape_characters('Activa tu periodo de prueba de 7 días con `/test` o contacta con @jocarduck para más información.'), parse_mode=ParseMode.MARKDOWN_V2) + text=helpers.telegram_escape_characters('No tienes permiso para usar este bot.'), parse_mode=ParseMode.MARKDOWN_V2) return ConversationHandler.END async def add_product_name(update: Update, context: CallbackContext): @@ -340,7 +358,7 @@ async def product_details(update: Update, context: ContextTypes.DEFAULT_TYPE) -> async def send_list(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: telegram_user_id = helpers.get_telegram_user_id(update) - if walladb.is_user_valid(telegram_user_id): + if walladb.is_user_active(telegram_user_id): if walladb.is_user_testing(telegram_user_id) or walladb.is_user_premium(telegram_user_id): query = update.callback_query if query.data == 'remove': @@ -373,28 +391,21 @@ async def remove_user_command(update: Update, context: ContextTypes.DEFAULT_TYPE async def status_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: telegram_user_id = helpers.get_telegram_user_id(update) - if walladb.is_user_valid(telegram_user_id): + if walladb.is_user_active(telegram_user_id): type = walladb.get_user_type(telegram_user_id) until = walladb.get_user_until(telegram_user_id) message = f"Tu cuenta es tipo: {type} y caduca el {helpers.get_spanish_date(until)}." await update.message.reply_markdown_v2(helpers.telegram_escape_characters(message)) -async def test_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: +async def start_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: telegram_user_id = helpers.get_telegram_user_id(update) telegram_user_name = helpers.get_telegram_user_name(update) #if not walladb.is_user_valid(telegram_user_id): if telegram_user_id < 0: message = "Este bot no se puede usar en grupos." elif walladb.get_user(telegram_user_id) == "NoName": - until = helpers.get_date_ahead(7) - walladb.add_test_user(telegram_user_id, telegram_user_name, until) - message = f"Periodo de prueba activado hasta el {helpers.get_spanish_date(until)}." - else: - message = "Ya has utilizado el periodo de prueba." - if walladb.is_user_testing(telegram_user_id): - message = "Ya estás en el periodo de prueba." - elif walladb.is_user_premium(telegram_user_id): - message = "Ya eres premium. No puedes volver al periodo de prueba." + walladb.enable_user(telegram_user_id, telegram_user_name) + message = f"Bienvenido a @wallamanta, usa el comando /help para más información." await update.message.reply_markdown_v2(helpers.telegram_escape_characters(message)) async def add_to_db_and_send(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: @@ -428,6 +439,36 @@ async def list_threads(update: Update, context: ContextTypes.DEFAULT_TYPE) -> No else: await update.message.reply_markdown_v2(helpers.telegram_escape_characters(f"{threads_string}")) +async def message_to_all(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: + if helpers.is_user_admin(update.message.chat_id): + message = update.message.text.split('/message_to_all ')[1] + helpers.send_message_to_all(message) + await update.message.reply_markdown_v2(helpers.telegram_escape_characters(f"Messages sent to all users")) + +async def code(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: + code = update.message.text.split('/code ')[1] + telegram_user_id = update.message.chat_id + if len(code) != 12: + await update.message.reply_markdown_v2(helpers.telegram_escape_characters(f"El código no tiene el formato correcto")) + else: + code = walladb.get_code(code) + if code: + until = helpers.get_date_ahead(code['days']) + try: + if code['type'] == 'premium': + walladb.add_premium_user(telegram_user_id, until) + await update.message.reply_markdown_v2(helpers.telegram_escape_characters(f"Activado periodo premium hasta el {until}")) + elif code['type'] == 'testing': + walladb.add_test_user(telegram_user_id, until) + await update.message.reply_markdown_v2(helpers.telegram_escape_characters(f"Activado periodo testing hasta el {until}")) + walladb.use_code(code) + except Exception as e: + error_code = helpers.get_random_string(8) + logging.info(f"Error trying to checkout code {code}: {e}. {error_code}") + await update.message.reply_markdown_v2(helpers.telegram_escape_characters(f"Ha habido un error, contacta con @jocarduck y dale el siguiente código de error: `{error_code}`")) + else: + await update.message.reply_markdown_v2(helpers.telegram_escape_characters(f"El código no es válido o ya ha sido usado")) + def count_threads(): time.sleep(10) while True: @@ -447,17 +488,19 @@ def main()->None: p = threading.Thread(target=account_checker.work, args=(3600, )) p.start() - p = threading.Thread(target=count_threads) - p.start() + #p = threading.Thread(target=count_threads) + #p.start() # on different commands - answer in Telegram + application.add_handler(CommandHandler("start", start_command)) application.add_handler(CommandHandler("help", help_command)) application.add_handler(CommandHandler("admin", admin_command)) application.add_handler(CommandHandler("add_premium_user", add_premium_user_command)) application.add_handler(CommandHandler("remove_user", remove_user_command)) application.add_handler(CommandHandler("status", status_command)) - application.add_handler(CommandHandler("test", test_command)) application.add_handler(CommandHandler("list_threads", list_threads)) + application.add_handler(CommandHandler("message_to_all", message_to_all)) + application.add_handler(CommandHandler("code", code)) #application.add_handler(CallbackQueryHandler("list", send_list())) #application.add_handler(CallbackQueryHandler(pattern="list", callback=send_list())) diff --git a/wallamanta/worker.py b/wallamanta/worker.py index 51629a9..4786961 100644 --- a/wallamanta/worker.py +++ b/wallamanta/worker.py @@ -1,5 +1,5 @@ -import time import requests +from random import randint import logging import helpers import walladb @@ -13,6 +13,7 @@ logging.basicConfig( logger = logging.getLogger(__name__) def request(product_name, steps=15, latitude=constants.LATITUDE, longitude=constants.LONGITUDE, distance='0', condition='all', min_price=0, max_price=10000000, category="", subcategories=[]): + headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'} distance = str(int(distance) * 1000) url = (f"https://api.wallapop.com/api/v3/general/search?keywords={product_name}" f"&order_by=newest&latitude={latitude}" @@ -22,8 +23,8 @@ def request(product_name, steps=15, latitude=constants.LATITUDE, longitude=const f"&max_sale_price={max_price}" f"&filters_source=quick_filters&language=es_ES") - if condition != "all": - url = url + f"&condition={condition}" # new, as_good_as_new, good, fair, has_given_it_all + #if condition != "all": + # url = url + f"&condition={condition}" # new, as_good_as_new, good, fair, has_given_it_all if category != "": url = url + f"&category_ids={category}" @@ -37,39 +38,45 @@ def request(product_name, steps=15, latitude=constants.LATITUDE, longitude=const search_objects = list() for step in range(steps): + #helpers.random_wait() tries = 5 for _ in range(tries): - response = requests.get(url+f"&step={step}") try: + if randint(0, 1) == 0: + response = requests.get(url+f"&step={step}", timeout=1, headers=headers) + else: + response = requests.get(url+f"&step={step}", timeout=1, headers=headers, proxies=dict(https=f'socks5://{constants.PROXY_SOCKS}')) if response.status_code == 200: search_objects = search_objects + response.json()['search_objects'] break else: logging.info(f"\'{product_name}\' -> Wallapop returned status {response.status_code}. Illegal parameters or Wallapop service is down. Retrying...") except Exception as e: - logging.info("Error while querying Wallapop, try #{_}: " + e) - time.sleep(3) + logging.info(f"Error while querying Wallapop, try #{_}: {e}") + for __ in range(_): + helpers.random_wait() return search_objects def first_run(product): - logging.info(f"First run for {product['product_name']} for {walladb.get_user(product['telegram_user_id'])} ({walladb.get_user(product['telegram_user_id'])})") + steps = 1 + #logging.info(f"First run for {product['product_name']} for {walladb.get_user(product['telegram_user_id'])} ({walladb.get_user(product['telegram_user_id'])})") list = [] if not helpers.is_valid_request(product): return list if product['category'] == '': - articles = request(product['product_name'], 15, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], product['category']) + articles = request(product['product_name'], steps, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], product['category']) for article in articles: list.insert(0, article['id']) else: if '0' in product['category'].split(','): - articles = request(product['product_name'], 15, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price']) + articles = request(product['product_name'], steps, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price']) for article in articles: list.insert(0, article['id']) else: for category in product['category'].split(','): if product['subcategory'] == '' or not helpers.has_subcategory(category): - articles = request(product['product_name'], 15, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category) + articles = request(product['product_name'], steps, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category) for article in articles: list.insert(0, article['id']) else: @@ -77,33 +84,35 @@ def first_run(product): for subcategory in product['subcategory'].split(','): if helpers.is_subcategory(category, subcategory): subcategories.append(subcategory) - articles = request(product['product_name'], 15, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category, subcategories) + articles = request(product['product_name'], steps, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category, subcategories) for article in articles: list.insert(0, article['id']) return list -def work(product, list): - logging.info(f"Searching {product['product_name']}") +def work(product, searches_list): + list = searches_list.copy() + steps = 1 + logging.info(f"Searching {product['product_name']} ({product['id']}) - ({walladb.get_user(product['telegram_user_id'])})") articles_list = [] if product['category'] == '': - articles_list.append(request(product['product_name'], 1, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'])) + articles_list.append(request(product['product_name'], steps, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'])) else: if '0' in product['category'].split(','): - articles_list.append(request(product['product_name'], 1, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'])) + articles_list.append(request(product['product_name'], steps, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'])) else: for category in product['category'].split(','): if product['subcategory'] == '' or not helpers.has_subcategory(category): - articles_list.append(request(product['product_name'], 1, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category)) + articles_list.append(request(product['product_name'], steps, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category)) else: subcategories = [] for subcategory in product['subcategory'].split(','): if helpers.is_subcategory(category, subcategory): subcategories.append(subcategory) - articles_list.append(request(product['product_name'], 1, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category, subcategories)) + articles_list.append(request(product['product_name'], steps, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category, subcategories)) for articles in articles_list: for article in articles: if not article['id'] in list: - logging.info(f"Found article {article['title']} for {walladb.get_user(product['telegram_user_id'])} ({product['telegram_user_id']})") + logging.info(f"Found article {article['title']} for {product['product_name']} ({product['id']}) - ({walladb.get_user(product['telegram_user_id'])})") try: if not has_excluded_words(article['title'].lower(), article['description'].lower(), product['title_description_exclude']) and not is_title_key_word_excluded(article['title'].lower(), product['title_exclude']): helpers.send_article(article, product) @@ -112,7 +121,7 @@ def work(product, list): logging.info("---------- EXCEPTION -----------") logging.info(f"{product['product_name']} worker crashed. {e}") logging.info(f"{product['product_name']}: Trying to parse {article['id']}: {article['title']} .\n") - return list[:600] + return list[:100] def has_excluded_words(title, description, excluded_words): if len(excluded_words) > 0: @@ -129,4 +138,4 @@ def is_title_key_word_excluded(title, excluded_words): logging.info("Checking '" + word + "' for title: '" + title) if word.lower().lstrip().rstrip() in title.lower(): return True - return False \ No newline at end of file + return False