diff --git a/data/fonts/Roboto-Bold.ttf b/data/fonts/Roboto-Bold.ttf new file mode 100644 index 0000000..43da14d Binary files /dev/null and b/data/fonts/Roboto-Bold.ttf differ diff --git a/data/images/wallamanta_logo.png b/data/images/wallamanta_logo.png new file mode 100644 index 0000000..ee0a81b Binary files /dev/null and b/data/images/wallamanta_logo.png differ diff --git a/wallamanta/__pycache__/constants.cpython-39.pyc b/wallamanta/__pycache__/constants.cpython-39.pyc new file mode 100644 index 0000000..90b48d2 Binary files /dev/null and b/wallamanta/__pycache__/constants.cpython-39.pyc differ diff --git a/wallamanta/constants.py b/wallamanta/constants.py new file mode 100644 index 0000000..06ded67 --- /dev/null +++ b/wallamanta/constants.py @@ -0,0 +1,36 @@ +import os + +#TELEGRAM_ESCAPE_CHARACTERS = ['_', '*', '[', ']', '(', ')', '~', '>', '#', '+', '-', '=', '|', '{', '}', '.', '!'] +TELEGRAM_ESCAPE_CHARACTERS = ['_', '*', '[', ']', '(', ')', '~', '>', '#', '+', '-', '=', '|', '{', '}', '.', '!'] +ADMIN_IDS = [10101691] +TELEGRAM_TOKEN = os.getenv("TELEGRAM_TOKEN") +DB = "/app/data/wallamanta.db" +LATITUDE = os.getenv("LATITUDE") +LONGITUDE = os.getenv("LONGITUDE") +SLEEP_TIME = int(os.getenv("SLEEP_TIME")) + +CATEGORIES = {'coches': 100, + 'motos': 14000, + 'motor y accesorios': 12800, + 'moda y accesorios': 12465, + 'inmobiliaria': 200, + 'tv audio y foto': 12545, + 'móviles y telefonía': 16000, + 'informática y electrónica': 15000, + 'informática': 15000, + 'electrónica': 15000, + 'deporte y ocio': 12579, + 'bicicletas': 17000, + 'consolas y videojuegos': 12900, + 'consolas': 12900, + 'videojuegos': 12900, + 'hogar y jardín': 12467, + 'electrodomésticos': 13100, + 'cine libros y música': 12463, + 'niños y bebés': 12461, + 'coleccionismo': 18000, + 'construcción y reformas': 19000, + 'industria y agricultura': 20000, + 'empleo': 21000, + 'servicios': 13200, + 'otros': 12485} \ No newline at end of file diff --git a/wallamanta/helpers.py b/wallamanta/helpers.py index 3ca4c50..6bb4291 100644 --- a/wallamanta/helpers.py +++ b/wallamanta/helpers.py @@ -1,11 +1,22 @@ +import time +import random +import requests +import logging +import os +import constants + +from PIL import Image, ImageDraw, ImageFont from datetime import datetime, timedelta -#TELEGRAM_ESCAPE_CHARACTERS = ['_', '*', '[', ']', '(', ')', '~', '>', '#', '+', '-', '=', '|', '{', '}', '.', '!'] -TELEGRAM_ESCAPE_CHARACTERS = ['_', '*', '[', ']', '(', ')', '~', '>', '#', '+', '-', '=', '|', '{', '}', '.', '!'] -ADMIN_IDS = [10101691] +# Enable logging +logging.basicConfig( + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO +) + +logger = logging.getLogger(__name__) def telegram_escape_characters(text): - for character in TELEGRAM_ESCAPE_CHARACTERS: + for character in constants.TELEGRAM_ESCAPE_CHARACTERS: try: text = text.replace(character, f'\\{character}') except: @@ -13,11 +24,86 @@ def telegram_escape_characters(text): return text def is_user_admin(telegram_user_id): - return telegram_user_id in ADMIN_IDS + return telegram_user_id in constants.ADMIN_IDS def get_telegram_user_id(update): return update.message.chat_id def get_date_ahead(add_days): date_ahead = datetime.today() + timedelta(days=add_days) - return date_ahead.strftime("%d/%m/%Y") \ No newline at end of file + return date_ahead.strftime("%d/%m/%Y") + +def random_wait(): + time.sleep(random.random()) + +def download_image(article): + r = requests.get(article['images'][0]['original']) + if r.status_code == 200: + image = open(f"/app/data/images/products/{article['id']}.jpg", "wb") + image.write(r.content) + image.close() + +def create_image(article): + download_image(article) + currency = '?' + if article['currency'] == 'EUR': + currency = '€' + price = str(article['price']) + currency + wallamanta_text = "@wallamanta_bot" + width = 1280 + height = 800 + baseheight = int(height * 0.85) + # límite de ancho para la parte izquierda (producto) + wlimit = int(((width / 3) * 2) - 80) + # límite de ancho para los logos (homelabers y amazon) + wlogo = int(width * 0.2) + # fuente y tamaño + font = ImageFont.truetype("/app/data/fonts/Roboto-Bold.ttf", 90) + wallamanta_text_font = ImageFont.truetype("/app/data/fonts/Roboto-Bold.ttf", 40) + # inicializamos canvas + image = Image.new('RGBA', (width, height), (255, 255, 255)) + # logo homelabers, redimensionamos y ponemos en la parte derecha arriba + #logo_image = Image.open("/app/data/images/logo.png").convert("RGBA") + #hlogo = int((float(logo_image.size[1]) * float(lpercent))) + #lpercent = wlogo / float(logo_image.size[0]) + #logo_image = logo_image.resize((wlogo, hlogo), Image.Resampling.LANCZOS) + #image.paste(logo_image, (int((width / 6) * 5 - logo_image.size[0] / 2), int(height * 0.1)), logo_image) + # logo wallamanta, redimensionamos y ponemos en la parte derecha abajo + #wallamanta_logo = Image.open("/app/data/images/wallamanta_logo.png").convert("RGBA") + #lpercent = wlogo / float(wallamanta_logo.size[0]) + #hlogo = int((float(wallamanta_logo.size[1]) * float(lpercent))) + #wallamanta_logo = wallamanta_logo.resize((wlogo, hlogo), Image.Resampling.LANCZOS) + #image.paste(wallamanta_logo, (int((width / 6) * 5 - wallamanta_logo.size[0] / 2), int(height - height * 0.2)), wallamanta_logo) + draw = ImageDraw.Draw(image) + # escribimos @wallamanta_bot + wtext, htext = draw.textsize(wallamanta_text, font=wallamanta_text_font) + draw.text(((width / 6) * 5 - wtext / 2, int(height - height * 0.2)), wallamanta_text, "#13C1AC", font=wallamanta_text_font) + # escribimos el precio + wtext, htext = draw.textsize(price, font=font) + draw.text(((width / 6) * 5 - wtext / 2, height / 2 - htext / 2), price, (0, 0, 0), font=font) + # dibujamos rectángulo verde externo, con un margen externo y ancho determinado + draw.rectangle([15, 15, width - 15, height - 15], width = 15, outline="#13C1AC") + # ponemos la imagen del producto en la parte izquierda y se redimensiona dependiendo de lo ancho + product_image = Image.open(f"/app/data/images/products/{article['id']}.jpg") + hpercent = (baseheight / float(product_image.size[1])) + wsize = int((float(product_image.size[0]) * float(hpercent))) + if wsize < wlimit: + product_image = product_image.resize((wsize, baseheight), Image.Resampling.LANCZOS) + else: + wpercent = wlimit / float(product_image.size[0]) + hsize = int((float(product_image.size[1]) * float(wpercent))) + product_image = product_image.resize((wlimit, hsize), Image.Resampling.LANCZOS) + image.paste(product_image, (int((width/3)-(product_image.size[0]/2)), int((height/2) - (product_image.size[1]/2)))) + # guardamos la imagen con otro nombre + image.save(f"/app/data/images/products/{article['id']}_composed.png", quality=95) + +def send_article(article, product): + create_image(article) + text = f"*{telegram_escape_characters(article['title'])}*\n\n*Descripción*: {telegram_escape_characters(article['description'])}\\nn*Precio*: {telegram_escape_characters(str(article['price']))} {telegram_escape_characters(article['currency'])}\n\n[IR AL ANUNCIO](https://es\.wallapop\.com/item/{telegram_escape_characters(article['web_slug'])})" + url = f"https://api.telegram.org/bot{constants.TELEGRAM_TOKEN}/sendPhoto?chat_id={product['telegram_user_id']}&caption={text}&parse_mode=MarkdownV2" + files = {'photo':open(f"/app/data/images/products/{article['id']}_composed.png", 'rb')} + logging.info(requests.post(url, files=files).content) + +def get_category_id(category): + ret = constants.CATEGORIES.get(category, '') + return ret \ No newline at end of file diff --git a/wallamanta/requirements.txt b/wallamanta/requirements.txt index 0248115..47cd6d5 100644 --- a/wallamanta/requirements.txt +++ b/wallamanta/requirements.txt @@ -1,3 +1,4 @@ python-telegram-bot==20.1 requests==2.28.1 -prettytable==3.6.0 \ No newline at end of file +prettytable==3.6.0 +Pillow==9.4.0 \ No newline at end of file diff --git a/wallamanta/walladb.py b/wallamanta/walladb.py index 58c9990..0661bd8 100644 --- a/wallamanta/walladb.py +++ b/wallamanta/walladb.py @@ -1,11 +1,7 @@ import sqlite3 import os import logging - -LATITUDE = os.getenv("LATITUDE") -LONGITUDE = os.getenv("LONGITUDE") - -DB = "/app/data/wallamanta.db" +import constants # Enable logging logging.basicConfig( @@ -21,16 +17,16 @@ def dict_factory(cursor, row): return d def setup_db(): - con = sqlite3.connect(DB) + con = sqlite3.connect(constants.DB) cur = con.cursor() cur.execute("CREATE TABLE IF NOT EXISTS users(telegram_user_id, active, type, until)") cur.execute("CREATE TABLE IF NOT EXISTS products(product_name, distance, \ - latitude, longitude, condition, min_price, max_price, \ + latitude, longitude, condition, min_price, max_price, category, \ title_exclude, title_description_exclude, telegram_user_id)") con.close() def is_user_valid(telegram_user_id): - con = sqlite3.connect(DB) + con = sqlite3.connect(constants.DB) cur = con.cursor() res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id={telegram_user_id} AND active=True") ret = res.fetchone() != None @@ -38,7 +34,7 @@ def is_user_valid(telegram_user_id): return ret def is_user_premium(telegram_user_id): - con = sqlite3.connect(DB) + con = sqlite3.connect(constants.DB) cur = con.cursor() res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id={telegram_user_id} AND active=True AND type='premium'") ret = res.fetchone() != None @@ -46,7 +42,7 @@ def is_user_premium(telegram_user_id): return ret def is_user_testing(telegram_user_id): - con = sqlite3.connect(DB) + con = sqlite3.connect(constants.DB) cur = con.cursor() res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id={telegram_user_id} AND active=True AND type='testing'") ret = res.fetchone() != None @@ -55,7 +51,7 @@ def is_user_testing(telegram_user_id): def add_premium_user(telegram_user_id, until): found = False - con = sqlite3.connect(DB) + con = sqlite3.connect(constants.DB) cur = con.cursor() res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id={telegram_user_id}") if res.fetchone() is None: @@ -70,7 +66,7 @@ def add_premium_user(telegram_user_id, until): def add_test_user(telegram_user_id, until): found = False - con = sqlite3.connect(DB) + con = sqlite3.connect(constants.DB) cur = con.cursor() res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id={telegram_user_id}") if res.fetchone() is None: @@ -82,7 +78,7 @@ def add_test_user(telegram_user_id, until): return not found def remove_valid_user(telegram_user_id): - con = sqlite3.connect(DB) + con = sqlite3.connect(constants.DB) cur = con.cursor() res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id={telegram_user_id}") if res.fetchone() != None: @@ -91,17 +87,33 @@ def remove_valid_user(telegram_user_id): con.close() def get_user_list(): - con = sqlite3.connect(DB) + con = sqlite3.connect(constants.DB) cur = con.cursor() res = cur.execute(f"SELECT * FROM users") ret = res.fetchall() con.close() return ret +def get_user_type(telegram_user_id): + con = sqlite3.connect(constants.DB) + cur = con.cursor() + res = cur.execute(f"SELECT type FROM users WHERE telegram_user_id={telegram_user_id}") + ret = res.fetchone() + con.close() + return ret[0] + +def get_user_until(telegram_user_id): + con = sqlite3.connect(constants.DB) + cur = con.cursor() + res = cur.execute(f"SELECT until FROM users WHERE telegram_user_id={telegram_user_id}") + ret = res.fetchone() + con.close() + return ret[0] + def get_product(product): product_name = product.get('product_name') telegram_user_id = product.get('telegram_user_id') - con = sqlite3.connect(DB) + con = sqlite3.connect(constants.DB) con.row_factory = dict_factory cur = con.cursor() res = cur.execute(f"SELECT * FROM products WHERE telegram_user_id={telegram_user_id} \ @@ -111,7 +123,7 @@ def get_product(product): return ret def get_products_from_user(telegram_user_id): - con = sqlite3.connect(DB) + con = sqlite3.connect(constants.DB) con.row_factory = dict_factory cur = con.cursor() res = cur.execute(f"SELECT * FROM products WHERE telegram_user_id={telegram_user_id}") @@ -120,7 +132,7 @@ def get_products_from_user(telegram_user_id): return ret def get_all_products(): - con = sqlite3.connect(DB) + con = sqlite3.connect(constants.DB) con.row_factory = dict_factory cur = con.cursor() res = cur.execute(f"SELECT * FROM products") @@ -132,31 +144,32 @@ def add_product(product): condition = 'all' product_name = product.get('product_name') distance = product.get('distance', 0) - latitude = product.get('latitude', LATITUDE) - longitude = product.get('longitude', LONGITUDE) + latitude = product.get('latitude', constants.LATITUDE) + longitude = product.get('longitude', constants.LONGITUDE) min_price = product.get('min_price') max_price = product.get('max_price') title_exclude = product.get('title_exclude', '') title_description_exclude = product.get('title_description_exclude', '') + category = product.get('category', '') telegram_user_id = product.get('telegram_user_id') params = (product_name, \ distance, latitude, longitude, condition, min_price, \ - max_price, title_exclude, title_description_exclude, telegram_user_id) + max_price, category, title_exclude, title_description_exclude, telegram_user_id) logging.info(f"Trying to add: {product_name}, {title_exclude}, {title_description_exclude}, {telegram_user_id}") - con = sqlite3.connect(DB) + con = sqlite3.connect(constants.DB) cur = con.cursor() res = cur.execute(f"SELECT * FROM products WHERE telegram_user_id={product.get('telegram_user_id')} \ AND product_name='{product.get('product_name')}'") if res.fetchone() is None: - cur.execute("INSERT INTO products VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", params) + cur.execute("INSERT INTO products VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", params) con.commit() con.close() def remove_product(product): removed = False if get_product(product): - con = sqlite3.connect(DB) + con = sqlite3.connect(constants.DB) cur = con.cursor() res = cur.execute(f"DELETE FROM products WHERE telegram_user_id={product.get('telegram_user_id')} \ AND product_name='{product.get('product_name')}'") @@ -166,7 +179,7 @@ def remove_product(product): return removed def count_user_products(telegram_user_id): - con = sqlite3.connect(DB) + con = sqlite3.connect(constants.DB) cur = con.cursor() res = cur.execute(f"SELECT Count() FROM products WHERE telegram_user_id={telegram_user_id}") ret = res.fetchone()[0] diff --git a/wallamanta/wallamanta.py b/wallamanta/wallamanta.py index 797d844..eeeb978 100644 --- a/wallamanta/wallamanta.py +++ b/wallamanta/wallamanta.py @@ -5,17 +5,12 @@ import logging import prettytable import helpers import walladb +import constants from worker import Worker from telegram import Update from telegram.ext import Application, CommandHandler, ContextTypes -TELEGRAM_CHANNEL_ID = os.getenv("TELEGRAM_CHANNEL_ID") -TELEGRAM_TOKEN = os.getenv("TELEGRAM_TOKEN") -LATITUDE = os.getenv("LATITUDE") -LONGITUDE = os.getenv("LONGITUDE") -SLEEP_TIME = os.getenv("SLEEP_TIME") - # Enable logging logging.basicConfig( format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO @@ -33,24 +28,33 @@ def save_json_file(products): async def help_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: if walladb.is_user_valid(helpers.get_telegram_user_id(update)): - message = """Añade un producto con `/add producto;precio_mínimo;precio_máximo,excluir_título(opcional, separado por comas);excluir_descripción_y_título(opciona, separado por comas);latitud(opcional);longitud(opcional),distancia(opcional)`\n - Ejemplo: `/add placa base itx;0;150`\n - Ejemplo 2: `/add cpu;10;30;;intel,core 2 duo,celeron;;;100`\n - Los campos opcionales que se dejen vacíos tomarán el valor configurado en el archivo `.env`\n - Lista los productos con `/list` o obtén la información de uno en concreto con `/list nombre del producto`\n - Borra un producto con `/remove nombre del producto`""" + message = """Añade un producto con `/add producto;precio_mínimo;precio_máximo,categoría,excluir_título(opcional, separado por comas); +excluir_descripción_y_título(opciona, separado por comas);latitud(opcional);longitud(opcional),distancia(opcional)`\n +Ejemplo: `/add placa base itx;0;150`\n +Ejemplo 2: `/add cpu;10;30;;intel,core 2 duo,celeron;;;100`\n +Los campos opcionales que se dejen vacíos tomarán el valor configurado en el archivo `.env`\n +Lista los productos con `/list` o obtén la información de uno en concreto con `/list nombre del producto`\n +Borra un producto con `/remove nombre del producto`\n +`/status` muestra tu tipo de membresía y fecha de caducidad""" else: message = """Activa tu periodo de prueba de 7 días con `/test` o contacta con @jocarduck para más información.""" await update.message.reply_markdown_v2(helpers.telegram_escape_characters(message)) +async def categories_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: + if walladb.is_user_valid(helpers.get_telegram_user_id(update)): + message = '' + for category in constants.CATEGORIES: + message = f"{message}, {category}" + await update.message.reply_markdown_v2(f"```{helpers.telegram_escape_characters(message)}```") + async def add_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: telegram_user_id = helpers.get_telegram_user_id(update) if walladb.is_user_valid(telegram_user_id): number_of_products = walladb.count_user_products(telegram_user_id) - message = """Tienes que pasar el número correcto de parámetros: `/add producto;precio_mínimo;precio_máximo,excluir_título(opcional, separado por comas);excluir_descripción_y_título(opciona, separado por comas);latitud(opcional);longitud(opcional),distancia(opcional)`\n - Ejemplo: `/add placa base itx;0;150`\n - Ejemplo 2: `/add cpu;10;30;;intel,core 2 duo,celeron;;;100`\n - Los campos opcionales que se dejen vacíos tomarán el valor configurado en el archivo `.env`""" + message = """Tienes que pasar el número correcto de parámetros: `/add producto;precio_mínimo;precio_máximo,categoría,excluir_título(opcional, separado por comas);excluir_descripción_y_título(opciona, separado por comas);latitud(opcional);longitud(opcional),distancia(opcional)`\n +Ejemplo: `/add placa base itx;0;150`\n +Ejemplo 2: `/add cpu;10;30;;intel,core 2 duo,celeron;;;100`\n +Los campos opcionales que se dejen vacíos tomarán el valor configurado en el archivo `.env`""" valid = False if walladb.is_user_testing(telegram_user_id): valid = True @@ -72,15 +76,17 @@ async def add_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> Non args = args[1].split(";") product['product_name'], product['min_price'], product['max_price'] = args[0:3] if len(args) > 3 and args[3]: - product['title_exclude'] = args[3] + product['category'] = helpers.get_category_id(args[3]) if len(args) > 4 and args[4]: - product['title_description_exclude'] = args[4] + product['title_exclude'] = args[4] if len(args) > 5 and args[5]: - product['latitude'] = args[5] + product['title_description_exclude'] = args[5] if len(args) > 6 and args[6]: - product['longitude'] = args[6] + product['latitude'] = args[6] if len(args) > 7 and args[7]: - product['distance'] = args[7] + product['longitude'] = args[7] + if len(args) > 8 and args[8]: + product['distance'] = args[8] logging.info(f'Adding: {product}') if not walladb.get_product(product): @@ -156,12 +162,14 @@ async def remove_user_command(update: Update, context: ContextTypes.DEFAULT_TYPE await update.message.reply_markdown_v2(helpers.telegram_escape_characters(f"{telegram_user_id} desactivado.")) async def status_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: - # TODO: Obtener estado del usuario y fecha de caducidad de suscripción en el caso de haberla - message = '' - await update.message.reply_markdown_v2(helpers.telegram_escape_characters(message)) + telegram_user_id = helpers.get_telegram_user_id(update) + if walladb.is_user_valid(telegram_user_id): + type = walladb.get_user_type(telegram_user_id) + until = walladb.get_user_until(telegram_user_id) + message = f"Tu cuenta es tipo: {type} y caduca el {until}." + await update.message.reply_markdown_v2(helpers.telegram_escape_characters(message)) async def test_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: - # TODO: Activar usuario en modo testing y responder con fecha de caducidad telegram_user_id = helpers.get_telegram_user_id(update) if not walladb.is_user_valid(telegram_user_id): until = helpers.get_date_ahead(7) @@ -186,10 +194,11 @@ def main()->None: """Start the bot.""" # Create the Application and pass it your bot's token. - application = Application.builder().token(TELEGRAM_TOKEN).build() + application = Application.builder().token(constants.TELEGRAM_TOKEN).build() # on different commands - answer in Telegram application.add_handler(CommandHandler("help", help_command)) + application.add_handler(CommandHandler("categories", categories_command)) application.add_handler(CommandHandler("add", add_command)) application.add_handler(CommandHandler("remove", remove_command)) application.add_handler(CommandHandler("list", list_command)) diff --git a/wallamanta/worker.py b/wallamanta/worker.py index 6d657ff..72ec4db 100644 --- a/wallamanta/worker.py +++ b/wallamanta/worker.py @@ -6,13 +6,7 @@ import logging import json import helpers import walladb - -TELEGRAM_CHANNEL_ID = os.getenv("TELEGRAM_CHANNEL_ID") -TELEGRAM_TOKEN = os.getenv("TELEGRAM_TOKEN") -LATITUDE = os.getenv("LATITUDE") -LONGITUDE = os.getenv("LONGITUDE") -SLEEP_TIME = int(os.getenv("SLEEP_TIME")) - +import constants # Enable logging logging.basicConfig( @@ -24,16 +18,14 @@ logger = logging.getLogger(__name__) class Worker: def is_valid_request(self, product): - is_valid = True - if not walladb.get_product(product): - is_valid = False - if not walladb.is_user_valid(product['telegram_user_id']): - is_valid = False - if not walladb.is_user_premium(product['telegram_user_id']): - is_valid = False + is_valid = False + if walladb.get_product(product): + if walladb.is_user_valid(product['telegram_user_id']): + if walladb.is_user_premium(product['telegram_user_id']) or walladb.is_user_testing(product['telegram_user_id']): + is_valid = True return is_valid - def request(self, product_name, n_articles, latitude=LATITUDE, longitude=LONGITUDE, distance='0', condition='all', min_price=0, max_price=10000000): + def request(self, product_name, n_articles, latitude=constants.LATITUDE, longitude=constants.LONGITUDE, distance='0', condition='all', min_price=0, max_price=10000000, category=""): url = (f"http://api.wallapop.com/api/v3/general/search?keywords={product_name}" f"&order_by=newest&latitude={latitude}" f"&longitude={longitude}" @@ -41,14 +33,17 @@ class Worker: f"&min_sale_price={min_price}" f"&max_sale_price={max_price}" f"&filters_source=quick_filters&language=es_ES") - + if condition != "all": url = url + f"&condition={condition}" # new, as_good_as_new, good, fair, has_given_it_all + if category != "": + url = url + f"&category_ids={category}" + for step in range(15): while True: - time.sleep(0.5) - response = requests.get(url+ f"&step={step+1}") + helpers.random_wait() + response = requests.get(url+f"&step={step+1}") try: if response.status_code == 200: break @@ -62,45 +57,41 @@ class Worker: return json_data['search_objects'] def first_run(self, product): + for i in range(5): + helpers.random_wait() list = [] if not self.is_valid_request(product): return list - articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price']) + articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], product['category']) for article in articles: list.insert(0, article['id']) return list def work(self, product, list): + helpers.random_wait() # Random wait to make requests separated in time in order to prevent API rate limit exec_times = [] - bot = telegram.Bot(token = TELEGRAM_TOKEN) while True: if not self.is_valid_request(product): + logging.info(f"{product['product_name']} not valid anymore, exiting worker") break # Exits and ends worker thread start_time = time.time() - articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price']) + articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], product['category']) for article in articles: if not article['id'] in list: logging.info(f"Found article {article['title']}") try: if not self.has_excluded_words(article['title'].lower(), article['description'].lower(), product['title_description_exclude']) and not self.is_title_key_word_excluded(article['title'].lower(), product['title_exclude']): try: - text = f"*Artículo*: {helpers.telegram_escape_characters(article['title'])}\n*Descripción*: {helpers.telegram_escape_characters(article['description'])}\n*Precio*: {helpers.telegram_escape_characters(str(article['price']))} {helpers.telegram_escape_characters(article['currency'])}\n[Ir al anuncio](https://es\.wallapop\.com/item/{helpers.telegram_escape_characters(article['web_slug'])})" - url = f"https://api.telegram.org/bot{TELEGRAM_TOKEN}/sendMessage?chat_id={TELEGRAM_CHANNEL_ID}&text={text}&parse_mode=MarkdownV2" - logging.info(text) - logging.info(requests.get(url).json()) + helpers.send_article(article, product) except: - text = f"*Artículo*: {helpers.telegram_escape_characters(article['title'])}\n*Descripción*: {helpers.telegram_escape_characters(article['description'])}\n*Precio*: {helpers.telegram_escape_characters(str(article['price']))} {helpers.telegram_escape_characters(article['currency'])}\n[Ir al anuncio](https://es\.wallapop\.com/item/{helpers.telegram_escape_characters(article['web_slug'])})" - url = f"https://api.telegram.org/bot{TELEGRAM_TOKEN}/sendMessage?chat_id={TELEGRAM_CHANNEL_ID}&text={text}&parse_mode=MarkdownV2" - logging.info(text) - logging.info(requests.get(url).json()) + helpers.send_article(article, product) time.sleep(1) # Avoid Telegram flood restriction list.insert(0, article['id']) except Exception as e: logging.info("---------- EXCEPTION -----------") logging.info(f"{product['product_name']} worker crashed. {e}") logging.info(f"{product['product_name']}: Trying to parse {article['id']}: {article['title']} .\n") - - time.sleep(SLEEP_TIME) + time.sleep(constants.SLEEP_TIME) exec_times.append(time.time() - start_time) logging.info(f"\'{product['product_name']}\' node-> last: {exec_times[-1]} max: {self.get_max_time(exec_times)} avg: {self.get_average_time(exec_times)}")