Jocaru/wallamanta
1
0
Fork 0
Code Issues 15 Packages Projects Releases Wiki Activity

Added constants. Added image creation. Refactored send found article

Browse Source
This commit is contained in:
Joan
2023-03-08 22:46:19 +01:00
parent f7b6512500
commit 67b1ee50ea
9 changed files with 224 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
data/fonts/Roboto-Bold.ttf Normal file
View File

Binary file not shown.

BIN
data/images/wallamanta_logo.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 5.5 KiB

BIN
wallamanta/__pycache__/constants.cpython-39.pyc Normal file
View File

Binary file not shown.

36
wallamanta/constants.py Normal file
View File

@@ -0,0 +1,36 @@
import os
#TELEGRAM_ESCAPE_CHARACTERS = ['_', '*', '[', ']', '(', ')', '~', '>', '#', '+', '-', '=', '|', '{', '}', '.', '!']
TELEGRAM_ESCAPE_CHARACTERS = ['_', '*', '[', ']', '(', ')', '~', '>', '#', '+', '-', '=', '|', '{', '}', '.', '!']
ADMIN_IDS = [10101691]
TELEGRAM_TOKEN = os.getenv("TELEGRAM_TOKEN")
DB = "/app/data/wallamanta.db"
LATITUDE = os.getenv("LATITUDE")
LONGITUDE = os.getenv("LONGITUDE")
SLEEP_TIME = int(os.getenv("SLEEP_TIME"))
CATEGORIES = {'coches': 100,
'motos': 14000,
'motor y accesorios': 12800,
'moda y accesorios': 12465,
'inmobiliaria': 200,
'tv audio y foto': 12545,
'móviles y telefonía': 16000,
'informática y electrónica': 15000,
'informática': 15000,
'electrónica': 15000,
'deporte y ocio': 12579,
'bicicletas': 17000,
'consolas y videojuegos': 12900,
'consolas': 12900,
'videojuegos': 12900,
'hogar y jardín': 12467,
'electrodomésticos': 13100,
'cine libros y música': 12463,
'niños y bebés': 12461,
'coleccionismo': 18000,
'construcción y reformas': 19000,
'industria y agricultura': 20000,
'empleo': 21000,
'servicios': 13200,
'otros': 12485}

96
wallamanta/helpers.py
View File

@@ -1,11 +1,22 @@
import time
import random
import requests
import logging
import os
import constants
from PIL import Image, ImageDraw, ImageFont
from datetime import datetime, timedelta
#TELEGRAM_ESCAPE_CHARACTERS = ['_', '*', '[', ']', '(', ')', '~', '>', '#', '+', '-', '=', '|', '{', '}', '.', '!']
TELEGRAM_ESCAPE_CHARACTERS = ['_', '*', '[', ']', '(', ')', '~', '>', '#', '+', '-', '=', '|', '{', '}', '.', '!']
ADMIN_IDS = [10101691]
# Enable logging
logging.basicConfig(
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO
)
logger = logging.getLogger(__name__)
def telegram_escape_characters(text):
for character in TELEGRAM_ESCAPE_CHARACTERS:
for character in constants.TELEGRAM_ESCAPE_CHARACTERS:
try:
text = text.replace(character, f'\\{character}')
except:
@@ -13,7 +24,7 @@ def telegram_escape_characters(text):
return text
def is_user_admin(telegram_user_id):
return telegram_user_id in ADMIN_IDS
return telegram_user_id in constants.ADMIN_IDS
def get_telegram_user_id(update):
return update.message.chat_id
@@ -21,3 +32,78 @@ def get_telegram_user_id(update):
def get_date_ahead(add_days):
date_ahead = datetime.today() + timedelta(days=add_days)
return date_ahead.strftime("%d/%m/%Y")
def random_wait():
time.sleep(random.random())
def download_image(article):
r = requests.get(article['images'][0]['original'])
if r.status_code == 200:
image = open(f"/app/data/images/products/{article['id']}.jpg", "wb")
image.write(r.content)
image.close()
def create_image(article):
download_image(article)
currency = '?'
if article['currency'] == 'EUR':
currency = '€'
price = str(article['price']) + currency
wallamanta_text = "@wallamanta_bot"
width = 1280
height = 800
baseheight = int(height * 0.85)
# límite de ancho para la parte izquierda (producto)
wlimit = int(((width / 3) * 2) - 80)
# límite de ancho para los logos (homelabers y amazon)
wlogo = int(width * 0.2)
# fuente y tamaño
font = ImageFont.truetype("/app/data/fonts/Roboto-Bold.ttf", 90)
wallamanta_text_font = ImageFont.truetype("/app/data/fonts/Roboto-Bold.ttf", 40)
# inicializamos canvas
image = Image.new('RGBA', (width, height), (255, 255, 255))
# logo homelabers, redimensionamos y ponemos en la parte derecha arriba
#logo_image = Image.open("/app/data/images/logo.png").convert("RGBA")
#hlogo = int((float(logo_image.size[1]) * float(lpercent)))
#lpercent = wlogo / float(logo_image.size[0])
#logo_image = logo_image.resize((wlogo, hlogo), Image.Resampling.LANCZOS)
#image.paste(logo_image, (int((width / 6) * 5 - logo_image.size[0] / 2), int(height * 0.1)), logo_image)
# logo wallamanta, redimensionamos y ponemos en la parte derecha abajo
#wallamanta_logo = Image.open("/app/data/images/wallamanta_logo.png").convert("RGBA")
#lpercent = wlogo / float(wallamanta_logo.size[0])
#hlogo = int((float(wallamanta_logo.size[1]) * float(lpercent)))
#wallamanta_logo = wallamanta_logo.resize((wlogo, hlogo), Image.Resampling.LANCZOS)
#image.paste(wallamanta_logo, (int((width / 6) * 5 - wallamanta_logo.size[0] / 2), int(height - height * 0.2)), wallamanta_logo)
draw = ImageDraw.Draw(image)
# escribimos @wallamanta_bot
wtext, htext = draw.textsize(wallamanta_text, font=wallamanta_text_font)
draw.text(((width / 6) * 5 - wtext / 2, int(height - height * 0.2)), wallamanta_text, "#13C1AC", font=wallamanta_text_font)
# escribimos el precio
wtext, htext = draw.textsize(price, font=font)
draw.text(((width / 6) * 5 - wtext / 2, height / 2 - htext / 2), price, (0, 0, 0), font=font)
# dibujamos rectángulo verde externo, con un margen externo y ancho determinado
draw.rectangle([15, 15, width - 15, height - 15], width = 15, outline="#13C1AC")
# ponemos la imagen del producto en la parte izquierda y se redimensiona dependiendo de lo ancho
product_image = Image.open(f"/app/data/images/products/{article['id']}.jpg")
hpercent = (baseheight / float(product_image.size[1]))
wsize = int((float(product_image.size[0]) * float(hpercent)))
if wsize < wlimit:
product_image = product_image.resize((wsize, baseheight), Image.Resampling.LANCZOS)
else:
wpercent = wlimit / float(product_image.size[0])
hsize = int((float(product_image.size[1]) * float(wpercent)))
product_image = product_image.resize((wlimit, hsize), Image.Resampling.LANCZOS)
image.paste(product_image, (int((width/3)-(product_image.size[0]/2)), int((height/2) - (product_image.size[1]/2))))
# guardamos la imagen con otro nombre
image.save(f"/app/data/images/products/{article['id']}_composed.png", quality=95)
def send_article(article, product):
create_image(article)
text = f"*{telegram_escape_characters(article['title'])}*\n\n*Descripción*: {telegram_escape_characters(article['description'])}\\nn*Precio*: {telegram_escape_characters(str(article['price']))} {telegram_escape_characters(article['currency'])}\n\n[IR AL ANUNCIO](https://es\.wallapop\.com/item/{telegram_escape_characters(article['web_slug'])})"
url = f"https://api.telegram.org/bot{constants.TELEGRAM_TOKEN}/sendPhoto?chat_id={product['telegram_user_id']}&caption={text}&parse_mode=MarkdownV2"
files = {'photo':open(f"/app/data/images/products/{article['id']}_composed.png", 'rb')}
logging.info(requests.post(url, files=files).content)
def get_category_id(category):
ret = constants.CATEGORIES.get(category, '')
return ret

1
wallamanta/requirements.txt
View File

@@ -1,3 +1,4 @@
python-telegram-bot==20.1
requests==2.28.1
prettytable==3.6.0
Pillow==9.4.0

61
wallamanta/walladb.py
View File

@@ -1,11 +1,7 @@
import sqlite3
import os
import logging
LATITUDE = os.getenv("LATITUDE")
LONGITUDE = os.getenv("LONGITUDE")
DB = "/app/data/wallamanta.db"
import constants
# Enable logging
logging.basicConfig(
@@ -21,16 +17,16 @@ def dict_factory(cursor, row):
return d
def setup_db():
con = sqlite3.connect(DB)
con = sqlite3.connect(constants.DB)
cur = con.cursor()
cur.execute("CREATE TABLE IF NOT EXISTS users(telegram_user_id, active, type, until)")
cur.execute("CREATE TABLE IF NOT EXISTS products(product_name, distance, \
latitude, longitude, condition, min_price, max_price, \
latitude, longitude, condition, min_price, max_price, category, \
title_exclude, title_description_exclude, telegram_user_id)")
con.close()
def is_user_valid(telegram_user_id):
con = sqlite3.connect(DB)
con = sqlite3.connect(constants.DB)
cur = con.cursor()
res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id={telegram_user_id} AND active=True")
ret = res.fetchone() != None
@@ -38,7 +34,7 @@ def is_user_valid(telegram_user_id):
return ret
def is_user_premium(telegram_user_id):
con = sqlite3.connect(DB)
con = sqlite3.connect(constants.DB)
cur = con.cursor()
res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id={telegram_user_id} AND active=True AND type='premium'")
ret = res.fetchone() != None
@@ -46,7 +42,7 @@ def is_user_premium(telegram_user_id):
return ret
def is_user_testing(telegram_user_id):
con = sqlite3.connect(DB)
con = sqlite3.connect(constants.DB)
cur = con.cursor()
res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id={telegram_user_id} AND active=True AND type='testing'")
ret = res.fetchone() != None
@@ -55,7 +51,7 @@ def is_user_testing(telegram_user_id):
def add_premium_user(telegram_user_id, until):
found = False
con = sqlite3.connect(DB)
con = sqlite3.connect(constants.DB)
cur = con.cursor()
res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id={telegram_user_id}")
if res.fetchone() is None:
@@ -70,7 +66,7 @@ def add_premium_user(telegram_user_id, until):
def add_test_user(telegram_user_id, until):
found = False
con = sqlite3.connect(DB)
con = sqlite3.connect(constants.DB)
cur = con.cursor()
res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id={telegram_user_id}")
if res.fetchone() is None:
@@ -82,7 +78,7 @@ def add_test_user(telegram_user_id, until):
return not found
def remove_valid_user(telegram_user_id):
con = sqlite3.connect(DB)
con = sqlite3.connect(constants.DB)
cur = con.cursor()
res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id={telegram_user_id}")
if res.fetchone() != None:
@@ -91,17 +87,33 @@ def remove_valid_user(telegram_user_id):
con.close()
def get_user_list():
con = sqlite3.connect(DB)
con = sqlite3.connect(constants.DB)
cur = con.cursor()
res = cur.execute(f"SELECT * FROM users")
ret = res.fetchall()
con.close()
return ret
def get_user_type(telegram_user_id):
con = sqlite3.connect(constants.DB)
cur = con.cursor()
res = cur.execute(f"SELECT type FROM users WHERE telegram_user_id={telegram_user_id}")
ret = res.fetchone()
con.close()
return ret[0]
def get_user_until(telegram_user_id):
con = sqlite3.connect(constants.DB)
cur = con.cursor()
res = cur.execute(f"SELECT until FROM users WHERE telegram_user_id={telegram_user_id}")
ret = res.fetchone()
con.close()
return ret[0]
def get_product(product):
product_name = product.get('product_name')
telegram_user_id = product.get('telegram_user_id')
con = sqlite3.connect(DB)
con = sqlite3.connect(constants.DB)
con.row_factory = dict_factory
cur = con.cursor()
res = cur.execute(f"SELECT * FROM products WHERE telegram_user_id={telegram_user_id} \
@@ -111,7 +123,7 @@ def get_product(product):
return ret
def get_products_from_user(telegram_user_id):
con = sqlite3.connect(DB)
con = sqlite3.connect(constants.DB)
con.row_factory = dict_factory
cur = con.cursor()
res = cur.execute(f"SELECT * FROM products WHERE telegram_user_id={telegram_user_id}")
@@ -120,7 +132,7 @@ def get_products_from_user(telegram_user_id):
return ret
def get_all_products():
con = sqlite3.connect(DB)
con = sqlite3.connect(constants.DB)
con.row_factory = dict_factory
cur = con.cursor()
res = cur.execute(f"SELECT * FROM products")
@@ -132,31 +144,32 @@ def add_product(product):
condition = 'all'
product_name = product.get('product_name')
distance = product.get('distance', 0)
latitude = product.get('latitude', LATITUDE)
longitude = product.get('longitude', LONGITUDE)
latitude = product.get('latitude', constants.LATITUDE)
longitude = product.get('longitude', constants.LONGITUDE)
min_price = product.get('min_price')
max_price = product.get('max_price')
title_exclude = product.get('title_exclude', '')
title_description_exclude = product.get('title_description_exclude', '')
category = product.get('category', '')
telegram_user_id = product.get('telegram_user_id')
params = (product_name, \
distance, latitude, longitude, condition, min_price, \
max_price, title_exclude, title_description_exclude, telegram_user_id)
max_price, category, title_exclude, title_description_exclude, telegram_user_id)
logging.info(f"Trying to add: {product_name}, {title_exclude}, {title_description_exclude}, {telegram_user_id}")
con = sqlite3.connect(DB)
con = sqlite3.connect(constants.DB)
cur = con.cursor()
res = cur.execute(f"SELECT * FROM products WHERE telegram_user_id={product.get('telegram_user_id')} \
AND product_name='{product.get('product_name')}'")
if res.fetchone() is None:
cur.execute("INSERT INTO products VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", params)
cur.execute("INSERT INTO products VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", params)
con.commit()
con.close()
def remove_product(product):
removed = False
if get_product(product):
con = sqlite3.connect(DB)
con = sqlite3.connect(constants.DB)
cur = con.cursor()
res = cur.execute(f"DELETE FROM products WHERE telegram_user_id={product.get('telegram_user_id')} \
AND product_name='{product.get('product_name')}'")
@@ -166,7 +179,7 @@ def remove_product(product):
return removed
def count_user_products(telegram_user_id):
con = sqlite3.connect(DB)
con = sqlite3.connect(constants.DB)
cur = con.cursor()
res = cur.execute(f"SELECT Count() FROM products WHERE telegram_user_id={telegram_user_id}")
ret = res.fetchone()[0]

59
wallamanta/wallamanta.py
View File

@@ -5,17 +5,12 @@ import logging
import prettytable
import helpers
import walladb
import constants
from worker import Worker
from telegram import Update
from telegram.ext import Application, CommandHandler, ContextTypes
TELEGRAM_CHANNEL_ID = os.getenv("TELEGRAM_CHANNEL_ID")
TELEGRAM_TOKEN = os.getenv("TELEGRAM_TOKEN")
LATITUDE = os.getenv("LATITUDE")
LONGITUDE = os.getenv("LONGITUDE")
SLEEP_TIME = os.getenv("SLEEP_TIME")
# Enable logging
logging.basicConfig(
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO
@@ -33,24 +28,33 @@ def save_json_file(products):
async def help_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
if walladb.is_user_valid(helpers.get_telegram_user_id(update)):
message = """Añade un producto con `/add producto;precio_mínimo;precio_máximo,excluir_título(opcional, separado por comas);excluir_descripción_y_título(opciona, separado por comas);latitud(opcional);longitud(opcional),distancia(opcional)`\n
Ejemplo: `/add placa base itx;0;150`\n
Ejemplo 2: `/add cpu;10;30;;intel,core 2 duo,celeron;;;100`\n
Los campos opcionales que se dejen vacíos tomarán el valor configurado en el archivo `.env`\n
Lista los productos con `/list` o obtén la información de uno en concreto con `/list nombre del producto`\n
Borra un producto con `/remove nombre del producto`"""
message = """Añade un producto con `/add producto;precio_mínimo;precio_máximo,categoría,excluir_título(opcional, separado por comas);
excluir_descripción_y_título(opciona, separado por comas);latitud(opcional);longitud(opcional),distancia(opcional)`\n
Ejemplo: `/add placa base itx;0;150`\n
Ejemplo 2: `/add cpu;10;30;;intel,core 2 duo,celeron;;;100`\n
Los campos opcionales que se dejen vacíos tomarán el valor configurado en el archivo `.env`\n
Lista los productos con `/list` o obtén la información de uno en concreto con `/list nombre del producto`\n
Borra un producto con `/remove nombre del producto`\n
`/status` muestra tu tipo de membresía y fecha de caducidad"""
else:
message = """Activa tu periodo de prueba de 7 días con `/test` o contacta con @jocarduck para más información."""
await update.message.reply_markdown_v2(helpers.telegram_escape_characters(message))
async def categories_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
if walladb.is_user_valid(helpers.get_telegram_user_id(update)):
message = ''
for category in constants.CATEGORIES:
message = f"{message}, {category}"
await update.message.reply_markdown_v2(f"```{helpers.telegram_escape_characters(message)}```")
async def add_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
telegram_user_id = helpers.get_telegram_user_id(update)
if walladb.is_user_valid(telegram_user_id):
number_of_products = walladb.count_user_products(telegram_user_id)
message = """Tienes que pasar el número correcto de parámetros: `/add producto;precio_mínimo;precio_máximo,excluir_título(opcional, separado por comas);excluir_descripción_y_título(opciona, separado por comas);latitud(opcional);longitud(opcional),distancia(opcional)`\n
Ejemplo: `/add placa base itx;0;150`\n
Ejemplo 2: `/add cpu;10;30;;intel,core 2 duo,celeron;;;100`\n
Los campos opcionales que se dejen vacíos tomarán el valor configurado en el archivo `.env`"""
message = """Tienes que pasar el número correcto de parámetros: `/add producto;precio_mínimo;precio_máximo,categoría,excluir_título(opcional, separado por comas);excluir_descripción_y_título(opciona, separado por comas);latitud(opcional);longitud(opcional),distancia(opcional)`\n
Ejemplo: `/add placa base itx;0;150`\n
Ejemplo 2: `/add cpu;10;30;;intel,core 2 duo,celeron;;;100`\n
Los campos opcionales que se dejen vacíos tomarán el valor configurado en el archivo `.env`"""
valid = False
if walladb.is_user_testing(telegram_user_id):
valid = True
@@ -72,15 +76,17 @@ async def add_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> Non
args = args[1].split(";")
product['product_name'], product['min_price'], product['max_price'] = args[0:3]
if len(args) > 3 and args[3]:
product['title_exclude'] = args[3]
product['category'] = helpers.get_category_id(args[3])
if len(args) > 4 and args[4]:
product['title_description_exclude'] = args[4]
product['title_exclude'] = args[4]
if len(args) > 5 and args[5]:
product['latitude'] = args[5]
product['title_description_exclude'] = args[5]
if len(args) > 6 and args[6]:
product['longitude'] = args[6]
product['latitude'] = args[6]
if len(args) > 7 and args[7]:
product['distance'] = args[7]
product['longitude'] = args[7]
if len(args) > 8 and args[8]:
product['distance'] = args[8]
logging.info(f'Adding: {product}')
if not walladb.get_product(product):
@@ -156,12 +162,14 @@ async def remove_user_command(update: Update, context: ContextTypes.DEFAULT_TYPE
await update.message.reply_markdown_v2(helpers.telegram_escape_characters(f"{telegram_user_id} desactivado."))
async def status_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
# TODO: Obtener estado del usuario y fecha de caducidad de suscripción en el caso de haberla
message = ''
telegram_user_id = helpers.get_telegram_user_id(update)
if walladb.is_user_valid(telegram_user_id):
type = walladb.get_user_type(telegram_user_id)
until = walladb.get_user_until(telegram_user_id)
message = f"Tu cuenta es tipo: {type} y caduca el {until}."
await update.message.reply_markdown_v2(helpers.telegram_escape_characters(message))
async def test_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
# TODO: Activar usuario en modo testing y responder con fecha de caducidad
telegram_user_id = helpers.get_telegram_user_id(update)
if not walladb.is_user_valid(telegram_user_id):
until = helpers.get_date_ahead(7)
@@ -186,10 +194,11 @@ def main()->None:
"""Start the bot."""
# Create the Application and pass it your bot's token.
application = Application.builder().token(TELEGRAM_TOKEN).build()
application = Application.builder().token(constants.TELEGRAM_TOKEN).build()
# on different commands - answer in Telegram
application.add_handler(CommandHandler("help", help_command))
application.add_handler(CommandHandler("categories", categories_command))
application.add_handler(CommandHandler("add", add_command))
application.add_handler(CommandHandler("remove", remove_command))
application.add_handler(CommandHandler("list", list_command))

49
wallamanta/worker.py
View File

@@ -6,13 +6,7 @@ import logging
import json
import helpers
import walladb
TELEGRAM_CHANNEL_ID = os.getenv("TELEGRAM_CHANNEL_ID")
TELEGRAM_TOKEN = os.getenv("TELEGRAM_TOKEN")
LATITUDE = os.getenv("LATITUDE")
LONGITUDE = os.getenv("LONGITUDE")
SLEEP_TIME = int(os.getenv("SLEEP_TIME"))
import constants
# Enable logging
logging.basicConfig(
@@ -24,16 +18,14 @@ logger = logging.getLogger(__name__)
class Worker:
def is_valid_request(self, product):
is_valid = False
if walladb.get_product(product):
if walladb.is_user_valid(product['telegram_user_id']):
if walladb.is_user_premium(product['telegram_user_id']) or walladb.is_user_testing(product['telegram_user_id']):
is_valid = True
if not walladb.get_product(product):
is_valid = False
if not walladb.is_user_valid(product['telegram_user_id']):
is_valid = False
if not walladb.is_user_premium(product['telegram_user_id']):
is_valid = False
return is_valid
def request(self, product_name, n_articles, latitude=LATITUDE, longitude=LONGITUDE, distance='0', condition='all', min_price=0, max_price=10000000):
def request(self, product_name, n_articles, latitude=constants.LATITUDE, longitude=constants.LONGITUDE, distance='0', condition='all', min_price=0, max_price=10000000, category=""):
url = (f"http://api.wallapop.com/api/v3/general/search?keywords={product_name}"
f"&order_by=newest&latitude={latitude}"
f"&longitude={longitude}"
@@ -45,10 +37,13 @@ class Worker:
if condition != "all":
url = url + f"&condition={condition}" # new, as_good_as_new, good, fair, has_given_it_all
if category != "":
url = url + f"&category_ids={category}"
for step in range(15):
while True:
time.sleep(0.5)
response = requests.get(url+ f"&step={step+1}")
helpers.random_wait()
response = requests.get(url+f"&step={step+1}")
try:
if response.status_code == 200:
break
@@ -62,45 +57,41 @@ class Worker:
return json_data['search_objects']
def first_run(self, product):
for i in range(5):
helpers.random_wait()
list = []
if not self.is_valid_request(product):
return list
articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'])
articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], product['category'])
for article in articles:
list.insert(0, article['id'])
return list
def work(self, product, list):
helpers.random_wait() # Random wait to make requests separated in time in order to prevent API rate limit
exec_times = []
bot = telegram.Bot(token = TELEGRAM_TOKEN)
while True:
if not self.is_valid_request(product):
logging.info(f"{product['product_name']} not valid anymore, exiting worker")
break # Exits and ends worker thread
start_time = time.time()
articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'])
articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], product['category'])
for article in articles:
if not article['id'] in list:
logging.info(f"Found article {article['title']}")
try:
if not self.has_excluded_words(article['title'].lower(), article['description'].lower(), product['title_description_exclude']) and not self.is_title_key_word_excluded(article['title'].lower(), product['title_exclude']):
try:
text = f"*Artículo*: {helpers.telegram_escape_characters(article['title'])}\n*Descripción*: {helpers.telegram_escape_characters(article['description'])}\n*Precio*: {helpers.telegram_escape_characters(str(article['price']))} {helpers.telegram_escape_characters(article['currency'])}\n[Ir al anuncio](https://es\.wallapop\.com/item/{helpers.telegram_escape_characters(article['web_slug'])})"
url = f"https://api.telegram.org/bot{TELEGRAM_TOKEN}/sendMessage?chat_id={TELEGRAM_CHANNEL_ID}&text={text}&parse_mode=MarkdownV2"
logging.info(text)
logging.info(requests.get(url).json())
helpers.send_article(article, product)
except:
text = f"*Artículo*: {helpers.telegram_escape_characters(article['title'])}\n*Descripción*: {helpers.telegram_escape_characters(article['description'])}\n*Precio*: {helpers.telegram_escape_characters(str(article['price']))} {helpers.telegram_escape_characters(article['currency'])}\n[Ir al anuncio](https://es\.wallapop\.com/item/{helpers.telegram_escape_characters(article['web_slug'])})"
url = f"https://api.telegram.org/bot{TELEGRAM_TOKEN}/sendMessage?chat_id={TELEGRAM_CHANNEL_ID}&text={text}&parse_mode=MarkdownV2"
logging.info(text)
logging.info(requests.get(url).json())
helpers.send_article(article, product)
time.sleep(1) # Avoid Telegram flood restriction
list.insert(0, article['id'])
except Exception as e:
logging.info("---------- EXCEPTION -----------")
logging.info(f"{product['product_name']} worker crashed. {e}")
logging.info(f"{product['product_name']}: Trying to parse {article['id']}: {article['title']} .\n")
time.sleep(SLEEP_TIME)
time.sleep(constants.SLEEP_TIME)
exec_times.append(time.time() - start_time)
logging.info(f"\'{product['product_name']}\' node-> last: {exec_times[-1]} max: {self.get_max_time(exec_times)} avg: {self.get_average_time(exec_times)}")