Added multiple categories and subcategories possibility

This commit is contained in:
2023-03-19 00:52:48 +01:00
parent 4e509790ec
commit e88dd77ba2
5 changed files with 380 additions and 111 deletions

View File

@@ -10,26 +10,183 @@ LATITUDE = os.getenv("LATITUDE")
LONGITUDE = os.getenv("LONGITUDE")
SLEEP_TIME = int(os.getenv("SLEEP_TIME"))
CATEGORIES = {'🚗 Coches 🚗': 100,
'🏍️ Motos 🏍️': 14000,
'🚙 Motor 🚙': 12800,
'👗 Moda 👗': 12465,
'🏘️ Inmobiliaria 🏘️': 200,
'📺 TV, audio y foto 📺': 12545,
'📱 Móviles 📱': 16000,
'🖥 Informática 🖥️': 15000,
'🏅 Deportes y ocio 🏅': 12579,
'🚴 Bicicletas 🚴': 17000,
'🎮 Consolas y videojuegos 🎮': 12900,
'🏡 Hogar y jardín 🏡': 12467,
'Electrodomésticos': 13100,
'🎥 Cine, libros y música 🎥': 12463,
'🧒 Niños y bebés 🧒': 12461,
'Coleccionismo': 18000,
'Construcción y reformas': 19000,
'Industria y agricultura': 20000,
'Empleo': 21000,
'Servicios': 13200,
'Otros': 12485,
'Todas': 0}
CATEGORIES = {
100: ["🚗 Coches 🚗"],
14000: ["🏍️ Motos 🏍️"],
12800: ["🚙 Motor 🚙",
{10164: "GPS y electrónica",
10165: "Herramientas",
10167: "Recambios de coches y furgonetas",
10168: "Recambios de motos y cuatriclicos",
10166: "Otros"}],
12465: ["👗 Moda 👗",
{10150: "Abrigos y chaquetas",
10436: "Accesorios",
10151: "Bañadores y bikinis",
10437: "Belleza y cosmética",
10152: "Bolsos, maletas y carteras",
10153: "Bufandas y guantes",
10154: "Calzado",
9567: "Gafas",
10156: "Joyas",
10158: "Prendas de vestir",
10159: "Ropa deportiva",
10160: "Ropa interior y calcetines",
10161: "Ropa premamá",
10162: "Sombreros y gorros",
10163: "Trajes, fiestas y bodas",
9656: "Otros"}],
200: ["🏘️ Inmobiliaria 🏘️"],
12545: ["📺 TV, audio y foto 📺",
{10198: "Auriculares y cascos",
10199: "Cámaras de vigilancia",
10200: "Cámaras y fotografía",
10201: "Drones",
10203: "Pilas y cargadores",
10204: "Proyectores y accesorios",
10205: "Reproductores",
10206: "Televisores y accesorios",
10207: "Vídeo y accesorios",
10202: "Otros"}],
16000: ["📱 Móviles 📱",
{9375: "Accesorios",
10170: "Cables",
9388: "Piezas y recambios",
9487: "Smartwatches",
9374: "Tablets",
10175: "Teléfonos antiguos",
9447: "Teléfonos móviles",
9370: "Otros"}],
15000: ["🖥 Informática 🖥️",
{10130: "Cables",
10131: "Cargadores y baterías",
10132: "Impresoras y accesorios",
10133: "Monitores",
10134: "Ordenadores y accesorios",
10136: "Realidad virtual y aumentada",
10137: "Software",
10135: "Otros"}],
12579: ["🏅 Deportes y ocio 🏅",
{10094: "Baloncesto",
10095: "Balonmano",
10096: "Estáticas y elípticas",
10097: "Fitness, running y yoga",
10098: "Fútbol",
10099: "Golf",
10100: "Juegos recreativos y de mesa",
10101: "Manualidades",
10102: "Montaña y esquí",
10103: "Natación y accesorios piscina",
10105: "Otros deportes",
10106: "Patinetes y patinaje",
10107: "Rugby",
10108: "Tenis y pádel",
10109: "Vóley",
10104: "Otros"}],
17000: ["🚴 Bicicletas 🚴",
{10055: "Accesorios para bicicletas",
10056: "Bicicletas y triciclos",
10058: "Piezas y recambios de bici",
10059: "Protección y vestimenta",
10057: "Otros"}],
12900: ["🎮 Consolas y videojuegos 🎮",
{10087: "Accesorios de consolas",
10088: "Consolas",
10089: "Manuales y guías",
10090: "Merchandising de videojuegos",
10092: "Recambios de consolas",
10093: "Videojuegos",
10091: "Otros"}],
12467: ["🏡 Hogar y jardín 🏡",
{10127: "Almacenaje",
10118: "Artículos para mascotas",
10119: "Baño",
10120: "Cocina, comedor y bar",
10121: "Colchones y ropa de cama",
10122: "Decoración",
10123: "Iluminación interior",
10124: "Jardín y exteriores",
10125: "Mobiliario para la casa",
10126: "Otros"}],
13100: ["Electrodomésticos",
{10110: "Climatización",
10111: "Electrodomésticos de cocina",
10112: "Lavandería y plancha",
10113: "Pequeños electrodomésticos",
10114: "Piezas y recambios",
10115: "Vitrocerámica",
10441: "Otros"}],
12463: ["🎥 Cine, libros y música 🎥",
{10060: "CDs y Vinilos",
10061: "Cómics y novelas gráficas",
10062: "Equipo profesional de sonido",
10063: "Instrumentos musicales",
10064: "Libros",
10066: "Partituras y libretos",
10067: "Películas y Series",
10068: "Pósters y merchandising",
10069: "Revistas",
10070: "Tocadiscos",
10065: "Otros"}],
12461: ["🧒 Niños y bebés 🧒",
{10177: "Accesorios de baño",
10178: "Alimentación del bebé",
10179: "Artículos de maternidad",
10180: "Artículos escolares",
10182: "Cunas y camas",
10183: "Disfraces infantiles",
10184: "Juguetes, juegos y peluches",
10185: "Mobiliario infantil",
10187: "Ropa infantil",
10188: "Seguridad y cuidado",
10181: "Transporte del bebé",
10189: "Tronas y andadores",
10186: "Otros"}],
18000: ["Coleccionismo",
{10071: "Antigüedades",
10072: "Artesanías y decoración",
10073: "Artículos de escritorio",
10074: "Banderas",
10075: "Coches y motocicletas",
10076: "Coleccionismo deportivo",
10077: "Coleccionismo militar",
10078: "Filatelia y sellos",
10079: "Imanes",
10080: "Llaveros",
10081: "Monedas y billetes",
10082: "Muñecos",
10083: "Naipes",
10085: "Postales y suvenires",
10086: "Relojes",
10084: "Otros"}],
19000: ["Construcción y reformas",
{10138: "Balcones",
10139: "Baños",
10140: "Cocinas",
10141: "Electricidad e Iluminación",
10142: "Escaleras y andamios",
10143: "Ferretería",
10144: "Herramientas y maquinaria",
10145: "Madera y otros materiales",
10147: "Pavimentos y revestimientos",
10148: "Pinturas y barnices",
10149: "Puertas y ventanas",
10146: "Otros"}],
20000: ["Industria y agricultura",
{10128: "Agricultura",
10129: "Industria"}],
21000: ["Empleo",
{10116: "Busco empleo",
10117: "Ofertas de empleo"}],
13200: ["Servicios",
{10190: "Clases y entrenamiento",
10191: "Cuidadoras y niñeras",
10192: "Mudanzas y transporte",
10194: "Reparaciones",
10195: "Servicios de limpieza",
10196: "Servicios del hogar",
10197: "Terapia y crecimiento personal",
10193: "Otros"}],
12485: ["Otros"],
0: ["Todas"]
}

View File

@@ -136,59 +136,76 @@ async def send_article(article, product):
#logging.info(requests.post(url, files=files).content)
logging.info(response)
def get_category_id(category):
ret = constants.CATEGORIES.get(category, '')
return ret
def get_category_name(category):
category_name = ''
for i_category in constants.CATEGORIES:
if constants.CATEGORIES[i_category] == category:
category_name = i_category
break
return category_name
category = int(category)
return constants.CATEGORIES[category][0]
def get_subcategory_name(subcategory):
subcategory = int(subcategory)
subcategory_name = ''
for category in constants.CATEGORIES:
if has_subcategory(category):
if constants.CATEGORIES[category][1].get(subcategory):
subcategory_name = constants.CATEGORIES[category][1].get(subcategory)
return subcategory_name
def has_subcategory(category):
category = int(category)
return len(constants.CATEGORIES[category]) > 1
def is_subcategory(category, subcategory):
category = int(category)
subcategory = int(subcategory)
if has_subcategory(category):
return subcategory in constants.CATEGORIES[category][1]
else:
return False
def create_categories_keyboard(categories):
categories = categories.split(',')
count = 0
category_line = []
keyboard = []
for category in categories:
if has_subcategory(category):
category_line.append(InlineKeyboardButton(constants.CATEGORIES[int(category)][0], callback_data=f"{int(category)}"))
count = count + 1
if count % 3 == 0:
keyboard.append(category_line)
category_line = []
if count % 3 != 0:
keyboard.append(category_line)
keyboard.append([InlineKeyboardButton("Volver", callback_data="finish")])
return keyboard
def create_subcategory_keyboard(category):
category = int(category)
count = 0
subcategory_line = []
keyboard = []
for subcategory in constants.CATEGORIES[category][1]:
subcategory_line.append(InlineKeyboardButton(constants.CATEGORIES[category][1][subcategory], callback_data=f"{subcategory}"))
count = count + 1
if count % 3 == 0:
keyboard.append(subcategory_line)
subcategory_line = []
if count % 3 != 0:
keyboard.append(subcategory_line)
keyboard.append([InlineKeyboardButton("Volver", callback_data="finish")])
return keyboard
def create_category_keyboard():
keyboard = [
[
InlineKeyboardButton("🖥 Informática 🖥️ ", callback_data='15000'),
InlineKeyboardButton("🚙 Motor 🚙", callback_data='12800'),
InlineKeyboardButton("🚗 Coches 🚗", callback_data='100'),
],
[
InlineKeyboardButton("🏍️ Motos 🏍️", callback_data='14000'),
InlineKeyboardButton("👗 Moda 👗", callback_data='12465'),
InlineKeyboardButton("🏘️ Inmobiliaria 🏘️", callback_data='200'),
],
[
InlineKeyboardButton("📺 TV, audio y foto 📺", callback_data='12545'),
InlineKeyboardButton("📱 Móviles 📱", callback_data='16000'),
InlineKeyboardButton("🏅 Deportes y ocio 🏅", callback_data='12579'),
],
[
InlineKeyboardButton("🚴 Bicicletas 🚴", callback_data='17000'),
InlineKeyboardButton("🎮 Consolas y videojuegos 🎮", callback_data='12900'),
InlineKeyboardButton("🏡 Hogar y jardín 🏡", callback_data='12467'),
],
[
InlineKeyboardButton("Electrodomésticos", callback_data='13100'),
InlineKeyboardButton("🎥 Cine, libros y música 🎥", callback_data='12463'),
InlineKeyboardButton("🧒 Niños y bebés 🧒", callback_data='12461'),
],
[
InlineKeyboardButton("Coleccionismo", callback_data='18000'),
InlineKeyboardButton("Construcción y reformas", callback_data='19000'),
InlineKeyboardButton("Industria y agricultura", callback_data='20000'),
],
[
InlineKeyboardButton("Empleo", callback_data='21000'),
InlineKeyboardButton("Servicios", callback_data='13200'),
InlineKeyboardButton("Otros", callback_data='12485'),
],
[
InlineKeyboardButton("Todos", callback_data='0'),
],
]
count = 0
category_line = []
keyboard = []
for category in constants.CATEGORIES:
category_line.append(InlineKeyboardButton(constants.CATEGORIES[category][0], callback_data=f"{category}"))
count = count + 1
if count % 3 == 0:
keyboard.append(category_line)
category_line = []
if count % 3 != 0:
keyboard.append(category_line)
return keyboard
def create_continue_keyboard():
@@ -196,6 +213,10 @@ def create_continue_keyboard():
[
InlineKeyboardButton("Finalizar", callback_data='add'),
InlineKeyboardButton("Descartar", callback_data='remove')
],
[
InlineKeyboardButton("Añadir categoría", callback_data='add_category'),
InlineKeyboardButton("Añadir subcategoría", callback_data='add_subcategory')
],[
InlineKeyboardButton("Exclusión de título", callback_data='title_exclude'),
InlineKeyboardButton("Exclusión de descripción", callback_data='title_description_exclude')

View File

@@ -21,7 +21,7 @@ def setup_db():
cur = con.cursor()
cur.execute("CREATE TABLE IF NOT EXISTS users(telegram_user_id, active, type, until, telegram_name)")
cur.execute("CREATE TABLE IF NOT EXISTS products(product_name, distance, \
latitude, longitude, condition, min_price, max_price, category, \
latitude, longitude, condition, min_price, max_price, category, subcategory, \
title_exclude, title_description_exclude, telegram_user_id)")
con.close()
@@ -65,17 +65,12 @@ def is_user_testing(telegram_user_id):
con.close()
return ret
def add_premium_user(telegram_user_id, telegram_name, until):
def add_premium_user(telegram_user_id, until):
found = False
con = sqlite3.connect(constants.DB)
cur = con.cursor()
params = (telegram_name,)
res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id=?", params)
if res.fetchone() is None:
params = (telegram_user_id, True, 'premium', until, telegram_name.first_name)
cur.execute("INSERT INTO users VALUES (?, ?, ?, ?, ?)", params)
con.commit()
else:
if res.fetchone() != None:
params = (until, telegram_user_id)
cur.execute(f"UPDATE users SET active = True, type = 'premium', until = ? WHERE telegram_user_id=?", params)
con.commit()
@@ -193,6 +188,7 @@ def add_product(product):
title_exclude = product.get('title_exclude', '')
title_description_exclude = product.get('title_description_exclude', '')
category = product.get('category', '')
subcategory = product.get('subcategory', '')
if category == '0':
category = ''
telegram_user_id = product.get('telegram_user_id')
@@ -205,8 +201,8 @@ def add_product(product):
if res.fetchone() is None:
params = (product_name, \
distance, latitude, longitude, condition, min_price, \
max_price, category, title_exclude, title_description_exclude, telegram_user_id)
cur.execute("INSERT INTO products VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", params)
max_price, category, subcategory, title_exclude, title_description_exclude, telegram_user_id)
cur.execute("INSERT INTO products VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", params)
con.commit()
con.close()

View File

@@ -175,8 +175,10 @@ async def continue_or_finish(update: Update, context: CallbackContext):
markup = InlineKeyboardMarkup(helpers.create_continue_keyboard())
last_step = context.user_data.get('last_step', '')
query = update.callback_query
try:
qd = query.data
logging.info(f"Query Data: {qd}")
except:
qd = ''
@@ -187,13 +189,58 @@ async def continue_or_finish(update: Update, context: CallbackContext):
context.user_data.clear()
await context.bot.send_message(chat_id=update.effective_chat.id, text=f'Borrado')
return ConversationHandler.END
if last_step == 'category':
category = int(query.data)
if qd == 'add_subcategory':
await context.bot.send_message(chat_id=update.effective_chat.id,
text=f"Categoría escogida: {helpers.get_category_name(category)}")
context.user_data['category'] = category
context.user_data['last_step'] = ''
text='Escoge la categoría', reply_markup=InlineKeyboardMarkup(helpers.create_categories_keyboard(context.user_data['category'])))
context.user_data['last_step'] = 'choose_subcategory'
return CONTINUE_OR_FINISH
if qd == 'finish_subcategory':
pass
if qd == 'add_category':
await context.bot.send_message(chat_id=update.effective_chat.id,
text='Categoría', reply_markup=InlineKeyboardMarkup(helpers.create_category_keyboard()))
context.user_data['last_step'] = 'category'
return CONTINUE_OR_FINISH
if last_step == "choose_subcategory":
category = query.data
if category != 'finish':
await context.bot.send_message(chat_id=update.effective_chat.id,
text='Escoge la subcategoría', reply_markup=InlineKeyboardMarkup(helpers.create_subcategory_keyboard(category)))
context.user_data['last_step'] = 'subcategory'
return CONTINUE_OR_FINISH
else:
context.user_data['last_step'] = ''
if last_step == 'subcategory':
subcategory = query.data
if subcategory != 'finish':
text = f"✅Subcategoría escogida: {helpers.get_subcategory_name(subcategory)}"
if context.user_data.get('subcategory'):
if subcategory not in context.user_data['subcategory'].split(','):
context.user_data['subcategory'] = context.user_data['subcategory'] + ',' + subcategory
else:
text = f"❌La subcategoría {helpers.get_subcategory_name(subcategory)} ya estaba añadida❌"
else:
context.user_data['subcategory'] = subcategory
await context.bot.send_message(chat_id=update.effective_chat.id,
text=text)
logging.info(f"{context.user_data['subcategory']}")
context.user_data['last_step'] = ''
if last_step == 'category':
category = query.data
if category != 'finish':
text = f"✅Categoría escogida: {helpers.get_category_name(category)}"
if context.user_data.get('category'):
if category not in context.user_data['category'].split(','):
context.user_data['category'] = context.user_data['category'] + ',' + category
else:
text = f"❌La categoría {helpers.get_category_name(category)} ya estaba añadida❌"
else:
context.user_data['category'] = category
await context.bot.send_message(chat_id=update.effective_chat.id,
text=text)
logging.info(f"{context.user_data['category']}")
context.user_data['last_step'] = ''
if qd == 'title_exclude':
await context.bot.send_message(chat_id=update.effective_chat.id,
@@ -250,6 +297,9 @@ async def continue_or_finish(update: Update, context: CallbackContext):
if context.user_data['last_step'] == '':
await context.bot.send_message(chat_id=update.effective_chat.id, text=f'Añadir?', reply_markup=markup)
return CONTINUE_OR_FINISH
async def add_subcategory(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
pass
async def cancel(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
logging.info('Conversation cancelled')
@@ -304,7 +354,7 @@ async def add_premium_user_command(update: Update, context: ContextTypes.DEFAULT
telegram_user_id = update.message.text.split('/add_premium_user ')[1].split(' ')[0]
days = update.message.text.split('/add_premium_user ')[1].split(' ')[1]
until = helpers.get_date_ahead(int(days))
if not walladb.add_premium_user(telegram_user_id, '', until):
if not walladb.add_premium_user(telegram_user_id, until):
products = walladb.get_products_from_user(telegram_user_id)
for product in products:

View File

@@ -25,7 +25,7 @@ class Worker:
is_valid = True
return is_valid
def request(self, product_name, n_articles, latitude=constants.LATITUDE, longitude=constants.LONGITUDE, distance='0', condition='all', min_price=0, max_price=10000000, category=""):
def request(self, product_name, n_articles, latitude=constants.LATITUDE, longitude=constants.LONGITUDE, distance='0', condition='all', min_price=0, max_price=10000000, category="", subcategories=[]):
url = (f"http://api.wallapop.com/api/v3/general/search?keywords={product_name}"
f"&order_by=newest&latitude={latitude}"
f"&longitude={longitude}"
@@ -40,6 +40,12 @@ class Worker:
if category != "":
url = url + f"&category_ids={category}"
if len(subcategories) > 0:
url_subcategories = ''
for subcategory in subcategories:
url_subcategories = url_subcategories + f"{subcategory},"
url = url + f"&object_type_ids={url_subcategories[:-1]}"
for step in range(15):
while True:
helpers.random_wait()
@@ -63,9 +69,29 @@ class Worker:
list = []
if not self.is_valid_request(product):
return list
articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], product['category'])
for article in articles:
list.insert(0, article['id'])
if product['category'] == '':
articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], product['category'])
for article in articles:
list.insert(0, article['id'])
else:
if '0' in product['category'].split(','):
articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'])
for article in articles:
list.insert(0, article['id'])
else:
for category in product['category'].split(','):
if product['subcategory'] == '' or not helpers.has_subcategory(category):
articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category)
for article in articles:
list.insert(0, article['id'])
else:
subcategories = []
for subcategory in product['subcategory'].split(','):
if helpers.is_subcategory(category, subcategory):
subcategories.append(subcategory)
articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category, subcategories)
for article in articles:
list.insert(0, article['id'])
return list
async def work(self, product, list):
@@ -76,23 +102,41 @@ class Worker:
logging.info(f"{product['product_name']} not valid anymore, exiting worker")
break # Exits and ends worker thread
start_time = time.time()
articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], product['category'])
for article in articles:
if not article['id'] in list:
logging.info(f"Found article {article['title']}")
try:
if not self.has_excluded_words(article['title'].lower(), article['description'].lower(), product['title_description_exclude']) and not self.is_title_key_word_excluded(article['title'].lower(), product['title_exclude']):
try:
await helpers.send_article(article, product)
except:
await helpers.send_article(article, product)
await asyncio.sleep(1)
#time.sleep(1) # Avoid Telegram flood restriction
list.insert(0, article['id'])
except Exception as e:
logging.info("---------- EXCEPTION -----------")
logging.info(f"{product['product_name']} worker crashed. {e}")
logging.info(f"{product['product_name']}: Trying to parse {article['id']}: {article['title']} .\n")
articles_list = []
if product['category'] == '':
articles_list.append(self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], product['category']))
else:
if '0' in product['category'].split(','):
articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'])
for article in articles:
list.insert(0, article['id'])
else:
for category in product['category'].split(','):
if product['subcategory'] == '' or not helpers.has_subcategory(category):
articles_list.append(self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category))
else:
subcategories = []
for subcategory in product['subcategory'].split(','):
if helpers.is_subcategory(category, subcategory):
subcategories.append(subcategory)
articles_list.append(self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'], category, subcategories))
for articles in articles_list:
for article in articles:
if not article['id'] in list:
logging.info(f"Found article {article['title']}")
try:
if not self.has_excluded_words(article['title'].lower(), article['description'].lower(), product['title_description_exclude']) and not self.is_title_key_word_excluded(article['title'].lower(), product['title_exclude']):
try:
await helpers.send_article(article, product)
except:
await helpers.send_article(article, product)
await asyncio.sleep(1)
#time.sleep(1) # Avoid Telegram flood restriction
list.insert(0, article['id'])
except Exception as e:
logging.info("---------- EXCEPTION -----------")
logging.info(f"{product['product_name']} worker crashed. {e}")
logging.info(f"{product['product_name']}: Trying to parse {article['id']}: {article['title']} .\n")
#time.sleep(constants.SLEEP_TIME)
await asyncio.sleep(constants.SLEEP_TIME)
exec_times.append(time.time() - start_time)
@@ -147,6 +191,7 @@ class Worker:
def run(product):
worker = Worker()
list = worker.first_run(product)
time.sleep(constants.SLEEP_TIME)
while True:
try:
loop = asyncio.new_event_loop()