Jocaru/wallamanta
1
0
Fork 0
Code Issues 15 Packages Projects Releases Wiki Activity

Added database and deleted json. Renamed main file. Modified functions to work with database. And more things...

Browse Source
This commit is contained in:
Joan Cano
2023-03-06 00:22:07 +01:00
parent 20778a9e96
commit 17e9cd05ec
6 changed files with 163 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
wallamanta/Dockerfile
View File

@@ -6,4 +6,4 @@ RUN pip install -r /app/requirements.txt
WORKDIR /app
CMD [ "python", "/app/alert.py" ]
CMD [ "python", "/app/wallamanta.py" ]

102
wallamanta/walladb.py
View File

@@ -1,12 +1,32 @@
import sqlite3
import os
import logging
LATITUDE = os.getenv("LATITUDE")
LONGITUDE = os.getenv("LONGITUDE")
DB = "/app/data/wallamanta.db"
# Enable logging
logging.basicConfig(
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO
)
logger = logging.getLogger(__name__)
def dict_factory(cursor, row):
d = {}
for idx, col in enumerate(cursor.description):
d[col[0]] = row[idx]
return d
def setup_db():
con = sqlite3.connect(DB)
cur = con.cursor()
cur.execute("CREATE TABLE IF NOT EXISTS users(telegram_user_id, active)")
cur.execute("CREATE TABLE IF NOT EXISTS products(product_name, distance, latitude, longitude, condition, min_price, max_price, title_keyword_exclude, exclude, telegram_user_id)")
cur.execute("CREATE TABLE IF NOT EXISTS products(product_name, distance, \
latitude, longitude, condition, min_price, max_price, \
title_exclude, title_description_exclude, telegram_user_id)")
con.close()
def is_user_valid(telegram_user_id):
@@ -18,6 +38,7 @@ def is_user_valid(telegram_user_id):
return ret
def add_valid_user(telegram_user_id):
found = False
con = sqlite3.connect(DB)
cur = con.cursor()
res = cur.execute(f"SELECT * FROM users WHERE telegram_user_id={telegram_user_id}")
@@ -27,7 +48,9 @@ def add_valid_user(telegram_user_id):
else:
cur.execute(f"UPDATE users SET active = True WHERE telegram_user_id={telegram_user_id}")
con.commit()
found = True
con.close()
return found
def remove_valid_user(telegram_user_id):
con = sqlite3.connect(DB)
@@ -46,28 +69,69 @@ def get_user_list():
con.close()
return ret
def get_product(args, telegram_user_id):
product_name = args[0]
def get_product(product):
product_name = product.get('product_name')
telegram_user_id = product.get('telegram_user_id')
con = sqlite3.connect(DB)
con.row_factory = sqlite3.Row
con.row_factory = dict_factory
cur = con.cursor()
res = cur.execute(f"SELECT * FROM products WHERE telegram_user_id={telegram_user_id} AND product_name='{product_name}'")
res = cur.execute(f"SELECT * FROM products WHERE telegram_user_id={telegram_user_id} \
AND product_name='{product_name}'")
ret = res.fetchone()
con.close()
return ret
def add_product(args, telegram_user_id):
title_exclude, title_description_exclude, latitude, longitude, distance = [], [], LATITUDE, LONGITUDE, "0"
product_name, min_price, max_price = args[0:3]
if len(args) > 3 and args[3]:
title_exclude = args[3].split(",")
if len(args) > 4 and args[4]:
title_description_exclude = args[4].split(",")
if len(args) > 5 and args[5]:
latitude = args[5]
if len(args) > 6 and args[6]:
longitude = args[6]
if len(args) > 7 and args[7]:
distance = args[7]
def get_products_from_user(telegram_user_id):
con = sqlite3.connect(DB)
con.row_factory = dict_factory
cur = con.cursor()
res = cur.execute(f"SELECT * FROM products WHERE telegram_user_id={telegram_user_id}")
ret = res.fetchall()
con.close()
return ret
def get_all_products():
con = sqlite3.connect(DB)
con.row_factory = dict_factory
cur = con.cursor()
res = cur.execute(f"SELECT * FROM products")
ret = res.fetchall()
con.close()
return ret
def add_product(product):
condition = 'all'
product_name = product.get('product_name')
distance = product.get('distance', 0)
latitude = product.get('latitude', LATITUDE)
longitude = product.get('longitude', LONGITUDE)
min_price = product.get('min_price')
max_price = product.get('max_price')
title_exclude = product.get('title_exclude', '')
title_description_exclude = product.get('title_description_exclude', '')
telegram_user_id = product.get('telegram_user_id')
params = (product_name, \
distance, latitude, longitude, condition, min_price, \
max_price, title_exclude, title_description_exclude, telegram_user_id)
logging.info(f"Trying to add: {product_name}, {title_exclude}, {title_description_exclude}, {telegram_user_id}")
con = sqlite3.connect(DB)
cur = con.cursor()
res = cur.execute(f"SELECT * FROM products WHERE telegram_user_id={product.get('telegram_user_id')} \
AND product_name='{product.get('product_name')}'")
if res.fetchone() is None:
cur.execute("INSERT INTO products VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", params)
con.commit()
con.close()
def remove_product(product):
removed = False
if get_product(product):
con = sqlite3.connect(DB)
cur = con.cursor()
res = cur.execute(f"DELETE FROM products WHERE telegram_user_id={product.get('telegram_user_id')} \
AND product_name='{product.get('product_name')}'")
con.commit()
con.close()
removed = True
return removed

77
wallamanta/alert.py → wallamanta/wallamanta.py
View File

@@ -42,7 +42,8 @@ async def help_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> No
await update.message.reply_markdown_v2(helpers.telegram_escape_characters(message))
async def add_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
if walladb.is_user_valid(helpers.get_telegram_user_id(update)):
telegram_user_id = helpers.get_telegram_user_id(update)
if walladb.is_user_valid(telegram_user_id):
message = """Tienes que pasar el número correcto de parámetros: `/add producto;precio_mínimo;precio_máximo,excluir_título(opcional, separado por comas);excluir_descripción_y_título(opciona, separado por comas);latitud(opcional);longitud(opcional),distancia(opcional)`\n
Ejemplo: `/add placa base itx;0;150`\n
Ejemplo 2: `/add cpu;10;30;;intel,core 2 duo,celeron;;;100`\n
@@ -52,45 +53,57 @@ async def add_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> Non
if len(args) == 1:
pass
elif len(args[1].split(";")) > 2:
product = dict()
product['telegram_user_id'] = telegram_user_id
args = args[1].split(";")
logging.info(f'Adding: {args}')
if not walladb.get_product(args, helpers.get_telegram_user_id(update)):
walladb.add_product(args, helpers.get_telegram_user_id(update))
p = threading.Thread(target=Worker.run, args=(walladb.get_product(args, helpers.get_telegram_user_id(update)), ))
product['product_name'], product['min_price'], product['max_price'] = args[0:3]
if len(args) > 3 and args[3]:
product['title_exclude'] = args[3]
if len(args) > 4 and args[4]:
product['title_description_exclude'] = args[4]
if len(args) > 5 and args[5]:
product['latitude'] = args[5]
if len(args) > 6 and args[6]:
product['longitude'] = args[6]
if len(args) > 7 and args[7]:
product['distance'] = args[7]
logging.info(f'Adding: {product}')
if not walladb.get_product(product):
walladb.add_product(product)
p = threading.Thread(target=Worker.run, args=(walladb.get_product(product), ))
p.start()
message = f"Añadido {walladb.get_product(args, helpers.get_telegram_user_id(update))['product_name']} a seguimiento"
message = f"Añadido {walladb.get_product(product)['product_name']} a seguimiento"
else:
message = f"{walladb.get_product(args, helpers.get_telegram_user_id(update))['product_name']} ya está en seguimiento!"
message = f"{walladb.get_product(product)['product_name']} ya está en seguimiento!"
await update.message.reply_markdown_v2(helpers.telegram_escape_characters(message))
async def remove_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
if walladb.is_user_valid(helpers.get_telegram_user_id(update)):
telegram_user_id = helpers.get_telegram_user_id(update)
if walladb.is_user_valid(telegram_user_id):
product_to_remove = update.message.text[len('/remove '):]
message = f"{product_to_remove} no está en seguimiento!"
products = parse_json_file()
for product in products:
if product['product_name'] == product_to_remove:
products.remove(product)
message = f"{product_to_remove} borrado!"
save_json_file(products)
if walladb.remove_product({'product_name' : product_to_remove, \
'telegram_user_id' : telegram_user_id}):
message = f"{product_to_remove} borrado!"
await update.message.reply_text(message)
async def list_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
if walladb.is_user_valid(helpers.get_telegram_user_id(update)):
products = parse_json_file()
telegram_user_id = helpers.get_telegram_user_id(update)
if walladb.is_user_valid(telegram_user_id):
products = walladb.get_products_from_user(telegram_user_id)
args = update.message.text.split("/list ")
found = False
if len(args) > 1:
table = prettytable.PrettyTable(['Campo', 'Valor'])
table.align['Campo'] = 'l'
table.align['Valor'] = 'r'
for product in products:
if product['product_name'] == args[1]:
for key in product:
table.add_row([key, product[key]])
found = True
break
product = walladb.get_product({'product_name':args[1],'telegram_user_id':telegram_user_id})
if product:
table = prettytable.PrettyTable(['Campo', 'Valor'])
table.align['Campo'] = 'l'
table.align['Valor'] = 'r'
for key in product:
table.add_row([key, product[key]])
found = True
if not found:
table = prettytable.PrettyTable(['Producto', 'Mín', 'Máx'])
table.align['Producto'] = 'l'
@@ -107,8 +120,16 @@ async def admin_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> N
async def add_user_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
if helpers.is_user_admin(update.message.chat_id):
telegram_user_id = update.message.text.split('/add_user ')[1]
walladb.add_valid_user(telegram_user_id)
await update.message.reply_markdown_v2(helpers.telegram_escape_characters(f"{telegram_user_id} añadido."))
if walladb.add_valid_user(telegram_user_id):
products = walladb.get_products_from_user(telegram_user_id)
for product in products:
logging.info(product)
p = threading.Thread(target=Worker.run, args=(product, ))
p.start()
await update.message.reply_markdown_v2(helpers.telegram_escape_characters(f"{telegram_user_id} re-activado. Re-activando productos."))
else:
await update.message.reply_markdown_v2(helpers.telegram_escape_characters(f"{telegram_user_id} añadido."))
async def remove_user_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
if helpers.is_user_admin(update.message.chat_id):
@@ -118,7 +139,7 @@ async def remove_user_command(update: Update, context: ContextTypes.DEFAULT_TYPE
def main()->None:
walladb.setup_db()
products = parse_json_file()
products = walladb.get_all_products()
for product in products:
logging.info(product)

57
wallamanta/worker.py
View File

@@ -5,6 +5,7 @@ import os
import logging
import json
import helpers
import walladb
TELEGRAM_CHANNEL_ID = os.getenv("TELEGRAM_CHANNEL_ID")
TELEGRAM_TOKEN = os.getenv("TELEGRAM_TOKEN")
@@ -34,16 +35,17 @@ class Worker:
if condition != "all":
url = url + f"&condition={condition}" # new, as_good_as_new, good, fair, has_given_it_all
while True:
response = requests.get(url)
try:
if response.status_code == 200:
break
else:
logging.info(f"\'{product_name}\' -> Wallapop returned status {response.status_code}. Illegal parameters or Wallapop service is down. Retrying...")
except Exception as e:
logging.info("Exception: " + e)
time.sleep(3)
for step in range(15):
while True:
response = requests.get(url+ f"&step={step+1}")
try:
if response.status_code == 200:
break
else:
logging.info(f"\'{product_name}\' -> Wallapop returned status {response.status_code}. Illegal parameters or Wallapop service is down. Retrying...")
except Exception as e:
logging.info("Exception: " + e)
time.sleep(3)
json_data = response.json()
return json_data['search_objects']
@@ -59,22 +61,17 @@ class Worker:
exec_times = []
bot = telegram.Bot(token = TELEGRAM_TOKEN)
while True:
f = open("/app/data/products.json")
products = json.load(f)
found = False
for fproduct in products:
if fproduct['product_name'] == product['product_name']:
found = True
break
if not found:
break # Exits worker if product not in list anymore
if not walladb.get_product(product):
break # Exits worker if product not in DB anymore
if not walladb.is_user_valid(product['telegram_user_id']):
break # Exits worker if user is not active anymore
start_time = time.time()
articles = self.request(product['product_name'], 0, product['latitude'], product['longitude'], product['distance'], product['condition'], product['min_price'], product['max_price'])
for article in articles:
if not article['id'] in list:
logging.info(f"Found article {article['title']}")
try:
if not self.has_excluded_words(article['title'].lower(), article['description'].lower(), product['exclude']) and not self.is_title_key_word_excluded(article['title'].lower(), product['title_keyword_exclude']):
if not self.has_excluded_words(article['title'].lower(), article['description'].lower(), product['title_description_exclude']) and not self.is_title_key_word_excluded(article['title'].lower(), product['title_exclude']):
try:
text = f"*Artículo*: {helpers.telegram_escape_characters(article['title'])}\n*Descripción*: {helpers.telegram_escape_characters(article['description'])}\n*Precio*: {helpers.telegram_escape_characters(str(article['price']))} {helpers.telegram_escape_characters(article['currency'])}\n[Ir al anuncio](https://es\.wallapop\.com/item/{helpers.telegram_escape_characters(article['web_slug'])})"
url = f"https://api.telegram.org/bot{TELEGRAM_TOKEN}/sendMessage?chat_id={TELEGRAM_CHANNEL_ID}&text={text}&parse_mode=MarkdownV2"
@@ -97,18 +94,20 @@ class Worker:
logging.info(f"\'{product['product_name']}\' node-> last: {exec_times[-1]} max: {self.get_max_time(exec_times)} avg: {self.get_average_time(exec_times)}")
def has_excluded_words(self, title, description, excluded_words):
for word in excluded_words:
logging.info("EXCLUDER: Checking '" + word + "' for title: '" + title)
if word in title or word in description:
logging.info("EXCLUDE!")
return True
if len(excluded_words) > 0:
for word in excluded_words.split(","):
logging.info("EXCLUDER: Checking '" + word + "' for title: '" + title)
if word in title or word in description:
logging.info("EXCLUDE!")
return True
return False
def is_title_key_word_excluded(self, title, excluded_words):
for word in excluded_words:
logging.info("Checking '" + word + "' for title: '" + title)
if word in title:
return True
if len(excluded_words) > 0:
for word in excluded_words.split(","):
logging.info("Checking '" + word + "' for title: '" + title)
if word in title:
return True
return False
def get_average_time(self, exec_times):