diff --git a/Wallamonitor b/Wallamonitor deleted file mode 160000 index 32d171d..0000000 --- a/Wallamonitor +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 32d171d577c68efe185b7b1e29e2316972849afe diff --git a/docker-compose.yml b/docker-compose.yml index 0b92edf..a2215dc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: "3" services: wallamanta-bot: - build: Wallamonitor + build: wallamanta container_name: wallamanta-bot restart: unless-stopped environment: @@ -10,4 +10,4 @@ services: - TELEGRAM_TOKEN=${TELEGRAM_TOKEN} - LATITUDE=${LATITUDE} - LONGITUDE=${LONGITUDE} - - SLEEP_TIME=${SLEEP_TIME} \ No newline at end of file + - SLEEP_TIME=${SLEEP_TIME} diff --git a/wallamanta/Dockerfile b/wallamanta/Dockerfile new file mode 100644 index 0000000..2304443 --- /dev/null +++ b/wallamanta/Dockerfile @@ -0,0 +1,9 @@ +FROM python:3.7 + +RUN mkdir /app +ADD . /app +RUN pip install -r /app/requirements.txt + +WORKDIR /app + +CMD [ "python", "/app/alert.py" ] \ No newline at end of file diff --git a/wallamanta/__init__.py b/wallamanta/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/wallamanta/alert.py b/wallamanta/alert.py new file mode 100644 index 0000000..237807b --- /dev/null +++ b/wallamanta/alert.py @@ -0,0 +1,92 @@ +import json +import os +import threading +import logging + +from worker import Worker +from telegram import ForceReply, Update +from telegram.ext import Application, CommandHandler, ContextTypes, MessageHandler, filters + +TELEGRAM_CHANNEL_ID = os.getenv("TELEGRAM_CHANNEL_ID") +TELEGRAM_TOKEN = os.getenv("TELEGRAM_TOKEN") +LATITUDE = os.getenv("LATITUDE") +LONGITUDE = os.getenv("LONGITUDE") +SLEEP_TIME = os.getenv("SLEEP_TIME") + +# Enable logging +logging.basicConfig( + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO +) + +logger = logging.getLogger(__name__) + +def parse_json_file(): + f = open("args.json") + return json.load(f) + +async def help_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: + """Send a message when the command /help is issued.""" + message = "Add with /add product;min_price;max_price" + await update.message.reply_text(message) + +async def add_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: + parsed = update.message.text.split(";") + logging.info(parsed) + if len(parsed) != 3: + message = "You must put the correct number of arguments: /add product;min_price;max_price" + else: + argument = {"product_name": f"{parsed[0][5:]}", #removes "/add " + "distance": "0", + "latitude": f"{LATITUDE}", + "longitude": f"{LONGITUDE}", + "condition": "all", + "min_price": f"{parsed[1]}", + "max_price": f"{parsed[2]}", + "title_keyword_exclude" : [], + "exclude": [] + } + p = threading.Thread(target=Worker.run, args=(argument, )) + p.start() + message = f"Added {parsed[0][5:]}" + await update.message.reply_text(message) + +async def remove_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: + await update.message.reply_text("Help!") + +async def list_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: + args = parse_json_file() + product_list = "" + for product in args: + product_list = f"{product_list}\n{product['product_name']};{product['min_price']}-{product['max_price']}" + await update.message.reply_text(product_list) + +async def echo(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: + """Echo the user message.""" + await update.message.reply_text(update.message.text) + +def main()->None: + args = parse_json_file() + + for argument in args: + logging.info(argument) + p = threading.Thread(target=Worker.run, args=(argument, )) + p.start() + + """Start the bot.""" + # Create the Application and pass it your bot's token. + application = Application.builder().token(TELEGRAM_TOKEN).build() + + # on different commands - answer in Telegram + application.add_handler(CommandHandler("help", help_command)) + application.add_handler(CommandHandler("add", add_command)) + application.add_handler(CommandHandler("remove", remove_command)) + application.add_handler(CommandHandler("list", list_command)) + + # on non command i.e message - echo the message on Telegram + #application.add_handler(MessageHandler(filters.TEXT & ~filters.COMMAND, echo)) + + # Run the bot until the user presses Ctrl-C + application.run_polling() + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/wallamanta/args.json b/wallamanta/args.json new file mode 100644 index 0000000..cdcd410 --- /dev/null +++ b/wallamanta/args.json @@ -0,0 +1,24 @@ +[ + { + "product_name": "zapatillas", + "distance": "0", + "latitude": "40.4165", + "longitude": "-3.70256", + "condition": "all", + "min_price": "0", + "max_price": "75", + "title_keyword_exclude" : [], + "exclude": [] + }, + { + "product_name": "placa base", + "distance": "0", + "latitude": "40.4165", + "longitude": "-3.70256", + "condition": "all", + "min_price": "0", + "max_price": "75", + "title_keyword_exclude" : [], + "exclude": [] + } +] diff --git a/wallamanta/error_log.txt b/wallamanta/error_log.txt new file mode 100644 index 0000000..60b44f1 --- /dev/null +++ b/wallamanta/error_log.txt @@ -0,0 +1 @@ +grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse 9jd5lyeq726k: portatil toshiba satelite pro i3 r50 .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzvlmp1dg46l: torre pc acer para piezas sin el disco duro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 8j3xn10dmlj9: Samsung Galaxy J5 2015 , SM-J500FN . Dorado .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse pzp1p4nql9z3: Memoria ram kingston hyperx ddr2 4 gb a 1.066 MH .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 8z8gxdxyol63: MÓVILES HUAWEI P8 LITE .grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse 9jd5lyeq726k: portatil toshiba satelite pro i3 r50 .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse e6530nvvpgzo: Dos módulos de memoria RAM .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse qjwdo3ly5wzo: Torre AMD Athlon 64 X2 Dual Core 6000+ .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzy72wddwvz5: Memoria ram Kingston 3gb DDR2,800mhz y 667mhz .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 36enl0qm3y6d: Servicio Técnico Apple Valencia .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse x6q90e52oozy: GALAXY J3 (2016) 8GB negro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse p617rwnr7565: Memoria Ram DDR3 1600 mHz (2 módulos x 4GB) .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse pj9g19o1d06e: Ram ddr4 1gb .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzvlmp1dg46l: torre pc acer para piezas sin el disco duro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 8j3xn10dmlj9: Samsung Galaxy J5 2015 , SM-J500FN . Dorado .grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse mznv5n09ok6n: torre ordenador i5 8GB SSD 240GB HDMI .grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse nzxyk71xg1j2: ASUS PH-GT1030-O2G GT 1030 2GB GDDR5 .grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse wzvlmpw7k46l: Ordenador portatil HP Probook (560) .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse e6530nvvpgzo: Dos módulos de memoria RAM .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse qjwdo3ly5wzo: Torre AMD Athlon 64 X2 Dual Core 6000+ .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzy72wddwvz5: Memoria ram Kingston 3gb DDR2,800mhz y 667mhz .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 36enl0qm3y6d: Servicio Técnico Apple Valencia .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse x6q90e52oozy: GALAXY J3 (2016) 8GB negro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse p617rwnr7565: Memoria Ram DDR3 1600 mHz (2 módulos x 4GB) .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse pj9g19o1d06e: Ram ddr4 1gb .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzvlmp1dg46l: torre pc acer para piezas sin el disco duro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 8j3xn10dmlj9: Samsung Galaxy J5 2015 , SM-J500FN . Dorado .grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse nzxyk71xg1j2: ASUS PH-GT1030-O2G GT 1030 2GB GDDR5 .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse e6530nvvpgzo: Dos módulos de memoria RAM .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse qjwdo3ly5wzo: Torre AMD Athlon 64 X2 Dual Core 6000+ .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzy72wddwvz5: Memoria ram Kingston 3gb DDR2,800mhz y 667mhz .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 36enl0qm3y6d: Servicio Técnico Apple Valencia .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse x6q90e52oozy: GALAXY J3 (2016) 8GB negro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse p617rwnr7565: Memoria Ram DDR3 1600 mHz (2 módulos x 4GB) .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse pj9g19o1d06e: Ram ddr4 1gb .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzvlmp1dg46l: torre pc acer para piezas sin el disco duro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 8j3xn10dmlj9: Samsung Galaxy J5 2015 , SM-J500FN . Dorado .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse e6530nvvpgzo: Dos módulos de memoria RAM .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse qjwdo3ly5wzo: Torre AMD Athlon 64 X2 Dual Core 6000+ .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzy72wddwvz5: Memoria ram Kingston 3gb DDR2,800mhz y 667mhz . \ No newline at end of file diff --git a/wallamanta/requirements.txt b/wallamanta/requirements.txt new file mode 100644 index 0000000..e0761c6 --- /dev/null +++ b/wallamanta/requirements.txt @@ -0,0 +1,2 @@ +python-telegram-bot==20.1 +requests==2.28.1 \ No newline at end of file diff --git a/wallamanta/telegram_handler.py b/wallamanta/telegram_handler.py new file mode 100644 index 0000000..4539cf7 --- /dev/null +++ b/wallamanta/telegram_handler.py @@ -0,0 +1,18 @@ + +import telegram +import os + +TELEGRAM_CHANNEL_ID = os.getenv("TELEGRAM_CHANNEL_ID") +TELEGRAM_TOKEN = os.getenv("TELEGRAM_TOKEN") +LATITUDE = os.getenv("LATITUDE") +LONGITUDE = os.getenv("LONGITUDE") +SLEEP_TIME = os.getenv("SLEEP_TIME") + +class TelegramHandler: + + def run(): + updater = Updater(TELEGRAM_TOKEN) + dispatcher = updater.dispatcher + dispatcher.add_handler()) + updater.start_polling() + updater.idle() diff --git a/wallamanta/worker.py b/wallamanta/worker.py new file mode 100644 index 0000000..4c6da53 --- /dev/null +++ b/wallamanta/worker.py @@ -0,0 +1,126 @@ +import time +import requests +import telegram +import os +import logging + +TELEGRAM_CHANNEL_ID = os.getenv("TELEGRAM_CHANNEL_ID") +TELEGRAM_TOKEN = os.getenv("TELEGRAM_TOKEN") +LATITUDE = os.getenv("LATITUDE") +LONGITUDE = os.getenv("LONGITUDE") +SLEEP_TIME = int(os.getenv("SLEEP_TIME")) + +# Enable logging +logging.basicConfig( + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO +) + +logger = logging.getLogger(__name__) + +class Worker: + + def request(self, product_name, n_articles, latitude=LATITUDE, longitude=LONGITUDE, distance='0', condition='all', min_price=0, max_price=10000000): + url = (f"http://api.wallapop.com/api/v3/general/search?keywords={product_name}" + f"&order_by=newest&latitude={latitude}" + f"&longitude={longitude}" + f"&distance={distance}" + f"&min_sale_price={min_price}" + f"&max_sale_price={max_price}" + f"&filters_source=quick_filters&language=es_ES") + + if condition != "all": + url = url + f"&condition={condition}" # new, as_good_as_new, good, fair, has_given_it_all + + while True: + response = requests.get(url) + try: + if response.status_code == 200: + break + else: + logging.info(f"\'{product_name}\' -> Wallapop returned status {response.status_code}. Illegal parameters or Wallapop service is down. Retrying...") + except Exception as e: + logging.info("Exception: " + e) + time.sleep(3) + + json_data = response.json() + return json_data['search_objects'] + + def first_run(self, args): + list = [] + articles = self.request(args['product_name'], 0, args['latitude'], args['longitude'], args['distance'], args['condition'], args['min_price'], args['max_price']) + for article in articles: + list.insert(0, article['id']) + return list + + def work(self, args, list): + exec_times = [] + bot = telegram.Bot(token = TELEGRAM_TOKEN) + + while True: + start_time = time.time() + articles = self.request(args['product_name'], 0, args['latitude'], args['longitude'], args['distance'], args['condition'], args['min_price'], args['max_price']) + for article in articles: + if not article['id'] in list: + logging.info("Found article {}".format(article['title'])) + try: + if not self.has_excluded_words(article['title'].lower(), article['description'].lower(), args['exclude']) and not self.is_title_key_word_excluded(article['title'].lower(), args['title_keyword_exclude']): + try: + text = f"*Artículo*: {article['title']}\n*Descripción*: {article['description']}\n*Precio*: {article['price']} {article['currency']}\n[Ir al anuncio](https://es.wallapop.com/item/{article['web_slug']})".replace(".", "\.") + url = f"https://api.telegram.org/bot{TELEGRAM_TOKEN}/sendMessage?chat_id={TELEGRAM_CHANNEL_ID}&text={text}&parse_mode=MarkdownV2" + logging.info(requests.get(url).json()) + except: + text = f"*Artículo*: {article['title']}\n*Descripción*: {article['description']}\n*Precio*: {article['price']} {article['currency']}\n[Ir al anuncio](https://es.wallapop.com/item/{article['web_slug']})".replace(".", "\.") + url = f"https://api.telegram.org/bot{TELEGRAM_TOKEN}/sendMessage?chat_id={TELEGRAM_CHANNEL_ID}&text={text}&parse_mode=MarkdownV2" + requests.get(url) + time.sleep(1) # Avoid Telegram flood restriction + list.insert(0, article['id']) + except Exception as e: + logging.info("---------- EXCEPTION -----------") + logging.info(f"{args['product_name']} worker crashed. {e}") + logging.info(f"{args['product_name']}: Trying to parse {article['id']}: {article['title']} .\n") + + time.sleep(SLEEP_TIME) + exec_times.append(time.time() - start_time) + logging.info(f"\'{args['product_name']}\' node-> last: {exec_times[-1]} max: {self.get_max_time(exec_times)} avg: {self.get_average_time(exec_times)}") + + def has_excluded_words(self, title, description, excluded_words): + for word in excluded_words: + logging.info("EXCLUDER: Checking '" + word + "' for title: '" + title) + if word in title or word in description: + logging.info("EXCLUDE!") + return True + return False + + def is_title_key_word_excluded(self, title, excluded_words): + for word in excluded_words: + logging.info("Checking '" + word + "' for title: '" + title) + if word in title: + return True + return False + + def get_average_time(self, exec_times): + sum = 0 + for i in exec_times: + sum = sum + i + + return sum / len(exec_times) + + def get_max_time(self, exec_times): + largest = 0 + for i in exec_times: + if i > largest: + largest = i + return largest + + def run(args): + worker = Worker() + list = worker.first_run(args) + while True: + try: + logging.info(f"Wallapop monitor worker started. Checking for new items containing: \'{args['product_name']}\' with given parameters periodically") + worker.work(args, list) + except Exception as e: + logging.info(f"Exception: {e}") + logging.info(f"{args['product_name']} worker crashed. Restarting worker...") + time.sleep(10) +