Added wallamanta code

docker-compose.yml

@@ -2,7 +2,7 @@ version: "3"
 
 services:
   wallamanta-bot:
-    build: Wallamonitor
+    build: wallamanta
     container_name: wallamanta-bot
     restart: unless-stopped
     environment:

wallamanta/Dockerfile

@@ -0,0 +1,9 @@
+FROM python:3.7
+
+RUN mkdir /app
+ADD . /app
+RUN pip install -r /app/requirements.txt
+
+WORKDIR /app
+
+CMD [ "python", "/app/alert.py" ]

wallamanta/alert.py

@@ -0,0 +1,92 @@
+import json
+import os
+import threading
+import logging
+
+from worker import Worker
+from telegram import ForceReply, Update
+from telegram.ext import Application, CommandHandler, ContextTypes, MessageHandler, filters
+
+TELEGRAM_CHANNEL_ID = os.getenv("TELEGRAM_CHANNEL_ID") 
+TELEGRAM_TOKEN = os.getenv("TELEGRAM_TOKEN")
+LATITUDE = os.getenv("LATITUDE")
+LONGITUDE = os.getenv("LONGITUDE")
+SLEEP_TIME = os.getenv("SLEEP_TIME")
+
+# Enable logging
+logging.basicConfig(
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO
+)
+
+logger = logging.getLogger(__name__)
+
+def parse_json_file():
+    f = open("args.json")
+    return json.load(f)
+
+async def help_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
+    """Send a message when the command /help is issued."""
+    message = "Add with /add product;min_price;max_price"
+    await update.message.reply_text(message)
+
+async def add_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
+    parsed = update.message.text.split(";")
+    logging.info(parsed)
+    if len(parsed) != 3:
+        message = "You must put the correct number of arguments: /add product;min_price;max_price"
+    else:
+        argument = {"product_name": f"{parsed[0][5:]}", #removes "/add "
+                    "distance": "0", 
+                    "latitude": f"{LATITUDE}",
+                    "longitude": f"{LONGITUDE}",
+                    "condition": "all",
+                    "min_price": f"{parsed[1]}",
+                    "max_price": f"{parsed[2]}",
+                    "title_keyword_exclude" : [],
+                    "exclude": []
+                    }
+        p = threading.Thread(target=Worker.run, args=(argument, ))
+        p.start()
+        message = f"Added {parsed[0][5:]}"
+    await update.message.reply_text(message)
+
+async def remove_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
+    await update.message.reply_text("Help!")
+
+async def list_command(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
+    args = parse_json_file()
+    product_list = ""
+    for product in args:
+        product_list = f"{product_list}\n{product['product_name']};{product['min_price']}-{product['max_price']}"
+    await update.message.reply_text(product_list)
+
+async def echo(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None:
+    """Echo the user message."""
+    await update.message.reply_text(update.message.text)
+
+def main()->None:
+    args = parse_json_file()
+
+    for argument in args:
+        logging.info(argument)
+        p = threading.Thread(target=Worker.run, args=(argument, ))
+        p.start()
+
+    """Start the bot."""
+    # Create the Application and pass it your bot's token.
+    application = Application.builder().token(TELEGRAM_TOKEN).build()
+
+    # on different commands - answer in Telegram
+    application.add_handler(CommandHandler("help", help_command))
+    application.add_handler(CommandHandler("add", add_command))
+    application.add_handler(CommandHandler("remove", remove_command))
+    application.add_handler(CommandHandler("list", list_command))
+
+    # on non command i.e message - echo the message on Telegram
+    #application.add_handler(MessageHandler(filters.TEXT & ~filters.COMMAND, echo))
+
+    # Run the bot until the user presses Ctrl-C
+    application.run_polling()
+
+if __name__ == "__main__":
+    main()

wallamanta/args.json

@@ -0,0 +1,24 @@
+[
+  {
+    "product_name": "zapatillas",
+    "distance": "0", 
+    "latitude": "40.4165",
+    "longitude": "-3.70256",
+    "condition": "all",
+    "min_price": "0",
+    "max_price": "75",
+    "title_keyword_exclude" : [],
+    "exclude": []
+  },
+  {
+    "product_name": "placa base",
+    "distance": "0", 
+    "latitude": "40.4165",
+    "longitude": "-3.70256",
+    "condition": "all",
+    "min_price": "0",
+    "max_price": "75",
+    "title_keyword_exclude" : [],
+    "exclude": []
+  }
+]

wallamanta/error_log.txt

@@ -0,0 +1 @@
+grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse 9jd5lyeq726k: portatil toshiba satelite pro i3 r50 .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzvlmp1dg46l: torre pc acer para piezas sin el disco duro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 8j3xn10dmlj9: Samsung Galaxy J5 2015 , SM-J500FN . Dorado .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse pzp1p4nql9z3: Memoria ram kingston hyperx ddr2 4 gb a 1.066 MH .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 8z8gxdxyol63: MÓVILES HUAWEI P8 LITE .grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse 9jd5lyeq726k: portatil toshiba satelite pro i3 r50 .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse e6530nvvpgzo: Dos módulos de memoria RAM  .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse qjwdo3ly5wzo: Torre AMD Athlon 64 X2 Dual Core 6000+ .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzy72wddwvz5: Memoria ram Kingston 3gb DDR2,800mhz y 667mhz  .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 36enl0qm3y6d: Servicio Técnico Apple Valencia .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse x6q90e52oozy: GALAXY J3 (2016) 8GB negro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse p617rwnr7565: Memoria Ram DDR3 1600 mHz (2 módulos x 4GB) .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse pj9g19o1d06e: Ram ddr4 1gb .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzvlmp1dg46l: torre pc acer para piezas sin el disco duro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 8j3xn10dmlj9: Samsung Galaxy J5 2015 , SM-J500FN . Dorado .grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse mznv5n09ok6n: torre ordenador i5 8GB SSD 240GB HDMI  .grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse nzxyk71xg1j2: ASUS PH-GT1030-O2G GT 1030 2GB GDDR5 .grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse wzvlmpw7k46l: Ordenador portatil HP Probook (560) .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse e6530nvvpgzo: Dos módulos de memoria RAM  .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse qjwdo3ly5wzo: Torre AMD Athlon 64 X2 Dual Core 6000+ .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzy72wddwvz5: Memoria ram Kingston 3gb DDR2,800mhz y 667mhz  .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 36enl0qm3y6d: Servicio Técnico Apple Valencia .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse x6q90e52oozy: GALAXY J3 (2016) 8GB negro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse p617rwnr7565: Memoria Ram DDR3 1600 mHz (2 módulos x 4GB) .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse pj9g19o1d06e: Ram ddr4 1gb .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzvlmp1dg46l: torre pc acer para piezas sin el disco duro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 8j3xn10dmlj9: Samsung Galaxy J5 2015 , SM-J500FN . Dorado .grafica worker crashed. 'title_key_word_exclude'grafica: Trying to parse nzxyk71xg1j2: ASUS PH-GT1030-O2G GT 1030 2GB GDDR5 .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse e6530nvvpgzo: Dos módulos de memoria RAM  .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse qjwdo3ly5wzo: Torre AMD Athlon 64 X2 Dual Core 6000+ .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzy72wddwvz5: Memoria ram Kingston 3gb DDR2,800mhz y 667mhz  .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 36enl0qm3y6d: Servicio Técnico Apple Valencia .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse x6q90e52oozy: GALAXY J3 (2016) 8GB negro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse p617rwnr7565: Memoria Ram DDR3 1600 mHz (2 módulos x 4GB) .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse pj9g19o1d06e: Ram ddr4 1gb .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzvlmp1dg46l: torre pc acer para piezas sin el disco duro .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse 8j3xn10dmlj9: Samsung Galaxy J5 2015 , SM-J500FN . Dorado .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse e6530nvvpgzo: Dos módulos de memoria RAM  .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse qjwdo3ly5wzo: Torre AMD Athlon 64 X2 Dual Core 6000+ .ram worker crashed. 'title_key_word_exclude'ram: Trying to parse wzy72wddwvz5: Memoria ram Kingston 3gb DDR2,800mhz y 667mhz  .

wallamanta/requirements.txt

@@ -0,0 +1,2 @@
+python-telegram-bot==20.1
+requests==2.28.1

wallamanta/telegram_handler.py

@@ -0,0 +1,18 @@
+
+import telegram
+import os
+
+TELEGRAM_CHANNEL_ID = os.getenv("TELEGRAM_CHANNEL_ID") 
+TELEGRAM_TOKEN = os.getenv("TELEGRAM_TOKEN")
+LATITUDE = os.getenv("LATITUDE")
+LONGITUDE = os.getenv("LONGITUDE")
+SLEEP_TIME = os.getenv("SLEEP_TIME")
+
+class TelegramHandler:
+
+    def run():
+        updater = Updater(TELEGRAM_TOKEN)
+        dispatcher = updater.dispatcher
+        dispatcher.add_handler())
+        updater.start_polling()
+        updater.idle()

wallamanta/worker.py

@@ -0,0 +1,126 @@
+import time
+import requests
+import telegram
+import os
+import logging
+
+TELEGRAM_CHANNEL_ID = os.getenv("TELEGRAM_CHANNEL_ID") 
+TELEGRAM_TOKEN = os.getenv("TELEGRAM_TOKEN")
+LATITUDE = os.getenv("LATITUDE")
+LONGITUDE = os.getenv("LONGITUDE")
+SLEEP_TIME = int(os.getenv("SLEEP_TIME"))
+
+# Enable logging
+logging.basicConfig(
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s', level=logging.INFO
+)
+
+logger = logging.getLogger(__name__)
+
+class Worker:
+
+    def request(self, product_name, n_articles, latitude=LATITUDE, longitude=LONGITUDE, distance='0', condition='all', min_price=0, max_price=10000000):
+        url = (f"http://api.wallapop.com/api/v3/general/search?keywords={product_name}"
+                    f"&order_by=newest&latitude={latitude}"
+                    f"&longitude={longitude}"
+                    f"&distance={distance}"
+                    f"&min_sale_price={min_price}" 
+                    f"&max_sale_price={max_price}" 
+                    f"&filters_source=quick_filters&language=es_ES")
+                    
+        if condition != "all":
+            url = url + f"&condition={condition}" # new, as_good_as_new, good, fair, has_given_it_all
+
+        while True:
+            response = requests.get(url)
+            try:
+                if response.status_code == 200:
+                    break
+                else:
+                    logging.info(f"\'{product_name}\' -> Wallapop returned status {response.status_code}. Illegal parameters or Wallapop service is down. Retrying...")
+            except Exception as e:
+                logging.info("Exception: " + e)
+                time.sleep(3)
+
+        json_data = response.json()
+        return json_data['search_objects']
+
+    def first_run(self, args):
+        list = []
+        articles = self.request(args['product_name'], 0, args['latitude'], args['longitude'], args['distance'], args['condition'], args['min_price'], args['max_price'])
+        for article in articles: 
+            list.insert(0, article['id'])
+        return list
+
+    def work(self, args, list):
+        exec_times = []
+        bot = telegram.Bot(token = TELEGRAM_TOKEN)
+        
+        while True:
+            start_time = time.time()
+            articles = self.request(args['product_name'], 0, args['latitude'], args['longitude'], args['distance'], args['condition'], args['min_price'], args['max_price'])
+            for article in articles:
+                if not article['id'] in list:
+                    logging.info("Found article {}".format(article['title']))
+                    try:
+                        if not self.has_excluded_words(article['title'].lower(), article['description'].lower(), args['exclude']) and not self.is_title_key_word_excluded(article['title'].lower(), args['title_keyword_exclude']):
+                            try:
+                                text =  f"*Artículo*: {article['title']}\n*Descripción*: {article['description']}\n*Precio*: {article['price']} {article['currency']}\n[Ir al anuncio](https://es.wallapop.com/item/{article['web_slug']})".replace(".", "\.")
+                                url = f"https://api.telegram.org/bot{TELEGRAM_TOKEN}/sendMessage?chat_id={TELEGRAM_CHANNEL_ID}&text={text}&parse_mode=MarkdownV2"
+                                logging.info(requests.get(url).json())
+                            except:
+                                text =  f"*Artículo*: {article['title']}\n*Descripción*: {article['description']}\n*Precio*: {article['price']} {article['currency']}\n[Ir al anuncio](https://es.wallapop.com/item/{article['web_slug']})".replace(".", "\.")
+                                url = f"https://api.telegram.org/bot{TELEGRAM_TOKEN}/sendMessage?chat_id={TELEGRAM_CHANNEL_ID}&text={text}&parse_mode=MarkdownV2"
+                                requests.get(url)
+                            time.sleep(1) # Avoid Telegram flood restriction
+                        list.insert(0, article['id'])   
+                    except Exception as e:
+                        logging.info("---------- EXCEPTION -----------")
+                        logging.info(f"{args['product_name']} worker crashed. {e}")
+                        logging.info(f"{args['product_name']}: Trying to parse {article['id']}: {article['title']} .\n")
+
+            time.sleep(SLEEP_TIME)
+            exec_times.append(time.time() - start_time)
+            logging.info(f"\'{args['product_name']}\' node-> last: {exec_times[-1]} max: {self.get_max_time(exec_times)} avg: {self.get_average_time(exec_times)}")
+
+    def has_excluded_words(self, title, description, excluded_words):
+        for word in excluded_words:
+            logging.info("EXCLUDER: Checking '" + word + "' for title: '" + title)
+            if word in title or word in description:
+                logging.info("EXCLUDE!")
+                return True
+        return False
+    
+    def is_title_key_word_excluded(self, title, excluded_words):
+        for word in excluded_words:
+            logging.info("Checking '" + word + "' for title: '" + title)
+            if word in title:
+                return True
+        return False
+
+    def get_average_time(self, exec_times):
+        sum = 0
+        for i in exec_times:
+            sum = sum + i
+        
+        return sum / len(exec_times)
+
+    def get_max_time(self, exec_times):
+        largest = 0
+        for i in exec_times:
+            if i > largest:
+                largest = i
+        return largest
+        
+    def run(args):
+        worker = Worker()
+        list = worker.first_run(args)
+        while True:
+            try:
+                logging.info(f"Wallapop monitor worker started. Checking for new items containing: \'{args['product_name']}\' with given parameters periodically")
+                worker.work(args, list)
+            except Exception as e:
+                logging.info(f"Exception: {e}")
+                logging.info(f"{args['product_name']} worker crashed. Restarting worker...")
+                time.sleep(10)
+