diff --git a/wallamanta/helpers.py b/wallamanta/helpers.py index 7e936fe..3b8a208 100644 --- a/wallamanta/helpers.py +++ b/wallamanta/helpers.py @@ -62,7 +62,7 @@ def random_wait(): time.sleep(random.random()) def download_image(article, product_id): - r = requests.get(article['images'][0]['original']) + r = requests.get(article['images'][0]['urls']['small']) if r.status_code == 200: with open(f"/app/data/images/products/{article['id']}_{product_id}.jpg", "wb") as image: image.write(r.content) @@ -70,9 +70,9 @@ def download_image(article, product_id): def create_image(article, product_id): download_image(article, product_id) currency = '?' - if article['currency'] == 'EUR': + if article['price']['currency'] == 'EUR': currency = '€' - price = str(article['price']) + currency + price = str(article['price']['amount']) + currency wallamanta_text = "@wallamanta_bot" width = 1280 height = 800 @@ -123,14 +123,10 @@ def remove_image(article, product_id): logging.info(f"Tried to delete {article} image. Couldn't find it.") def get_publish_date(article): - article_date = article['creation_date'] - formato_fecha = "%Y-%m-%dT%H:%M:%S.%f%z" - return datetime.strptime(article_date, formato_fecha).astimezone(pytz.timezone("Europe/Madrid")).strftime("%d/%m/%Y - %H:%M:%S") + return datetime.fromtimestamp(article['created_at']/1000, pytz.timezone("Europe/Madrid")).strftime("%d/%m/%Y - %H:%M:%S") def get_modified_date(article): - article_date = article['modification_date'] - formato_fecha = "%Y-%m-%dT%H:%M:%S.%f%z" - return datetime.strptime(article_date, formato_fecha).astimezone(pytz.timezone("Europe/Madrid")).strftime("%d/%m/%Y - %H:%M:%S") + return datetime.fromtimestamp(article['modified_at']/1000, pytz.timezone("Europe/Madrid")).strftime("%d/%m/%Y - %H:%M:%S") def get_random_string(length): result_str = ''.join(random.choice(string.ascii_letters) for i in range(length)) @@ -177,11 +173,11 @@ def send_article(article, product): user_ships = f"📦 *Envío:* ✅" else: user_ships = f"📦 *Envío:* ❌" - if article['currency'] == 'EUR': + if article['price']['currency'] == 'EUR': currency = '€' else: currency = '?' - price = f"*💰 Precio*: {telegram_escape_characters(str(article['price']))} {telegram_escape_characters(currency)}" + price = f"*💰 Precio*: {telegram_escape_characters(str(article['price']['amount']))} {telegram_escape_characters(currency)}" text = f"{title}\n\n{description}\n\n{found_by}\n\n{created_at}\n{modified_at}\n\n{location}\n\n{user_ships}\n\n{price}" #url = f"https://api.telegram.org/bot{constants.TELEGRAM_TOKEN}/sendPhoto?chat_id={product['telegram_user_id']}&caption={text}&parse_mode=MarkdownV2" #files = {'photo':open(f"/app/data/images/products/{article['id']}_composed.png", 'rb')} @@ -360,13 +356,13 @@ def send_to_nr(article, product): event = Event( "ProductFound", { "article_name": article['title'], - "article_price": article['price'], + "article_price": article['price']['amount'], "article_web_slug": article['web_slug'], "product_name": product['product_name'], "telegram_user_id": product['telegram_user_id'], "telegram_name": walladb.get_user(product['telegram_user_id']), "category_id": article['category_id'], - "seller_id": article['seller_id'], + "seller_id": article['user_id'], "environment": constants.NR_ENV } ) diff --git a/wallamanta/worker.py b/wallamanta/worker.py index 4786961..e35dab3 100644 --- a/wallamanta/worker.py +++ b/wallamanta/worker.py @@ -12,19 +12,36 @@ logging.basicConfig( logger = logging.getLogger(__name__) -def request(product_name, steps=15, latitude=constants.LATITUDE, longitude=constants.LONGITUDE, distance='0', condition='all', min_price=0, max_price=10000000, category="", subcategories=[]): - headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'} - distance = str(int(distance) * 1000) - url = (f"https://api.wallapop.com/api/v3/general/search?keywords={product_name}" +BASE_URL = "https://api.wallapop.com/api/v3/search?" + +def request(product_name, steps=5, latitude=constants.LATITUDE, longitude=constants.LONGITUDE, distance='0', condition='all', min_price=0, max_price=0, category="", subcategories=[]): + headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36', + 'Accept-Language': 'es-ES', + 'Accept': 'application/json, text/plain, */*', + 'Accept-Encoding': 'gzip, deflate, br', + 'Connection': 'keep-alive', + 'Referer': 'https://es.wallapop.com/', + 'Sec-Fetch-Dest': 'empty', + 'Sec-Fetch-Mode': 'cors', + 'Priority': 'u=0', + 'Sec-Fetch-Site': 'same-site', + 'X-DeviceOS': '0' + } + distance = str(int(distance)) + url = (f"{BASE_URL}keywords={product_name}" f"&order_by=newest&latitude={latitude}" f"&longitude={longitude}" - f"&distance={distance}" - f"&min_sale_price={min_price}" - f"&max_sale_price={max_price}" - f"&filters_source=quick_filters&language=es_ES") + f"&source=quick_filter&show_multiple_sections=false") - #if condition != "all": - # url = url + f"&condition={condition}" # new, as_good_as_new, good, fair, has_given_it_all + if min_price != 0: + url = url + f"&min_sale_price={min_price}" + + if max_price != 0: + url = url + f"&max_sale_price={max_price}" + + if distance != '0': + url = url + f"&distance_in_km={distance}" if category != "": url = url + f"&category_ids={category}" @@ -37,17 +54,20 @@ def request(product_name, steps=15, latitude=constants.LATITUDE, longitude=const search_objects = list() + next_page = '' for step in range(steps): - #helpers.random_wait() + if next_page != '': + url = BASE_URL + f"&next_page={next_page}" tries = 5 for _ in range(tries): try: if randint(0, 1) == 0: - response = requests.get(url+f"&step={step}", timeout=1, headers=headers) + response = requests.get(url, timeout=1, headers=headers) else: - response = requests.get(url+f"&step={step}", timeout=1, headers=headers, proxies=dict(https=f'socks5://{constants.PROXY_SOCKS}')) + response = requests.get(url, timeout=1, headers=headers, proxies=dict(https=f'socks5://{constants.PROXY_SOCKS}')) if response.status_code == 200: - search_objects = search_objects + response.json()['search_objects'] + search_objects = search_objects + response.json()['data']['section']['payload']['items'] + next_page = response.json()['meta']['next_page'] break else: logging.info(f"\'{product_name}\' -> Wallapop returned status {response.status_code}. Illegal parameters or Wallapop service is down. Retrying...")