Jocaru/wallamanta
1
0
Fork 0
Code Issues 15 Packages Projects Releases Wiki Activity

Wallapop API changes

Browse Source
This commit is contained in:
Joan
2024-10-07 14:01:27 +02:00
parent eb5ec8cba1
commit 20b5eaaea1
2 changed files with 43 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
wallamanta/helpers.py
View File

@@ -62,7 +62,7 @@ def random_wait():
time.sleep(random.random())
def download_image(article, product_id):
r = requests.get(article['images'][0]['original'])
r = requests.get(article['images'][0]['urls']['small'])
if r.status_code == 200:
with open(f"/app/data/images/products/{article['id']}_{product_id}.jpg", "wb") as image:
image.write(r.content)
@@ -70,9 +70,9 @@ def download_image(article, product_id):
def create_image(article, product_id):
download_image(article, product_id)
currency = '?'
if article['currency'] == 'EUR':
if article['price']['currency'] == 'EUR':
currency = '€'
price = str(article['price']) + currency
price = str(article['price']['amount']) + currency
wallamanta_text = "@wallamanta_bot"
width = 1280
height = 800
@@ -123,14 +123,10 @@ def remove_image(article, product_id):
logging.info(f"Tried to delete {article} image. Couldn't find it.")
def get_publish_date(article):
article_date = article['creation_date']
formato_fecha = "%Y-%m-%dT%H:%M:%S.%f%z"
return datetime.strptime(article_date, formato_fecha).astimezone(pytz.timezone("Europe/Madrid")).strftime("%d/%m/%Y - %H:%M:%S")
return datetime.fromtimestamp(article['created_at']/1000, pytz.timezone("Europe/Madrid")).strftime("%d/%m/%Y - %H:%M:%S")
def get_modified_date(article):
article_date = article['modification_date']
formato_fecha = "%Y-%m-%dT%H:%M:%S.%f%z"
return datetime.strptime(article_date, formato_fecha).astimezone(pytz.timezone("Europe/Madrid")).strftime("%d/%m/%Y - %H:%M:%S")
return datetime.fromtimestamp(article['modified_at']/1000, pytz.timezone("Europe/Madrid")).strftime("%d/%m/%Y - %H:%M:%S")
def get_random_string(length):
result_str = ''.join(random.choice(string.ascii_letters) for i in range(length))
@@ -177,11 +173,11 @@ def send_article(article, product):
user_ships = f"📦 *Envío:* ✅"
else:
user_ships = f"📦 *Envío:* ❌"
if article['currency'] == 'EUR':
if article['price']['currency'] == 'EUR':
currency = '€'
else:
currency = '?'
price = f"*💰 Precio*: {telegram_escape_characters(str(article['price']))} {telegram_escape_characters(currency)}"
price = f"*💰 Precio*: {telegram_escape_characters(str(article['price']['amount']))} {telegram_escape_characters(currency)}"
text = f"{title}\n\n{description}\n\n{found_by}\n\n{created_at}\n{modified_at}\n\n{location}\n\n{user_ships}\n\n{price}"
#url = f"https://api.telegram.org/bot{constants.TELEGRAM_TOKEN}/sendPhoto?chat_id={product['telegram_user_id']}&caption={text}&parse_mode=MarkdownV2"
#files = {'photo':open(f"/app/data/images/products/{article['id']}_composed.png", 'rb')}
@@ -360,13 +356,13 @@ def send_to_nr(article, product):
event = Event(
"ProductFound", {
"article_name": article['title'],
"article_price": article['price'],
"article_price": article['price']['amount'],
"article_web_slug": article['web_slug'],
"product_name": product['product_name'],
"telegram_user_id": product['telegram_user_id'],
"telegram_name": walladb.get_user(product['telegram_user_id']),
"category_id": article['category_id'],
"seller_id": article['seller_id'],
"seller_id": article['user_id'],
"environment": constants.NR_ENV
}
)

48
wallamanta/worker.py
View File

@@ -12,19 +12,36 @@ logging.basicConfig(
logger = logging.getLogger(__name__)
def request(product_name, steps=15, latitude=constants.LATITUDE, longitude=constants.LONGITUDE, distance='0', condition='all', min_price=0, max_price=10000000, category="", subcategories=[]):
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36'}
distance = str(int(distance) * 1000)
url = (f"https://api.wallapop.com/api/v3/general/search?keywords={product_name}"
BASE_URL = "https://api.wallapop.com/api/v3/search?"
def request(product_name, steps=5, latitude=constants.LATITUDE, longitude=constants.LONGITUDE, distance='0', condition='all', min_price=0, max_price=0, category="", subcategories=[]):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
'Accept-Language': 'es-ES',
'Accept': 'application/json, text/plain, */*',
'Accept-Encoding': 'gzip, deflate, br',
'Connection': 'keep-alive',
'Referer': 'https://es.wallapop.com/',
'Sec-Fetch-Dest': 'empty',
'Sec-Fetch-Mode': 'cors',
'Priority': 'u=0',
'Sec-Fetch-Site': 'same-site',
'X-DeviceOS': '0'
}
distance = str(int(distance))
url = (f"{BASE_URL}keywords={product_name}"
f"&order_by=newest&latitude={latitude}"
f"&longitude={longitude}"
f"&distance={distance}"
f"&min_sale_price={min_price}"
f"&max_sale_price={max_price}"
f"&filters_source=quick_filters&language=es_ES")
f"&source=quick_filter&show_multiple_sections=false")
#if condition != "all":
# url = url + f"&condition={condition}" # new, as_good_as_new, good, fair, has_given_it_all
if min_price != 0:
url = url + f"&min_sale_price={min_price}"
if max_price != 0:
url = url + f"&max_sale_price={max_price}"
if distance != '0':
url = url + f"&distance_in_km={distance}"
if category != "":
url = url + f"&category_ids={category}"
@@ -37,17 +54,20 @@ def request(product_name, steps=15, latitude=constants.LATITUDE, longitude=const
search_objects = list()
next_page = ''
for step in range(steps):
#helpers.random_wait()
if next_page != '':
url = BASE_URL + f"&next_page={next_page}"
tries = 5
for _ in range(tries):
try:
if randint(0, 1) == 0:
response = requests.get(url+f"&step={step}", timeout=1, headers=headers)
response = requests.get(url, timeout=1, headers=headers)
else:
response = requests.get(url+f"&step={step}", timeout=1, headers=headers, proxies=dict(https=f'socks5://{constants.PROXY_SOCKS}'))
response = requests.get(url, timeout=1, headers=headers, proxies=dict(https=f'socks5://{constants.PROXY_SOCKS}'))
if response.status_code == 200:
search_objects = search_objects + response.json()['search_objects']
search_objects = search_objects + response.json()['data']['section']['payload']['items']
next_page = response.json()['meta']['next_page']
break
else:
logging.info(f"\'{product_name}\' -> Wallapop returned status {response.status_code}. Illegal parameters or Wallapop service is down. Retrying...")