Advertisement
Guest User

parsing

a guest
Dec 24th, 2024
71
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 1.89 KB | None | 0 0
  1. # Lets call it function A
  2. async def get_sold_by_link(link):
  3.     headers = {
  4.         #http headers
  5.     }
  6.     soup = None
  7.     try:
  8.         async with SEMAPHORE:
  9.             async with aiohttp.ClientSession(headers=headers, timeout=TIMEOUT) as session:
  10.                 async with session.get(link, proxy=await anext(PROXIES), ssl=False) as response:
  11.                     text = await response.text()
  12.                     soup = BeautifulSoup(text, features="html.parser")
  13.     except Exception as e:
  14.         return
  15.     try:
  16.         sell_count = int(soup.select_one(".goods-sell-count").text.strip().split("\n")[0].split()[-1])
  17.     except Exception:
  18.         return
  19.     return sell_count
  20.  
  21. #Function B
  22.  
  23. async def get_price(good_id):
  24.     headers = {
  25.         #http headers
  26.     }
  27.     try:
  28.         async with SEMAPHORE:
  29.             async with aiohttp.ClientSession(headers=headers, timeout=TIMEOUT) as session:
  30.                 async with session.get(
  31.               f"https://plati.market/asp/price_options.asp?p={good_id}&n=0&c=RUB", proxy=await anext(PROXIES), ssl=False) as response:
  32.                     text = await response.text()
  33.                     data = json.loads(text)
  34.                     try:
  35.                         return int(data["amount"])
  36.                     except Exception as e:
  37.                         return
  38.     except Exception as e:
  39.         return
  40.    
  41.    
  42. async def main():
  43.     for counter, ref in enumerate(config.hrefs, start=1):
  44.         try:
  45.             reviews = await get_reviews(ref)
  46.         except Exception as e:
  47.             continue
  48.         tasks = []
  49.         for c, review in enumerate(reviews):
  50.             tasks.append(process_review(review))
  51.             if c % 700 == 0: # code to slow everything down. Now it's 700 for the testing, but works in range 10-20
  52.                 await asyncio.gather(*tasks)
  53.                 tasks = []
  54.         await asyncio.gather(*tasks)
  55.    
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement