Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- # Lets call it function A
- async def get_sold_by_link(link):
- headers = {
- #http headers
- }
- soup = None
- try:
- async with SEMAPHORE:
- async with aiohttp.ClientSession(headers=headers, timeout=TIMEOUT) as session:
- async with session.get(link, proxy=await anext(PROXIES), ssl=False) as response:
- text = await response.text()
- soup = BeautifulSoup(text, features="html.parser")
- except Exception as e:
- return
- try:
- sell_count = int(soup.select_one(".goods-sell-count").text.strip().split("\n")[0].split()[-1])
- except Exception:
- return
- return sell_count
- #Function B
- async def get_price(good_id):
- headers = {
- #http headers
- }
- try:
- async with SEMAPHORE:
- async with aiohttp.ClientSession(headers=headers, timeout=TIMEOUT) as session:
- async with session.get(
- f"https://plati.market/asp/price_options.asp?p={good_id}&n=0&c=RUB", proxy=await anext(PROXIES), ssl=False) as response:
- text = await response.text()
- data = json.loads(text)
- try:
- return int(data["amount"])
- except Exception as e:
- return
- except Exception as e:
- return
- async def main():
- for counter, ref in enumerate(config.hrefs, start=1):
- try:
- reviews = await get_reviews(ref)
- except Exception as e:
- continue
- tasks = []
- for c, review in enumerate(reviews):
- tasks.append(process_review(review))
- if c % 700 == 0: # code to slow everything down. Now it's 700 for the testing, but works in range 10-20
- await asyncio.gather(*tasks)
- tasks = []
- await asyncio.gather(*tasks)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement