Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import time
- import aiohttp
- import asyncio
- async def get_page(session, category: str, page_id: int) -> str:
- if page_id:
- url = f'https://www.ozon.ru/brand/{category}/?page={page_id}'
- else:
- url = f'https://www.ozon.ru/brand/{category}/'
- print(f'get url: {url}')
- async with session.get(url) as response:
- return await response.text()
- async def load_data():
- category_list = ['adidas-144082850', 'puma-87235756']
- tasks = []
- async with aiohttp.ClientSession() as session:
- for category in category_list:
- for page_id in range(50):
- tasks.append(get_page(session, category, page_id))
- results = await asyncio.gather(*tasks)
- for text in results:
- pass
- # обрабатываем полученный текст, сохраняем в файл/базу
- if __name__ == '__main__':
- start_time = time.time()
- loop = asyncio.get_event_loop()
- loop.run_until_complete(load_data())
- print("--- %s seconds ---" % (time.time() - start_time))
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement