Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import asyncio
- from pyppeteer import launch
- link = "https://stackoverflow.com/questions/tagged/web-scraping"
- async def fetch(url):
- browser = await launch(headless=True,autoClose=False)
- page = await browser.newPage()
- await page.goto(url)
- linkstorage = []
- elements = await page.querySelectorAll('.summary .question-hyperlink')
- for element in elements:
- linkstorage.append(await page.evaluate('(element) => element.href', element))
- results = await asyncio.gather(*[browse_all_links(link, page) for link in linkstorage])
- return results
- async def browse_all_links(link, page):
- await page.goto(link)
- title = await page.querySelectorEval('.question-hyperlink','(e => e.innerText)')
- print(title)
- if __name__ == '__main__':
- loop = asyncio.get_event_loop()
- future = asyncio.ensure_future(fetch(link))
- loop.run_until_complete(future)
- loop.close()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement