html_scrape

async def get_page(url):
    async with semaphore:
        timeout = aiohttp.ClientTimeout(connect=15)
        async with aiohttp.ClientSession(timeout=timeout, headers=headers) as session:
            proxy = await get_proxy(session)
            # Get podcast html
            while True:
                try:
                    async with session.get(url, proxy=proxy) as r:
                        print('URL: {} STATUS: {}'.format(url, r.status))
                        if r.status != 200:
                            await asyncio.sleep(1)
                            proxy = await get_proxy(session)
                            continue
                        main_html = await r.text()
                        break
                except Exception:
                    await asyncio.sleep(1)
                    proxy = await get_proxy(session)
                    continue
            # Get podcast listeners html
            # Get podbay json
            # Get podcastrss html
            # Get getrssfeed html
            result = {
                'main_html': main_html,
                'listeners_html': listeners_html,
                'podbay_json': podbay_json,
                'podcastrss_html': podcastrss_html,
                'getrssfeed_html': getrssfeed_html
            }
            pub = await aioredis.create_redis('redis://localhost')
            await pub.publish_json('chan:1', result)
            pub.close()


if __name__ == "__main__":
    urls = []
    loop = asyncio.get_event_loop()
    tasks = [asyncio.ensure_future(get_page(url)) for url in urls]
    loop.run_until_complete(asyncio.wait(tasks))