ProcessPoolExecutor_playwright.py

import multiprocessing
import os
from concurrent.futures import ProcessPoolExecutor

from playwright.sync_api import sync_playwright
from playwright_stealth import stealth_sync


def find_res(city, shared_list):
    res = []
    try:
        with sync_playwright() as pw:
            proxy = {
                'server': os.getenv('PROXY_SERVER'),
                'username': os.getenv('PROXY_USERNAME'),
                'password': os.getenv('PROXY_PASSWORD')
            }
            browser = pw.firefox.launch(headless=True, slow_mo=200)
            context = browser.new_context(proxy=proxy)
            stealth_sync(context)
            page = context.new_page()
            resp = page.goto(f"https://site.com/{city}")
            if resp.status == 200:
                shared_list.append(resp.url)
                res.append(resp.text())

    except Exception as e:
        print(f"Exception: {e}")
        return []

    return res


def process_city(args):
    city, shared_list = args
    return find_res(city, shared_list)


def main():
    total_res = []
    cities = ['New York', 'Los Angeles', 'Chicago', 'Houston', 'Phoenix', 'Philadelphia', 'San Antonio', 'San Diego',]
    with multiprocessing.Manager() as manager:
        shared_list = manager.list()
        with ProcessPoolExecutor(max_workers=10) as executor:
            for res in executor.map(process_city, [(city, shared_list) for city in cities]):
                total_res.append(res)

    print(total_res)


if __name__ == '__main__':
    main()