seleniumwire_playground

from seleniumwire.utils import decode as decodesw
from seleniumwire import webdriver
from urllib.parse import unquote
import json

def show_requests_url(driver,target_url):
    driver.get(target_url)
    urls = []
    for request in driver.requests:
        urls.append({"url":request.url})
    return urls

def show_responses(driver, target_url):
    driver.get(target_url)
    responses = []
    for request in driver.requests:
        try:
            data = decodesw(
                request.response.body,
                request.response.headers.get('Content-Encoding', 'identity')
            )
            resp = json.loads(data.decode("utf-8"))
            responses.append(resp)
        except:
            pass
    return responses


def main():
    keywords = ['https://www.google-analytics.com']
    # uncomment this if you want full responses
    # options = webdriver.FirefoxOptions()
    options = webdriver.ChromeOptions()
    options.add_argument("--headless")
    driver = webdriver.Chrome(seleniumwire_options={"disable_encoding":True},options=options)
    target_url = "https://pergikuliner.com/restaurants?page=1"
    # target_url = "https://www.google.com"
    urls = show_requests_url(driver,target_url)
    responses = show_responses(driver,target_url)

    for url in urls:
        for kw in keywords:
            if kw in url['url']:
                print(unquote(url['url']))
    with open('json_responses.json','w') as fo:
        json.dump(responses,fo)

    driver.close()

if __name__ == "__main__":
    main()