Scholarship_scraper.py on ec2

import json
import string
import time
from tqdm import tqdm
import ast

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.chrome.options import Options
from pyvirtualdisplay import Display


#Important
display = Display(visible=0, size=(800, 600))
display.start()


options = Options()
options.add_argument("--start-maximized")
options.add_argument("--headless")
options.add_argument('--no-sandbox')
options.add_argument('--disable-dev-shm-usage')


print("starting")

driver=webdriver.Chrome(chrome_options=options)


with open("milestone_2.txt", encoding='cp1252') as f:
    names_and_links = ast.literal_eval(f.read())




result = []

for item in tqdm(names_and_links[0:]):
    obj = {}
    name = ""
    sponsor = ""
    data = {}
    required_materials_list = []

    obj["Eligibility Requirements"] = {}
    obj["Qualified Institutions & Level of Study"] = {}
    obj["Application Requirements"] = {}
    obj["General Information"] = {}
    obj["Deadlines & Fees"] = {}
    obj["Contact Information"] = {}
    obj["Url"] = item["Link"]


    driver.get(item["Link"])
    driver.implicitly_wait(60)
    time.sleep(4.3)


    try:
        name = driver.find_element_by_class_name("scholarshipDetails").find_element_by_tag_name("div").text.split("\n")[1].strip()

    except:
        pass


    try:
        sponsor = driver.find_element_by_class_name("scholarshipDetails").find_element_by_class_name("copySm").find_element_by_class_name("right").text.strip()

    except Exception as e:
        pass



    try:
        eligibility_requirements = driver.find_element_by_class_name("eligibilityView").find_elements_by_class_name("searchResultField")


        for item in eligibility_requirements:
            if ":" in item.text:
                key = ", ".join(item.text.split(":")[0].strip().split("\n"))
                value = ", ".join(item.text.split(":")[1].strip().split("\n"))
                data = {key:value}
                obj["Eligibility Requirements"].update(data)


    except Exception as e:
        pass



    try:
        qualified_institutions = driver.find_element_by_class_name("qualifiedInstitutionsView").find_elements_by_class_name("searchResultField")

        for item in qualified_institutions:
            if ":" in item.text:
                key = ", ".join(item.text.split(":")[0].strip().split("\n"))
                value = " ".join(item.text.split(":")[1:]).strip()

                if "Award may be used for" in key or "Award may be used at" in key:
                    data = {key:value.split("\n")}
                    obj["Qualified Institutions & Level of Study"].update(data)

                else:
                    data = {key: ", ".join(value.split("\n"))}
                    obj["Qualified Institutions & Level of Study"].update(data)


    except Exception as e:
        pass





    try:
        app_requirements = driver.find_element_by_class_name("applicationRequirementsView").find_elements_by_class_name("searchResultField")


        for item in app_requirements:
            if ":" in item.text:
                key = ", ".join(item.text.split(":")[0].strip().split("\n"))
                value = item.text.split(":")[1].strip().split("\n")
                data = {key:value}

                obj["Application Requirements"].update(data)

    except Exception as e:
        pass




    try:
        required_materials = driver.find_element_by_class_name("applicationRequirementsView").find_element_by_class_name("searchResultField").text.strip().split("\n")

        for req_mat in required_materials:
            required_materials_list.append(req_mat.strip())

        obj["Application Requirements"].update({"Required Materials": required_materials_list})

    except:
        pass










    try:
        gen_info = driver.find_element_by_class_name("scholarshipProfileGeneralInformationView").find_elements_by_class_name("searchResultField")


        for item in gen_info:
            if ":" in item.text:
                key = ", ".join(item.text.split(":")[0].strip().split("\n"))
                value = " ".join(item.text.split(":")[1:]).strip()

                if "Type of award" in key:
                    data = {key:value.split("\n")}
                    obj["General Information"].update(data)

                else:
                    data = {key: ", ".join(value.split("\n"))}
                    obj["General Information"].update(data)


    except Exception as e:
        pass




    try:
        deadlines = driver.find_element_by_class_name("scholarshipProfileDeadlinesView").find_elements_by_class_name("searchResultField")


        for item in deadlines:
            if ":" in item.text:
                key = ", ".join(item.text.split(":")[0].strip().split("\n"))
                value = " ".join(item.text.split(":")[1:]).strip()

                data = {key: ", ".join(value.split("\n"))}
                obj["Deadlines & Fees"].update(data)


    except Exception as e:
        pass



    try:
        contact_info = driver.find_element_by_class_name("scholarshipProfileContactInformationView").find_elements_by_class_name("searchResultField")


        for item in contact_info:
            if ":" in item.text:
                key = ", ".join(item.text.split(":")[0].strip().split("\n"))
                value = " ".join(item.text.split(":")[1:]).strip()

                data = {key: ", ".join(value.split("\n"))}
                obj["Contact Information"].update(data)


    except Exception as e:
        pass





    obj["Name"] = name

    obj["Sponsor"] = sponsor


    result.append(obj)




with open("result.json", "w", encoding='cp1252') as f:
    json.dump(result, f, indent=4)