from selenium import webdriver
from webdriver_manager.chrome import ChromeDriverManager
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
import time, json

def fedsdatacenter():
    options = webdriver.ChromeOptions()
    options.add_argument("--start-maximized")
    #options.add_argument("--headless")
    data=[]
    driver = webdriver.Chrome(ChromeDriverManager().install(), options=options)
    driver.get('https://www.fedsdatacenter.com/federal-pay-rates/output.php?n=&a=&l=&o=&y=2021&sEcho=4&iColumns=9&sColumns=%2C%2C%2C%2C%2C%2C%2C%2C&iDisplayStart=0&iDisplayLength=100&mDataProp_0=0&bSortable_0=true&mDataProp_1=1&bSortable_1=true&mDataProp_2=2&bSortable_2=true&mDataProp_3=3&bSortable_3=true&mDataProp_4=4&bSortable_4=true&mDataProp_5=5&bSortable_5=true&mDataProp_6=6&bSortable_6=true&mDataProp_7=7&bSortable_7=true&mDataProp_8=8&bSortable_8=true&iSortCol_0=0&sSortDir_0=asc&iSortingCols=1&_=1')
    json_data = driver.find_element(By.TAG_NAME,'body').text
    data_dict = json.loads(json_data)
    for x in range (1, int(data_dict["iTotalRecords"])):
        try:
            driver.get(f'https://www.fedsdatacenter.com/federal-pay-rates/output.php?n=&a=&l=&o=&y=2021&sEcho=4&iColumns=9&sColumns=%2C%2C%2C%2C%2C%2C%2C%2C&iDisplayStart=0&iDisplayLength=100&mDataProp_0=0&bSortable_0=true&mDataProp_1=1&bSortable_1=true&mDataProp_2=2&bSortable_2=true&mDataProp_3=3&bSortable_3=true&mDataProp_4=4&bSortable_4=true&mDataProp_5=5&bSortable_5=true&mDataProp_6=6&bSortable_6=true&mDataProp_7=7&bSortable_7=true&mDataProp_8=8&bSortable_8=true&iSortCol_0=0&sSortDir_0=asc&iSortingCols=1&_={x}')
            json_data = driver.find_element(By.TAG_NAME,'body').text
            data_dict = json.loads(json_data)
            data.append(data_dict["aaData"])
        except:
            pass
    print(data)
if __name__ == '__main__':
    fedsdatacenter()