Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import getpass
- import sys
- from bs4 import BeautifulSoup
- from selenium import webdriver
- from selenium.webdriver.support.ui import Select
- LOGIN_URL = 'https://auth.vt.edu/login?service=https%3a%2f%2firweb.ir.vt.edu%2fwebtest%2fAuthenticated%2fGradeDistribution.aspx'
- URL = 'https://irweb.ir.vt.edu/webtest/Authenticated/GradeDistribution.aspx'
- driver = None
- page = None
- soup = None
- table = None
- table1 = None
- course_list = []
- total_pages = 0
- page_number = 0
- terms = {}
- def login(username, password):
- term_value = []
- term_name = []
- print "Driver is opening........"
- global driver
- driver = webdriver.Firefox()
- driver.get(LOGIN_URL)
- print "Driver is maximizing....."
- driver.maximize_window()
- user_element = driver.find_element_by_id("username")
- pass_element = driver.find_element_by_id("password")
- user_element.send_keys(username)
- pass_element.send_keys(password)
- print "Logging in......"
- pass_element.submit()
- select = driver.find_element_by_xpath("//select[@id='ctl00_ContentPlaceHolder1_ddlTerm']") # get the select element
- options = select.find_elements_by_tag_name("option") # get all the options into a list
- for option in options: # iterate over the options, place attribute value in list
- term_value.append(str(option.get_attribute("value")))
- term_name.append(str(option.text))
- global terms
- terms = dict(zip(term_name, term_value))
- def showSelectionsMenu():
- hide_selection = driver.find_element_by_class_name('hideSelection')
- is_hidden = "none" in hide_selection.get_attribute("style") #true when hide selection is hidden and show selection is active
- if is_hidden:
- driver.find_element_by_class_name('showSelection').click() #click on show selections
- def chooseTerm(term):
- showSelectionsMenu()
- select_term = Select(driver.find_element_by_id('ctl00_ContentPlaceHolder1_ddlTerm'))
- value = terms[term]
- select_term.select_by_value(value)
- driver.implicitly_wait(5)
- showSelectionsMenu()
- submit = driver.find_element_by_id("ctl00_ContentPlaceHolder1_Submit")
- submit.click()
- update()
- # checkVar()
- def addCourses():
- print "Storing table information......."
- grade_table = table1[32].contents[0] #Table of all grade data located at index 32
- for row in grade_table.find_all('tr'):
- course = []
- for cell in row:
- if (str(cell.text) == '' or str(cell.text) == 'Subject'):
- course.append("dontAdd")
- else:
- course.append(str(cell.text))
- if course[1] != "dontAdd":
- if len(course) > 14: #If row longer 15 then it has extra spaces in the beginning after the first page
- course.pop(0) #Pop off the blank cell in the beginning of course
- course_list.append(course)
- ## x = 1
- ## for row in table[42:43][0].contents[0].contents[0].contents[0]: # location of table in the document
- ## # print x
- ## course = []
- ## length = len(row.contents)
- ## n = 1
- ## if length == 14: # The first page has 14 elements and indexes at 0 whereas everything else has 15 elements and indexes at 1
- ## n = 0
- ## # print "Length of row: " + str(length)
- ## for i in range(n, length):
- ## #print "Column number: " + str(i) + " " + row.contents[i].text
- ## if (str(row.contents[i].text) == '' or str(row.contents[i].text) == 'Subject'):
- ## course.append("dontAdd")
- ## else:
- ## course.append(str(row.contents[i].text))
- ## if (course[0] != "dontAdd"):
- ## # print "course[0]: " + course[0]
- ## course_list.append(course)
- ## x += 1
- # checkVar()
- def nextPage():
- next_element = driver.find_element_by_id("ctl00_ContentPlaceHolder1_ReportViewer_ctl05_ctl00_Next_ctl00_ctl00")
- next_element.click()
- ready = driver.find_elements_by_id("ctl00_ContentPlaceHolder1_ReportViewer_ctl05_ctl00_CurrentPage")[0].is_enabled()
- while not ready:
- ready = driver.find_elements_by_id("ctl00_ContentPlaceHolder1_ReportViewer_ctl05_ctl00_CurrentPage")[
- 0].is_enabled()
- update()
- # checkVar()
- def addAllCourses():
- print "Page " + str(page_number) + " of " + str(total_pages)
- addCourses()
- for n in range(total_pages - 1):
- nextPage()
- print "Page " + str(page_number) + " of " + str(total_pages)
- addCourses()
- def finish():
- print "Driver is closing........."
- driver.quit()
- def checkVar():
- print "Driver Status: " + str(driver)
- print "Page Status: " + str(sys.getsizeof(page))
- print "Page #: " + str(page_number)
- print "Soup: " + str(sys.getsizeof(soup))
- print "Table: " + str(sys.getsizeof(table))
- def update():
- #Update the soup and table
- global soup
- soup = BeautifulSoup(page, 'html.parser')
- global table
- table = soup.findAll('tr')
- global table1
- table1 = soup.find_all('table')
- #Update the page information
- global page
- page = driver.page_source
- page_t = driver.find_elements_by_id("ctl00_ContentPlaceHolder1_ReportViewer_ctl05_ctl00_TotalPages")[0].text
- global total_pages
- total_pages = int(page_t)
- page_n = driver.find_elements_by_id("ctl00_ContentPlaceHolder1_ReportViewer_ctl05_ctl00_CurrentPage")[0].get_attribute('value')
- global page_number
- page_number = int(page_n)
- def go():
- username = raw_input("Username: ")
- password = getpass.win_getpass('Password: ')
- login(username, password)
- print "Log In Successful!"
- n = 0
- for key in terms.viewkeys():
- print n
- n+=1
- print key
- term = raw_input("Enter Term: ")
- chooseTerm(term)
- addAllCourses()
- response = raw_input("Do you want to see the data(Y/N): ")
- if (response == "Y"):
- print 'Populating table information.......'
- for course in course_list:
- print course
- finished = 0
- finished = raw_input("Type 1 to quit:")
- while (int(finished) != 1):
- finished = raw_input("Type 1 to quit:")
- term = raw_input("Enter Term: ")
- chooseTerm(term)
- finish()
- print "Finished!!!!!!!"
- go()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement