Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #!python3.6
- #coding: utf8
- """
- MIT License
- Copyright (c) 2016-2018 Siidheesh Theivasigamani
- Permission is hereby granted, free of charge, to any person obtaining a copy
- of this software and associated documentation files (the "Software"), to deal
- in the Software without restriction, including without limitation the rights
- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- copies of the Software, and to permit persons to whom the Software is
- furnished to do so, subject to the following conditions:
- The above copyright notice and this permission notice shall be included in all
- copies or substantial portions of the Software.
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
- SOFTWARE.
- *** Workbin Downloader by siid ***
- Changelog:
- v0.0.7: 31 January 2018
- Applied quick fix to have multiple courses with the same name, e.g. two EE2012 mods on IVLE, to be put into the same folder
- v0.0.6: 12 January 2018
- Fixed name formatting flaw where "GEH1036/GEK1505" translated into file path GEH1036/GEK1505/.. and for other modules with a "/" in the name as well.
- v0.0.5: 25 September 2017
- Multiple workbins from each module are now supported. Previously, only the module's first listed workbin was assumed to exist.
- v0.0.4: 25 September 2017
- Fixed sanitisation of file and folder names, added uploadtime into file names to account for "updated" files with the same name
- v0.0.3: 23 August 2017
- Added user, pass, and token validation
- v0.0.2: 17 August 2017
- Removed need for human input/interaction
- v0.0.1: August 2016
- Basic functionality
- Description:
- Downloads workbins of currently active mods into the script's current directory.
- *** Needs your own NUS IVLE LAPI key and IVLE credentials ***
- When it works, it works. WHen it doesn't, it doesn't :)
- """
- from __future__ import print_function
- from http.server import BaseHTTPRequestHandler, HTTPServer
- from urllib.parse import parse_qs
- from pathlib import Path
- import webbrowser
- import urllib.request
- import json
- import re
- import os
- import sys
- import pprint
- pp = pprint.PrettyPrinter(indent=4)
- import ssl
- ssl._create_default_https_context = ssl._create_unverified_context
- import requests
- import mechanicalsoup
- browser = mechanicalsoup.StatefulBrowser()
- # YOUR NUS LAPI KEY GOES HERE
- # get from https://ivle.nus.edu.sg/LAPI/default.aspx
- key = "WPhIannlffrfUwf1P0LWr"
- IVLE_USER = ""
- IVLE_PASS = ""
- token = None
- #redundant now that MechanicalSoup is used, change it if another localhost server is used with the same port, as the request HAS to fail and throw an exception (see bottom)
- PORT = 8323
- modules = {}
- workbin_tree = {}
- rootdir = os.path.dirname(os.path.abspath(__file__))
- stats = {"count":0,"dl":[],"errcount":0}
- def enum_annonce(cid):
- #res = urllib.request.urlopen("https://ivle.nus.edu.sg/api/Lapi.svc/Webcasts?APIKey=%s&AuthToken=%s&CourseID=%s&Duration=0&MediaChannelID=&TitleOnly=false" %(key,token,cid))
- res = urllib.request.urlopen("https://ivle.nus.edu.sg/api/Lapi.svc/Announcements_Unread?APIKey=%s&AuthToken=%sTitleOnly=False" %(key,token))
- return json.loads(res.read().decode("utf-8"))
- def download_file(file_id,target_file):
- print(target_file.encode('utf-8').strip(), end="")
- res = urllib.request.urlopen("https://ivle.nus.edu.sg/api/downloadfile.ashx?APIKey=%s&AuthToken=%s&ID=%s&target=workbin" %(key,token,file_id))
- CHUNK = 20 * 1024 * 1024
- path = os.path.normcase(rootdir+target_file)
- try:
- with open(path, 'wb') as f:
- while True:
- chunk = res.read(CHUNK)
- if not chunk: break
- f.write(chunk)
- print(".", end="")
- #print("%s downloaded" % target_file)
- print("\tdone!")
- if target_file not in stats["dl"]:
- stats["dl"].append(target_file)
- stats["count"] += 1
- except Exception as ex:
- template = "\texception of type {0} occurred. Arguments:{1!r}"
- print(template.format(type(ex).__name__, ex.args))
- stats["errcount"] += 1
- def manage_dir(tree,curr=""):
- p = Path(rootdir+curr)
- if not p.exists() or not p.is_dir():
- p.mkdir()
- for entry in tree:
- if type(tree[entry]) is not dict:
- if not Path(rootdir+curr+"/"+tree[entry]).is_file():
- download_file(entry,curr+"/"+tree[entry])
- #else:
- #print("skipped %s" % curr+"/"+tree[entry])
- else:
- manage_dir(tree[entry],curr+"/"+re.sub('[^\w_.)( -]', '_', entry))
- def dig_folder(curr):
- tree = {}
- if "Files" in curr:
- for file in curr["Files"]:
- name,ext = re.sub('[^\w_.)( -]', '_', file["FileName"]).split(".", 1)
- tree[file["ID"]] = name + "_" + file["UploadTime"][file["UploadTime"].index("(") + 1:file["UploadTime"].rindex("+")] + "." + ext
- if "Folders" in curr:
- for folder in curr["Folders"]:
- tree[re.sub('[^\w_.)( -]', '_', folder["FolderName"])] = dig_folder(folder)
- return tree
- def update_workbins():
- print("\n******** Enumerating modules and workbins ********\n")
- res = urllib.request.urlopen("https://ivle.nus.edu.sg/api/Lapi.svc/Modules?APIKey=%s&AuthToken=%s&Duration=0&IncludeAllInfo=false" % (key,token))
- data = json.loads(res.read().decode("utf-8"))["Results"]
- for entry in data:
- if entry["isActive"]=="Y":
- cid = entry["ID"]
- """annonce = enum_annonce(cid)
- if annonce['Results'] and annonce['Results'][0]:
- print("Unread announcements for", code, "!")"""
- code = entry["CourseCode"]
- res = urllib.request.urlopen("https://ivle.nus.edu.sg/api/Lapi.svc/Workbins?APIKey=%s&AuthToken=%s&CourseID=%s&Duration=0&WorkbinID=&TitleOnly=false" % (key,token,cid))
- data1 = json.loads(res.read().decode("utf-8"))
- if not data1["Results"] or not data1["Results"][0]:
- print("Empty workbin", code)
- continue
- #if code not in workbin_tree:
- print("Found workbin", code)
- if len(data1["Results"]) <= 1:
- workbin_tree[code] = dig_folder(data1["Results"][0])
- else:
- workbin_tree[code] = {}
- for wbin in data1["Results"]:
- workbin_tree[code][re.sub('[^\w_.)( -]', '_', wbin['Title'])] = dig_folder(wbin)
- print("\n******** Updating filesystem - Downloading new files, missing files ********\n")
- #pprint.pprint(workbin_tree)
- manage_dir(workbin_tree)
- if stats["errcount"]>0:
- print("Errors encountered:",stats["errcount"],'\n')
- if stats["count"]>0:
- print("\n******** Done! ********\n")
- print("Files downloaded:",stats["count"],'\n')
- print('\n'.join(stats["dl"]),'\n')
- else:
- print("\n******** No changes made! ********\n")
- if len(sys.argv) == 1:
- print("Press enter to quit")
- input()
- quit()
- if not IVLE_USER or not IVLE_PASS:
- print("ERROR: You need to edit the script and enter your IVLE user and pass in the IVLE_USER and IVLE_PASS variables at the top before the script can work.")
- elif key is not None and len(key) > 0:
- browser.open("https://ivle.nus.edu.sg/api/login/?apikey="+key+"&url=http://localhost:"+str(PORT)+"/")
- browser.select_form('#frm')
- browser["userid"] = IVLE_USER
- browser["password"] = IVLE_PASS
- try:
- resp = browser.submit_selected()
- print("ERROR: Your login credentials were incorrect.")
- except requests.exceptions.ConnectionError as e:
- url = e.request.url
- parts = url.split('=')
- if len(parts) == 2 and parts[0] == 'http://localhost:'+str(PORT)+'/?token':
- token = parts[1]
- update_workbins()
- else:
- print("ERROR: You need to get an IVLE LAPI key from https://ivle.nus.edu.sg/LAPI/default.aspx first. Press enter to open the webpage")
- input()
- webbrowser.open("https://ivle.nus.edu.sg/LAPI/default.aspx", new=0, autoraise=True)
Add Comment
Please, Sign In to add comment