from grab import Grab
import sys
import requests
import os

import urllib3
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)

from time import sleep
import random

path = '/tmp/site/'



login = 'some_login'
password = 'some_passwd'

g = Grab(log_file='out.html')
g.go('http://some_site/authentication/')
g.doc.set_input('username', login)
g.doc.set_input('password', password)
g.doc.submit()
g.go('http://some_site/link/')
sel = g.doc.select('//a[contains(@class, "course-content__topic-item-link")]')
count = 0

for elem in sel:
    directory = path + str(count) + '/'
    if not os.path.exists(directory):
        os.makedirs(directory)
    url = g.make_url_absolute(elem.attr('href'))
    print(url)
    g.go(url)
    try:
        video = g.make_url_absolute(g.doc.select('//video[contains(@class, "video-js")]//source').attr('src'))
        filename = video.split('/')[-1].split('#')[0].split('?')[0]
        afilename = directory + filename
        try:
            r = requests.get(video, stream=True, verify=False)
            with open(afilename, "wb") as file:
                for chunk in r.iter_content(chunk_size=255):

                    # writing one chunk at a time to pdf file
                    if chunk:
                        file.write(chunk)
            file.close()
        except:
            pass
        else:
            pass

    except IndexError:
        print('no video')
    else:
        print(video)
        print(filename)



    slides = g.doc.select('//div[contains(@class, "online-content__preview-slider")]//img')
    for slide in slides:
        img = g.make_url_absolute(slide.attr('src')).rstrip()
        print(img)
        filename = img.split('/')[-1].split('#')[0].split('?')[0]
        afilename = directory + filename
        print(afilename)
        data = requests.get(img, verify=False).content
        with open(afilename, 'wb') as f:
            f.write(data)

    #input("Press Enter to continue...")

    print(count)
    count += 1
    print('count rised:', count)