import requests
from bs4 import BeautifulSoup
import re
import lxml
from pathlib import Path
import sys
import time

url = "https://github.com/wangzhengya/allitebooks/tree/master/results"
html = requests.get(url)
soup = BeautifulSoup(html.content, 'lxml')
sources_list = []

for link in soup.findAll('a', attrs={'href': re.compile('/wangzhengya/allitebooks/blob/master/results/')}):
    source = ('https://github.com' + link.get('href'))
    sources_list.append(source)


def downloader(url, folder):

    path_to_folder = Path.home() / 'downloads' / 'pdf-books' / folder
    path_to_folder.mkdir(exist_ok=True, parents=True)
    pdf_file = url.split('/')[-1]

    check_for_existance_FILE = Path.joinpath(path_to_folder, pdf_file)

    if check_for_existance_FILE.is_file() is False:
        try:
            pdf = requests.get(url)
            with open(Path.joinpath(path_to_folder, pdf_file), 'wb') as FILE:
                FILE.write(pdf.content)

            print(str(pdf_file) + ' copied to ' + str(folder))

            time.sleep(2)

        except:
            e = sys.exc_info()[0]
            print("An exception occurred:")
            print(e)

    else:
        print('File already downloaded -> ' + str(check_for_existance_FILE))

def get_file_urls_from_source(source):
    folder = source.split('/')[-1].split('.')[-2]
    pdf_urls = requests.get(source)
    soup = BeautifulSoup(pdf_urls.content, 'lxml')

    for link in soup.findAll('td', attrs={'class': re.compile('blob-code blob-code-inner js-file-line')}):
        pdf = (link.get_text())

        downloader(pdf, folder)


for i, source in enumerate(sources_list):
    print(i+1, source)
    get_file_urls_from_source(source)