import re
import os
import time
import datetime
import urllib
from urllib.parse import urlparse

from url_normalize import url_normalize

import requests
import psycopg2
from bs4 import BeautifulSoup

import database

import scheduler


def getPictures(soup, url):
    links = []
    for link in soup.findAll('img', attrs={'src': re.compile("^http://")}):
        links.append(url_normalize(link.get('src')))
    for link in soup.findAll('img', attrs={'src': re.compile("^(?!www\.|(?:http|ftp)s?://|[A-Za-z]:\\|//).*")}):
        links.append(url + link.get('src'))
    return links


def getLinks(soup, url):
    links = []
    for link in soup.findAll('a', attrs={'href': re.compile("^http://")}):
        links.append(url_normalize(link.get('href')))
        # for link in soup.findAll('a', attrs={'href': re.compile("^(/|.*" + url + ")")}):
    for link in soup.findAll('a', attrs={'href': re.compile("^(?!www\.|(?:http|ftp)s?://|[A-Za-z]:\\|//).*")}):
        links.append(url + link.get('href'))
    return links


def initWorker(driver, url, active, lock, cache):
    db_conn = psycopg2.connect("host=localhost dbname=crawldb user=postgres password=admin")
    # db_conn = psycopg2.connect("host=localhost dbname=crawldb user=mkozmelj")
    r = requests.get(url)
    if r.status_code == 200:
        # print(r.status_code, url)
        nov_url = url
        driver.get(url)
        time.sleep(10)
        soup = BeautifulSoup(driver.page_source, 'html.parser')
        if len(soup.find_all('base')) == 1:
            nov_url = soup.find_all('base')[0]['href']
        links = getLinks(soup, nov_url)
        pictures = getPictures(soup, nov_url)
        driver.quit()
        files = []
        for i in links:
            if re.search(r"(\.gov\.si)+", i):
                try:
                    # Če stran nima robots.txt pride do errorja
                    if cache.allowed(i, '*') is True:
                        if re.compile("^.+\.(?:doc|docx|pdf|ppt|pptx)($|\n)", re.VERBOSE).search(i):
                            files.append(i)
                        else:
                            database.add_page(urlparse(i).netloc, i, "FRONTIER",
                                              None, None, datetime.datetime.now(),
                                              db_conn, lock)
                            database.add_link(url, i, db_conn, lock)
                except Exception as e:
                    print("Napaka pri dodajanju v bazo: ")
                    print(e)
                    # Todo Pri shranjevanju v bazo je potrebno preverit če je mogoče duplikat.
                    #  V primeru da bomo računali podobnost bo najbolje računat hašh kar tukaj.
        database.add_page(urlparse(nov_url).netloc, url, "HTML", str(soup), r.status_code, r.headers['Date'], db_conn,
                          lock)
        #print("stevilo slik: ", len(pictures))
        for i in pictures:
            if re.search(r"(\.gov\.si)+", i):
                if re.compile("^.+\.(?:jpg | gif | png | bmp | tiff)($ | \n)", re.VERBOSE).search(i):
                    filename = os.path.basename(urlparse(i).path)
                    if database.check_image(filename, url, db_conn, lock) == -1:
                        try:
                            with urllib.request.urlopen(i) as response:
                                data = response.read(100000)  # 1 MB
                                extension = os.path.splitext(filename)[1][1:]
                                # content_type = response.info().get_content_type()
                                database.add_image(url, filename, extension, data, datetime.datetime.now(), db_conn,
                                                   lock)
                        except:
                            print("Napaka pri dodajanju slike v bazo")
                            pass
                    #else:
                        #print("Slika je ze v bazi!")
        for j in files:
            filename = os.path.basename(urlparse(j).path)
            if database.check_file(filename, url, db_conn, lock) == -1 and filename:
                with urllib.request.urlopen(j) as response:
                    data = response.read(100000)  # 1 MB
                    # content_type = response.info().get_content_type()
                    extension = os.path.splitext(filename)[1][1:]
                    # content_type = response.info().get_content_maintype()
                    database.add_page_data(url, extension.upper(), data, db_conn, lock)


    elif r.status_code == 404:
        print(r.status_code, url)
        driver.get(url)
        time.sleep(10)
        soup = BeautifulSoup(driver.page_source, 'html.parser')
        database.add_page(urlparse(url).netloc, url, "HTML", str(soup), r.status_code, datetime.datetime.now(),
                          db_conn, lock)
    db_conn.commit()
    db_conn.close()
    driver.quit()
    # print("koncujem task",url)
    active.value -= 1