from bs4 import BeautifulSoup
import re
import time
import collections

# ---- Definitions ----#
#Amount of documents
amount_of_documents = 1000

#Directory of raw HTML documents
directory_of_raw_documents = "raw_documents/"

#Directory of parsed documents
directory_of_parsed_documents = "parsed_documents/"

#Punctuation list to remove
punctuation_list = ['\\','/',':',',','...','(',')','.','?','!','\"',';','\'','’']

#File name list to remove
file_name_remove_list = ['_','-',' '] + punctuation_list

#File name to map document ids to document titles
mapping_file_name = "document_id_mapping.txt"

#File to find the index of file names and urls
index_of_file_names_and_urls = open('index-task1.txt', 'r')

#List of headers to ignore
ignore_list = ['See also']
# ---- Code ----#

#grabs document names from index-task1
def grab_document_names_from_index():
    position = 0
    document_dict = collections.OrderedDict()
    for lines in index_of_file_names_and_urls:
        position = position + 1
        file_url = lines.split(str(position) + ".txt - https://en.wikipedia.org/wiki/")
        file_url = file_url[1]
        file_url = file_url.split()
        file_url = file_url[0]
        file_url = ''.join(c for c in file_url if c not in file_name_remove_list) + str(position) + ".txt"
        document_dict[file_url] = position
    return document_dict




def open_document():
    list_of_document_titles = []
    document_index = collections.OrderedDict()
    document_index = grab_document_names_from_index()
    position = 0
    for entry in document_index:
        position = position+1
        with open(directory_of_raw_documents + str(document_index[entry]) + ".txt", "r", encoding='utf-8') as document:
            html = document.read()
            soup = BeautifulSoup(html, "html.parser")
            body = soup.find('div', id='bodyContent')

            print("Processing document " + "(" + str(position) + "/" + str(amount_of_documents) + ")")

            #Remove tables
            for table in body.find_all('table'): table.clear()

            #Remove content table
            for div in body.find_all('div', {'id' : 'toc'}): div.decompose()

            #Remove hatnote
            for class_item in body.find_all('div',{'class':'hatnote'}): class_item.decompose()

            #Remove images
            for img in body.find_all('img'): img.decompose()

            #Remove references
            for class_item in body.find_all('ol', {'class':'references'}): class_item.decompose()

            #Remove thumbs
            for div in body.find_all('div', {'class':'thumb'}): div.decompose()

            #Remove navigations
            for div in body.find_all('div', {'class': 'navbox'}): div.decompose()

            #Remove references by other names
            for div in body.find_all('div', {'class': 'reflist'}): div.decompose()

            #Remove reference heading
            for span in body.find_all('span',{'id': 'References'}): span.decompose()

            #Remove hidden div
            for div in body.find_all('div',{'id': 'mw-hidden-catlinks'}): div.decompose()

            #Remove site hub
            for div in body.find_all('div',{'id': 'siteSub'}): div.decompose()

            #Remove footer
            for div in body.find_all('div', {'class': 'printfooter'}): div.decompose()

            #Remove jump-to-nav
            for div in body.find_all('div', {'id': 'jump-to-nav'}): div.decompose()

            #Remove bottom navigation
            for div in body.find_all('div', {'role':'navigation'}): div.decompose()

            #Remove normal catlinks
            for div in body.find_all('div', {'id': 'mw-normal-catlinks'}): div.decompose()

            # remove See_Also
            previous_h2 = ""
            try:
                for element in body.find_all(['h2', 'ul']):
                    if (element.name == 'h2'):
                        previous_h2 = element('span')[0].get_text()
                        if(previous_h2 in ignore_list):
                            print(element.find_previous_sibling('ul'))

            except(AttributeError, TypeError) as e:  # error where there is no previous h2
                continue
            #Grab content
            results = body.get_text()
            results = results.replace('\n',' ')
            results = results.replace('—', ' ')
            results = results.replace('–',' ')
            results = ' '.join(results.split())
            results = re.sub('\[(.*?)\]','',results, flags=re.DOTALL) #removes [edit] and any citations
            results = remove_punctuation(results) #removes punctuation
            results = results.lower() #makes results lower

            #Save content to file
            print("Saved to " + entry)
            #saves the document
            output_file = open(directory_of_parsed_documents + entry, 'w', encoding="utf-8")
            output_file.write(results)
            output_file.close()

#removes punctuation from provided string
#but preserves punctuation between digits (I am allowing ALL punctuation)
def remove_punctuation(string):
    results = ""
    previous = ""
    next = ""
    for i, c in enumerate(string):
        if (i > 0):
            previous = string[i - 1]
        if (i + 1 != len(string)):
            next = string[i + 1]
        if (not previous.isdigit() or not next.isdigit()):  # if the previous and the next one are not digits
            if(previous.isdigit() and c == "/"):
                results +=c
            if (next.isdigit() and c == "/"):
                results += c
            if (c not in punctuation_list):
                results += c
        else:
            results += c
    return results

open_document()