from bs4 import BeautifulSoup
import re
import time
import collections

# ---- Definitions ----#
#Amount of documents
amount_of_documents = 1000

#Directory of raw HTML documents
directory_of_raw_documents = "raw_documents/"

#Directory of parsed documents
directory_of_parsed_documents = "parsed_documents/"

#List of h2s to ignore
ignore_list = ['See_also','Notes','References','Related_journals','Notes_and_references','External_links','Further_reading','Sources','Footnotes','Related_pages','Quotations','Other_websites']

#Punctuation list to remove
punctuation_list = ['\'',':',',','...','(',')','.','?','!','"',';']

#File name list to remove
file_name_remove_list = ['_','-',' '] + punctuation_list

#File name to map document ids to document titles
mapping_file_name = "document_id_mapping.txt"

#File to find the index of file names and urls
index_of_file_names_and_urls = open('index-task1.txt', 'r')
# ---- Code ----#

#grabs document names from index-task1
def grab_document_names_from_index():
    position = 0
    document_dict = collections.OrderedDict()
    for lines in index_of_file_names_and_urls:
        position = position + 1
        file_url = lines.split(str(position) + " - https://en.wikipedia.org/wiki/")
        file_url = file_url[1]
        file_url = file_url.split()
        file_url = file_url[0]
        file_url = ''.join(c for c in file_url if c not in file_name_remove_list) + ".txt"
        document_dict[file_url] = position
    return document_dict




def open_document():
    list_of_document_titles = []
    document_index = collections.OrderedDict()
    document_index = grab_document_names_from_index()
    position = 0
    for entry in document_index:
        position = position+1
        with open(directory_of_raw_documents + str(document_index[entry]), "r") as document:
            html = document.read()
            soup = BeautifulSoup(html, "html.parser")
            body = soup.find('div', id='bodyContent')

            print("Processing document " + "(" + str(position) + "/" + str(amount_of_documents) + ")")

            #Clear tables
            for table in body.find_all('table'): table.clear()

            #clear content table
            for div in body.find_all('div', {'id' : 'toc'}): div.decompose()

            #Grab content
            results = ""
            for item in body.find_all(['h2','h3','p','li']):
                try:
                    if item.name != 'h2': #We are not in the h2, therefore, lets find the id of the previous h2
                        parent_h2_id = item.find_previous_sibling('h2').find('span')['id']
                    else:
                        parent_h2_id = item.find('span')['id']#If we are in the h2, get the ID.
                except (AttributeError, TypeError) as e: #error where there is no previous h2
                    parent_h2_id = None
                if parent_h2_id and parent_h2_id in ignore_list:
                    continue
                else:
                    results += item.get_text() + "\n"
                    #results = results.replace("[edit]", "")
            results = results.encode('utf-8') #encodes it to stop weird errors
            results = re.sub('\[(.*?)\]','',results, flags=re.DOTALL) #removes [edit] and any citations
            results = remove_punctuation(results) #removes punctuation
            results = results.lower() #makes results lower


            print("Saved to " + entry)
            #saves the document
            output_file = open(directory_of_parsed_documents + entry, 'w')
            output_file.write(results)
            output_file.close()

#removes punctuation from provided string
#but preserves punctuation between digits (I am allowing ALL punctuation)
def remove_punctuation(string):
    results = ""
    previous = ""
    next = ""
    for i, c in enumerate(string):
        if (i > 0):
            previous = string[i - 1]
        if (i + 1 != len(string)):
            next = string[i + 1]
        if (not previous.isdigit() or not next.isdigit()):  # if the previous and the next one are not digits
            if (c not in punctuation_list):
                results += c
        else:
            results += c
    return results

open_document()