# -*- coding: utf-8 -*-

from bs4 import BeautifulSoup
import re
import time
import collections

# ---- Definitions ----#
#Amount of documents
amount_of_documents = 1000

#Directory of raw HTML documents
directory_of_raw_documents = "raw_documents/"

#Directory of parsed documents
directory_of_parsed_documents = "parsed_documents/"

#List of h2s to ignore
ignore_list = ['Notes','References','Related_journals','Notes_and_references','External_links','Further_reading','Sources','Footnotes','Quotations','Other_websites']

#Punctuation list to remove
punctuation_list = ['\\','/',':',',','...','(',')','.','?','!','\"',';','\'']

#File name list to remove
file_name_remove_list = ['_','-',' '] + punctuation_list

#File name to map document ids to document titles
mapping_file_name = "document_id_mapping.txt"

#File to find the index of file names and urls
index_of_file_names_and_urls = open('index-task1.txt', 'r')
# ---- Code ----#

#grabs document names from index-task1
def grab_document_names_from_index():
    position = 0
    document_dict = collections.OrderedDict()
    for lines in index_of_file_names_and_urls:
        position = position + 1
        file_url = lines.split(str(position) + " - https://en.wikipedia.org/wiki/")
        file_url = file_url[1]
        file_url = file_url.split()
        file_url = file_url[0]
        file_url = ''.join(c for c in file_url if c not in file_name_remove_list) + ".txt"
        document_dict[file_url] = position
    return document_dict




def open_document():
    list_of_document_titles = []
    document_index = collections.OrderedDict()
    document_index = grab_document_names_from_index()
    position = 0
    for entry in document_index:
        position = position+1
        with open(directory_of_raw_documents + str(document_index[entry]), "r") as document:
            html = document.read()
            soup = BeautifulSoup(html, "html.parser")
            body = soup.find('div', id='bodyContent')

            print("Processing document " + "(" + str(position) + "/" + str(amount_of_documents) + ")")

            #Clear tables
            for table in body.find_all('table'): table.clear()

            #Clear content table
            for div in body.find_all('div', {'id' : 'toc'}): div.decompose()

            #Clear hatnote
            for class_item in body.find_all('div',{'class':'hatnote'}): class_item.decompose()

            #Clear images
            for img in body.find_all('img'): img.decompose()

            #Clear references
            for class_item in body.find_all('ol', {'class':'references'}): class_item.decompose()

            #Clear thumbs
            for div in body.find_all('div', {'class':'thumb'}): div.decompose()

            #Clear navigations
            for div in body.find_all('div', {'class': 'navbox'}): div.decompose()

            #Clear references by other names
            for div in body.find_all('div', {'class': 'reflist'}): div.decompose()

            #Clear reference heading
            for span in body.find_all('span',{'id': 'References'}): span.decompose()

            #Remove hidden div
            for div in body.find_all('div',{'id': 'mw-hidden-catlinks'}): div.decompose()

            #remove site hub
            for div in body.find_all('div',{'id': 'siteSub'}): div.decompose()

            #remove footer
            for div in body.find_all('div', {'class': 'printfooter'}): div.decompose()

            #remove jump-to-nav
            for div in body.find_all('div', {'id': 'jump-to-nav'}): div.decompose()

            #remove bottom navigation
            for div in body.find_all('div', {'role':'navigation'}): div.decompose()

            #Grab content
            results = ""
            #for item in body.find_all(['h2','h3','p','li']):
            #for item in body.find_all(['h2', 'h3', 'p', 'li']):
                #results += item.get_text() + " "
                #results = results.replace("[edit]", "")
            results = body.get_text()
            #results = results.encode('utf-8') #encodes it to stop weird errors
            results = results.replace('\n', ' ')
            results = ' '.join(results.split())
            results = re.sub('\[(.*?)\]','',results, flags=re.DOTALL) #removes [edit] and any citations
            results = remove_punctuation(results) #removes punctuation
            results = results.lower() #makes results lower


            print("Saved to " + entry)
            #saves the document
            output_file = open(directory_of_parsed_documents + entry, 'w')
            output_file.write(results)
            output_file.close()

#removes punctuation from provided string
#but preserves punctuation between digits (I am allowing ALL punctuation)
def remove_punctuation(string):
    results = ""
    previous = ""
    next = ""
    for i, c in enumerate(string):
        if (i > 0):
            previous = string[i - 1]
        if (i + 1 != len(string)):
            next = string[i + 1]
        if (not previous.isdigit() or not next.isdigit()):  # if the previous and the next one are not digits
            if (c not in punctuation_list):
                results += c
        else:
            results += c
    return results

#open_document()
string = 'major role — international and — international long-lived—many users’ consumer’s'
print(remove_punctuation(string))