# -*- coding: utf-8 -*-
import scrapy
import sys
from scrapy.selector import HtmlXPathSelector
from bs4 import BeautifulSoup

if sys.version_info[0] == 3:
    from urllib.request import urlopen
else:
    # Not Python 3 - today, it is most likely to be Python 2
    # But note that this might need an update when Python 4
    # might be around one day
    from urllib import urlopen

import csv


def extract_texts_between_tags(
        response,
        html_tag):
    """
    Extracts texts between html tag
    :param response: response from scrapy
    :param html_tag: html tag to extract text from
    :output: list of strings ?
    """
    xpath = "//" + html_tag + "/text()"
    texts = response.xpath(xpath).extract()
    texts_to_return = []
    for text in texts:
        # strip text
        text = text.strip()
        if text:
            texts_to_return.append(text)
    return texts_to_return


class WysdomCrawlerSpider(scrapy.Spider):
    name = 'wysdom_crawler'
    # TODO: we want this to be configurable
    MAX_DEPTH = 2
    file_name = "test_file_name"
    csv_headers = ['Tag Phrase', 'Referral URL', 'Link out URL']

    def __init__(self, category=None, *args, **kwargs):
        super(WysdomCrawlerSpider, self).__init__(*args, **kwargs)
        self.is_csv = False
        if self.is_csv:
            self.create_csv_file(csv_headers)
            # TODO add more tags
            self.tags = ['p', 'h1', 'h2', 'h3', 'h4', 'th', 'li']
            self.delimiter = ","
        else:
            self.tags = ['p', 'h1', 'h2', 'div', 'li']
            self.delimiter = " "

        self.start_urls = ['https://www.td.com/ca/en/personal-banking/how-to/']

    def append_text_to_file(self, file_name, parsed_text):
        """
        Appends text to file
        :param file_name: string name of .txt file
        :param parse_text:
        """
        with open('file_name' + '.txt', 'a') as f:
            f.write(parsed_text.encode("utf-8"))

    def create_csv_file(self, csv_headers):
        """
        Create new CSV file with headers in init
        :param headers: list of strings for headers
        """
        with open(self.file_name + '.csv', 'wb') as csvfile:
            filewriter = csv.writer(csvfile,
                                    delimiter=',',
                                    quotechar='|',
                                    quoting=csv.QUOTE_MINIMAL)
            filewriter.writerow(csv_headers)



    def append_text_to_csv(self, add_row):
        """
        Appends text to csv file as spider crawls
        :param add_row: list of row with phrase, referral url and linkout url
        """
        with open(self.file_name + '.csv', 'wb') as csvfile:
            filewriter.writerow(add_row)



    def get_response_links(self, response):
        """
        Grabs all links from HTML
        :param response: response from scrapy
        """
        result = []
        for link in response.xpath('//a[@href]/@href').extract():
            link = link.strip()
            if link == "":
                continue
            if not (link.startswith("http://") or link.startswith("https://")):
                continue
            result.append(link)
        return result


    def start_requests(self):
        """
        Generates initial request for spider
        """
        headers = {
            'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:48.0) Gecko/20100101 Firefox/48.0'}
        for url in self.start_urls:
            yield scrapy.Request(url, headers=headers, callback=self.parse)


    def parse(self, response):
        """
        Recursively crawls 3 layers deep by handling requests for URLs
        :param response: response from scrapy
        """
        #time.sleep(5)

        # {"p": ["<p>sup</p>", "<p>hi</p>"]}
        parsed_text = {}
        for tag in self.tags:
            extracted_text = extract_texts_between_tags(response, tag)
            parsed_text[tag] = extracted_text
            print extracted_text

            if self.is_csv:
                # phrase, referral url, linkout url
                add_row = [extracted_text , scrapy.Response.url, scrapy.Response.request.url]
                self.append_text_to_csv(self.file_name, extracted_text, tag_headers)
            else:
                self.append_text_to_file(self.file_name, self.delimiter.join(extracted_text))

        print parsed_text
        depth = response.meta["depth"]
        print depth
        if depth < self.MAX_DEPTH:
            for link in self.get_response_links(response):
                yield scrapy.Request(link, callback=self.parse)