'''This program scrapes craigslist. Still much to be done
to make more robust but this is a stable build on its own

Author: https://www.fiverr.com/n3tr1x
'''
print('Loading..')
import os
import sys
import traceback
import re
import random
import time
import datetime
import re
from sys import path as PATH, stdout as STDOUT, argv  # MARK


print("\t\t\t\t\t\t==== SCGL BOT v1.0 ====")
print("== CHECKING MODULES ==")


from re import sub as REPLACE  # [MARK]

from mysql.connector import connect as CONNECT
from pandas import read_sql as RSQL
from settings import *


from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.select import Select
from selenium.common.exceptions import *

driver = None  # MARK


if TYPE.upper() == 'FIREFOX':
    from selenium.webdriver import Firefox
    from selenium.webdriver.firefox.firefox_binary import FirefoxBinary
    binary = None
    binary_location = r'/usr/bin/firefox-esr'
    binary = FirefoxBinary(binary_location)
    driver = Firefox(firefox_binary=binary)


if TYPE.upper() == 'CHROME':
    from selenium.webdriver import Chrome
    path_to_cdriver = 'chromedriver'
    driver = Chrome(path_to_cdriver)


print('')
print('')


# WebWait object
web_wait_object = WebDriverWait(driver, 20)  # MARK


def wait_load():
    """Throws in random waits to not appear as a bot to
 craigslist anti-scraping software"""
    return time.sleep(random.randint(1, 3))


BLOCKED_STUB = "This IP has been automatically blocked."


def load_url(url, callabl, name='element', counts=5, **args):
    '''
  Loads a URL and or check for the callable using webwait
: Args:
   - url - The url to load or '' to just wait until element is found
   - callabl - The callable object that WebWait calls
   - name - The name to be printed as message
   - count - The amount of retries after an element is not found
'''
    try:
        if url != '':
            driver.get(url)
        while counts > 0:
            try:
                web_wait_object.until(callabl)
                return True
            except:
                cr_miscellany(**args)
                counts -= 1
                print("\n[DEBUG] ", name)
                print('ERROR URL: %s' % (url))
                print('Trying again..', counts, 'more tries left')
        return False
    except (RemoteDriverServerException, TimeoutException) as e:
        print('[Error]', e)
        exit()


def url_changed(url):
    '''
 Checks if URL of the selenium driver has changed
'''
    return (url not in driver.current_url)


def switch_to_email_tab(user):
    print('Searching for the email tab for user:', user)
    for handle in driver.window_handles:
        driver.switch_to.window(handle)
        if 'Workspace Webmail' in driver.title:
            try:
                if user in driver.find_element_by_xpath('//span[@id="username"]').text:
                    print('Found the tab for user:', user)
                    return True
                else:
                    continue
            except:
                return False
    return False


def switch_to_post_tab(link, func=None):
    print('Searching for the post tab for link: ', link)
    for handle in driver.window_handles:
        driver.switch_to.window(handle)
        if link in driver.current_url:
            try:
                func()
            except:
                pass
            break


LOGIN_CONTROLLER = dict()


def email_login(username='sales39@containerking.com', password='falcon00', name='I', ID_MSG='POST/EDIT/DELETE'):
    current_url = driver.current_url
    if ('craigslist.org' in current_url) and (not 'You should receive an email shortly' in driver.page_source):
        print("No verification mail needed. Returning.. :) ")
        return ''
    else:
        print("Email verification required.. ")

    if not (username in LOGIN_CONTROLLER.keys()):
        LOGIN_CONTROLLER[username] = False

    if not LOGIN_CONTROLLER[username]:
        print('You are not logged in with: ' + username)
        driver.get('https://sso.godaddy.com/login?app=email&realm=pass')
        print("\n==Please wait while " + name + " check your mail==")
        if not load_url('',
                        EC.presence_of_element_located((By.ID, 'password')),
                        'Cannot find password field'):
            print('An error occured, Please retry')

        uname = driver.find_element_by_id('username')
        uname.clear()
        uname.send_keys(username)
        pword = driver.find_element_by_id('password')
        pword.clear()
        pword.send_keys(password)
        driver.find_element_by_xpath('//*[@id="submitBtn"]').click()

    if not switch_to_email_tab(username):
        LOGIN_CONTROLLER[username] = False
        return email_login(username, password, name, ID_MSG)

    if not load_url('',
                    EC.presence_of_element_located(
                        (By.XPATH, '//*[@id="foldertree_INBOX"]//span')),
                    'Cannot find Inbox button'):
        print('An error occured, Please retry')

    switch_to_post_tab('post.craigslist', lambda: driver.close())
    switch_to_email_tab(username)
    LOGIN_CONTROLLER[username] = True
    elem = driver.find_element_by_xpath('//*[@id="foldertree_INBOX"]//span')
    if not elem.text in 'Inbox':
        print('Godaddy updated their code, Please update email login code')
    elem.click()
    if not load_url('',
                    EC.presence_of_element_located(
                        (By.XPATH, '//*[@id="tbody_mailindex"]')),
                    'Inbox display'):
        print('An error occured, Please retry')

    inbox = None
    messages = []
    while len(messages) <= 1:
        time.sleep(1)
        inbox = driver.find_element_by_id("tbody_mailindex")
        messages = inbox.find_elements_by_tag_name('tr')
    print("Looking for unread messages with the keyword: " + ID_MSG)
    print("\nThere are " + str(len(messages)) + " emails.")
    for message in messages:
        tmp_id = message.get_attribute('id')
        # print(tmp_id)
        temp = None
        try:
            temp = driver.find_element_by_xpath(
                '//tr[@id="' + tmp_id + '"]/td[3]/div/span')
        except Exception as e:
            # print(e)
            continue
        print(temp.text)
        if not (ID_MSG in temp.text):
            #print ("A tpe continue in email")
            continue
        if (not 'bold' in message.get_attribute('class')):
            #print ("B type")
            continue
        # print('here')
        temp.click()
        if not load_url('', EC.presence_of_element_located((By.XPATH, '//*[@id="wmMessage"]')), 'Inbox message check', counts=2):
            print('An error occured, Please retry')
            return email_login(username, password, name, ID_MSG)

        wm = driver.find_element_by_id('wmMessage')
        links = wm.find_elements_by_tag_name('a')
        for link in links:
            try:
                if 'post.craigslist' in link.get_attribute('href'):
                    href = link.get_attribute('href')
                    print('[INFO] Found verification link: ' + href)
                    link.click()
                    wait_load()
                    return href
            except TypeError as e:
                pass
            except:
                pass
    return False


def cr_miscellany(**args):
    # dynamically assign variables
    for a in args:
        exec("%s = '%s'" % (a, args[a]))
    try:
        del a, args  # clear variables assigned
    except:
        pass

    current_url = driver.current_url
    if current_url.endswith('pn') or current_url.endswith('tou'):
        if not load_url('', EC.presence_of_element_located((By.XPATH,
                                                            '//*[@class="previewButtons"]')),
                        'Accept Terms and Conditions', counts=1):
            # signify phone number verification needed
            raise Exception('Phone number verification stuff encountered')
        # Click the accept terms and conditions button
        try:
            driver.find_element_by_xpath(
                '//section[@class="body"]//section[@class="previewButtons"]//button[@type="submit"]').click()
        except:
            pass
        wait_load()

        # necessary in case you're logged out
        if not load_url('', EC.visibility_of_element_located((By.XPATH, '//*[@name="an"]')), 'Continue with credit card', counts=1):
            print('An error occured, Please retry')
            return
        try:
            driver.find_element_by_xpath(
                '//form[1]//input[@value="ccard"]').click()
            wait_load()
        except:
            pass
        try:
            driver.find_element_by_xpath(
                '//form[1]//button[@type="submit"]').click()
        except:
            pass

    # deal with confirmation after billing page
    current_url = driver.current_url
    if (('craigslist.org' in current_url) and ('You should receive an email shortly'
                                               in driver.page_source)) or current_url.endswith('mailoop'):
        print("Email verification required.. ")
        state = email_login(email, password, ID_MSG=title)
        if state == False:
            raise Exception('Error occured while verifying email')
        wait_load()
        try:
            for handle in driver.window_handles:
                driver.switch_to.window(handle)
                if state in current_url:
                    break
        except Exception as e:
            print(e)

    if BLOCKED_STUB in driver.page_source:
        print('Attempting to use pre-coded algorithm to correct IP block')

        driver.back()
        time.sleep(2)
        driver.forward()


def complete_post(url, infos, scrnsht, email, password, title):
    SUCCESSFUL_PURCHASE = False
    if len(infos) is 0:
        print('Empty credit card info sent')

    # attempt login incase login wasn't persisted
    craigslist_login(email, password)

    # click continue on billing page
    if not load_url('', EC.presence_of_element_located((By.NAME, 'go')), 'Continue with credit card button', counts=5, email=email, password=password, ID_MSG=title):
        print('An error occured, Please retry')
    driver.find_element_by_name('go').click()

    if not load_url('', EC.presence_of_element_located((By.XPATH, '//*[@id="ccinfo"]')), 'Credit card information'):
        print('An error occured, Please retry')
        driver.find_element_by_name('go').click()
    try:
        for info in infos:
            if info == 'pcard':
                continue
            if not load_url('', EC.presence_of_element_located((By.NAME, info)), 'Providing credit card information: ' + info):
                print('An error occured, Please retry')

            if (info in 'cardCountry'):
                ccntr = driver.find_elements_by_name(info)
                for ctr in ccntr:
                    if ctr.get_attribute('value') == 'US':
                        print('[INFO] Filling: ' + info + ": " + infos[info])
                        ctr.click()
                        print('[INFO] Filled: ' + info + ": " + infos[info])
                        break
                continue
            if (info in 'cardState'):
                cst = driver.find_element_by_name(info)
                st = infos[info]
                stag = Select(cst)
                stag.select_by_visible_text(infos[info])
                while True:
                    try:
                        print('[INFO] Filling: ' + info + ": " + infos[info])
                        stag.first_selected_option
                        print('[INFO] Filled: ' + info + ": " + infos[info])
                        break
                    except:
                        continue
                continue
            print('Filling: ' + info + ": " + str(infos[info]))
            driver.find_element_by_name(info).send_keys(infos[info])
            print('Filled: ' + info + ": " + str(infos[info]))
    except Exception as e:
        tb = sys.exc_info()[2]
        print(tb)
        return False
    purchase_button = driver.find_element_by_xpath(
        '//div[@id="presubmit"]/button')
    try:
        purchase_button.click()
        wait_load()
        if not load_url('', EC.presence_of_element_located((By.XPATH, '//div[@id="presubmit"]/button')), 'Using guess to determine if post was successful', counts=1):
            SUCCESSFUL_PURCHASE = True
            print(
                'Counting the post as success. Checks passed.. If this is wrong, please contact admin')
    except Exception as e:
        print(e)
        return False
    #driver.get_screenshot_as_file( str(time()) + '_SUCCESS_' + infos['cardLastName'])
    if SUCCESSFUL_PURCHASE:
        invoice_elem = driver.find_elements_by_xpath(
            '//table[@id="postingInvoice"]//td')
        for inv_elem in invoice_elem:
            inv_text = inv_elem.text
            m = re.search(POSTING_REGEX, inv_text)
            if m:
                posting_id = m.group(1)
                return posting_id
        print('[DEBUG] Posting ID not found.')
    else:
        return False


def breakloop():
    eval('break')


def is_logged_in(username):
    return (username == CRAIGSACC_CONTROLLER[0])


CRAIGSACC_CONTROLLER = ['']


def craigslist_login(username, password):
    try:
        if (not is_logged_in(username)):
            try:
                driver.find_element_by_partial_link_text('log in').click()
            except:
                driver.get('https://accounts.craigslist.org/login/home')
            if load_url('', EC.presence_of_element_located((By.ID,
                                                            'paginator')), '', counts=0):
                try:
                    driver.find_element_by_partial_link_text('log out').click()
                except:
                    pass
        else:
            print('You are already logged into current city')
            return

        driver.find_element_by_id('inputEmailHandle').send_keys(username)
        driver.find_element_by_id('inputPassword').send_keys(password)
        wait_load()
        driver.find_element_by_xpath(
            '//*[@class="accountform-actions"]/button').click()
        if not load_url('', EC.presence_of_element_located((By.ID,
                                                            'paginator')), '', counts=5):
            print('Login unsuccessful')
            raise Exception
        CRAIGSACC_CONTROLLER[0] = username
    except:
        print("Encountered an error while I was logging in, Proceeding with error..")
        traceback.print_exception(*sys.exc_info())


def getinput(prompt, typeCheck=False, obj=None, timeout=30.0):
    prompt = prompt + "\n\t\t/QUIT to cancel command prompt\n<> "
    retval = False
    try:
        while True:
            inp = input(prompt)
            if '/QUIT' not in inp.strip().upper():
                return retval
            if typeCheck:
                try:
                    retval = obj(inp)
                    break
                except:
                    continue
            else:
                retval = inp
                break
    finally:
        return retval


def prettify(inp):
    if inp is None:
        return ''
    inp = REPLACE(r'[\r]{1,}[\n]{1,}', '\n', inp)
    return inp


def craigslist_handler(url, typ, cat, market, subarea, title, body, randomize, postal, price, email, password, images, pn, sale_man, sale_mod, sale_size):
    # images='/root/Documents/slar/crawler/A.jpeg<==>/root/Documents/slar/crawler/B.jpeg<==>/root/Documents/slar/crawler/C.jpeg'
    craigslist_login(email, password)
    """Main workhorse of the script.  This function uses Selenium to automate
    posting in the supplied location and category. """
    if not load_url(url, EC.presence_of_element_located((By.ID, 'post')),
                    "I'm unable to find the link to post to classifieds\n", counts=5):
        print('An error occured, Please retry')

    # if not load_url('https://www.craigslist.org/about/sites',
    #		EC.presence_of_element_located((By.PARTIAL_LINK_TEXT, market)),
    #		 'location url'):
    #    print('An error occured, Please retry')
    #
    try:
        driver.find_element_by_partial_link_text('post to classifieds').click()
    except:
        try:
            driver.find_element_by_xpath(
                '//*[@class="page-container"]//div[@id="leftbar"]//ul[@id="postlks"]//li//a[@id="post"]').click()
        except:
            driver.find_element_by_id('post').click()
    wait_load()
    if not load_url('', EC.presence_of_element_located((By.CLASS_NAME, 'picker')),
                    "Unable to select type", counts=5):
        print('An error occured, Please retry')

    select_opts = driver.find_elements_by_xpath(
        '//form[@class="picker"]//ul[@class="selection-list"]//label')
    print("[INFO] Selecting a type")
    for opt in select_opts:
        if typ.strip().lower() in opt.find_element_by_class_name('right-side').text.lower():
            time.sleep(3)
            opt.find_element_by_tag_name('input').click()
            break
    wait_load()
    if driver.current_url.endswith('type'):
        time.sleep(5)
        if driver.current_url.endswith('type'):
            driver.find_element_by_name('go').click()
    print("[INFO] Selected type: " + typ)

    while True:
        if(driver.current_url.endswith('cat')):
            break
    print("[INFO] Selecting a category")
    select_opts = driver.find_elements_by_xpath(
        '//form[@class="picker"]//ul[@class="selection-list"]//label')
    for opt in select_opts:
        if cat.strip().lower() in opt.find_element_by_class_name('right-side').text.lower():
            wait_load()
            opt.find_element_by_tag_name('input').click()
            break
    wait_load()
    if driver.current_url.endswith('cat'):
        time.sleep(5)
        if driver.current_url.endswith('cat'):
            driver.find_element_by_name('go').click()
    print("[INFO] Selected category: " + cat.strip())

    MARKET = False
    msg = "Subarea is required"
    count = 2
    print(subarea)
    try:
        while True:
            if driver.current_url.endswith('edit'):
                break
            if not load_url('', EC.presence_of_element_located((By.CLASS_NAME, 'picker')),
                            msg, counts=count):
                break
            if(driver.current_url.endswith('subarea')) and (not MARKET):
                print('[INFO] Selecting a subarea')
                if not subarea:
                    elems = driver.find_elements_by_xpath(
                        '//form[@class="picker"]//ul[@class="selection-list"]//label//input')
                    print('Subarea is required but isn\'t provided')
                    print('Using pre-coded guess algorithm to pick one subarea')
                    elems[0].click()
                    MARKET = True
                    continue
                select_opts = driver.find_elements_by_xpath(
                    '//form[@class="picker"]//ul[@class="selection-list"]//label')
                for opt in select_opts:
                    if subarea.strip().lower() in opt.text.lower():
                        opt.find_element_by_tag_name('input').click()
                        MARKET = True
                        break
                if not MARKET:
                    elems = driver.find_elements_by_xpath(
                        '//form[@class="picker"]//ul[@class="selection-list"]//label//input')
                    if len(elems) > 0:
                        elems[0].click()
                msg = "Using pre-coded guess algorithm to check for any unexpected input from craigslist"
                count = 1
                print("[INFO] Selected subarea: " + subarea)
            else:
                if driver.current_url.endswith('edit'):
                    break
                if driver.current_url.endswith('subarea') and MARKET:
                    continue
                elems = driver.find_elements_by_xpath(
                    '//form[@class="picker"]//ul[@class="selection-list"]//label//input')
                if len(elems) > 0:
                    elems[0].click()
                else:
                    break
    except:
        pass

    wait_load()
    while True:
        if(driver.current_url.endswith('edit')):
            break
    print('[INFO] Filling post information')
    if not load_url('', EC.presence_of_element_located((By.ID, 'postingForm')),
                    "Unable to get the posting form", counts=5):
        print('An error occured, Please retry')

    driver.find_element_by_name('PostingTitle').send_keys(title)
    driver.find_element_by_name('Ask').send_keys(price)
    driver.find_element_by_name('postal').send_keys(postal)
    driver.find_element_by_name('GeographicArea').send_keys(market)
    driver.find_element_by_name('PostingBody').send_keys(prettify(body))
    driver.find_element_by_name('sale_manufacturer').send_keys(sale_man)
    driver.find_element_by_name('sale_model').send_keys(sale_mod)
    driver.find_element_by_name('sale_size').send_keys(sale_size)
    driver.find_element_by_name('wantamap').click()
    try:
        driver.find_element_by_name('FromEMail').send_keys(prettify(email))
        driver.find_element_by_name('ConfirmEMail').send_keys(prettify(email))
    except:
        pass
    driver.find_element_by_xpath(
        '//*[@id="contact_phone"]').send_keys(pn)
    time.sleep(2)
    print('[INFO] Sending post information')
    driver.find_element_by_name('go').click()
    wait_load()
    '''
    while True:
      if(driver.current_url.endswith('geoverify')):
        break
    wait_load()
    print('[INFO] Confirming post location')
    if not load_url('', EC.presence_of_element_located((By.ID, 'leafletForm')),
    	"Unable to get the posting form", counts=5):
    	print('An error occured, Please retry')

    driver.find_element_by_id('leafletForm').find_element_by_xpath('//button[@class="continue bigbutton"]').click()
    print("[INFO] Post location confirmed ")
    wait_load()
    '''
    try:
        if load_url('', EC.presence_of_element_located((By.ID, 'uploader')),
                    "Unable to get the posting images", counts=2):
            try:
                driver.find_element_by_id('classic').click()
            except:
                pass
            if not load_url('', EC.presence_of_element_located((By.XPATH, '//form[@class="add"]//input[@name="file"]')),
                            "Unable to get the posting form", counts=5):
                print('An error occured while processing upload, Please retry')
            if images:
                image_arr = images.split('<==>')
                if int(randomize) == 1:
                    random.shuffle(image_arr)
                for image in image_arr:
                    image = image.strip()
                    if not image.startswith('/'):
                        image = '/' + image
                    if not (os.path.exists(image)):
                        print('[DEBUG] Image: ' + image +
                              ' does not exist.. Proceeding')
                        continue
                    try:
                        inp = driver.find_element_by_xpath(
                            '//input[@type="file"]')
                        print('Uploading image: ' + image)
                        inp.clear()
                        inp.send_keys(image)
                        if not load_url(driver.current_url + '?s=editimage', EC.presence_of_element_located((By.XPATH, '//form[@class="add"]')), "Upload button..", counts=6):
                            print('An error occured while locating upload button')
                        driver.find_element_by_xpath(
                            '//form[@class="add"]').submit()
                        if not load_url('', EC.presence_of_element_located((By.XPATH, '//div[@class="imgwrap"]//img')), "Uploading image..", counts=6):
                            print(
                                'An error occured while processing upload, Please retry')
                        print('Image: ' + image + ' uploaded')
                    except StaleElementReferenceException:
                        print('Stale reference')
                    except Exception as e:
                        print('Error: ' + str(e) + ' occured while uploading.')
                        tb = sys.exc_info()[2]
                        print("[DEBUG] More info:\n\t\t " + tb)
        driver.find_element_by_xpath(
            '//button[@class="done bigbutton"]').click()
        wait_load()
    except:
        driver.find_element_by_xpath(
            '//button[@class="done bigbutton"]').click()
        wait_load()
        print('[DEBUG] Image(s) not uploaded. Proceeding that way')
        traceback.print_exception(*sys.exc_info())
    print('Publishing..')
    if not load_url('', EC.presence_of_element_located((By.NAME, 'go')), "Publish button", counts=5):
        print('An error occured while processing upload, Please retry')
    driver.find_element_by_name('go').click()
    print("\nI've published a draft to post the ad: " + title)
    print("Proceeding to verify the post.. Please be patient")


def publish_post(url, typ, cat, market, subarea, title, body, randomize, postal, price,
                 email='sales39@containerking.com', password='falcon00', info={}, scrnsht=str(time.time()) + 'screenshot.png',
                 images=False, pn='', sale_man='', sale_mod='', sale_size=''):
    # [TODO] handle_unfinished(info, scrnsht)
    try:
        status = False
        post_time = time.time()
        craigslist_handler(url, typ, cat, market, subarea, title, body, randomize,
                           postal, price, email, password, images, pn, sale_man,
                           sale_mod, sale_size)
        # [TODO] put_job(href + '[===]' + email + '[===]' + password + '[===]' + title)
        state = email_login(email, password)
        if state == False:
            completed_time = time.time()
            return status, (completed_time - post_time)
        wait_load()
        try:
            for handle in driver.window_handles:
                driver.switch_to.window(handle)
                if state in driver.current_url:
                    break
        except Exception as e:
            print(e)
        status = complete_post(state, info, scrnsht, email, password, title)
        completed_time = time.time()
        return status, (completed_time - post_time)
    except:
        traceback.print_exception(*sys.exc_info())
        return False, None


def cleanup_tabs():
    for handle in driver.window_handles:
        driver.switch_to.window(handle)
        if 'post.craigslist' in driver.current_url:
            print('[INFO] Closing: ' + driver.title)
            driver.close()


def close_tabs():
    for handle in driver.window_handles:
        driver.switch_to.window(handle)
        driver.close()


# all post objects go in here
POSTS_CONTROLLER = {}


def check_market(post_title, post_id, market_url, city, limit=5):
    if limit == 0:
        limit = 5
    print('[INFO] Checking market if ' + str(post_title) +
          " (" + str(post_id) + ') post is in ' + city + ' top ' + str(limit))
    if not load_url(market_url, EC.presence_of_element_located((By.ID, 'sortable-results')), "Markets result info", counts=5):
        print('An error occured while checking the market: ' +
              market_url + ', Please retry')
    total_res = len(driver.find_elements_by_xpath(
        '//div[@id="sortable-results"]/ul[@class="rows"]/li[@class="result-row"]/p[@class="result-info"]'))

    print('[INFO] I can see ' + str(total_res) +
          ' ads on ' + city + '\'s page')

    XP = '//div[@id="sortable-results"]/ul[@class="rows"]/li[@class="result-row"][position() < ' + \
        str(limit + 1) + ']'
    top_posts = driver.find_elements_by_xpath(XP)
    print('[INFO] Looking for the post: ' + str(post_title) +
          " (" + str(post_id) + ') in ' + city)
    for i in range(0, len(top_posts)):
        if i >= limit:
            break
        top_post = top_posts[i].get_attribute('data-pid')
        if top_post in str(post_id):
            print(post_id + ' in top: ' + str(limit))
            return True
        print (top_post)
    return False


SERVER_CONNECTOR = [None]


def update_posts_db(city, state, cat, time, pid, posting_id, ad_id):
    try:
        if not SERVER_CONNECTOR[0]:
            print('[DEBUG] Cannot find the server connector object')
        cur = SERVER_CONNECTOR[0].cursor()
        cur.execute("INSERT INTO CraigslistPoster.posts(`client`, `postingID`, `post_id`, `ad`, `market_city`, `market_state`, `category`, `POST_TIME`) VALUES(%s, %s, %s, %s, %s, %s, %s, %s)",
                    (current_client[0], str(posting_id), str(pid), str(ad_id), str(city), str(state), str(cat), time))
        #print('Post db updated %s'%(str(postingID)))
        SERVER_CONNECTOR[0].commit()
    except:
        traceback.print_exception(*sys.exc_info())


def handle_dump(pdump):
    return publish_post(url=pdump['ad_url'], typ=pdump['typ'],
                        cat=pdump['cat'], market=pdump['market'], subarea=pdump['subarea'], title=pdump['title'],
                        body=pdump['body'], randomize=pdump['random'],
                        postal=pdump['postal'], price=pdump['price'], email=pdump['email'],
                        password=pdump['password'], images=pdump['images'], info=pdump['info'],
                        pn=pdump['pn'], sale_man=pdump['sale_man'], sale_mod=pdump['sale_mod'],
                        sale_size=pdump['sale_size'])


def check_posts_db(pid, city, delay):
    try:
        if not SERVER_CONNECTOR[0]:
            print('[DEBUG] Cannot find the server connector object')
        cur = SERVER_CONNECTOR[0].cursor(buffered=True)
        cur.execute(
            "SELECT * FROM `posts` WHERE `market_city` LIKE '%%%s%%'" % (city))
        NON_EMPTY = False
        for x in cur:
            NON_EMPTY = True
        if not NON_EMPTY:
            print('Not yet registered.. Proceeding')
            return True
        cur.execute(
            "SELECT POST_TIME, postingID FROM CraigslistPoster.posts WHERE `post_id` = %s ORDER BY `POST_TIME` DESC LIMIT 1", (str(pid),))
        for then, pid in cur:
            now = datetime.datetime.now()
            if ((now - then).total_seconds()) > delay:
                return pid
            else:
                print('%d minutes left..' %
                      ((delay / 60) - ((now - then).total_seconds() / 60)))
                return False
    except:
        traceback.print_exception(*sys.exc_info())


# Unsuccessful ads
US_ADS = []

# Skipped ads
SK_ADS = []

# Successful ads
SU_ADS = []

# Incomplete ads
INC_ADS = []


AD_IN_PROGRESS = [False]


def handle_market(ads):
    status = None
    t_o_c = None
    murl = ads['m_url']
    mlim = ads['m_lim']
    delay = ads['del'] * 60
    city = ads['m_cit']
    state = ads['m_sta']
    post_id = ads['pid']
    ad_id = ads['ad_id']
    INC_ADS.append("City: %s; PID: %s" % (city, str(post_id)))
    try:
        postingID = check_posts_db(post_id, city, delay)
        if postingID:
            if (not check_market(ads['title'], postingID, murl, city, mlim)) or (postingID is True):
                while AD_IN_PROGRESS[0]:
                    print('[INFO] An ad in progress.. Queueing')
                    time.sleep(5)
                AD_IN_PROGRESS[0] = True
                status, t_o_c = handle_dump(ads)
            else:
                print('[INFO] %s\'s not ready. Running to the next city :)' %
                      (city))
                INC_ADS.pop()
                SK_ADS.append("City: %s; PID: %s" % (city, str(post_id)))
                return
        else:
            print('[INFO] %s\'s not ready. Running to the next city :)' %
                  (city))
            INC_ADS.pop()
            SK_ADS.append("City: %s; PID: %s" % (city, str(post_id)))
            return
        if not status:
            print('[ERROR] An error occured while posting to: ' + city)
            AD_IN_PROGRESS[0] = False
            INC_ADS.pop()
            US_ADS.append("City: %s; PID: %s" % (city, str(post_id)))
            return
        else:
            # status here is posting_id [TODO] Clean up later
            update_posts_db(
                city, state, ads['cat'], datetime.datetime.now(), post_id, status, ad_id)
            AD_IN_PROGRESS[0] = False
    except:
        traceback.print_exception(*sys.exc_info())
        INC_ADS.pop()
        US_ADS.append("City: %s; PID: %s" % (city, str(post_id)))
        return
    print('Ad posted to: {0} in: {1}'.format(city, t_o_c))
    INC_ADS.pop()
    SU_ADS.append("City: %s; PID: %s" % (city, str(post_id)))


def manage_posts(ads):
    print('%d ads/market recorded for: %s' % (len(ads), current_client[0]))
    if (len(ads) == 0) or (not authorize_post()):
        print("[INFO] Posts not authorized! Moving on..")
        return
    for i in range(0, len(ads)):
        print('======================= ad: %d ========================' %
              (i + 1), end="\n\n")
        try:
            handle_market(ads[i])
            print("Leaving " + str(ads[i]['m_cit']) +
                  " Delay time: " + str(ads[i]['del']) + "mins")
        except:
            pass


def authorize_post():
    if ('Prepaid Ads'.lower() in user_data[0]['paymentPlan'][0].lower()) and (user_data[0]['purchasedAds'][0] <= 0):
        return False
    return True


def decrease_posts_left():
    ads_left = user_data[0]['purchasedAds'][0]
    ads_left_temp = ads_left - 1
    try:
        if not SERVER_CONNECTOR[0]:
            print('[DEBUG] Cannot find the server connector object')
        cur = SERVER_CONNECTOR[0].cursor()
        cur.execute("UPDATE users SET purchasedAds = %d WHERE client = %s",
                    (ads_left_temp, current_client[0]))
        print('User db updated %s:%d'%(current_client[0], ads_left_temp))
        SERVER_CONNECTOR[0].commit()
        ads_left = ads_left_temp
    except:
        print('[ERROR] Unable to update purchased Ads')
        traceback.print_exception(*sys.exc_info())


def read_data_from_sql_db(connection, sql):
    df = RSQL(con=connection, sql=sql)
    dataframe_dict = df.to_dict()
    return dataframe_dict


def prettify(inp):
    if inp is None:
        return ''
    inp = inp.strip()
    inp = inp.replace('\r', '')
    return inp


def handle_ads_data(dic):
    total_ads = len(dic['search_url'].values())
    if total_ads < 0:
        return False
    ad_dict = {'typ': ['type'], 'cat': ['category'], 'market': ['tags'], 'subarea': ['market_area'], 'title': ['title'],
               'body': ['description'], 'password': ['password'], 'price': ['price'], 'postal': ['postalcode'],
               'email': ['email'], 'images': ['images'], 'sale_man': ['tag1'], 'sale_mod': ['tag2'],
               'sale_size': ['tag3'], 'pid': ['market', int], 'del': ['delay', float], 'm_url': ['search_url'],
               'm_lim': ['post_above', int], 'm_cit': ['market_city'], 'm_sta': ['market_state'],
               'random': ['randomize_images', int], 'ad_url': ['post_url'], 'pn': ['phone_no'], 'ad_id': ['id', int]}
    return parse_db_item(dic, total_ads, ad_dict)


def handle_cc_data(dic, ads):
    total_ads = len(dic['number'].values())
    if total_ads < 0:
        return False
    ad_dict = {'cardNumber': ['number', int], 'cvNumber': ['cvv', int], 'expMonth': ['exp_month', int],
               'expYear': ['exp_year', int], 'cardFirstName': ['firstname'], 'cardLastName': ['lastname'],
               'cardAddress': ['address'], 'cardCity': ['city'], 'cardState': ['state'], 'cardPostal': ['postalcode', int],
               'contactPhone': ['phone'], 'pcard': ['primarycard', bool]}
    ret = parse_db_item(dic, total_ads, ad_dict)
    for i in range(0, len(ret)):
        if ret[i]['pcard'] == False:
            continue
        ret[i]['cardCountry'] = "US"
        ret[i]['contactName'] = ret[i]['cardFirstName'] + \
            " " + ret[i]['cardLastName']
        cardinfo = ret[i]
        for i in range(0, len(ads)):
            ads[i]["info"] = cardinfo
        break
    return ads


def parse_db_item(dic, total_ads, parse_dict):
    ret = list()
    for i in range(0, total_ads):
        retval = dict()
        for item in parse_dict:
            parse_val = parse_dict[item]
            item_idx = parse_val[0]
            item_val = None
            try:
                item_val = dic[item_idx][i]
            except:
                pass
            if item_val is None:
                if len(parse_val) > 1:
                    if parse_val[1] is int:
                        item_val = 0
                    if parse_val[1] is float:
                        item_val = 0.0
                else:
                    item_val = ''
            if isinstance(item_val, str):
                item_val = item_val.strip()
            if len(parse_val) > 1:
                try:
                    parse_val[1](item_val)
                    item_val = parse_val[1](item_val)
                except:
                    pass
            retval[item] = item_val
        ret.append(retval)
    return ret


# provide info about unsuccessful, successful and incomplete ads
def account():
    print('UNSUCCESSFUL ADS: %d.' % (len(US_ADS)))
    for i in range(0, len(US_ADS)):
                # print("\t\t%s"%(US_ADS[i]))
        pass

    print('INCOMPLETE ADS: %d.' % (len(INC_ADS)))
    for i in range(0, len(INC_ADS)):
        # print("\t\t%s"%(INC_ADS[i]))
        pass

    print('SKIPPED ADS: %d.' % (len(SK_ADS)))
    for i in range(0, len(SK_ADS)):
        # print("\t\t%s"%(SK_ADS[i]))
        pass

    print('SUCCESSFUL ADS: %d.' % (len(SU_ADS)))
    for i in range(0, len(SU_ADS)):
        # print("\t\t%s"%(SU_ADS[i]))
        pass


def initAll():
    global US_ADS, SK_ADS, SU_ADS, INC_ADS
    # Unsuccessful ads
    US_ADS = []

    # Skipped ads
    SK_ADS = []

    # Successful ads
    SU_ADS = []

    # Incomplete ads
    INC_ADS = []


'''
Get the arguments from the shell
'''


def get_args():
    parser = argparse.ArgumentParser(
        description='Run craigslist till they give up')
    group = parser.add_argument_group('The following are required')
    group.add_argument('--client', '-c', metavar="CLIENT", type=str, nargs=1,
                       help="specify the clients name\n", required=True)
    args = parser.parse_args()
    return args


import argparse
current_client = ['']
user_data = [{}]

def main():
    initAll()
    print('')
    print('[INFO] Trying to reach the server')
    conn = CONNECT(user=DB_USER, password=DB_PASS,
                   database=DB_NAME, port=DB_PORT, host=DB_HOST)
    print('Server reached..')
    print('')
    SERVER_CONNECTOR[0] = conn
    current_client[0] = get_args().client[0]
    container_data = read_data_from_sql_db(
        conn, sql="SELECT ads.id, ads.randomize_images, ads.market, ads.client, ads.type, ads.category, ads.tags, ads.tag1, ads.tag2, ads.tag3, ads.title, ads.description, ads.price, ads.postalcode, ads.email, ads.password, ads.images, markets.market_city, markets.market_state, markets.phone_no, markets.delay, markets.post_above, markets.post_url, markets.search_url, markets.market_area FROM ads INNER JOIN markets ON ads.market = markets.id WHERE ads.active = '1' and ads.client = '%s'" % (current_client[0]))
    payments_data = read_data_from_sql_db(
        conn, sql="SELECT * FROM CraigslistPoster.payments WHERE client = '%s'" % (current_client[0]))
    user_data[0] = read_data_from_sql_db(conn, sql="SELECT * FROM users WHERE client = '%s'" % (current_client[0]))
    ads = handle_ads_data(container_data)
    ads = handle_cc_data(payments_data, ads)
    manage_posts(ads)
    account()


if __name__ == "__main__":
    try:
        main()
        driver.quit()
    except:
        driver.quit()

    # email_login()