import manhattan as mh
import datetime as dt
import matplotlib.pyplot as plt
import numpy as np
import QuantLib as ql
import pytz
import pandas as pd
import time

est = pytz.timezone('America/New_York')
utc = pytz.utc

begining_day = dt.datetime(2017, 6, 26)
ending_day = dt.datetime.today()

def get_symbols():
    lines = open('sp.txt', 'r').readlines()
    symbols = []
    for line in lines[1:]:
        symbols.append(line.split(',')[0])

    return symbols


def generate_dataframe():
    american_calendar = ql.UnitedStates(ql.UnitedStates.NYSE)
    days = mh.trading_days_between(begining_day, ending_day, american_calendar)

    sp = pd.DataFrame()

    data = {}
    stocks = {}
    symbols = get_symbols()

    for s in symbols:
        data[s] = pd.DataFrame()
        stocks[s] = mh.find_product('SIAC', s)
        if stocks[s] is None:
            stocks[s] = mh.find_product('UTP', s)


    for d in days:
        print(d)

        for s in symbols:
            try:
                data[s] = data[s].append(mh.get_snapshot(d + dt.timedelta(hours=9, minutes=30), d + dt.timedelta(hours=15), stocks[s], freq=60))
            except:
                pass


    for s in symbols:
        data[s].to_pickle('stock_data/{}.pkl'.format(s))




if __name__ == '__main__':
    generate_dataframe()