# Marcel Ochsendorf 10.02.2018
# ProDevMo
# Imports collected dataset form the rep_server

# TOOL REQUIREMENTS :
# -> needs a json filelist {files:["file1.csv", "file2.csv"] to get all files to download
# -> each download url need a filename at the end with this format <xxx>_<yyy>_<LABEL>_<zzz>.csv
# -> the label name in the filename has to be matched with
# -> each csv file must have a header TIME, X, Y, Z (4 collums)



#TODO
# -> replace urls with var
# ->
import requests
import json
from collections import namedtuple
import os
import sys
import csv
from StringIO import StringIO
import shutil



downloaded_file_container = namedtuple("downloaded_file_container", "csv_content dl_path filename label len")


TRAIN_DATA_BASE_DIR = "./train_data"
if not os.path.exists(TRAIN_DATA_BASE_DIR):
    os.makedirs(TRAIN_DATA_BASE_DIR)
else:
    shutil.rmtree(TRAIN_DATA_BASE_DIR)


# TRASH DIR FOR UNCONVERTED FILES
INVALID_FILE_FOLDER = TRAIN_DATA_BASE_DIR+"/__trash__"
if not os.path.exists(INVALID_FILE_FOLDER):
    os.makedirs(INVALID_FILE_FOLDER)


#LABEL 1 NOTHING SOME IDLE DATA IF YOU HOLD THE PHONE IN YOUR HAND
LABEL_NOTHING = "idle"
LABEL_NOTHING_PATH = TRAIN_DATA_BASE_DIR+"/nothing/"
label_counter_nothing = 0
if not os.path.exists(LABEL_NOTHING_PATH):
    os.makedirs(LABEL_NOTHING_PATH)

#LABEL 2 PUSH SOME IDLE DATA IF YOU PUSH YOUR FORM AWAY FROM YOU
LABEL_PUSH = "push"
LABEL_PUSH_PATH = TRAIN_DATA_BASE_DIR+"/push/"
label_counter_push = 0
if not os.path.exists(LABEL_PUSH_PATH):
    os.makedirs(LABEL_PUSH_PATH)

#LABEL 3 PULL SOME IDLE DATA IF YOU PUSH YOUR FORM AWAY FROM YOU
LABEL_PULL = "pull"
LABEL_PULL_PATH = TRAIN_DATA_BASE_DIR+"/pull/"
label_counter_pull = 0
if not os.path.exists(LABEL_PULL_PATH):
    os.makedirs(LABEL_PULL_PATH)

#LABEL 4 PULL SOME IDLE DATA IF YOU PUSH YOUR FORM AWAY FROM YOU
LABEL_ROTATE_LEFT = "left"
LABEL_ROTATE_LEFT_PATH = TRAIN_DATA_BASE_DIR+"/rotate_left/"
label_counter_roate_left = 0
if not os.path.exists(LABEL_ROTATE_LEFT_PATH):
    os.makedirs(LABEL_ROTATE_LEFT_PATH)

#LABEL 5 PULL SOME IDLE DATA IF YOU PUSH YOUR FORM AWAY FROM YOU
LABEL_ROTATE_RIGHT = "right"
LABEL_ROTATE_RIGHT_PATH = TRAIN_DATA_BASE_DIR+"/rotate_right/"
label_counter_rotate_right = 0
if not os.path.exists(LABEL_ROTATE_RIGHT_PATH):
    os.makedirs(LABEL_ROTATE_RIGHT_PATH)


#DOWNLOAD FILE LIST FROM SERVER
response = requests.get("http://marcelochsendorf.com:3040/")
if(response.status_code != 200):
    print("get failed 0 ")
    exit()
requested_filelist = json.loads(response.text)



#print (requested_filelist['files'])



# DOWNLOAD ALL FILES AND CHECK IF LEN IS GOOD AND CHECK FOR LABEL
SENSOR_READ_TIME_INTERVAL = 0.1 #REFRESH RATE IN THE SENSOR IN SECONDS
MIN_CSV_LEN = 15 #ignores files under x entries
csv_found_min_len = sys.maxint # after going though all csv files,
squares = [] # DOWNLOADED CSV FILE LIST
for i in range(len(requested_filelist['files'])):
    #print(requested_filelist['files'])
    path__, filename_to_dl = os.path.split(requested_filelist['files'][i])
    response_csv = requests.get("http://marcelochsendorf.com:3040/save/" + filename_to_dl)
    if(response_csv.status_code != 200):
        print("get failed 1 ")
        exit()
    print(filename_to_dl)
    sys.stdout.flush()
    fn_basename, fn_timestamp,fn_action, fn_counter = filename_to_dl.split("_")
    #HERE CHECK FOR LABEL NAME IN FILE its for ignoring invaluf file labes you can specify on top of the file
    if(fn_action == LABEL_NOTHING):
        label_counter_nothing = label_counter_nothing + 1
    elif (fn_action == LABEL_PUSH):
        label_counter_push = label_counter_push + 1
    elif (fn_action == LABEL_PULL):
        label_counter_pull = label_counter_pull + 1
    elif (fn_action == LABEL_ROTATE_LEFT):
        label_counter_roate_left = label_counter_roate_left + 1
    elif (fn_action == LABEL_ROTATE_RIGHT):
        label_counter_rotate_right = label_counter_rotate_right + 1
    else:
        print("skip file - invalid label . " + filename_to_dl + "(" + fn_action + ")")
        continue


    row_len_csv = 0
    csv_content = csv.reader(StringIO(response_csv.text))
    for row in csv_content:
        row_len_csv += 1
    #appen if long enought
    # -1 to ignore the csv header
    if(row_len_csv-1 >= MIN_CSV_LEN):
        m = downloaded_file_container(csv_content=response_csv.text, dl_path="http://marcelochsendorf.com:3040/save/" + filename_to_dl, filename=filename_to_dl, label=fn_action, len = row_len_csv-1)
        squares.append(m)
        if(row_len_csv < csv_found_min_len):
            csv_found_min_len = row_len_csv
    else:
        print("ignored csv file")




print("MIN DATASET LEN : " + str(csv_found_min_len))
print("DL " + str(label_counter_nothing) + " nothing labeled files")
print("DL " + str(label_counter_pull) + " pull labeled files")
print("DL " + str(label_counter_push) + " push labeled files")
print("DL " + str(label_counter_roate_left) + " roate_left labeled files")
print("DL " + str(label_counter_rotate_right) + " roate_right labeled files")


#csv parsen
#länge bestimmen speichern
#timestamp adden
#und nach kategoriew suchen per keyword und dann in ornern einfügen
    # path = path/to/file
    # filename = foobar.txt

#SAVEIN ALL DOWNLOADED FILES TO THE RIGHT LOCATION
comvert_file_counter = 0
for files_to_convert in squares:
    #print(x.dl_path)
    time_step = 0.0
    filepath = INVALID_FILE_FOLDER
    #DETERMAN FILEPATH VIA LABEL
    if(files_to_convert.label == LABEL_NOTHING):
        filepath = LABEL_NOTHING_PATH
    elif (files_to_convert.label == LABEL_PUSH):
        filepath = LABEL_PUSH_PATH
    elif (files_to_convert.label == LABEL_PULL):
        filepath = LABEL_PULL_PATH
    elif (files_to_convert.label == LABEL_ROTATE_LEFT):
        filepath = LABEL_ROTATE_LEFT_PATH
    elif (files_to_convert.label == LABEL_ROTATE_RIGHT):
        filepath = LABEL_ROTATE_RIGHT_PATH
    else:
        print("skip file - invalid label to determan path ." + files_to_convert.label)
        continue

    #APPEND FILENAME if invalid chose a counting number
    if(files_to_convert.filename == ""):
        filepath =filepath + str(comvert_file_counter)
        comvert_file_counter = comvert_file_counter+1
    else:
        filepath =filepath+ files_to_convert.filename # append the original filename


    with open(filepath, 'w') as csvfile:
        fieldnames = ['TIME_STEP','ACCEL_X', 'ACCEL_Y', 'ACCEL_Z']
        writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
        writer.writeheader()

        csv_dl_content = csv.reader(StringIO(files_to_convert.csv_content))
        is_csv_head = True
        read_row_counter = 0
        for read_row in csv_dl_content:
            #SKIP CSV HEADER
            if(is_csv_head):
                is_csv_head = False
                continue
            #TODO adde nur csv_found_min_len elemente
            if(read_row_counter >= csv_found_min_len):
                break
            read_row_counter = read_row_counter + 1
            writer.writerow({'TIME_STEP': str(time_step), 'ACCEL_X': read_row[1],'ACCEL_Y': read_row[2],'ACCEL_Z': read_row[3]})
            time_step = time_step + SENSOR_READ_TIME_INTERVAL

i = 0