Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- # Marcel Ochsendorf 10.02.2018
- # ProDevMo
- # Imports collected dataset form the rep_server
- # TOOL REQUIREMENTS :
- # -> needs a json filelist {files:["file1.csv", "file2.csv"] to get all files to download
- # -> each download url need a filename at the end with this format <xxx>_<yyy>_<LABEL>_<zzz>.csv
- # -> the label name in the filename has to be matched with
- # -> each csv file must have a header TIME, X, Y, Z (4 collums)
- #TODO
- # -> replace urls with var
- # ->
- import requests
- import json
- from collections import namedtuple
- import os
- import sys
- import csv
- from StringIO import StringIO
- import shutil
- downloaded_file_container = namedtuple("downloaded_file_container", "csv_content dl_path filename label len")
- TRAIN_DATA_BASE_DIR = "./train_data"
- if not os.path.exists(TRAIN_DATA_BASE_DIR):
- os.makedirs(TRAIN_DATA_BASE_DIR)
- else:
- shutil.rmtree(TRAIN_DATA_BASE_DIR)
- # TRASH DIR FOR UNCONVERTED FILES
- INVALID_FILE_FOLDER = TRAIN_DATA_BASE_DIR+"/__trash__"
- if not os.path.exists(INVALID_FILE_FOLDER):
- os.makedirs(INVALID_FILE_FOLDER)
- #LABEL 1 NOTHING SOME IDLE DATA IF YOU HOLD THE PHONE IN YOUR HAND
- LABEL_NOTHING = "idle"
- LABEL_NOTHING_PATH = TRAIN_DATA_BASE_DIR+"/nothing/"
- label_counter_nothing = 0
- if not os.path.exists(LABEL_NOTHING_PATH):
- os.makedirs(LABEL_NOTHING_PATH)
- #LABEL 2 PUSH SOME IDLE DATA IF YOU PUSH YOUR FORM AWAY FROM YOU
- LABEL_PUSH = "push"
- LABEL_PUSH_PATH = TRAIN_DATA_BASE_DIR+"/push/"
- label_counter_push = 0
- if not os.path.exists(LABEL_PUSH_PATH):
- os.makedirs(LABEL_PUSH_PATH)
- #LABEL 3 PULL SOME IDLE DATA IF YOU PUSH YOUR FORM AWAY FROM YOU
- LABEL_PULL = "pull"
- LABEL_PULL_PATH = TRAIN_DATA_BASE_DIR+"/pull/"
- label_counter_pull = 0
- if not os.path.exists(LABEL_PULL_PATH):
- os.makedirs(LABEL_PULL_PATH)
- #LABEL 4 PULL SOME IDLE DATA IF YOU PUSH YOUR FORM AWAY FROM YOU
- LABEL_ROTATE_LEFT = "left"
- LABEL_ROTATE_LEFT_PATH = TRAIN_DATA_BASE_DIR+"/rotate_left/"
- label_counter_roate_left = 0
- if not os.path.exists(LABEL_ROTATE_LEFT_PATH):
- os.makedirs(LABEL_ROTATE_LEFT_PATH)
- #LABEL 5 PULL SOME IDLE DATA IF YOU PUSH YOUR FORM AWAY FROM YOU
- LABEL_ROTATE_RIGHT = "right"
- LABEL_ROTATE_RIGHT_PATH = TRAIN_DATA_BASE_DIR+"/rotate_right/"
- label_counter_rotate_right = 0
- if not os.path.exists(LABEL_ROTATE_RIGHT_PATH):
- os.makedirs(LABEL_ROTATE_RIGHT_PATH)
- #DOWNLOAD FILE LIST FROM SERVER
- response = requests.get("http://marcelochsendorf.com:3040/")
- if(response.status_code != 200):
- print("get failed 0 ")
- exit()
- requested_filelist = json.loads(response.text)
- #print (requested_filelist['files'])
- # DOWNLOAD ALL FILES AND CHECK IF LEN IS GOOD AND CHECK FOR LABEL
- SENSOR_READ_TIME_INTERVAL = 0.1 #REFRESH RATE IN THE SENSOR IN SECONDS
- MIN_CSV_LEN = 15 #ignores files under x entries
- csv_found_min_len = sys.maxint # after going though all csv files,
- squares = [] # DOWNLOADED CSV FILE LIST
- for i in range(len(requested_filelist['files'])):
- #print(requested_filelist['files'])
- path__, filename_to_dl = os.path.split(requested_filelist['files'][i])
- response_csv = requests.get("http://marcelochsendorf.com:3040/save/" + filename_to_dl)
- if(response_csv.status_code != 200):
- print("get failed 1 ")
- exit()
- print(filename_to_dl)
- sys.stdout.flush()
- fn_basename, fn_timestamp,fn_action, fn_counter = filename_to_dl.split("_")
- #HERE CHECK FOR LABEL NAME IN FILE its for ignoring invaluf file labes you can specify on top of the file
- if(fn_action == LABEL_NOTHING):
- label_counter_nothing = label_counter_nothing + 1
- elif (fn_action == LABEL_PUSH):
- label_counter_push = label_counter_push + 1
- elif (fn_action == LABEL_PULL):
- label_counter_pull = label_counter_pull + 1
- elif (fn_action == LABEL_ROTATE_LEFT):
- label_counter_roate_left = label_counter_roate_left + 1
- elif (fn_action == LABEL_ROTATE_RIGHT):
- label_counter_rotate_right = label_counter_rotate_right + 1
- else:
- print("skip file - invalid label . " + filename_to_dl + "(" + fn_action + ")")
- continue
- row_len_csv = 0
- csv_content = csv.reader(StringIO(response_csv.text))
- for row in csv_content:
- row_len_csv += 1
- #appen if long enought
- # -1 to ignore the csv header
- if(row_len_csv-1 >= MIN_CSV_LEN):
- m = downloaded_file_container(csv_content=response_csv.text, dl_path="http://marcelochsendorf.com:3040/save/" + filename_to_dl, filename=filename_to_dl, label=fn_action, len = row_len_csv-1)
- squares.append(m)
- if(row_len_csv < csv_found_min_len):
- csv_found_min_len = row_len_csv
- else:
- print("ignored csv file")
- print("MIN DATASET LEN : " + str(csv_found_min_len))
- print("DL " + str(label_counter_nothing) + " nothing labeled files")
- print("DL " + str(label_counter_pull) + " pull labeled files")
- print("DL " + str(label_counter_push) + " push labeled files")
- print("DL " + str(label_counter_roate_left) + " roate_left labeled files")
- print("DL " + str(label_counter_rotate_right) + " roate_right labeled files")
- #csv parsen
- #laenge bestimmen speichern
- #timestamp adden
- #und nach kategoriew suchen per keyword und dann in ornern einfügen
- # path = path/to/file
- # filename = foobar.txt
- #SAVEIN ALL DOWNLOADED FILES TO THE RIGHT LOCATION
- comvert_file_counter = 0
- for files_to_convert in squares:
- #print(x.dl_path)
- time_step = 0.0
- filepath = INVALID_FILE_FOLDER
- #DETERMAN FILEPATH VIA LABEL
- if(files_to_convert.label == LABEL_NOTHING):
- filepath = LABEL_NOTHING_PATH
- elif (files_to_convert.label == LABEL_PUSH):
- filepath = LABEL_PUSH_PATH
- elif (files_to_convert.label == LABEL_PULL):
- filepath = LABEL_PULL_PATH
- elif (files_to_convert.label == LABEL_ROTATE_LEFT):
- filepath = LABEL_ROTATE_LEFT_PATH
- elif (files_to_convert.label == LABEL_ROTATE_RIGHT):
- filepath = LABEL_ROTATE_RIGHT_PATH
- else:
- print("skip file - invalid label to determan path ." + files_to_convert.label)
- continue
- #APPEND FILENAME if invalid chose a counting number
- if(files_to_convert.filename == ""):
- filepath =filepath + str(comvert_file_counter)
- comvert_file_counter = comvert_file_counter+1
- else:
- filepath =filepath+ files_to_convert.filename # append the original filename
- with open(filepath, 'w') as csvfile:
- fieldnames = ['TIME_STEP','ACCEL_X', 'ACCEL_Y', 'ACCEL_Z']
- writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
- writer.writeheader()
- csv_dl_content = csv.reader(StringIO(files_to_convert.csv_content))
- is_csv_head = True
- read_row_counter = 0
- for read_row in csv_dl_content:
- #SKIP CSV HEADER
- if(is_csv_head):
- is_csv_head = False
- continue
- #TODO adde nur csv_found_min_len elemente
- if(read_row_counter >= csv_found_min_len):
- break
- read_row_counter = read_row_counter + 1
- writer.writerow({'TIME_STEP': str(time_step), 'ACCEL_X': read_row[1],'ACCEL_Y': read_row[2],'ACCEL_Z': read_row[3]})
- time_step = time_step + SENSOR_READ_TIME_INTERVAL
- i = 0
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement