dave 2

ileList = glob.glob('/home/ec2-user/dev/two_angle/clutter_100k/*')
progress = 0
simulation_results  = {}


for f in fileList:
  if not progress % 100:
    print "Finished loading %d keys" % progress
  progress += 1
  try:
    with open(f) as fid:
      result = json.load(fid)
  except IOError:
    print "bad file"
    continue

  try:
    bag_id = result['input']['experimental']['bag_id']
    if not simulation_results.has_key(bag_id):
      simulation_results[bag_id] = []

    simulation_results[bag_id].append(result)
  except KeyError:
    print "bad file"
    continue

print "data org for clutter"
outputs = []
progress = 0
for bag_id in simulation_results:
  if not progress % 100:
    print "Finished writing %d simulations" % progress
  progress += 1
  tmp_out = simulation_results[bag_id]['output']
  tmp1 = re.sub(' \n',' ', tmp_out)
  outputs.append(tmp1)

fileName = '/home/ec2-user/clutterData/clutter_100k_2angle.txt'
threat_file = open(fileName, 'w+')
threat_file.write("".join(str(j) for j in outputs) )
threat_file.close()