Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- fileName = '/mnt/eftdata2/pos_ptlf_201309%(i)02d.txt'
- names = [fileName % {'i':i} for i in range(5,12)]
- strDate = '2013-09-%(j)02d'
- dates = [strDate %{'j':j} for j in range(5,12)]
- # Connect to the IPython cluster
- # set 7 cluster engines, 1 cluster preprocess 1 daily data (total 7 daily dataset per week)
- c = Client(profile='titaClusters')
- c[:].run('chunk.py')
- c[:].block = True
- numC = len(c)
- dat01 = c[0].apply_sync(extract_data,names[0])
- clear_clients(c[0])
- dat02 = c[1].apply_sync(extract_data,names[1])
- clear_clients(c[1])
- dat03 = c[2].apply_sync(extract_data,names[2])
- clear_clients(c[2])
- dat04 = c[3].apply_sync(extract_data,names[3])
- clear_clients(c[3])
- dat05 = c[4].apply_sync(extract_data,names[4])
- clear_clients(c[4])
- dat06 = c[5].apply_sync(extract_data,names[5])
- clear_clients(c[5])
- dat07 = c[6].apply_sync(extract_data,names[6])
- clear_clients(c[6])
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement