Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- def iterateDirectories(thispath,maxim,fname):
- saved_files=[0]
- stop=False
- # Saved files: Index start at 0 but file saving starts at 1
- info_report = defaultdict(lambda: [])
- ind=0
- for subdir, dirs, files in os.walk(thispath):
- if(stop==False):
- found=False
- #print("Scanning: " + subdir)
- for file in files:
- file_name = file.partition('_')[0]
- #print os.path.join(subdir, file)
- filepath = subdir + os.sep + file
- if filepath.endswith(".txt"):
- try:
- thisReport = processFile(file,subdir)
- found=True
- except MemoryError as err:
- print("MemoryError: ")
- dfs_in_memory()
- continue
- except:
- print("EXCEPTION AT ITERATE DIRS")
- continue
- if(not is_empty_frame(thisReport)):
- info_report[subdir].append(thisReport)
- #After traversal of one directory, join all dframes and save to persistent storage
- #Then delete all DF's to save memory usage
- #uid = saved_files[-1:] #Last element
- if(found==True): #uid[0] != "False" and
- #uid = uid[0] + 1
- #dfs_in_memory()
- #saveall_df(info_report[subdir],uid) #Save and delete
- #saved_files.append(uid)
- ind+=1
- try:
- append_csv(info_report[subdir],fname)
- except:
- print("ERROR: Appending failed. One more attempt..")
- try:
- append_csv(info_report[subdir],fname)
- except:
- print("Error: Given up appending line.")
- info_report[subdir] = [] #Remove values.
- #print("File Uid: " + str(uid) + " saved.")
- print("Subdir: " + str(subdir))
- print(str(ind)+ " Files saved so far")
- if(ind==maxim):
- print("=========================END SCAN [Max_Bound] =========================")
- stop=True
- return True
- #dfs_in_memory()
- else:
- pass
- print("=========================END SCAN=========================")
- return True
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement