Advertisement
Guest User

Untitled

a guest
Mar 30th, 2017
398
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 3.03 KB | None | 0 0
  1. def writer(start, stop, num):
  2. v_id_count = 0
  3. v_Count = 0
  4. out = open('C:\chunks\part_'+str(num)+'.txt', 'w')
  5. for i in islice(dirs,start,stop):
  6. id = i
  7. v_id_count += 1
  8. p = '{}\{}'.format(path,i)
  9. files = [z for z in os.listdir(p)]
  10. for file in files:
  11. file_path = '{}\{}'.format(p, file)
  12. f = open(file_path, 'r').read()
  13. soup = BeautifulSoup(f, 'html.parser')
  14. # cur = con.cursor()
  15. v_Count_loop = 0
  16. try:
  17. for v in range(get_count(soup)):
  18. string = '{};{};{};{};{};{};{}'.format(id, get_status_date(soup), get_status(soup), get_z_name(soup),get_d_name(soup),
  19. get_method(soup, v), get_description(soup, v))
  20. out.write(string + 'n')
  21. v_Count += 1
  22. v_Count_loop += 1
  23. print('Поток {}. Вставлено {} строк. Всего вставлено {}. Обработано id {}'.format(num, v_Count_loop, v_Count, v_inn_count))
  24. except Exception as e:
  25. print(e)
  26. out.close()
  27.  
  28. if __name__ == '__main__':
  29.  
  30. path = 'C:\html'
  31. dirs = [x for x in os.listdir(path)]
  32.  
  33. maxpoint = len(dirs)
  34.  
  35. params = []
  36.  
  37. #делим на диапазоны
  38. for i in range(1,11):
  39. if round(maxpoint/10 * (i-1)) == 0:
  40. st = round(maxpoint/10 * (i-1))
  41. else:
  42. st = round(maxpoint/10 * (i-1)) + 1
  43. ed = round(maxpoint/10*i)
  44. args = st, ed, i
  45. params.append(args)
  46.  
  47. # print(params[0])
  48.  
  49. p1 = Process(target = writer, args=params[0])
  50. p2 = Process(target = writer, args=params[1])
  51. p3 = Process(target = writer, args=params[2])
  52. p4 = Process(target = writer, args=params[3])
  53. p5 = Process(target = writer, args=params[4])
  54. p6 = Process(target = writer, args=params[5])
  55. p7 = Process(target = writer, args=params[6])
  56. p8 = Process(target = writer, args=params[7])
  57. p9 = Process(target = writer, args=params[8])
  58. p10 = Process(target = writer, args=params[9])
  59.  
  60. p1.start()
  61. p2.start()
  62. p3.start()
  63. p4.start()
  64. p5.start()
  65. p6.start()
  66. p7.start()
  67. p8.start()
  68. p9.start()
  69. p10.start()
  70.  
  71. p1.join()
  72. p2.join()
  73. p3.join()
  74. p4.join()
  75. p5.join()
  76. p6.join()
  77. p7.join()
  78. p8.join()
  79. p9.join()
  80. p10.join()
  81.  
  82. import os
  83. from multiprocessing import Pool, Lock
  84. lock = Lock()
  85.  
  86. def writer(args_):
  87. dir_, path_ = args_
  88. folder = os.path.join(path_, dir_)
  89. for file in os.listdir(folder):
  90. ff = os.path.join(folder, file)
  91.  
  92. if os.path.isfile(ff):
  93. lock.acquire()
  94. with open(ff, 'a') as f:
  95. f.write('')
  96. lock.release()
  97.  
  98. if __name__ == '__main__':
  99. path = os.path.realpath('TEMP')
  100. folders = (f for f in os.listdir(path) if os.path.isdir(os.path.join(path, f)))
  101.  
  102. with Pool(processes=10) as pool:
  103. pool.map(writer, ([f, path] for f in folders))
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement