API tools faq
paste
Guest User

Untitled

a guest
Jan 16th, 2019
100
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 10.16 KB | None | 0 0
raw download clone embed print report
  1. # coding: utf-8
  2.  
  3. #training file for LSTM rnn
  4. import random
  5. import time
  6. import shutil
  7. from tqdm import tqdm
  8. import os
  9. import numpy as np
  10. import pickle as pk
  11. import tensorflow as tf
  12. from Network import LSTM_Network
  13.  
  14.  
  15. epoch = 5
  16.  
  17. with open('/d2/data/apps/sn_pred/modelTraining/Preprocessing_part/sorted_train_data.pkl','rb') as f:
  18. sorted_train_data = pk.load(f)
  19.  
  20. with open('/d2/data/apps/sn_pred/modelTraining/Preprocessing_part/sorted_test_data.pkl','rb') as f:
  21. sorted_test_data = pk.load(f)
  22.  
  23. with open('/d2/data/apps/sn_pred/modelTraining/Preprocessing_part/mapping_labels.pkl','rb') as f:
  24. mapping_labels = pk.load(f)
  25. print("len_of_cate",len(mapping_labels))
  26.  
  27.  
  28.  
  29. # #sorted for minimal padding
  30. # sorted_train_data = sorted(train_data_in, key=lambda x: len(x))[2:]
  31.  
  32. # sorted_test_data = sorted(test_data_in, key=lambda x: len(x))
  33. # #saving the test for validation
  34. # np.save('sorted_test_data', sorted_test_data)
  35.  
  36. print("train_data", len(sorted_train_data))
  37. print("test_data", len(sorted_test_data))
  38.  
  39. def sequence_padding(seqs_):
  40. """
  41. Padding the seq with same length for RNN
  42. :param seqs_:
  43. taking batch of without padded seqs as argument ex : [[278698,3442], [194661] , [1098,2341,77]]
  44. :return:
  45. padded with zeros with max len of seq in batch ex : [ [278698,3442,0] , [194661,0,0] ,[1098,2341,77]]
  46.  
  47. :raise:
  48. if seq is not 2 dimension:
  49. TypeError: object of type 'int' has no len()
  50. """
  51.  
  52. max_sequence = max(list(map(len, seqs_)))
  53.  
  54. # getting Max length of sequence
  55. padded_sequence = [i + [0] * (max_sequence - len(i)) if len(i) < max_sequence else i for i in seqs_]
  56.  
  57. # padded_sequence = list(map(lambda x:x + [0]*(max_sequence-len(x)) if len(x)<max_sequence else x,seqs))
  58. # padded sequence with max length of sequence
  59. return padded_sequence
  60. # return padded sequence
  61.  
  62. def reshape_data(batch_seq_data):
  63. """
  64. Reshaping the sequences with np array format
  65.  
  66. :param batch_seq_data:
  67. taking batch of seq as argument
  68.  
  69. :return:
  70. batch of np array values
  71.  
  72. other_columns.append((data_loads.iloc[m]['u_cause'],
  73. data_loads.iloc[m]['subcategory'],
  74. data_loads.iloc[m]['category'],
  75. data_loads.iloc[m]['priority'],
  76. data_loads.iloc[m]['severity'],
  77. data_loads.iloc[m]['urgency'],
  78. data_loads.iloc[m]['location'],
  79. data_loads.iloc[m]['opened_by'],
  80. data_loads.iloc[m]['sys_created_by'],
  81. data_loads.iloc[m]['u_business_service'],
  82. data_loads.iloc[m]['u_vendor']))
  83. """
  84.  
  85. # inputs = padding([list(i[:-1]) for i in batch_seq_data], hj=j)
  86. # encoded_long_word_pad = sequence_padding([list(i[:-1]) for i in batch_seq_data])
  87.  
  88. padded_long_word = sequence_padding([m[0] for m in batch_seq_data])
  89. padd_short_word = sequence_padding([m[1] for m in batch_seq_data])
  90. padd_long_char = sequence_padding([m[2] for m in batch_seq_data])
  91. padd_short_char = sequence_padding([m[3] for m in batch_seq_data])
  92. other_columns = [m[4] for m in batch_seq_data]
  93.  
  94. u_causes = [m[0] for m in other_columns]
  95.  
  96. subcategory = [m[1] for m in other_columns]
  97. category = [m[2] for m in other_columns]
  98. priority = [m[3] for m in other_columns]
  99. severity = [m[4] for m in other_columns]
  100. urgency = [m[5] for m in other_columns]
  101. location = [m[6] for m in other_columns]
  102. opened_by = [m[7] for m in other_columns]
  103. sys_created_by = [m[8] for m in other_columns]
  104. u_business_service = [m[9] for m in other_columns]
  105. u_vendor = [m[10] for m in other_columns],
  106.  
  107. labels = [m[5] for m in batch_seq_data]
  108. return {
  109. 'padded_long_word' : np.array(padded_long_word),
  110. 'padd_short_word' : np.array(padd_short_word),
  111. 'padd_long_char' : np.array(padd_long_char),
  112. 'padd_short_char' : np.array(padd_short_char),
  113. 'oth_1' : np.array(u_causes),
  114. 'oth_2' : np.array(subcategory),
  115. 'oth_3' : np.array(category),
  116. 'oth_4' : np.array(priority),
  117. 'oth_5' : np.array(severity),
  118. 'oth_6' : np.array(urgency),
  119. 'oth_7' : np.array(location),
  120. 'oth_8' : np.array(opened_by),
  121. 'oth_9' : np.array(sys_created_by),
  122. 'oth_10' : np.array(u_business_service),
  123. 'oth_11' : np.array(u_vendor),
  124. 'lables' : np.array(labels)
  125. }
  126.  
  127. def evaluate_(model, batch_size=50):
  128. """
  129. Checking the test accuracy on testing data set
  130.  
  131. :param model:
  132. current lstm model
  133.  
  134. :param batch_size:
  135. batch size for test data set
  136.  
  137. :return:
  138. mean accuracy of test data set
  139.  
  140. :raise:
  141. if input shape is different from placeholder shape:
  142. ValueError: Cannot feed value of shape
  143.  
  144. """
  145. sess = tf.get_default_session()
  146.  
  147. # batch_data = test_data_in
  148. # batch_labesl = train_labels
  149.  
  150. iteration = len(sorted_test_data) // batch_size
  151.  
  152. accuracy = []
  153.  
  154. for i in range(iteration):
  155. batch_data_i = sorted_test_data[i * batch_size:(i + 1) * batch_size]
  156.  
  157. input_test = reshape_data(batch_data_i)
  158. padded_long_word_e = input_test['padded_long_word']
  159. padd_short_word_e = input_test['padd_short_word']
  160. padd_long_char_e = input_test['padd_long_char']
  161. padd_short_char_e = input_test['padd_short_char']
  162. oth_1 = input_test['oth_1']
  163. oth_2 = input_test['oth_2']
  164. oth_3 = input_test['oth_3']
  165. oth_4 = input_test['oth_4']
  166. oth_5 = input_test['oth_5']
  167. oth_6 = input_test['oth_6']
  168. oth_7 = input_test['oth_7']
  169. oth_8 = input_test['oth_8']
  170. oth_9 = input_test['oth_9']
  171. oth_10 = input_test['oth_10']
  172. oth_11 = input_test['oth_11'][0]
  173. labels_e = input_test['lables']
  174.  
  175. network_out = sess.run(model.output, feed_dict={model.placeholder['long_word_sente']: padded_long_word_e,
  176. model.placeholder['short_word_sente']: padd_short_word_e,
  177. model.placeholder['long_char_sente']: padd_long_char_e ,
  178. model.placeholder['short_char_sente']:padd_short_char_e,
  179. model.placeholder['oth_col_1']:oth_1,
  180. model.placeholder['oth_col_2']:oth_2,
  181. model.placeholder['oth_col_3']:oth_3,
  182. model.placeholder['oth_col_4']:oth_4,
  183. model.placeholder['oth_col_5']:oth_5,
  184. model.placeholder['oth_col_6']:oth_6,
  185. model.placeholder['oth_col_7']:oth_7,
  186. model.placeholder['oth_col_8']:oth_8,
  187. model.placeholder['oth_col_9']:oth_9,
  188. model.placeholder['oth_col_10']:oth_10,
  189. model.placeholder['oth_col_11']:oth_11,
  190. model.placeholder['label_s']:labels_e })
  191. accuracy.append(network_out['accuracy'])
  192. return np.mean(np.array(accuracy))
  193.  
  194. def train_model(model, batch_size=50):
  195. """
  196.  
  197. :param model:
  198. current lstm model
  199.  
  200. :param batch_size:
  201. batch size for training
  202.  
  203. :print:
  204. epoch
  205. iteration
  206. training_loss
  207. accuracy
  208.  
  209. """
  210. saver = tf.train.Saver()
  211. with tf.Session() as sess:
  212. sess.run(tf.global_variables_initializer())
  213.  
  214. iteration = len(sorted_train_data) // batch_size
  215. print("iteration", iteration)
  216. time.sleep(5)
  217.  
  218. for i in range(epoch):
  219.  
  220. for j in range(iteration):
  221. batch_data_j = sorted_train_data[j * batch_size:(j + 1) * batch_size]
  222.  
  223. input_train = reshape_data(batch_data_j)
  224. padded_long_word_t = input_train['padded_long_word']
  225. padd_short_word_t = input_train['padd_short_word']
  226. padd_long_char_t = input_train['padd_long_char']
  227. padd_short_char_t = input_train['padd_short_char']
  228. oth_1 = input_train['oth_1']
  229. oth_2 = input_train['oth_2']
  230. oth_3 = input_train['oth_3']
  231. oth_4 = input_train['oth_4']
  232. oth_5 = input_train['oth_5']
  233. oth_6 = input_train['oth_6']
  234. oth_7 = input_train['oth_7']
  235. oth_8 = input_train['oth_8']
  236. oth_9 = input_train['oth_9']
  237. oth_10 = input_train['oth_10']
  238. oth_11 = input_train['oth_11'][0]
  239.  
  240. #import ipdb; ipdb.set_trace();
  241.  
  242. labels_t = input_train['lables']
  243.  
  244. network_out, _ = sess.run([model.output, model.train],
  245. feed_dict={model.placeholder['long_word_sente']: padded_long_word_t,
  246. model.placeholder['short_word_sente']: padd_short_word_t,
  247. model.placeholder['long_char_sente']: padd_long_char_t ,
  248. model.placeholder['short_char_sente']:padd_short_char_t,
  249. model.placeholder['oth_col_1']:oth_1,
  250. model.placeholder['oth_col_2']:oth_2,
  251. model.placeholder['oth_col_3']:oth_3,
  252. model.placeholder['oth_col_4']:oth_4,
  253. model.placeholder['oth_col_5']:oth_5,
  254. model.placeholder['oth_col_6']:oth_6,
  255. model.placeholder['oth_col_7']:oth_7,
  256. model.placeholder['oth_col_8']:oth_8,
  257. model.placeholder['oth_col_9']:oth_9,
  258. model.placeholder['oth_col_10']:oth_10,
  259. model.placeholder['oth_col_11']:oth_11,
  260. model.placeholder['label_s']:labels_t })
  261. print({'epoch': i,
  262. 'iteration': j,
  263. 'training_loss': network_out['loss'],
  264. 'training_accuracy': network_out['accuracy']
  265. })
  266.  
  267. if j % 100 == 0:
  268. with open('iterres.txt', 'a') as f:
  269. f.write(
  270. str({'epoch': i, 'test_accuracy': evaluate_(model, batch_size=150), 'iteration': j}) + '\n')
  271.  
  272. os.system('mkdir ' + str(i) + 'epoch' + str(j))
  273. saver.save(sess, '.' + str(i) + 'epoch' + str(j) + '/' + str(i))
  274.  
  275. if float(evaluate_(model, batch_size=150))>0.76:
  276. shutil.make_archive(str(i) + 'epoch' + str(j),'zip','.' + str(i) + 'epoch' + str(j))
  277. os.system('rm -rf ' + str(i) + 'epoch' + str(j))
  278.  
  279. os.system('mkdir ' + str(i) + 'epoch' + str(j))
  280. saver.save(sess, '.' + str(i) + 'epoch' + str(j) + '/' + str(i))
  281.  
  282. print({'epoch': i, 'test_accuracy': evaluate_(model)})
  283. with open('epochandresult', 'a') as f:
  284. f.write(str({'epoch': i, 'test_accuracy': evaluate_(model)}) + '\n')
  285.  
  286.  
  287. if __name__ == "__main__":
  288.  
  289. model = LSTM_Network(dropout_value_= 0.5,
  290. word_vocab_size_= 30002,
  291. word_embedding_dim_= 150 ,
  292. forget_bias_= 1.0,
  293. rnn_num_units= 150,
  294. labels_nos = len(mapping_labels),
  295. char_vocab_size_ = 34 ,
  296. char_embedding_dim_= 50)
  297.  
  298. train_model(model)
  299.  
  300. #### Error: ValueError: Cannot feed value of shape (1, 50) for Tensor 'oth_col_11:0', which has shape '(?,)'
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!