API tools faq
paste
Guest User

Untitled

a guest
Mar 23rd, 2016
268
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 11.34 KB | None | 0 0
raw download clone embed print report
  1. #!/usr/bin/env python
  2.  
  3. #Data layer for video. Change flow_frames and RGB_frames to be the path to the flow and RGB frames.
  4.  
  5. import sys
  6. sys.path.append('/home/anilil/projects/lstm/lisa-caffe-public/python/')
  7. import caffe
  8. import io
  9. from PIL import Image
  10. import matplotlib.pyplot as plt
  11. import numpy as np
  12. import scipy.misc
  13. import time
  14. import pdb
  15. import glob
  16. import pickle as pkl
  17. import random
  18. import h5py
  19. from multiprocessing import Pool
  20. from threading import Thread
  21. import skimage.io
  22. import copy
  23. import os
  24.  
  25. flow_frames = '/media/anilil/Data/Datasets/UCf_scales/ori_mv_vis/Ori_MV/'
  26. RGB_frames = ''
  27. test_frames = 16
  28. train_frames = 16
  29. test_buffer = 3
  30. train_buffer = 24
  31.  
  32. def processImageCrop(im_info, transformer, flow):
  33. im_path = im_info[0]
  34. im_crop = im_info[1]
  35. im_reshape = im_info[2]
  36. im_flip = im_info[3]
  37. data_in = caffe.io.load_image(im_path)
  38. if (data_in.shape[0] < im_reshape[0]) | (data_in.shape[1] < im_reshape[1]):
  39. data_in = caffe.io.resize_image(data_in, im_reshape)
  40. if im_flip:
  41. data_in = caffe.io.flip_image(data_in, 1, flow)
  42. data_in = data_in[im_crop[0]:im_crop[2], im_crop[1]:im_crop[3], :]
  43. processed_image = transformer.preprocess('data_in',data_in)
  44. return processed_image
  45.  
  46. class ImageProcessorCrop(object):
  47. def __init__(self, transformer, flow):
  48. self.transformer = transformer
  49. self.flow = flow
  50. def __call__(self, im_info):
  51. return processImageCrop(im_info, self.transformer, self.flow)
  52.  
  53. class sequenceGeneratorVideo(object):
  54. def __init__(self, buffer_size, clip_length, num_videos, video_dict, video_order):
  55. self.buffer_size = buffer_size
  56. self.clip_length = clip_length
  57. self.N = self.buffer_size*self.clip_length
  58. self.num_videos = num_videos
  59. self.video_dict = video_dict
  60. self.video_order = video_order
  61. self.idx = 0
  62.  
  63. def __call__(self):
  64. label_r = []
  65. im_paths = []
  66. im_crop = []
  67. im_reshape = []
  68. im_flip = []
  69.  
  70. if self.idx + self.buffer_size >= self.num_videos:
  71. idx_list = range(self.idx, self.num_videos)
  72. idx_list.extend(range(0, self.buffer_size-(self.num_videos-self.idx)))
  73. else:
  74. idx_list = range(self.idx, self.idx+self.buffer_size)
  75.  
  76.  
  77. for i in idx_list:
  78. key = self.video_order[i]
  79. label = self.video_dict[key]['label']
  80. video_reshape = self.video_dict[key]['reshape']
  81. video_crop = self.video_dict[key]['crop']
  82. label_r.extend([label]*self.clip_length)
  83.  
  84. im_reshape.extend([(video_reshape)]*self.clip_length)
  85. r0 = int(random.random()*(video_reshape[0] - video_crop[0]))
  86. r1 = int(random.random()*(video_reshape[1] - video_crop[1]))
  87. im_crop.extend([(r0, r1, r0+video_crop[0], r1+video_crop[1])]*self.clip_length)
  88. f = random.randint(0,1)
  89. im_flip.extend([f]*self.clip_length)
  90. rand_frame = int(random.random()*(self.video_dict[key]['num_frames']-self.clip_length)+1+1)
  91. frames = []
  92.  
  93. for i in range(rand_frame,rand_frame+self.clip_length):
  94. frames.append(self.video_dict[key]['frames'] %i)
  95.  
  96. im_paths.extend(frames)
  97.  
  98.  
  99. im_info = zip(im_paths,im_crop, im_reshape, im_flip)
  100.  
  101. self.idx += self.buffer_size
  102. if self.idx >= self.num_videos:
  103. self.idx = self.idx - self.num_videos
  104.  
  105. return label_r, im_info
  106.  
  107. def advance_batch(result, sequence_generator, image_processor, pool):
  108.  
  109. label_r, im_info = sequence_generator()
  110. tmp = image_processor(im_info[0])
  111. result['data'] = pool.map(image_processor, im_info)
  112. result['label'] = label_r
  113. cm = np.ones(len(label_r))
  114. cm[0::16] = 0
  115. result['clip_markers'] = cm
  116.  
  117. class BatchAdvancer():
  118. def __init__(self, result, sequence_generator, image_processor, pool):
  119. self.result = result
  120. self.sequence_generator = sequence_generator
  121. self.image_processor = image_processor
  122. self.pool = pool
  123.  
  124. def __call__(self):
  125. return advance_batch(self.result, self.sequence_generator, self.image_processor, self.pool)
  126.  
  127. class videoRead(caffe.Layer):
  128.  
  129. def initialize(self):
  130. self.train_or_test = 'test'
  131. self.flow = False
  132. self.buffer_size = test_buffer #num videos processed per batch
  133. self.frames = test_frames #length of processed clip
  134. self.N = self.buffer_size*self.frames
  135. self.idx = 0
  136. self.channels = 3
  137. self.height = 227
  138. self.width = 227
  139. self.path_to_images = RGB_frames
  140. self.video_list = 'new_testlist.txt'
  141.  
  142. def setup(self, bottom, top):
  143. random.seed(10)
  144. self.initialize()
  145. f = open(self.video_list, 'r')
  146. f_lines = f.readlines()
  147. f.close()
  148.  
  149. video_dict = {}
  150. current_line = 0
  151. self.video_order = []
  152. for ix, line in enumerate(f_lines):
  153. video = line.split(' ')[0]
  154. l = int(line.split('/')[0])
  155. frames = sorted(glob.glob('%s%s/*.jpg' %(self.path_to_images, str(video))))
  156. num_frames = len(frames)
  157. video_dict[video] = {}
  158. video_dict[video]['frames'] = frames#[0].split('.')[0] + '.%04d.jpg'
  159. video_dict[video]['reshape'] = (256,256)
  160. video_dict[video]['crop'] = (227, 227)
  161. video_dict[video]['num_frames'] = num_frames
  162. video_dict[video]['label'] = l
  163. self.video_order.append(video)
  164.  
  165. self.video_dict = video_dict
  166. self.num_videos = len(video_dict.keys())
  167.  
  168. #set up data transformer
  169. shape = (self.N, self.channels, self.height, self.width)
  170.  
  171. self.transformer = caffe.io.Transformer({'data_in': shape})
  172. self.transformer.set_raw_scale('data_in', 255)
  173. if self.flow:
  174. image_mean = [128, 128, 128]
  175. self.transformer.set_is_flow('data_in', True)
  176. else:
  177. image_mean = [103.939, 116.779, 128.68]
  178. self.transformer.set_is_flow('data_in', False)
  179. channel_mean = np.zeros((3,227,227))
  180. for channel_index, mean_val in enumerate(image_mean):
  181. channel_mean[channel_index, ...] = mean_val
  182. self.transformer.set_mean('data_in', channel_mean)
  183. self.transformer.set_channel_swap('data_in', (2, 1, 0))
  184. self.transformer.set_transpose('data_in', (2, 0, 1))
  185.  
  186. self.thread_result = {}
  187. self.thread = None
  188. pool_size = 15
  189.  
  190. self.image_processor = ImageProcessorCrop(self.transformer, self.flow)
  191. self.sequence_generator = sequenceGeneratorVideo(self.buffer_size, self.frames, self.num_videos, self.video_dict, self.video_order)
  192.  
  193. self.pool = Pool(processes=pool_size)
  194. self.batch_advancer = BatchAdvancer(self.thread_result, self.sequence_generator, self.image_processor, self.pool)
  195. self.dispatch_worker()
  196. self.top_names = ['data', 'label','clip_markers']
  197. print 'Outputs:', self.top_names
  198. if len(top) != len(self.top_names):
  199. raise Exception('Incorrect number of outputs (expected %d, got %d)' %
  200. (len(self.top_names), len(top)))
  201. self.join_worker()
  202. for top_index, name in enumerate(self.top_names):
  203. if name == 'data':
  204. shape = (self.N, self.channels, self.height, self.width)
  205. elif name == 'label':
  206. shape = (self.N,)
  207. elif name == 'clip_markers':
  208. shape = (self.N,)
  209. top[top_index].reshape(*shape)
  210.  
  211. def reshape(self, bottom, top):
  212. pass
  213.  
  214. def forward(self, bottom, top):
  215.  
  216. if self.thread is not None:
  217. self.join_worker()
  218.  
  219. #rearrange the data: The LSTM takes inputs as [video0_frame0, video1_frame0,...] but the data is currently arranged as [video0_frame0, video0_frame1, ...]
  220. new_result_data = [None]*len(self.thread_result['data'])
  221. new_result_label = [None]*len(self.thread_result['label'])
  222. new_result_cm = [None]*len(self.thread_result['clip_markers'])
  223. for i in range(self.frames):
  224. for ii in range(self.buffer_size):
  225. old_idx = ii*self.frames + i
  226. new_idx = i*self.buffer_size + ii
  227. new_result_data[new_idx] = self.thread_result['data'][old_idx]
  228. new_result_label[new_idx] = self.thread_result['label'][old_idx]
  229. new_result_cm[new_idx] = self.thread_result['clip_markers'][old_idx]
  230.  
  231. for top_index, name in zip(range(len(top)), self.top_names):
  232. if name == 'data':
  233. for i in range(self.N):
  234. top[top_index].data[i, ...] = new_result_data[i]
  235. elif name == 'label':
  236. top[top_index].data[...] = new_result_label
  237. elif name == 'clip_markers':
  238. top[top_index].data[...] = new_result_cm
  239.  
  240. self.dispatch_worker()
  241.  
  242. def dispatch_worker(self):
  243. assert self.thread is None
  244. self.thread = Thread(target=self.batch_advancer)
  245. self.thread.start()
  246.  
  247. def join_worker(self):
  248. assert self.thread is not None
  249. self.thread.join()
  250. self.thread = None
  251.  
  252. def backward(self, top, propagate_down, bottom):
  253. pass
  254.  
  255. class videoReadTrain_flow(videoRead):
  256.  
  257. def initialize(self):
  258. self.train_or_test = 'train'
  259. self.flow = True
  260. self.buffer_size = train_buffer #num videos processed per batch
  261. self.frames = train_frames #length of processed clip
  262. self.N = self.buffer_size*self.frames
  263. self.idx = 0
  264. self.channels = 3
  265. self.height = 227
  266. self.width = 227
  267. self.path_to_images = flow_frames
  268. self.video_list = 'new_trainlist.txt'
  269.  
  270. class videoReadTest_flow(videoRead):
  271.  
  272. def initialize(self):
  273. self.train_or_test = 'test'
  274. self.flow = True
  275. self.buffer_size = test_buffer #num videos processed per batch
  276. self.frames = test_frames #length of processed clip
  277. self.N = self.buffer_size*self.frames
  278. self.idx = 0
  279. self.channels = 3
  280. self.height = 227
  281. self.width = 227
  282. self.path_to_images = flow_frames
  283. self.video_list = 'new_testlist.txt'
  284.  
  285. class videoReadTrain_RGB(videoRead):
  286.  
  287. def initialize(self):
  288. self.train_or_test = 'train'
  289. self.flow = False
  290. self.buffer_size = train_buffer #num videos processed per batch
  291. self.frames = train_frames #length of processed clip
  292. self.N = self.buffer_size*self.frames
  293. self.idx = 0
  294. self.channels = 3
  295. self.height = 227
  296. self.width = 227
  297. self.path_to_images = RGB_frames
  298. self.video_list = 'new_trainlist.txt'
  299.  
  300. class videoReadTest_RGB(videoRead):
  301.  
  302. def initialize(self):
  303. self.train_or_test = 'test'
  304. self.flow = False
  305. self.buffer_size = test_buffer #num videos processed per batch
  306. self.frames = test_frames #length of processed clip
  307. self.N = self.buffer_size*self.frames
  308. self.idx = 0
  309. self.channels = 3
  310. self.height = 227
  311. self.width = 227
  312. self.path_to_images = RGB_frames
  313. self.video_list = 'new_testlist.txt'
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!