API tools faq
paste
Guest User

Untitled

a guest
Jul 16th, 2018
86
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 13.37 KB | None | 0 0
raw download clone embed print report
  1. data_path1 = 'C:/tensorflow1/models/research/object_detection/images/csv1.tfrecords' # address to save the hdf5 file
  2.  
  3. with tf.Session() as sess:
  4. feature = {'image/encoded': tf.FixedLenFeature([], tf.string),
  5. 'image/object/class/label': tf.FixedLenFeature([], tf.int64)}
  6. # Create a list of filenames and pass it to a queue
  7. filename_queue = tf.train.string_input_producer([data_path1], num_epochs=1)
  8. # Define a reader and read the next record
  9. reader = tf.TFRecordReader()
  10. _, serialized_example = reader.read(filename_queue)
  11. # Decode the record read by the reader
  12. features = tf.parse_single_example(serialized_example, features=feature)
  13. # Convert the image data from string back to the numbers
  14. image = tf.decode_raw(features['image/encoded'], tf.int64)
  15.  
  16. # Cast label data into int32
  17. label = tf.cast(features['image/object/class/label'], tf.int32)
  18. # Reshape image data into the original shape
  19. image = tf.reshape(image, [150, 150, 1])
  20. images, labels = tf.train.shuffle_batch([image, label], batch_size=10, capacity=30, num_threads=1, min_after_dequeue=10)
  21.  
  22.  
  23.  
  24. with tf.Session() as sess:
  25. # Initialize all global and local variables
  26. init_op = tf.group(tf.global_variables_initializer(), tf.local_variables_initializer())
  27. # sess = tf.Session()
  28. sess.run(init_op)
  29. # Create a coordinator and run all QueueRunner objects
  30. coord = tf.train.Coordinator()
  31. threads = tf.train.start_queue_runners(coord=coord)
  32. for batch_index in range(5):
  33. img, lbl = sess.run([images, labels])
  34. img = img.astype(np.uint8)
  35. for j in range(6):
  36. plt.subplot(2, 3, j+1)
  37. plt.imshow(img[j, ...])
  38. plt.show()
  39. # Stop the threads
  40. coord.request_stop()
  41.  
  42. # Wait for threads to stop
  43. coord.join(threads)
  44. sess.close()
  45.  
  46. from __future__ import division
  47. from __future__ import print_function
  48. from __future__ import absolute_import
  49.  
  50. import os
  51. import io
  52. import pandas as pd
  53. import tensorflow as tf
  54.  
  55. from PIL import Image
  56. from object_detection.utils import dataset_util
  57. from collections import namedtuple, OrderedDict
  58.  
  59. flags = tf.app.flags
  60. flags.DEFINE_string('csv_input', '', 'Path to the CSV input')
  61. flags.DEFINE_string('output_path', '', 'Path to output TFRecord')
  62. FLAGS = flags.FLAGS
  63.  
  64.  
  65. # TO-DO replace this with label map
  66. def class_text_to_int(row_label):
  67. if row_label == 'car':
  68. return 1
  69. else:
  70. return 0
  71.  
  72.  
  73. def split(df, group):
  74. data = namedtuple('data', ['filename', 'object'])
  75. gb = df.groupby(group)
  76. return [data(filename, gb.get_group(x)) for filename, x in zip(gb.groups.keys(), gb.groups)]
  77.  
  78.  
  79. def create_tf_example(group, path):
  80. with tf.gfile.GFile(os.path.join(path, '{}'.format(group.filename)+'.jpg'), 'rb') as fid:
  81. print(fid)
  82. encoded_jpg = fid.read()
  83. encoded_jpg_io = io.BytesIO(encoded_jpg)
  84. image = Image.open(encoded_jpg_io)
  85. width, height = image.size
  86.  
  87. filename = group.filename.encode('utf8')
  88. image_format = b'jpg'
  89. xmins = []
  90. xmaxs = []
  91. ymins = []
  92. ymaxs = []
  93. classes_text = []
  94. classes = []
  95.  
  96. for index, row in group.object.iterrows():
  97. xmins.append(row['xmin'] / width)
  98. xmaxs.append(row['xmax'] / width)
  99. ymins.append(row['ymin'] / height)
  100. ymaxs.append(row['ymax'] / height)
  101. classes_text.append(row['class'].encode('utf8'))
  102. classes.append(class_text_to_int(row['class']))
  103.  
  104. tf_example = tf.train.Example(features=tf.train.Features(feature={
  105. 'image/height': dataset_util.int64_feature(height),
  106. 'image/width': dataset_util.int64_feature(width),
  107. 'image/filename': dataset_util.bytes_feature(filename),
  108. 'image/source_id': dataset_util.bytes_feature(filename),
  109. 'image/encoded': dataset_util.bytes_feature(encoded_jpg),
  110. 'image/format': dataset_util.bytes_feature(image_format),
  111. 'image/object/bbox/xmin': dataset_util.float_list_feature(xmins),
  112. 'image/object/bbox/xmax': dataset_util.float_list_feature(xmaxs),
  113. 'image/object/bbox/ymin': dataset_util.float_list_feature(ymins),
  114. 'image/object/bbox/ymax': dataset_util.float_list_feature(ymaxs),
  115. 'image/object/class/text': dataset_util.bytes_list_feature(classes_text),
  116. 'image/object/class/label': dataset_util.int64_list_feature(classes),
  117. }))
  118. return tf_example
  119.  
  120.  
  121. def main(_):
  122. writer = tf.python_io.TFRecordWriter('csv1.tfrecords')
  123. path = os.path.join(os.getcwd(), 'thermal')
  124. examples = pd.read_csv('thermal_labels.csv')
  125. grouped = split(examples, 'filename')
  126. for group in grouped:
  127. tf_example = create_tf_example(group, path)
  128. writer.write(tf_example.SerializeToString())
  129.  
  130. writer.close()
  131. output_path = os.path.join(os.getcwd(), 'csv1.tfrecords')
  132. print('Successfully created the TFRecords: {}'.format(output_path))
  133.  
  134.  
  135. if __name__ == '__main__':
  136. tf.app.run()
  137.  
  138. INFO:tensorflow:Error reported to Coordinator: <class 'tensorflow.python.framework.errors_impl.InvalidArgumentError'>, Input to DecodeRaw has length 224949 that is not a multiple of 8, the size of int64
  139. [[Node: DecodeRaw = DecodeRaw[little_endian=true, out_type=DT_INT64, _device="/job:localhost/replica:0/task:0/device:CPU:0"](ParseSingleExample/ParseSingleExample)]]
  140. ---------------------------------------------------------------------------
  141. OutOfRangeError Traceback (most recent call last)
  142. c:anaconda3envstf18libsite-packagestensorflowpythonclientsession.py in _do_call(self, fn, *args)
  143. 1321 try:
  144. -> 1322 return fn(*args)
  145. 1323 except errors.OpError as e:
  146.  
  147. c:anaconda3envstf18libsite-packagestensorflowpythonclientsession.py in _run_fn(feed_dict, fetch_list, target_list, options, run_metadata)
  148. 1306 return self._call_tf_sessionrun(
  149. -> 1307 options, feed_dict, fetch_list, target_list, run_metadata)
  150. 1308
  151.  
  152. c:anaconda3envstf18libsite-packagestensorflowpythonclientsession.py in _call_tf_sessionrun(self, options, feed_dict, fetch_list, target_list, run_metadata)
  153. 1408 self._session, options, feed_dict, fetch_list, target_list,
  154. -> 1409 run_metadata)
  155. 1410 else:
  156.  
  157. OutOfRangeError: RandomShuffleQueue '_37_shuffle_batch_2/random_shuffle_queue' is closed and has insufficient elements (requested 10, current size 0)
  158. [[Node: shuffle_batch_2 = QueueDequeueManyV2[component_types=[DT_INT64, DT_INT32], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/device:CPU:0"](shuffle_batch_2/random_shuffle_queue, shuffle_batch_2/n)]]
  159.  
  160. During handling of the above exception, another exception occurred:
  161.  
  162. OutOfRangeError Traceback (most recent call last)
  163. <ipython-input-16-b8d05859b312> in <module>()
  164. 8 threads = tf.train.start_queue_runners(coord=coord)
  165. 9 for batch_index in range(5):
  166. ---> 10 img, lbl = sess.run([images, labels])
  167. 11 img = img.astype(np.uint8)
  168. 12 for j in range(6):
  169.  
  170. c:anaconda3envstf18libsite-packagestensorflowpythonclientsession.py in run(self, fetches, feed_dict, options, run_metadata)
  171. 898 try:
  172. 899 result = self._run(None, fetches, feed_dict, options_ptr,
  173. --> 900 run_metadata_ptr)
  174. 901 if run_metadata:
  175. 902 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
  176.  
  177. c:anaconda3envstf18libsite-packagestensorflowpythonclientsession.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
  178. 1133 if final_fetches or final_targets or (handle and feed_dict_tensor):
  179. 1134 results = self._do_run(handle, final_targets, final_fetches,
  180. -> 1135 feed_dict_tensor, options, run_metadata)
  181. 1136 else:
  182. 1137 results = []
  183.  
  184. c:anaconda3envstf18libsite-packagestensorflowpythonclientsession.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
  185. 1314 if handle is None:
  186. 1315 return self._do_call(_run_fn, feeds, fetches, targets, options,
  187. -> 1316 run_metadata)
  188. 1317 else:
  189. 1318 return self._do_call(_prun_fn, handle, feeds, fetches)
  190.  
  191. c:anaconda3envstf18libsite-packagestensorflowpythonclientsession.py in _do_call(self, fn, *args)
  192. 1333 except KeyError:
  193. 1334 pass
  194. -> 1335 raise type(e)(node_def, op, message)
  195. 1336
  196. 1337 def _extend_graph(self):
  197.  
  198. OutOfRangeError: RandomShuffleQueue '_37_shuffle_batch_2/random_shuffle_queue' is closed and has insufficient elements (requested 10, current size 0)
  199. [[Node: shuffle_batch_2 = QueueDequeueManyV2[component_types=[DT_INT64, DT_INT32], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/device:CPU:0"](shuffle_batch_2/random_shuffle_queue, shuffle_batch_2/n)]]
  200.  
  201. Caused by op 'shuffle_batch_2', defined at:
  202. File "c:anaconda3envstf18librunpy.py", line 193, in _run_module_as_main
  203. "__main__", mod_spec)
  204. File "c:anaconda3envstf18librunpy.py", line 85, in _run_code
  205. exec(code, run_globals)
  206. File "c:anaconda3envstf18libsite-packagesipykernel_launcher.py", line 16, in <module>
  207. app.launch_new_instance()
  208. File "c:anaconda3envstf18libsite-packagestraitletsconfigapplication.py", line 658, in launch_instance
  209. app.start()
  210. File "c:anaconda3envstf18libsite-packagesipykernelkernelapp.py", line 486, in start
  211. self.io_loop.start()
  212. File "c:anaconda3envstf18libsite-packagestornadoplatformasyncio.py", line 127, in start
  213. self.asyncio_loop.run_forever()
  214. File "c:anaconda3envstf18libasynciobase_events.py", line 422, in run_forever
  215. self._run_once()
  216. File "c:anaconda3envstf18libasynciobase_events.py", line 1432, in _run_once
  217. handle._run()
  218. File "c:anaconda3envstf18libasyncioevents.py", line 145, in _run
  219. self._callback(*self._args)
  220. File "c:anaconda3envstf18libsite-packagestornadoplatformasyncio.py", line 117, in _handle_events
  221. handler_func(fileobj, events)
  222. File "c:anaconda3envstf18libsite-packagestornadostack_context.py", line 276, in null_wrapper
  223. return fn(*args, **kwargs)
  224. File "c:anaconda3envstf18libsite-packageszmqeventloopzmqstream.py", line 450, in _handle_events
  225. self._handle_recv()
  226. File "c:anaconda3envstf18libsite-packageszmqeventloopzmqstream.py", line 480, in _handle_recv
  227. self._run_callback(callback, msg)
  228. File "c:anaconda3envstf18libsite-packageszmqeventloopzmqstream.py", line 432, in _run_callback
  229. callback(*args, **kwargs)
  230. File "c:anaconda3envstf18libsite-packagestornadostack_context.py", line 276, in null_wrapper
  231. return fn(*args, **kwargs)
  232. File "c:anaconda3envstf18libsite-packagesipykernelkernelbase.py", line 283, in dispatcher
  233. return self.dispatch_shell(stream, msg)
  234. File "c:anaconda3envstf18libsite-packagesipykernelkernelbase.py", line 233, in dispatch_shell
  235. handler(stream, idents, msg)
  236. File "c:anaconda3envstf18libsite-packagesipykernelkernelbase.py", line 399, in execute_request
  237. user_expressions, allow_stdin)
  238. File "c:anaconda3envstf18libsite-packagesipykernelipkernel.py", line 208, in do_execute
  239. res = shell.run_cell(code, store_history=store_history, silent=silent)
  240. File "c:anaconda3envstf18libsite-packagesipykernelzmqshell.py", line 537, in run_cell
  241. return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
  242. File "c:anaconda3envstf18libsite-packagesIPythoncoreinteractiveshell.py", line 2662, in run_cell
  243. raw_cell, store_history, silent, shell_futures)
  244. File "c:anaconda3envstf18libsite-packagesIPythoncoreinteractiveshell.py", line 2785, in _run_cell
  245. interactivity=interactivity, compiler=compiler, result=result)
  246. File "c:anaconda3envstf18libsite-packagesIPythoncoreinteractiveshell.py", line 2903, in run_ast_nodes
  247. if self.run_code(code, result):
  248. File "c:anaconda3envstf18libsite-packagesIPythoncoreinteractiveshell.py", line 2963, in run_code
  249. exec(code_obj, self.user_global_ns, self.user_ns)
  250. File "<ipython-input-8-bd671b499233>", line 23, in <module>
  251. images, labels = tf.train.shuffle_batch([image, label], batch_size=10, capacity=30, num_threads=1, min_after_dequeue=10)
  252. File "c:anaconda3envstf18libsite-packagestensorflowpythontraininginput.py", line 1300, in shuffle_batch
  253. name=name)
  254. File "c:anaconda3envstf18libsite-packagestensorflowpythontraininginput.py", line 846, in _shuffle_batch
  255. dequeued = queue.dequeue_many(batch_size, name=name)
  256. File "c:anaconda3envstf18libsite-packagestensorflowpythonopsdata_flow_ops.py", line 483, in dequeue_many
  257. self._queue_ref, n=n, component_types=self._dtypes, name=name)
  258. File "c:anaconda3envstf18libsite-packagestensorflowpythonopsgen_data_flow_ops.py", line 3799, in queue_dequeue_many_v2
  259. component_types=component_types, timeout_ms=timeout_ms, name=name)
  260. File "c:anaconda3envstf18libsite-packagestensorflowpythonframeworkop_def_library.py", line 787, in _apply_op_helper
  261. op_def=op_def)
  262. File "c:anaconda3envstf18libsite-packagestensorflowpythonframeworkops.py", line 3392, in create_op
  263. op_def=op_def)
  264. File "c:anaconda3envstf18libsite-packagestensorflowpythonframeworkops.py", line 1718, in __init__
  265. self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
  266.  
  267. OutOfRangeError (see above for traceback): RandomShuffleQueue '_37_shuffle_batch_2/random_shuffle_queue' is closed and has insufficient elements (requested 10, current size 0)
  268. [[Node: shuffle_batch_2 = QueueDequeueManyV2[component_types=[DT_INT64, DT_INT32], timeout_ms=-1, _device="/job:localhost/replica:0/task:0/device:CPU:0"](shuffle_batch_2/random_shuffle_queue, shuffle_batch_2/n)]]
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!