API tools faq
paste
Advertisement
Guest User

Untitled

a guest
Jun 20th, 2019
542
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 14.22 KB | None | 0 0
raw download clone embed print report
  1. history = model.fit(train_images,
  2. train_masks,
  3. batch_size=params['batch_size'],
  4. epochs=params['epochs'],
  5. verbose=1,
  6. shuffle=True,
  7. validation_data = (val_images, val_masks))
  8.  
  9. ResourceExhaustedError Traceback (most recent call last)
  10. /export/share/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
  11. 1326 try:
  12. -> 1327 return fn(*args)
  13. 1328 except errors.OpError as e:
  14.  
  15. /export/share/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _run_fn(session, feed_dict, fetch_list, target_list, options, run_metadata)
  16. 1305 feed_dict, fetch_list, target_list,
  17. -> 1306 status, run_metadata)
  18. 1307
  19.  
  20. /export/share/anaconda3/lib/python3.6/contextlib.py in __exit__(self, type, value, traceback)
  21. 87 try:
  22. ---> 88 next(self.gen)
  23. 89 except StopIteration:
  24.  
  25. /export/share/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/errors_impl.py in raise_exception_on_not_ok_status()
  26. 465 compat.as_text(pywrap_tensorflow.TF_Message(status)),
  27. --> 466 pywrap_tensorflow.TF_GetCode(status))
  28. 467 finally:
  29.  
  30. ResourceExhaustedError: OOM when allocating tensor with shape[2,32,192,192,160]
  31. [[Node: training/Adam/gradients/conv3d_17/convolution_grad/Conv3DBackpropInputV2 = Conv3DBackpropInputV2[T=DT_FLOAT, _class=["loc:@conv3d_17/convolution"], data_format="NDHWC", padding="SAME", strides=[1, 1, 1, 1, 1], _device="/job:localhost/replica:0/task:0/gpu:0"](training/Adam/gradients/conv3d_17/convolution_grad/Shape, conv3d_17/kernel/read, training/Adam/gradients/conv3d_17/add_grad/Reshape)]]
  32.  
  33. During handling of the above exception, another exception occurred:
  34.  
  35. ResourceExhaustedError Traceback (most recent call last)
  36. <ipython-input-9-6fa9876b44a1> in <module>()
  37. ----> 1 train(images[0:params['n_images'],:,:,:], masks[0:params['n_images'],:,:,:,:], images[100:130,:,:,:], masks[100:130,:,:,:,:], params)
  38.  
  39. <ipython-input-7-6118d16d7eba> in train(train_images, train_masks, val_images, val_masks, params)
  40. 78 verbose=1,
  41. 79 shuffle=True,
  42. ---> 80 validation_data = (val_images, val_masks))
  43. 81 #callbacks=[model_checkpoint, earlystopping])
  44. 82 model.save('results/' + params2name(params) + '/weights.h5')
  45.  
  46. /export/share/anaconda3/lib/python3.6/site-packages/keras/engine/training.py in fit(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, **kwargs)
  47. 1648 initial_epoch=initial_epoch,
  48. 1649 steps_per_epoch=steps_per_epoch,
  49. -> 1650 validation_steps=validation_steps)
  50. 1651
  51. 1652 def evaluate(self, x=None, y=None,
  52.  
  53. /export/share/anaconda3/lib/python3.6/site-packages/keras/engine/training.py in _fit_loop(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch, steps_per_epoch, validation_steps)
  54. 1211 batch_logs['size'] = len(batch_ids)
  55. 1212 callbacks.on_batch_begin(batch_index, batch_logs)
  56. -> 1213 outs = f(ins_batch)
  57. 1214 if not isinstance(outs, list):
  58. 1215 outs = [outs]
  59.  
  60. /export/share/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py in __call__(self, inputs)
  61. 2350 session = get_session()
  62. 2351 updated = session.run(fetches=fetches, feed_dict=feed_dict,
  63. -> 2352 **self.session_kwargs)
  64. 2353 return updated[:len(self.outputs)]
  65. 2354
  66.  
  67. /export/share/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in run(self, fetches, feed_dict, options, run_metadata)
  68. 893 try:
  69. 894 result = self._run(None, fetches, feed_dict, options_ptr,
  70. --> 895 run_metadata_ptr)
  71. 896 if run_metadata:
  72. 897 proto_data = tf_session.TF_GetBuffer(run_metadata_ptr)
  73.  
  74. /export/share/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _run(self, handle, fetches, feed_dict, options, run_metadata)
  75. 1122 if final_fetches or final_targets or (handle and feed_dict_tensor):
  76. 1123 results = self._do_run(handle, final_targets, final_fetches,
  77. -> 1124 feed_dict_tensor, options, run_metadata)
  78. 1125 else:
  79. 1126 results = []
  80.  
  81. /export/share/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _do_run(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)
  82. 1319 if handle is None:
  83. 1320 return self._do_call(_run_fn, self._session, feeds, fetches, targets,
  84. -> 1321 options, run_metadata)
  85. 1322 else:
  86. 1323 return self._do_call(_prun_fn, self._session, handle, feeds, fetches)
  87.  
  88. /export/share/anaconda3/lib/python3.6/site-packages/tensorflow/python/client/session.py in _do_call(self, fn, *args)
  89. 1338 except KeyError:
  90. 1339 pass
  91. -> 1340 raise type(e)(node_def, op, message)
  92. 1341
  93. 1342 def _extend_graph(self):
  94.  
  95. ResourceExhaustedError: OOM when allocating tensor with shape[2,32,192,192,160]
  96. [[Node: training/Adam/gradients/conv3d_17/convolution_grad/Conv3DBackpropInputV2 = Conv3DBackpropInputV2[T=DT_FLOAT, _class=["loc:@conv3d_17/convolution"], data_format="NDHWC", padding="SAME", strides=[1, 1, 1, 1, 1], _device="/job:localhost/replica:0/task:0/gpu:0"](training/Adam/gradients/conv3d_17/convolution_grad/Shape, conv3d_17/kernel/read, training/Adam/gradients/conv3d_17/add_grad/Reshape)]]
  97.  
  98. Caused by op 'training/Adam/gradients/conv3d_17/convolution_grad/Conv3DBackpropInputV2', defined at:
  99. File "/export/share/anaconda3/lib/python3.6/runpy.py", line 193, in _run_module_as_main
  100. "__main__", mod_spec)
  101. File "/export/share/anaconda3/lib/python3.6/runpy.py", line 85, in _run_code
  102. exec(code, run_globals)
  103. File "/export/share/anaconda3/lib/python3.6/site-packages/ipykernel_launcher.py", line 16, in <module>
  104. app.launch_new_instance()
  105. File "/export/share/anaconda3/lib/python3.6/site-packages/traitlets/config/application.py", line 658, in launch_instance
  106. app.start()
  107. File "/export/share/anaconda3/lib/python3.6/site-packages/ipykernel/kernelapp.py", line 477, in start
  108. ioloop.IOLoop.instance().start()
  109. File "/export/share/anaconda3/lib/python3.6/site-packages/tornado/ioloop.py", line 888, in start
  110. handler_func(fd_obj, events)
  111. File "/export/share/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
  112. return fn(*args, **kwargs)
  113. File "/export/share/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py", line 450, in _handle_events
  114. self._handle_recv()
  115. File "/export/share/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py", line 480, in _handle_recv
  116. self._run_callback(callback, msg)
  117. File "/export/share/anaconda3/lib/python3.6/site-packages/zmq/eventloop/zmqstream.py", line 432, in _run_callback
  118. callback(*args, **kwargs)
  119. File "/export/share/anaconda3/lib/python3.6/site-packages/tornado/stack_context.py", line 277, in null_wrapper
  120. return fn(*args, **kwargs)
  121. File "/export/share/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 283, in dispatcher
  122. return self.dispatch_shell(stream, msg)
  123. File "/export/share/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 235, in dispatch_shell
  124. handler(stream, idents, msg)
  125. File "/export/share/anaconda3/lib/python3.6/site-packages/ipykernel/kernelbase.py", line 399, in execute_request
  126. user_expressions, allow_stdin)
  127. File "/export/share/anaconda3/lib/python3.6/site-packages/ipykernel/ipkernel.py", line 196, in do_execute
  128. res = shell.run_cell(code, store_history=store_history, silent=silent)
  129. File "/export/share/anaconda3/lib/python3.6/site-packages/ipykernel/zmqshell.py", line 533, in run_cell
  130. return super(ZMQInteractiveShell, self).run_cell(*args, **kwargs)
  131. File "/export/share/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2698, in run_cell
  132. interactivity=interactivity, compiler=compiler, result=result)
  133. File "/export/share/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2808, in run_ast_nodes
  134. if self.run_code(code, result):
  135. File "/export/share/anaconda3/lib/python3.6/site-packages/IPython/core/interactiveshell.py", line 2862, in run_code
  136. exec(code_obj, self.user_global_ns, self.user_ns)
  137. File "<ipython-input-9-6fa9876b44a1>", line 1, in <module>
  138. train(images[0:params['n_images'],:,:,:], masks[0:params['n_images'],:,:,:,:], images[100:130,:,:,:], masks[100:130,:,:,:,:], params)
  139. File "<ipython-input-7-6118d16d7eba>", line 80, in train
  140. validation_data = (val_images, val_masks))
  141. File "/export/share/anaconda3/lib/python3.6/site-packages/keras/engine/training.py", line 1627, in fit
  142. self._make_train_function()
  143. File "/export/share/anaconda3/lib/python3.6/site-packages/keras/engine/training.py", line 990, in _make_train_function
  144. loss=self.total_loss)
  145. File "/export/share/anaconda3/lib/python3.6/site-packages/keras/legacy/interfaces.py", line 87, in wrapper
  146. return func(*args, **kwargs)
  147. File "/export/share/anaconda3/lib/python3.6/site-packages/keras/optimizers.py", line 415, in get_updates
  148. grads = self.get_gradients(loss, params)
  149. File "/export/share/anaconda3/lib/python3.6/site-packages/keras/optimizers.py", line 73, in get_gradients
  150. grads = K.gradients(loss, params)
  151. File "/export/share/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py", line 2389, in gradients
  152. return tf.gradients(loss, variables, colocate_gradients_with_ops=True)
  153. File "/export/share/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py", line 542, in gradients
  154. grad_scope, op, func_call, lambda: grad_fn(op, *out_grads))
  155. File "/export/share/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py", line 348, in _MaybeCompile
  156. return grad_fn() # Exit early
  157. File "/export/share/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gradients_impl.py", line 542, in <lambda>
  158. grad_scope, op, func_call, lambda: grad_fn(op, *out_grads))
  159. File "/export/share/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_grad.py", line 80, in _Conv3DGrad
  160. data_format=data_format),
  161. File "/export/share/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gen_nn_ops.py", line 664, in conv3d_backprop_input_v2
  162. name=name)
  163. File "/export/share/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 767, in apply_op
  164. op_def=op_def)
  165. File "/export/share/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 2630, in create_op
  166. original_op=self._default_original_op, op_def=op_def)
  167. File "/export/share/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1204, in __init__
  168. self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
  169.  
  170. ...which was originally created as op 'conv3d_17/convolution', defined at:
  171. File "/export/share/anaconda3/lib/python3.6/runpy.py", line 193, in _run_module_as_main
  172. "__main__", mod_spec)
  173. [elided 18 identical lines from previous traceback]
  174. File "<ipython-input-9-6fa9876b44a1>", line 1, in <module>
  175. train(images[0:params['n_images'],:,:,:], masks[0:params['n_images'],:,:,:,:], images[100:130,:,:,:], masks[100:130,:,:,:,:], params)
  176. File "<ipython-input-7-6118d16d7eba>", line 54, in train
  177. model = unet(params,imsz)
  178. File "/DATA/jeaneliott/bladderectum/utils.py", line 115, in unet
  179. conv9 = Conv3D(params['n_feat_maps'], (3, 3, 3), activation='relu', padding='same')(up9)
  180. File "/export/share/anaconda3/lib/python3.6/site-packages/keras/engine/topology.py", line 603, in __call__
  181. output = self.call(inputs, **kwargs)
  182. File "/export/share/anaconda3/lib/python3.6/site-packages/keras/layers/convolutional.py", line 172, in call
  183. dilation_rate=self.dilation_rate)
  184. File "/export/share/anaconda3/lib/python3.6/site-packages/keras/backend/tensorflow_backend.py", line 3364, in conv3d
  185. data_format=tf_data_format)
  186. File "/export/share/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py", line 672, in convolution
  187. op=op)
  188. File "/export/share/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py", line 338, in with_space_to_batch
  189. return op(input, num_spatial_dims, padding)
  190. File "/export/share/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py", line 664, in op
  191. name=name)
  192. File "/export/share/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/nn_ops.py", line 146, in _non_atrous_convolution
  193. name=name)
  194. File "/export/share/anaconda3/lib/python3.6/site-packages/tensorflow/python/ops/gen_nn_ops.py", line 524, in conv3d
  195. data_format=data_format, name=name)
  196. File "/export/share/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py", line 767, in apply_op
  197. op_def=op_def)
  198. File "/export/share/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 2630, in create_op
  199. original_op=self._default_original_op, op_def=op_def)
  200. File "/export/share/anaconda3/lib/python3.6/site-packages/tensorflow/python/framework/ops.py", line 1204, in __init__
  201. self._traceback = self._graph._extract_stack() # pylint: disable=protected-access
  202.  
  203. ResourceExhaustedError (see above for traceback): OOM when allocating tensor with shape[2,32,192,192,160]
  204. [[Node: training/Adam/gradients/conv3d_17/convolution_grad/Conv3DBackpropInputV2 = Conv3DBackpropInputV2[T=DT_FLOAT, _class=["loc:@conv3d_17/convolution"], data_format="NDHWC", padding="SAME", strides=[1, 1, 1, 1, 1], _device="/job:localhost/replica:0/task:0/gpu:0"](training/Adam/gradients/conv3d_17/convolution_grad/Shape, conv3d_17/kernel/read, training/Adam/gradients/conv3d_17/add_grad/Reshape)]]
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!