API tools faq
paste
Advertisement
Guest User

Untitled

a guest
May 6th, 2018
248
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 12.87 KB | None | 0 0
raw download clone embed print report
  1. Traceback (most recent call last):
  2. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/site-packages/pyspark/cloudpickle.py", line 148, in dump
  3. return Pickler.dump(self, obj)
  4. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/pickle.py", line 409, in dump
  5. self.save(obj)
  6. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/pickle.py", line 476, in save
  7. f(self, obj) # Call unbound method with explicit self
  8. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/pickle.py", line 751, in save_tuple
  9. save(element)
  10. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/pickle.py", line 476, in save
  11. f(self, obj) # Call unbound method with explicit self
  12. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/site-packages/pyspark/cloudpickle.py", line 255, in save_function
  13. self.save_function_tuple(obj)
  14. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/site-packages/pyspark/cloudpickle.py", line 292, in save_function_tuple
  15. save((code, closure, base_globals))
  16. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/pickle.py", line 476, in save
  17. f(self, obj) # Call unbound method with explicit self
  18. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/pickle.py", line 736, in save_tuple
  19. save(element)
  20. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/pickle.py", line 476, in save
  21. f(self, obj) # Call unbound method with explicit self
  22. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/pickle.py", line 781, in save_list
  23. self._batch_appends(obj)
  24. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/pickle.py", line 805, in _batch_appends
  25. save(x)
  26. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/pickle.py", line 476, in save
  27. f(self, obj) # Call unbound method with explicit self
  28. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/site-packages/pyspark/cloudpickle.py", line 255, in save_function
  29. self.save_function_tuple(obj)
  30. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/site-packages/pyspark/cloudpickle.py", line 292, in save_function_tuple
  31. save((code, closure, base_globals))
  32. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/pickle.py", line 476, in save
  33. f(self, obj) # Call unbound method with explicit self
  34. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/pickle.py", line 736, in save_tuple
  35. save(element)
  36. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/pickle.py", line 476, in save
  37. f(self, obj) # Call unbound method with explicit self
  38. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/pickle.py", line 781, in save_list
  39. self._batch_appends(obj)
  40. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/pickle.py", line 808, in _batch_appends
  41. save(tmp[0])
  42. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/pickle.py", line 476, in save
  43. f(self, obj) # Call unbound method with explicit self
  44. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/site-packages/pyspark/cloudpickle.py", line 249, in save_function
  45. self.save_function_tuple(obj)
  46. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/site-packages/pyspark/cloudpickle.py", line 297, in save_function_tuple
  47. save(f_globals)
  48. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/pickle.py", line 476, in save
  49. f(self, obj) # Call unbound method with explicit self
  50. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/pickle.py", line 821, in save_dict
  51. self._batch_setitems(obj.items())
  52. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/pickle.py", line 852, in _batch_setitems
  53. save(v)
  54. File "/home/ubuntu/anaconda3/envs/insight36k/lib/python3.6/pickle.py", line 496, in save
  55. rv = reduce(self.proto)
  56. TypeError: can't pickle psycopg2.extensions.cursor objects
  57. ---------------------------------------------------------------------------
  58. TypeError Traceback (most recent call last)
  59. ~/anaconda3/envs/insight36k/lib/python3.6/site-packages/pyspark/cloudpickle.py in dump(self, obj)
  60. 147 try:
  61. --> 148 return Pickler.dump(self, obj)
  62. 149 except RuntimeError as e:
  63.  
  64. ~/anaconda3/envs/insight36k/lib/python3.6/pickle.py in dump(self, obj)
  65. 408 self.framer.start_framing()
  66. --> 409 self.save(obj)
  67. 410 self.write(STOP)
  68.  
  69. ~/anaconda3/envs/insight36k/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
  70. 475 if f is not None:
  71. --> 476 f(self, obj) # Call unbound method with explicit self
  72. 477 return
  73.  
  74. ~/anaconda3/envs/insight36k/lib/python3.6/pickle.py in save_tuple(self, obj)
  75. 750 for element in obj:
  76. --> 751 save(element)
  77. 752
  78.  
  79. ~/anaconda3/envs/insight36k/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
  80. 475 if f is not None:
  81. --> 476 f(self, obj) # Call unbound method with explicit self
  82. 477 return
  83.  
  84. ~/anaconda3/envs/insight36k/lib/python3.6/site-packages/pyspark/cloudpickle.py in save_function(self, obj, name)
  85. 254 if klass is None or klass is not obj:
  86. --> 255 self.save_function_tuple(obj)
  87. 256 return
  88.  
  89. ~/anaconda3/envs/insight36k/lib/python3.6/site-packages/pyspark/cloudpickle.py in save_function_tuple(self, func)
  90. 291 save(_make_skel_func)
  91. --> 292 save((code, closure, base_globals))
  92. 293 write(pickle.REDUCE)
  93.  
  94. ~/anaconda3/envs/insight36k/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
  95. 475 if f is not None:
  96. --> 476 f(self, obj) # Call unbound method with explicit self
  97. 477 return
  98.  
  99. ~/anaconda3/envs/insight36k/lib/python3.6/pickle.py in save_tuple(self, obj)
  100. 735 for element in obj:
  101. --> 736 save(element)
  102. 737 # Subtle. Same as in the big comment below.
  103.  
  104. ~/anaconda3/envs/insight36k/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
  105. 475 if f is not None:
  106. --> 476 f(self, obj) # Call unbound method with explicit self
  107. 477 return
  108.  
  109. ~/anaconda3/envs/insight36k/lib/python3.6/pickle.py in save_list(self, obj)
  110. 780 self.memoize(obj)
  111. --> 781 self._batch_appends(obj)
  112. 782
  113.  
  114. ~/anaconda3/envs/insight36k/lib/python3.6/pickle.py in _batch_appends(self, items)
  115. 804 for x in tmp:
  116. --> 805 save(x)
  117. 806 write(APPENDS)
  118.  
  119. ~/anaconda3/envs/insight36k/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
  120. 475 if f is not None:
  121. --> 476 f(self, obj) # Call unbound method with explicit self
  122. 477 return
  123.  
  124. ~/anaconda3/envs/insight36k/lib/python3.6/site-packages/pyspark/cloudpickle.py in save_function(self, obj, name)
  125. 254 if klass is None or klass is not obj:
  126. --> 255 self.save_function_tuple(obj)
  127. 256 return
  128.  
  129. ~/anaconda3/envs/insight36k/lib/python3.6/site-packages/pyspark/cloudpickle.py in save_function_tuple(self, func)
  130. 291 save(_make_skel_func)
  131. --> 292 save((code, closure, base_globals))
  132. 293 write(pickle.REDUCE)
  133.  
  134. ~/anaconda3/envs/insight36k/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
  135. 475 if f is not None:
  136. --> 476 f(self, obj) # Call unbound method with explicit self
  137. 477 return
  138.  
  139. ~/anaconda3/envs/insight36k/lib/python3.6/pickle.py in save_tuple(self, obj)
  140. 735 for element in obj:
  141. --> 736 save(element)
  142. 737 # Subtle. Same as in the big comment below.
  143.  
  144. ~/anaconda3/envs/insight36k/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
  145. 475 if f is not None:
  146. --> 476 f(self, obj) # Call unbound method with explicit self
  147. 477 return
  148.  
  149. ~/anaconda3/envs/insight36k/lib/python3.6/pickle.py in save_list(self, obj)
  150. 780 self.memoize(obj)
  151. --> 781 self._batch_appends(obj)
  152. 782
  153.  
  154. ~/anaconda3/envs/insight36k/lib/python3.6/pickle.py in _batch_appends(self, items)
  155. 807 elif n:
  156. --> 808 save(tmp[0])
  157. 809 write(APPEND)
  158.  
  159. ~/anaconda3/envs/insight36k/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
  160. 475 if f is not None:
  161. --> 476 f(self, obj) # Call unbound method with explicit self
  162. 477 return
  163.  
  164. ~/anaconda3/envs/insight36k/lib/python3.6/site-packages/pyspark/cloudpickle.py in save_function(self, obj, name)
  165. 248 #print("save global", islambda(obj), obj.__code__.co_filename, modname, themodule)
  166. --> 249 self.save_function_tuple(obj)
  167. 250 return
  168.  
  169. ~/anaconda3/envs/insight36k/lib/python3.6/site-packages/pyspark/cloudpickle.py in save_function_tuple(self, func)
  170. 296 # save the rest of the func data needed by _fill_function
  171. --> 297 save(f_globals)
  172. 298 save(defaults)
  173.  
  174. ~/anaconda3/envs/insight36k/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
  175. 475 if f is not None:
  176. --> 476 f(self, obj) # Call unbound method with explicit self
  177. 477 return
  178.  
  179. ~/anaconda3/envs/insight36k/lib/python3.6/pickle.py in save_dict(self, obj)
  180. 820 self.memoize(obj)
  181. --> 821 self._batch_setitems(obj.items())
  182. 822
  183.  
  184. ~/anaconda3/envs/insight36k/lib/python3.6/pickle.py in _batch_setitems(self, items)
  185. 851 save(k)
  186. --> 852 save(v)
  187. 853 write(SETITEM)
  188.  
  189. ~/anaconda3/envs/insight36k/lib/python3.6/pickle.py in save(self, obj, save_persistent_id)
  190. 495 if reduce is not None:
  191. --> 496 rv = reduce(self.proto)
  192. 497 else:
  193.  
  194. TypeError: can't pickle psycopg2.extensions.cursor objects
  195.  
  196. During handling of the above exception, another exception occurred:
  197.  
  198. PicklingError Traceback (most recent call last)
  199. <ipython-input-43-6fc5bd3e1447> in <module>()
  200. 16 x = xml_schema_rdd.map(update_patient_info)
  201. 17 # Note: commit has to come after rdd.map
  202. ---> 18 x.collect()
  203. 19 conn.commit()
  204. 20 cursor.close()
  205.  
  206. ~/anaconda3/envs/insight36k/lib/python3.6/site-packages/pyspark/rdd.py in collect(self)
  207. 807 """
  208. 808 with SCCallSiteSync(self.context) as css:
  209. --> 809 port = self.ctx._jvm.PythonRDD.collectAndServe(self._jrdd.rdd())
  210. 810 return list(_load_from_socket(port, self._jrdd_deserializer))
  211. 811
  212.  
  213. ~/anaconda3/envs/insight36k/lib/python3.6/site-packages/pyspark/rdd.py in _jrdd(self)
  214. 2453
  215. 2454 wrapped_func = _wrap_function(self.ctx, self.func, self._prev_jrdd_deserializer,
  216. -> 2455 self._jrdd_deserializer, profiler)
  217. 2456 python_rdd = self.ctx._jvm.PythonRDD(self._prev_jrdd.rdd(), wrapped_func,
  218. 2457 self.preservesPartitioning)
  219.  
  220. ~/anaconda3/envs/insight36k/lib/python3.6/site-packages/pyspark/rdd.py in _wrap_function(sc, func, deserializer, serializer, profiler)
  221. 2386 assert serializer, "serializer should not be empty"
  222. 2387 command = (func, profiler, deserializer, serializer)
  223. -> 2388 pickled_command, broadcast_vars, env, includes = _prepare_for_python_RDD(sc, command)
  224. 2389 return sc._jvm.PythonFunction(bytearray(pickled_command), env, includes, sc.pythonExec,
  225. 2390 sc.pythonVer, broadcast_vars, sc._javaAccumulator)
  226.  
  227. ~/anaconda3/envs/insight36k/lib/python3.6/site-packages/pyspark/rdd.py in _prepare_for_python_RDD(sc, command)
  228. 2372 # the serialized command will be compressed by broadcast
  229. 2373 ser = CloudPickleSerializer()
  230. -> 2374 pickled_command = ser.dumps(command)
  231. 2375 if len(pickled_command) > (1 << 20): # 1M
  232. 2376 # The broadcast will have same life cycle as created PythonRDD
  233.  
  234. ~/anaconda3/envs/insight36k/lib/python3.6/site-packages/pyspark/serializers.py in dumps(self, obj)
  235. 458
  236. 459 def dumps(self, obj):
  237. --> 460 return cloudpickle.dumps(obj, 2)
  238. 461
  239. 462
  240.  
  241. ~/anaconda3/envs/insight36k/lib/python3.6/site-packages/pyspark/cloudpickle.py in dumps(obj, protocol)
  242. 702
  243. 703 cp = CloudPickler(file,protocol)
  244. --> 704 cp.dump(obj)
  245. 705
  246. 706 return file.getvalue()
  247.  
  248. ~/anaconda3/envs/insight36k/lib/python3.6/site-packages/pyspark/cloudpickle.py in dump(self, obj)
  249. 160 msg = "Could not serialize object: %s: %s" % (e.__class__.__name__, emsg)
  250. 161 print_exec(sys.stderr)
  251. --> 162 raise pickle.PicklingError(msg)
  252. 163
  253. 164 def save_memoryview(self, obj):
  254.  
  255. PicklingError: Could not serialize object: TypeError: can't pickle psycopg2.extensions.cursor objects
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!