API tools faq
paste
Guest User

full_tb_pyspark

a guest
Jul 4th, 2019
252
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 10.51 KB | None | 0 0
raw download clone embed print report
  1. E py4j.protocol.Py4JJavaError: An error occurred while calling o95.showString.
  2. E : org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 0.0 failed 1 times, most recent failure: Lost task 0.0 in stage 0.0 (TID 0, localhost, executor driver): java.lang.UnsupportedOperationException
  3. E at org.apache.spark.sql.execution.datasources.parquet.VectorizedColumnReader.decodeDictionaryIds(VectorizedColumnReader.java:324)
  4. E at org.apache.spark.sql.execution.datasources.parquet.VectorizedColumnReader.readBatch(VectorizedColumnReader.java:185)
  5. E at org.apache.spark.sql.execution.datasources.parquet.VectorizedParquetRecordReader.nextBatch(VectorizedParquetRecordReader.java:263)
  6. E at org.apache.spark.sql.execution.datasources.parquet.VectorizedParquetRecordReader.nextKeyValue(VectorizedParquetRecordReader.java:161)
  7. E at org.apache.spark.sql.execution.datasources.RecordReaderIterator.hasNext(RecordReaderIterator.scala:39)
  8. E at org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.hasNext(FileScanRDD.scala:106)
  9. E at org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.nextIterator(FileScanRDD.scala:182)
  10. E at org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.hasNext(FileScanRDD.scala:106)
  11. E at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.scan_nextBatch$(Unknown Source)
  12. E at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
  13. E at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
  14. E at org.apache.spark.sql.execution.WholeStageCodegenExec$$anonfun$10$$anon$1.hasNext(WholeStageCodegenExec.scala:614)
  15. E at org.apache.spark.sql.execution.SparkPlan$$anonfun$2.apply(SparkPlan.scala:253)
  16. E at org.apache.spark.sql.execution.SparkPlan$$anonfun$2.apply(SparkPlan.scala:247)
  17. E at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$25.apply(RDD.scala:830)
  18. E at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$25.apply(RDD.scala:830)
  19. E at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
  20. E at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)
  21. E at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)
  22. E at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
  23. E at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)
  24. E at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)
  25. E at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
  26. E at org.apache.spark.scheduler.Task.run(Task.scala:109)
  27. E at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)
  28. E at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  29. E at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  30. E at java.lang.Thread.run(Thread.java:748)
  31. E
  32. E Driver stacktrace:
  33. E at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1599)
  34. E at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1587)
  35. E at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1586)
  36. E at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
  37. E at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
  38. E at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1586)
  39. E at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:831)
  40. E at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:831)
  41. E at scala.Option.foreach(Option.scala:257)
  42. E at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:831)
  43. E at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1820)
  44. E at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1769)
  45. E at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1758)
  46. E at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)
  47. E at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:642)
  48. E at org.apache.spark.SparkContext.runJob(SparkContext.scala:2027)
  49. E at org.apache.spark.SparkContext.runJob(SparkContext.scala:2048)
  50. E at org.apache.spark.SparkContext.runJob(SparkContext.scala:2067)
  51. E at org.apache.spark.sql.execution.SparkPlan.executeTake(SparkPlan.scala:363)
  52. E at org.apache.spark.sql.execution.CollectLimitExec.executeCollect(limit.scala:38)
  53. E at org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$collectFromPlan(Dataset.scala:3272)
  54. E at org.apache.spark.sql.Dataset$$anonfun$head$1.apply(Dataset.scala:2484)
  55. E at org.apache.spark.sql.Dataset$$anonfun$head$1.apply(Dataset.scala:2484)
  56. E at org.apache.spark.sql.Dataset$$anonfun$52.apply(Dataset.scala:3253)
  57. E at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:77)
  58. E at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3252)
  59. E at org.apache.spark.sql.Dataset.head(Dataset.scala:2484)
  60. E at org.apache.spark.sql.Dataset.take(Dataset.scala:2698)
  61. E at org.apache.spark.sql.Dataset.showString(Dataset.scala:254)
  62. E at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
  63. E at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
  64. E at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  65. E at java.lang.reflect.Method.invoke(Method.java:498)
  66. E at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)
  67. E at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
  68. E at py4j.Gateway.invoke(Gateway.java:282)
  69. E at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)
  70. E at py4j.commands.CallCommand.execute(CallCommand.java:79)
  71. E at py4j.GatewayConnection.run(GatewayConnection.java:214)
  72. E at java.lang.Thread.run(Thread.java:748)
  73. E Caused by: java.lang.UnsupportedOperationException
  74. E at org.apache.spark.sql.execution.datasources.parquet.VectorizedColumnReader.decodeDictionaryIds(VectorizedColumnReader.java:324)
  75. E at org.apache.spark.sql.execution.datasources.parquet.VectorizedColumnReader.readBatch(VectorizedColumnReader.java:185)
  76. E at org.apache.spark.sql.execution.datasources.parquet.VectorizedParquetRecordReader.nextBatch(VectorizedParquetRecordReader.java:263)
  77. E at org.apache.spark.sql.execution.datasources.parquet.VectorizedParquetRecordReader.nextKeyValue(VectorizedParquetRecordReader.java:161)
  78. E at org.apache.spark.sql.execution.datasources.RecordReaderIterator.hasNext(RecordReaderIterator.scala:39)
  79. E at org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.hasNext(FileScanRDD.scala:106)
  80. E at org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.nextIterator(FileScanRDD.scala:182)
  81. E at org.apache.spark.sql.execution.datasources.FileScanRDD$$anon$1.hasNext(FileScanRDD.scala:106)
  82. E at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.scan_nextBatch$(Unknown Source)
  83. E at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source)
  84. E at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43)
  85. E at org.apache.spark.sql.execution.WholeStageCodegenExec$$anonfun$10$$anon$1.hasNext(WholeStageCodegenExec.scala:614)
  86. E at org.apache.spark.sql.execution.SparkPlan$$anonfun$2.apply(SparkPlan.scala:253)
  87. E at org.apache.spark.sql.execution.SparkPlan$$anonfun$2.apply(SparkPlan.scala:247)
  88. E at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$25.apply(RDD.scala:830)
  89. E at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$25.apply(RDD.scala:830)
  90. E at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
  91. E at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)
  92. E at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)
  93. E at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
  94. E at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324)
  95. E at org.apache.spark.rdd.RDD.iterator(RDD.scala:288)
  96. E at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
  97. E at org.apache.spark.scheduler.Task.run(Task.scala:109)
  98. E at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345)
  99. E at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
  100. E at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
  101. E ... 1 more
  102.  
  103. /root/.local/share/virtualenvs/code-_Py8Si6I/lib/python3.5/site-packages/py4j/protocol.py:320: Py4JJavaError
Advertisement
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!