API tools faq
paste
Advertisement
Guest User

Apache Spark: error reading file via ftp

a guest
Mar 28th, 2016
395
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 8.79 KB | None | 0 0
raw download clone embed print report
  1. org.apache.spark.sql.catalyst.errors.package$TreeNodeException: execute, tree:
  2. TungstenAggregate(key=[], functions=[(count(1),mode=Final,isDistinct=false)], output=[count#1L])
  3. +- TungstenExchange SinglePartition, None
  4. +- TungstenAggregate(key=[], functions=[(count(1),mode=Partial,isDistinct=false)], output=[count#4L])
  5. +- Project
  6. +- Scan ExistingRDD[_1#0]
  7.  
  8. at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:49)
  9. at org.apache.spark.sql.execution.aggregate.TungstenAggregate.doExecute(TungstenAggregate.scala:80)
  10. at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$5.apply(SparkPlan.scala:132)
  11. at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$5.apply(SparkPlan.scala:130)
  12. at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:150)
  13. at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:130)
  14. at org.apache.spark.sql.execution.SparkPlan.executeCollect(SparkPlan.scala:166)
  15. at org.apache.spark.sql.execution.SparkPlan.executeCollectPublic(SparkPlan.scala:174)
  16. at org.apache.spark.sql.DataFrame$$anonfun$org$apache$spark$sql$DataFrame$$execute$1$1.apply(DataFrame.scala:1538)
  17. at org.apache.spark.sql.DataFrame$$anonfun$org$apache$spark$sql$DataFrame$$execute$1$1.apply(DataFrame.scala:1538)
  18. at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:56)
  19. at org.apache.spark.sql.DataFrame.withNewExecutionId(DataFrame.scala:2125)
  20. at org.apache.spark.sql.DataFrame.org$apache$spark$sql$DataFrame$$execute$1(DataFrame.scala:1537)
  21. at org.apache.spark.sql.DataFrame.org$apache$spark$sql$DataFrame$$collect(DataFrame.scala:1544)
  22. at org.apache.spark.sql.DataFrame$$anonfun$count$1.apply(DataFrame.scala:1554)
  23. at org.apache.spark.sql.DataFrame$$anonfun$count$1.apply(DataFrame.scala:1553)
  24. at org.apache.spark.sql.DataFrame.withCallback(DataFrame.scala:2138)
  25. at org.apache.spark.sql.DataFrame.count(DataFrame.scala:1553)
  26. at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:32)
  27. at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:37)
  28. at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:39)
  29. at $iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:41)
  30. at $iwC$$iwC$$iwC$$iwC.<init>(<console>:43)
  31. at $iwC$$iwC$$iwC.<init>(<console>:45)
  32. at $iwC$$iwC.<init>(<console>:47)
  33. at $iwC.<init>(<console>:49)
  34. at <init>(<console>:51)
  35. at .<init>(<console>:55)
  36. at .<clinit>(<console>)
  37. at .<init>(<console>:7)
  38. at .<clinit>(<console>)
  39. at $print(<console>)
  40. at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
  41. at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
  42. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  43. at java.lang.reflect.Method.invoke(Method.java:498)
  44. at org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:1065)
  45. at org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1346)
  46. at org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:840)
  47. at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:871)
  48. at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:819)
  49. at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:857)
  50. at org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:902)
  51. at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:814)
  52. at org.apache.spark.repl.SparkILoop.processLine$1(SparkILoop.scala:657)
  53. at org.apache.spark.repl.SparkILoop.innerLoop$1(SparkILoop.scala:665)
  54. at org.apache.spark.repl.SparkILoop.org$apache$spark$repl$SparkILoop$$loop(SparkILoop.scala:670)
  55. at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply$mcZ$sp(SparkILoop.scala:997)
  56. at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply(SparkILoop.scala:945)
  57. at org.apache.spark.repl.SparkILoop$$anonfun$org$apache$spark$repl$SparkILoop$$process$1.apply(SparkILoop.scala:945)
  58. at scala.tools.nsc.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:135)
  59. at org.apache.spark.repl.SparkILoop.org$apache$spark$repl$SparkILoop$$process(SparkILoop.scala:945)
  60. at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:1059)
  61. at org.apache.spark.repl.Main$.main(Main.scala:31)
  62. at org.apache.spark.repl.Main.main(Main.scala)
  63. at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
  64. at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
  65. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  66. at java.lang.reflect.Method.invoke(Method.java:498)
  67. at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:731)
  68. at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181)
  69. at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206)
  70. at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121)
  71. at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
  72. Caused by: org.apache.spark.sql.catalyst.errors.package$TreeNodeException: execute, tree:
  73. TungstenExchange SinglePartition, None
  74. +- TungstenAggregate(key=[], functions=[(count(1),mode=Partial,isDistinct=false)], output=[count#4L])
  75. +- Project
  76. +- Scan ExistingRDD[_1#0]
  77.  
  78. at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:49)
  79. at org.apache.spark.sql.execution.Exchange.doExecute(Exchange.scala:247)
  80. at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$5.apply(SparkPlan.scala:132)
  81. at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$5.apply(SparkPlan.scala:130)
  82. at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:150)
  83. at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:130)
  84. at org.apache.spark.sql.execution.aggregate.TungstenAggregate$$anonfun$doExecute$1.apply(TungstenAggregate.scala:86)
  85. at org.apache.spark.sql.execution.aggregate.TungstenAggregate$$anonfun$doExecute$1.apply(TungstenAggregate.scala:80)
  86. at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:48)
  87. ... 63 more
  88. Caused by: org.apache.hadoop.mapred.InvalidInputException: Input path does not exist: ftp://user:pwd@192.168.1.5/brecht-d-m/input.nt
  89. at org.apache.hadoop.mapred.FileInputFormat.singleThreadedListStatus(FileInputFormat.java:285)
  90. at org.apache.hadoop.mapred.FileInputFormat.listStatus(FileInputFormat.java:228)
  91. at org.apache.hadoop.mapred.FileInputFormat.getSplits(FileInputFormat.java:313)
  92. at org.apache.spark.rdd.HadoopRDD.getPartitions(HadoopRDD.scala:199)
  93. at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)
  94. at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)
  95. at scala.Option.getOrElse(Option.scala:120)
  96. at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)
  97. at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
  98. at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)
  99. at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)
  100. at scala.Option.getOrElse(Option.scala:120)
  101. at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)
  102. at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
  103. at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)
  104. at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)
  105. at scala.Option.getOrElse(Option.scala:120)
  106. at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)
  107. at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
  108. at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)
  109. at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)
  110. at scala.Option.getOrElse(Option.scala:120)
  111. at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)
  112. at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
  113. at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)
  114. at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)
  115. at scala.Option.getOrElse(Option.scala:120)
  116. at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)
  117. at org.apache.spark.rdd.MapPartitionsRDD.getPartitions(MapPartitionsRDD.scala:35)
  118. at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:239)
  119. at org.apache.spark.rdd.RDD$$anonfun$partitions$2.apply(RDD.scala:237)
  120. at scala.Option.getOrElse(Option.scala:120)
  121. at org.apache.spark.rdd.RDD.partitions(RDD.scala:237)
  122. at org.apache.spark.ShuffleDependency.<init>(Dependency.scala:91)
  123. at org.apache.spark.sql.execution.Exchange.prepareShuffleDependency(Exchange.scala:220)
  124. at org.apache.spark.sql.execution.Exchange$$anonfun$doExecute$1.apply(Exchange.scala:254)
  125. at org.apache.spark.sql.execution.Exchange$$anonfun$doExecute$1.apply(Exchange.scala:248)
  126. at org.apache.spark.sql.catalyst.errors.package$.attachTree(package.scala:48)
  127. ... 71 more
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!