API tools faq
paste
Guest User

Untitled

a guest
Nov 9th, 2017
140
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 8.56 KB | None | 0 0
raw download clone embed print report
  1. org.apache.spark.SparkException: Job aborted.
  2. at org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply$mcV$sp(FileFormatWriter.scala:147)
  3. at org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply(FileFormatWriter.scala:121)
  4. at org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply(FileFormatWriter.scala:121)
  5. at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:57)
  6. at org.apache.spark.sql.execution.datasources.FileFormatWriter$.write(FileFormatWriter.scala:121)
  7. at org.apache.spark.sql.execution.datasources.InsertIntoHadoopFsRelationCommand.run(InsertIntoHadoopFsRelationCommand.scala:101)
  8. at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult$lzycompute(commands.scala:58)
  9. at org.apache.spark.sql.execution.command.ExecutedCommandExec.sideEffectResult(commands.scala:56)
  10. at org.apache.spark.sql.execution.command.ExecutedCommandExec.doExecute(commands.scala:74)
  11. at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:114)
  12. at org.apache.spark.sql.execution.SparkPlan$$anonfun$execute$1.apply(SparkPlan.scala:114)
  13. at org.apache.spark.sql.execution.SparkPlan$$anonfun$executeQuery$1.apply(SparkPlan.scala:135)
  14. at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
  15. at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:132)
  16. at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:113)
  17. at org.apache.spark.sql.execution.QueryExecution.toRdd$lzycompute(QueryExecution.scala:87)
  18. at org.apache.spark.sql.execution.QueryExecution.toRdd(QueryExecution.scala:87)
  19. at org.apache.spark.sql.execution.datasources.DataSource.write(DataSource.scala:492)
  20. at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:215)
  21. at org.apache.spark.sql.DataFrameWriter.save(DataFrameWriter.scala:198)
  22. at org.apache.spark.sql.DataFrameWriter.csv(DataFrameWriter.scala:579)
  23. ... 46 elided
  24. Caused by: org.apache.spark.SparkException: Job aborted due to stage failure: Task 116 in stage 1.0 failed 1 times, most recent failure: Lost task 116.0 in stage 1.0 (TID 234, localhost, executor driver): org.postgresql.util.PSQLException: The connection attempt failed.
  25. at org.postgresql.core.v3.ConnectionFactoryImpl.openConnectionImpl(ConnectionFactoryImpl.java:275)
  26. at org.postgresql.core.ConnectionFactory.openConnection(ConnectionFactory.java:55)
  27. at org.postgresql.jdbc.PgConnection.<init>(PgConnection.java:219)
  28. at org.postgresql.Driver.makeConnection(Driver.java:407)
  29. at org.postgresql.Driver.connect(Driver.java:275)
  30. at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anonfun$createConnectionFactory$1.apply(JdbcUtils.scala:59)
  31. at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anonfun$createConnectionFactory$1.apply(JdbcUtils.scala:50)
  32. at org.apache.spark.sql.execution.datasources.jdbc.JDBCRDD.compute(JDBCRDD.scala:286)
  33. at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
  34. at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
  35. at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
  36. at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
  37. at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
  38. at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
  39. at org.apache.spark.scheduler.Task.run(Task.scala:99)
  40. at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:282)
  41. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  42. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  43. at java.lang.Thread.run(Thread.java:745)
  44. Caused by: java.net.SocketTimeoutException: connect timed out
  45. at java.net.PlainSocketImpl.socketConnect(Native Method)
  46. at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
  47. at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
  48. at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
  49. at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
  50. at java.net.Socket.connect(Socket.java:589)
  51. at org.postgresql.core.PGStream.<init>(PGStream.java:64)
  52. at org.postgresql.core.v3.ConnectionFactoryImpl.openConnectionImpl(ConnectionFactoryImpl.java:147)
  53. ... 18 more
  54. Driver stacktrace:
  55. at org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1435)
  56. at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1423)
  57. at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1422)
  58. at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)
  59. at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)
  60. at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1422)
  61. at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802)
  62. at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802)
  63. at scala.Option.foreach(Option.scala:257)
  64. at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:802)
  65. at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1650)
  66. at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1605)
  67. at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1594)
  68. at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)
  69. at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:628)
  70. at org.apache.spark.SparkContext.runJob(SparkContext.scala:1918)
  71. at org.apache.spark.SparkContext.runJob(SparkContext.scala:1931)
  72. at org.apache.spark.SparkContext.runJob(SparkContext.scala:1951)
  73. at org.apache.spark.sql.execution.datasources.FileFormatWriter$$anonfun$write$1.apply$mcV$sp(FileFormatWriter.scala:127)
  74. ... 66 more
  75. Caused by: org.postgresql.util.PSQLException: The connection attempt failed.
  76. at org.postgresql.core.v3.ConnectionFactoryImpl.openConnectionImpl(ConnectionFactoryImpl.java:275)
  77. at org.postgresql.core.ConnectionFactory.openConnection(ConnectionFactory.java:55)
  78. at org.postgresql.jdbc.PgConnection.<init>(PgConnection.java:219)
  79. at org.postgresql.Driver.makeConnection(Driver.java:407)
  80. at org.postgresql.Driver.connect(Driver.java:275)
  81. at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anonfun$createConnectionFactory$1.apply(JdbcUtils.scala:59)
  82. at org.apache.spark.sql.execution.datasources.jdbc.JdbcUtils$$anonfun$createConnectionFactory$1.apply(JdbcUtils.scala:50)
  83. at org.apache.spark.sql.execution.datasources.jdbc.JDBCRDD.compute(JDBCRDD.scala:286)
  84. at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
  85. at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
  86. at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38)
  87. at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)
  88. at org.apache.spark.rdd.RDD.iterator(RDD.scala:287)
  89. at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)
  90. at org.apache.spark.scheduler.Task.run(Task.scala:99)
  91. at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:282)
  92. ... 3 more
  93. Caused by: java.net.SocketTimeoutException: connect timed out
  94. at java.net.PlainSocketImpl.socketConnect(Native Method)
  95. at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
  96. at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
  97. at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
  98. at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
  99. at java.net.Socket.connect(Socket.java:589)
  100. at org.postgresql.core.PGStream.<init>(PGStream.java:64)
  101. at org.postgresql.core.v3.ConnectionFactoryImpl.openConnectionImpl(ConnectionFactoryImpl.java:147)
  102. ... 18 more
  103.  
  104. import java.sql.DriverManager
  105. import java.util.Properties
  106.  
  107. val jdbcUrl = s"jdbc:postgresql://XXXX:5432/XXXX"
  108.  
  109. val connectionProperties = new Properties()
  110. connectionProperties.put("user", "XXXX")
  111. connectionProperties.put("password", "XXXX")
  112. connectionProperties.put("partitionColumn", "channel")
  113. connectionProperties.put("lowerBound", "0")
  114. connectionProperties.put("upperBound", "1000")
  115. connectionProperties.put("numPartitions", "200")
  116.  
  117. val pushdown_query = "(select * from media) media_alias"
  118. val mediaOccDF = spark.read.jdbc(jdbcUrl, pushdown_query, connectionProperties).cache
  119.  
  120. println(mediaOccDF.count())
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!