Guest User

Untitled

a guest
Jul 26th, 2016
58
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.66 KB | None | 0 0
  1. lazy val providedDependencies = Seq(
  2. "org.apache.spark" %% "spark-core" % "2.0.0-preview",
  3. "org.apache.spark" %% "spark-sql" % "2.0.0-preview",
  4. "org.elasticsearch" %% "elasticsearch-spark" % "2.3.3"
  5. )
  6.  
  7. libraryDependencies ++= providedDependencies
  8.  
  9. import org.apache.spark.rdd.RDD
  10. import org.apache.spark.sql._
  11. import org.elasticsearch.spark._
  12.  
  13. case class Foo(Id : Int, Material_Id : Int, Quantity : Double)
  14.  
  15. object TestBatch {
  16. def main(args : Array[String]) {
  17.  
  18. val spark = SparkSession.builder.master("local[4]").appName("TestJob-Batch").getOrCreate()
  19. import spark.implicits._
  20.  
  21. val fooRDD = spark.createDataFrame(Seq(Foo(1, 2, 3.0))).rdd
  22.  
  23. val esConfig = Map(
  24. "es.nodes" -> "localhost",
  25. "es.port" -> "9200",
  26. "es.index.auto.create" -> "yes",
  27. "es.mapping.id" -> "id"
  28. )
  29.  
  30. fooRDD.saveToEs(resource = "spark/foo", cfg = esConfig)
  31.  
  32. System.exit(0)
  33. }
  34. }
  35.  
  36. java.lang.NoSuchMethodError: org.apache.spark.TaskContext.addOnCompleteCallback(Lscala/Function0;)V
  37. at org.elasticsearch.spark.rdd.EsRDDWriter.write(EsRDDWriter.scala:42)
  38. at org.elasticsearch.spark.rdd.EsSpark$$anonfun$doSaveToEs$1.apply(EsSpark.scala:84)
  39. at org.elasticsearch.spark.rdd.EsSpark$$anonfun$doSaveToEs$1.apply(EsSpark.scala:84)
  40. at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:70)
  41. at org.apache.spark.scheduler.Task.run(Task.scala:85)
  42. at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:274)
  43. at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
  44. at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
  45. at java.lang.Thread.run(Thread.java:745)
Add Comment
Please, Sign In to add comment