Advertisement
Guest User

Untitled

a guest
Jan 19th, 2017
89
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 6.73 KB | None | 0 0
  1. if(!logMessageJavaRDD.isEmpty()) {
  2. LogMessage max = logMessageJavaRDD.max(ComparatorLogMessage.class.newInstance());
  3. List<LogMessage> tempList = new ArrayList<LogMessage>();
  4. max.convertToHBLogMessage();
  5. tempList.add(max);
  6. JavaRDD<LogMessage> parallelize = ssc.sparkContext().parallelize(tempList);
  7. JavaRDD<LogMessage> union = logMessageJavaRDD.union(parallelize);
  8. return union;
  9. }else{
  10. return logMessageJavaRDD;
  11. }
  12.  
  13. }
  14.  
  15. java.io.NotSerializableException: DStream checkpointing has been enabled but the DStreams with their functions are not serializable
  16. org.apache.spark.streaming.api.java.JavaStreamingContext
  17. Serialization stack:
  18. - object not serializable (class: org.apache.spark.streaming.api.java.JavaStreamingContext, value: org.apache.spark.streaming.api.java.JavaStreamingContext@2ed40452)
  19. - field (class: org.necla.ngla.loganalyzer.stateful.Type9.Type9ViolationChecker$6, name: val$ssc, type: class org.apache.spark.streaming.api.java.JavaStreamingContext)
  20. - object (class org.necla.ngla.loganalyzer.stateful.Type9.Type9ViolationChecker$6, org.necla.ngla.loganalyzer.stateful.Type9.Type9ViolationChecker$6@4f9f32a6)
  21. - field (class: org.apache.spark.streaming.api.java.JavaDStreamLike$$anonfun$transform$1, name: transformFunc$1, type: interface org.apache.spark.api.java.function.Function)
  22. - object (class org.apache.spark.streaming.api.java.JavaDStreamLike$$anonfun$transform$1, <function1>)
  23. - field (class: org.apache.spark.streaming.dstream.DStream$$anonfun$transform$1$$anonfun$apply$21, name: cleanedF$2, type: interface scala.Function1)
  24. - object (class org.apache.spark.streaming.dstream.DStream$$anonfun$transform$1$$anonfun$apply$21, <function2>)
  25. - field (class: org.apache.spark.streaming.dstream.DStream$$anonfun$transform$2$$anonfun$5, name: cleanedF$3, type: interface scala.Function2)
  26. - object (class org.apache.spark.streaming.dstream.DStream$$anonfun$transform$2$$anonfun$5, <function2>)
  27. - field (class: org.apache.spark.streaming.dstream.TransformedDStream, name: transformFunc, type: interface scala.Function2)
  28. - object (class org.apache.spark.streaming.dstream.TransformedDStream, org.apache.spark.streaming.dstream.TransformedDStream@39983b43)
  29. - writeObject data (class: org.apache.spark.streaming.dstream.DStream)
  30. - object (class org.apache.spark.streaming.dstream.ForEachDStream, org.apache.spark.streaming.dstream.ForEachDStream@4645de69)
  31. - element of array (index: 0)
  32. - array (class [Ljava.lang.Object;, size 16)
  33. - field (class: scala.collection.mutable.ArrayBuffer, name: array, type: class [Ljava.lang.Object;)
  34. - object (class scala.collection.mutable.ArrayBuffer, ArrayBuffer(org.apache.spark.streaming.dstream.ForEachDStream@4645de69, org.apache.spark.streaming.dstream.ForEachDStream@6aeebb54, org.apache.spark.streaming.dstream.ForEachDStream@5f4cbea4, org.apache.spark.streaming.dstream.ForEachDStream@277831dd, org.apache.spark.streaming.dstream.ForEachDStream@4411b869))
  35. - writeObject data (class: org.apache.spark.streaming.dstream.DStreamCheckpointData)
  36. - object (class org.apache.spark.streaming.dstream.DStreamCheckpointData, [
  37. 0 checkpoint files
  38.  
  39. ])
  40. - writeObject data (class: org.apache.spark.streaming.dstream.DStream)
  41. - object (class org.apache.spark.streaming.kafka.DirectKafkaInputDStream, org.apache.spark.streaming.kafka.DirectKafkaInputDStream@28b5d9b8)
  42. - element of array (index: 0)
  43. - array (class [Ljava.lang.Object;, size 16)
  44. - field (class: scala.collection.mutable.ArrayBuffer, name: array, type: class [Ljava.lang.Object;)
  45. - object (class scala.collection.mutable.ArrayBuffer, ArrayBuffer(org.apache.spark.streaming.kafka.DirectKafkaInputDStream@28b5d9b8))
  46. - writeObject data (class: org.apache.spark.streaming.DStreamGraph)
  47. - object (class org.apache.spark.streaming.DStreamGraph, org.apache.spark.streaming.DStreamGraph@6a8a4078)
  48. - field (class: org.apache.spark.streaming.Checkpoint, name: graph, type: class org.apache.spark.streaming.DStreamGraph)
  49. - object (class org.apache.spark.streaming.Checkpoint, org.apache.spark.streaming.Checkpoint@4e49e681)
  50.  
  51. at org.apache.spark.streaming.StreamingContext.validate(StreamingContext.scala:574)
  52. at org.apache.spark.streaming.StreamingContext.liftedTree1$1(StreamingContext.scala:618)
  53. at org.apache.spark.streaming.StreamingContext.start(StreamingContext.scala:617)
  54. at org.apache.spark.streaming.api.java.JavaStreamingContext.start(JavaStreamingContext.scala:624)
  55. at org.necla.ngla.loganalyzer.stateful.Type9.Type9ViolationCheckerTest.execute(Type9ViolationCheckerTest.java:97)
  56. at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
  57. at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
  58. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  59. at java.lang.reflect.Method.invoke(Method.java:606)
  60. at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:44)
  61. at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
  62. at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:41)
  63. at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
  64. at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:28)
  65. at com.intellij.junit4.JUnit4TestRunnerUtil$IgnoreIgnoredTestJUnit4ClassRunner.runChild(JUnit4TestRunnerUtil.java:365)
  66. at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:50)
  67. at org.junit.runners.ParentRunner$3.run(ParentRunner.java:193)
  68. at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:52)
  69. at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:191)
  70. at org.junit.runners.ParentRunner.access$000(ParentRunner.java:42)
  71. at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:184)
  72. at org.junit.runners.ParentRunner.run(ParentRunner.java:236)
  73. at org.junit.runner.JUnitCore.run(JUnitCore.java:157)
  74. at com.intellij.junit4.JUnit4IdeaTestRunner.startRunnerWithArgs(JUnit4IdeaTestRunner.java:68)
  75. at com.intellij.rt.execution.junit.IdeaTestRunner$Repeater.startRunnerWithArgs(IdeaTestRunner.java:51)
  76. at com.intellij.rt.execution.junit.JUnitStarter.prepareStreamsAndStart(JUnitStarter.java:237)
  77. at com.intellij.rt.execution.junit.JUnitStarter.main(JUnitStarter.java:70)
  78. at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
  79. at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
  80. at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
  81. at java.lang.reflect.Method.invoke(Method.java:606)
  82. at com.intellij.rt.execution.application.AppMain.main(AppMain.java:147)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement