Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- <pyspark.streaming.dstream.DStream object at 0x7fb069210ed0>
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
- at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
- at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
- at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
- at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
- at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
- at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
- at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
- at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
- at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
- at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
- at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
- at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
- at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
- at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
- at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
- at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/util.py", line 62, in call
- r = self.func(t, *rdds)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/dstream.py", line 159, in <lambda>
- func = lambda t, rdd: old_func(rdd)
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 25, in SaveRecord
- datamap.saveAsNewAPIHadoopDataset(conf=conf,keyConverter=keyConv,valueConverter=valueConv)
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/rdd.py", line 1348, in saveAsNewAPIHadoopDataset
- keyConverter, valueConverter, True)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 538, in __call__
- self.target_id, self.name)
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/protocol.py", line 300, in get_return_value
- format(target_id, '.', name), value)
- Py4JJavaError: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset.
- : java.lang.ClassNotFoundException: org.apache.spark.examples.pythonconverters.StringToImmutableBytesWritableConverter
- at java.net.URLClassLoader.findClass(URLClassLoader.java:381)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
- at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
- at java.lang.Class.forName0(Native Method)
- at java.lang.Class.forName(Class.java:348)
- at org.apache.spark.util.Utils$.classForName(Utils.scala:173)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:46)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1$$anonfun$1.apply(PythonHadoopUtil.scala:45)
- at scala.util.Try$.apply(Try.scala:161)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:45)
- at org.apache.spark.api.python.Converter$$anonfun$getInstance$1.apply(PythonHadoopUtil.scala:44)
- at scala.Option.map(Option.scala:145)
- at org.apache.spark.api.python.Converter$.getInstance(PythonHadoopUtil.scala:44)
- at org.apache.spark.api.python.PythonRDD$.getKeyValueConverters(PythonRDD.scala:691)
- at org.apache.spark.api.python.PythonRDD$.convertRDD(PythonRDD.scala:704)
- at org.apache.spark.api.python.PythonRDD$.saveAsHadoopDataset(PythonRDD.scala:801)
- at org.apache.spark.api.python.PythonRDD.saveAsHadoopDataset(PythonRDD.scala)
- at sun.reflect.GeneratedMethodAccessor11.invoke(Unknown Source)
- at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
- at java.lang.reflect.Method.invoke(Method.java:498)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:231)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:379)
- at py4j.Gateway.invoke(Gateway.java:259)
- at py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:133)
- at py4j.commands.CallCommand.execute(CallCommand.java:79)
- at py4j.GatewayConnection.run(GatewayConnection.java:207)
- at java.lang.Thread.run(Thread.java:745)
- Traceback (most recent call last):
- File "/home/ubuntu/unix_practice/spark-example/sp_json.py", line 39, in <module>
- ssc.awaitTermination() # Wait for the computation to terminate
- File "/usr/local/spark/python/lib/pyspark.zip/pyspark/streaming/context.py", line 247, in awaitTermination
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 536, in __call__
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 364, in send_command
- File "/usr/local/spark/python/lib/py4j-0.8.2.1-src.zip/py4j/java_gateway.py", line 473, in send_command
- File "/usr/lib/python2.7/socket.py", line 430, in readline
- data = recv(1)
- KeyboardInterrupt
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement