Advertisement
Guest User

Untitled

a guest
Mar 26th, 2019
91
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 4.03 KB | None | 0 0
  1. ---------------------------------------------------------------------------
  2. Py4JJavaError Traceback (most recent call last)
  3. <ipython-input-1-d8306c542318> in <module>()
  4. 8 #We can create a SparkConf() object and use it to initialize the spark context
  5. 9 conf = SparkConf().setAppName("Collinear Points").setMaster("local[4]") #Initialize spark context using 4 local cores as workers
  6. ---> 10 sc = SparkContext(conf=conf)
  7. 11
  8. 12 from pyspark.rdd import RDD
  9. /usr/local/lib/python3.6/site-packages/pyspark/context.py in __init__(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, gateway, jsc, profiler_cls)
  10. 116 try:
  11. 117 self._do_init(master, appName, sparkHome, pyFiles, environment, batchSize, serializer,
  12. --> 118 conf, jsc, profiler_cls)
  13. 119 except:
  14. 120 # If an error occurs, clean up in order to allow future SparkContext creation:
  15. /usr/local/lib/python3.6/site-packages/pyspark/context.py in _do_init(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, jsc, profiler_cls)
  16. 178
  17. 179 # Create the Java SparkContext through Py4J
  18. --> 180 self._jsc = jsc or self._initialize_context(self._conf._jconf)
  19. 181 # Reset the SparkConf to the one actually used by the SparkContext in JVM.
  20. 182 self._conf = SparkConf(_jconf=self._jsc.sc().conf())
  21. /usr/local/lib/python3.6/site-packages/pyspark/context.py in _initialize_context(self, jconf)
  22. 268 Initialize SparkContext in function to allow subclass specific initialization
  23. 269 """
  24. --> 270 return self._jvm.JavaSparkContext(jconf)
  25. 271
  26. 272 @classmethod
  27. /usr/local/lib/python3.6/site-packages/py4j/java_gateway.py in __call__(self, *args)
  28. 1426 answer = self._gateway_client.send_command(command)
  29. 1427 return_value = get_return_value(
  30. -> 1428 answer, self._gateway_client, None, self._fqn)
  31. 1429
  32. 1430 for temp_arg in temp_args:
  33. /usr/local/lib/python3.6/site-packages/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name)
  34. 318 raise Py4JJavaError(
  35. 319 "An error occurred while calling {0}{1}{2}.\n".
  36. --> 320 format(target_id, ".", name), value)
  37. 321 else:
  38. 322 raise Py4JError(
  39. Py4JJavaError: An error occurred while calling None.org.apache.spark.api.java.JavaSparkContext.
  40. : java.lang.OutOfMemoryError: unable to create new native thread
  41. at java.lang.Thread.start0(Native Method)
  42. at java.lang.Thread.start(Thread.java:717)
  43. at org.apache.spark.storage.BlockManager$RemoteBlockTempFileManager.<init>(BlockManager.scala:1611)
  44. at org.apache.spark.storage.BlockManager.<init>(BlockManager.scala:213)
  45. at org.apache.spark.SparkEnv$.create(SparkEnv.scala:349)
  46. at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:175)
  47. at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:256)
  48. at org.apache.spark.SparkContext.<init>(SparkContext.scala:423)
  49. at org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58)
  50. at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
  51. at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
  52. at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
  53. at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
  54. at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:247)
  55. at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
  56. at py4j.Gateway.invoke(Gateway.java:238)
  57. at py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:80)
  58. at py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:69)
  59. at py4j.GatewayConnection.run(GatewayConnection.java:214)
  60. at java.lang.Thread.run(Thread.java:748)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement