Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- ---------------------------------------------------------------------------
- Py4JJavaError Traceback (most recent call last)
- <ipython-input-1-d8306c542318> in <module>()
- 8 #We can create a SparkConf() object and use it to initialize the spark context
- 9 conf = SparkConf().setAppName("Collinear Points").setMaster("local[4]") #Initialize spark context using 4 local cores as workers
- ---> 10 sc = SparkContext(conf=conf)
- 11
- 12 from pyspark.rdd import RDD
- /usr/local/lib/python3.6/site-packages/pyspark/context.py in __init__(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, gateway, jsc, profiler_cls)
- 116 try:
- 117 self._do_init(master, appName, sparkHome, pyFiles, environment, batchSize, serializer,
- --> 118 conf, jsc, profiler_cls)
- 119 except:
- 120 # If an error occurs, clean up in order to allow future SparkContext creation:
- /usr/local/lib/python3.6/site-packages/pyspark/context.py in _do_init(self, master, appName, sparkHome, pyFiles, environment, batchSize, serializer, conf, jsc, profiler_cls)
- 178
- 179 # Create the Java SparkContext through Py4J
- --> 180 self._jsc = jsc or self._initialize_context(self._conf._jconf)
- 181 # Reset the SparkConf to the one actually used by the SparkContext in JVM.
- 182 self._conf = SparkConf(_jconf=self._jsc.sc().conf())
- /usr/local/lib/python3.6/site-packages/pyspark/context.py in _initialize_context(self, jconf)
- 268 Initialize SparkContext in function to allow subclass specific initialization
- 269 """
- --> 270 return self._jvm.JavaSparkContext(jconf)
- 271
- 272 @classmethod
- /usr/local/lib/python3.6/site-packages/py4j/java_gateway.py in __call__(self, *args)
- 1426 answer = self._gateway_client.send_command(command)
- 1427 return_value = get_return_value(
- -> 1428 answer, self._gateway_client, None, self._fqn)
- 1429
- 1430 for temp_arg in temp_args:
- /usr/local/lib/python3.6/site-packages/py4j/protocol.py in get_return_value(answer, gateway_client, target_id, name)
- 318 raise Py4JJavaError(
- 319 "An error occurred while calling {0}{1}{2}.\n".
- --> 320 format(target_id, ".", name), value)
- 321 else:
- 322 raise Py4JError(
- Py4JJavaError: An error occurred while calling None.org.apache.spark.api.java.JavaSparkContext.
- : java.lang.OutOfMemoryError: unable to create new native thread
- at java.lang.Thread.start0(Native Method)
- at java.lang.Thread.start(Thread.java:717)
- at org.apache.spark.storage.BlockManager$RemoteBlockTempFileManager.<init>(BlockManager.scala:1611)
- at org.apache.spark.storage.BlockManager.<init>(BlockManager.scala:213)
- at org.apache.spark.SparkEnv$.create(SparkEnv.scala:349)
- at org.apache.spark.SparkEnv$.createDriverEnv(SparkEnv.scala:175)
- at org.apache.spark.SparkContext.createSparkEnv(SparkContext.scala:256)
- at org.apache.spark.SparkContext.<init>(SparkContext.scala:423)
- at org.apache.spark.api.java.JavaSparkContext.<init>(JavaSparkContext.scala:58)
- at sun.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method)
- at sun.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:62)
- at sun.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45)
- at java.lang.reflect.Constructor.newInstance(Constructor.java:423)
- at py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:247)
- at py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)
- at py4j.Gateway.invoke(Gateway.java:238)
- at py4j.commands.ConstructorCommand.invokeConstructor(ConstructorCommand.java:80)
- at py4j.commands.ConstructorCommand.execute(ConstructorCommand.java:69)
- at py4j.GatewayConnection.run(GatewayConnection.java:214)
- at java.lang.Thread.run(Thread.java:748)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement