API tools faq
paste
Guest User

SparkPi submit log

a guest
May 2nd, 2020
206
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Bash 6.84 KB | None | 0 0
raw download clone embed print report
  1. vagrant@kmaster:~/Documents/monarch_repos/spark-2.4.5-bin-hadoop2.7$ bin/spark-submit --master k8s://<ip>:<port> --deploy-mode cluster --name spark-pi --class org.apache.spark.examples.SparkPi --conf spark.executor.instances=2 --conf spark.kubernetes.container.image=kmaster:5000/spark:latest --conf spark.kubernetes.container.image.pullPolicy=IfNotPresent --conf spark.kubernetes.authenticate.driver.serviceAccountName=spark local:///opt/spark/examples/jars/spark-examples_2.11-2.4.5.jar 100
  2. log4j:WARN No appenders could be found for logger (io.fabric8.kubernetes.client.Config).
  3. log4j:WARN Please initialize the log4j system properly.
  4. log4j:WARN See http://logging.apache.org/log4j/1.2/faq.html#noconfig for more info.
  5. Using Spark's default log4j profile: org/apache/spark/log4j-defaults.properties
  6. Exception in thread "main" io.fabric8.kubernetes.client.KubernetesClientException: Operation: [create]  for kind: [Pod]  with name: [null]  in namespace: [default]  failed.
  7.     at io.fabric8.kubernetes.client.KubernetesClientException.launderThrowable(KubernetesClientException.java:64)
  8.     at io.fabric8.kubernetes.client.KubernetesClientException.launderThrowable(KubernetesClientException.java:72)
  9.     at io.fabric8.kubernetes.client.dsl.base.BaseOperation.create(BaseOperation.java:337)
  10.     at io.fabric8.kubernetes.client.dsl.base.BaseOperation.create(BaseOperation.java:330)
  11.     at org.apache.spark.deploy.k8s.submit.Client$$anonfun$run$2.apply(KubernetesClientApplication.scala:141)
  12.     at org.apache.spark.deploy.k8s.submit.Client$$anonfun$run$2.apply(KubernetesClientApplication.scala:140)
  13.     at org.apache.spark.util.Utils$.tryWithResource(Utils.scala:2543)
  14.     at org.apache.spark.deploy.k8s.submit.Client.run(KubernetesClientApplication.scala:140)
  15.     at org.apache.spark.deploy.k8s.submit.KubernetesClientApplication$$anonfun$run$5.apply(KubernetesClientApplication.scala:250)
  16.     at org.apache.spark.deploy.k8s.submit.KubernetesClientApplication$$anonfun$run$5.apply(KubernetesClientApplication.scala:241)
  17.     at org.apache.spark.util.Utils$.tryWithResource(Utils.scala:2543)
  18.     at org.apache.spark.deploy.k8s.submit.KubernetesClientApplication.run(KubernetesClientApplication.scala:241)
  19.     at org.apache.spark.deploy.k8s.submit.KubernetesClientApplication.start(KubernetesClientApplication.scala:204)
  20.     at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:845)
  21.     at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:161)
  22.     at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:184)
  23.     at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)
  24.     at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:920)
  25.     at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:929)
  26.     at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
  27. Caused by: java.net.SocketException: Broken pipe (Write failed)
  28.     at java.net.SocketOutputStream.socketWrite0(Native Method)
  29.     at java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:111)
  30.     at java.net.SocketOutputStream.write(SocketOutputStream.java:155)
  31.     at sun.security.ssl.OutputRecord.writeBuffer(OutputRecord.java:431)
  32.     at sun.security.ssl.OutputRecord.write(OutputRecord.java:417)
  33.     at sun.security.ssl.SSLSocketImpl.writeRecordInternal(SSLSocketImpl.java:894)
  34.     at sun.security.ssl.SSLSocketImpl.writeRecord(SSLSocketImpl.java:865)
  35.     at sun.security.ssl.AppOutputStream.write(AppOutputStream.java:123)
  36.     at okio.Okio$1.write(Okio.java:79)
  37.     at okio.AsyncTimeout$1.write(AsyncTimeout.java:180)
  38.     at okio.RealBufferedSink.flush(RealBufferedSink.java:224)
  39.     at okhttp3.internal.http2.Http2Writer.settings(Http2Writer.java:203)
  40.     at okhttp3.internal.http2.Http2Connection.start(Http2Connection.java:515)
  41.     at okhttp3.internal.http2.Http2Connection.start(Http2Connection.java:505)
  42.     at okhttp3.internal.connection.RealConnection.startHttp2(RealConnection.java:298)
  43.     at okhttp3.internal.connection.RealConnection.establishProtocol(RealConnection.java:287)
  44.     at okhttp3.internal.connection.RealConnection.connect(RealConnection.java:168)
  45.     at okhttp3.internal.connection.StreamAllocation.findConnection(StreamAllocation.java:257)
  46.     at okhttp3.internal.connection.StreamAllocation.findHealthyConnection(StreamAllocation.java:135)
  47.     at okhttp3.internal.connection.StreamAllocation.newStream(StreamAllocation.java:114)
  48.     at okhttp3.internal.connection.ConnectInterceptor.intercept(ConnectInterceptor.java:42)
  49.     at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
  50.     at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
  51.     at okhttp3.internal.cache.CacheInterceptor.intercept(CacheInterceptor.java:93)
  52.     at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
  53.     at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
  54.     at okhttp3.internal.http.BridgeInterceptor.intercept(BridgeInterceptor.java:93)
  55.     at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
  56.     at okhttp3.internal.http.RetryAndFollowUpInterceptor.intercept(RetryAndFollowUpInterceptor.java:126)
  57.     at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
  58.     at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
  59.     at io.fabric8.kubernetes.client.utils.BackwardsCompatibilityInterceptor.intercept(BackwardsCompatibilityInterceptor.java:119)
  60.     at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
  61.     at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
  62.     at io.fabric8.kubernetes.client.utils.ImpersonatorInterceptor.intercept(ImpersonatorInterceptor.java:68)
  63.     at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
  64.     at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
  65.     at io.fabric8.kubernetes.client.utils.HttpClientUtils.lambda$createHttpClient$3(HttpClientUtils.java:112)
  66.     at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
  67.     at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
  68.     at okhttp3.RealCall.getResponseWithInterceptorChain(RealCall.java:254)
  69.     at okhttp3.RealCall.execute(RealCall.java:92)
  70.     at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:411)
  71.     at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:372)
  72.     at io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleCreate(OperationSupport.java:241)
  73.     at io.fabric8.kubernetes.client.dsl.base.BaseOperation.handleCreate(BaseOperation.java:819)
  74.     at io.fabric8.kubernetes.client.dsl.base.BaseOperation.create(BaseOperation.java:334)
  75.     ... 17 more
  76. 20/05/03 00:37:45 INFO ShutdownHookManager: Shutdown hook called
  77. 20/05/03 00:37:45 INFO ShutdownHookManager: Deleting directory /tmp/spark-364e0e63-136e-40d2-9a13-1e26eaa4b7ad
Add Comment
Please, Sign In to add comment
Public Pastes
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!