Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- > sc <- spark_connect(master = "yarn-client", config = conf, version = '2.2.0')
- Sys.setenv(SPARK_HOME="/usr/lib/spark")
- options(rsparkling.sparklingwater.version = '2.0.3')
- # Configure cluster (c3.4xlarge 30G 16core 320disk)
- conf <- spark_config()
- conf$'sparklyr.shell.executor-memory' <- "20g"
- conf$'sparklyr.shell.driver-memory' <- "20g"
- conf$spark.executor.cores <- 16
- conf$spark.executor.memory <- "20G"
- conf$spark.yarn.am.cores <- 16
- conf$spark.yarn.am.memory <- "20G"
- conf$spark.executor.instances <- 8
- conf$spark.dynamicAllocation.enabled <- "false"
- conf$maximizeResourceAllocation <- "true"
- conf$spark.default.parallelism <- 32
- # Connect to cluster
- sc <- spark_connect(master = "yarn-client", config = conf, version = '2.2.0') # error shows after executing this line
- C:sparkspark-2.3.1-bin-hadoop2.7
- C:sparkspark-2.3.1-bin-hadoop2.7
Add Comment
Please, Sign In to add comment