spark.driver.extraLibraryPath /usr/hdp/current/hadoop-client/lib/native:/usr/hdp/current/hadoop-client/lib/native/Linux-amd64-64
spark.eventLog.dir hdfs:///spark2-history/
spark.eventLog.enabled true
spark.executor.extraLibraryPath /usr/hdp/current/hadoop-client/lib/native:/usr/hdp/current/hadoop-client/lib/native/Linux-amd64-64
spark.history.fs.logDirectory hdfs:///spark2-history/
spark.history.kerberos.keytab none
spark.history.kerberos.principal none
spark.history.provider org.apache.spark.deploy.history.FsHistoryProvider
spark.history.ui.port 18081
spark.yarn.historyServer.address ambari-cluster-002.c.pg-us-n-app-053847.internal:18081
spark.yarn.queue default

spark.shuffle.service.enabled true
spark.dynamicAllocation.enabled true
spark.dynamicAllocation.initialExecutors 2
spark.dynamicAllocation.maxExecutors 5
spark.dynamicAllocation.minExecutors 2

#spark.dynamicAllocation.initialExecutors 15
#spark.dynamicAllocation.maxExecutors 25
#spark.dynamicAllocation.minExecutors 10


#spark.driver.memory 4096
#spark.executor.memory 4096

spark.yarn.executor.memoryOverhead 1024
spark.yarn.driver.memoryOverhead 1024

spark.executor.cores 4
spark.driver.cores 2


spark.executor.extraJavaOptions -XX:+UseG1GC -XX:InitiatingHeapOccupancyPercent=35 -XX:ConcGCThreads=12 -XX:G1HeapRegionSize=16M

spark.speculation false
spark.serializer org.apache.spark.serializer.KryoSerializer

spark.sql.orc.enabled true
spark.sql.hive.convertMetastoreOrc true
spark.sql.orc.filterPushdown true
spark.sql.orc.char.enabled true
spark.sql.hive.convertMetastoreParquet false
#spark.sql.shuffle.partitions 400