Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- export SPARK_HOME= /data/local/opt/spark-2.4.3-bin-hadoop2.7
- cd $SPARK_HOME
- ## scala v 2.11 (!!)
- ./bin/spark-shell -v --master yarn-client --num-executors 20 --driver-memory 2g --executor-memory 2g \
- --jars /tmp/bdg-sequila-acc_2.11-0.1-spark-2.4.3-SNAPSHOT-assembly.jar \
- --conf spark.sql.extensions=org.biodatageeks.sequila.spark.BdgExtensions \
- --conf spark.hadoop.yarn.timeline-service.enabled=false \
- --conf spark.hadoop.hive.metastore.uris=thrift://cdh01.cl.ii.pw.edu.pl:9083 \
- --conf spark.hadoop.yarn.timeline-service.enabled=false \
- --conf spark.driver.extraJavaOptions=-Dhdp.version=3.1.0.0-78 \
- --conf spark.yarn.am.extraJavaOptions=-Dhdp.version=3.1.0.0-78 \
- --conf spark.hadoop.metastore.catalog.default=hive \
- --conf spark.jars.ivy=/data/local/cache/ivy2/repository/
- --packages org.biodatageeks:bdg-sequila-acc_2.12:0.1-spark-2.4.3-SNAPSHOT \
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement