Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- lazy val conf = ConfigFactory.load()
- ingestion{
- process {
- value = "sas"
- }
- sas {
- origin{
- value = "/route"
- }
- destination{
- value = "/route"
- }
- extension{
- value = ".sas7bdat"
- }
- file{
- value = "mytable"
- }
- month{
- value = "201010,201011"
- }
- table{
- value = "tbl"
- }
- }
- }
- spark2-submit --class com.bcp.coe.Main --master yarn --deploy-mode client --driver-memory 10G --driver-cores 8 --executor-memory 13G --executor-cores 4 --num-executors 10 --verbose --files properties.conf /home/s16746/ingestion-1.0-SNAPSHOT-jar-with-dependencies.jar --files application.conf
- com.typesafe.config.ConfigException$Missing: No configuration setting found for key 'ingestion'
- spark2-submit ...
- --conf spark.driver.extraClassPath=./
- --conf spark.executor.extraClassPath=./ // if you need to load config at executors
- ...
- spark2-submit ...
- --conf spark.driver.extraJavaOptions=-Dconfig.file=./application.conf
- --conf spark.executor.extraJavaOptions=-Dconfig.file=./application.conf
- ...
Add Comment
Please, Sign In to add comment