Guest User

Untitled

a guest
Nov 13th, 2018
119
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.08 KB | None | 0 0
  1. lazy val conf = ConfigFactory.load()
  2.  
  3. ingestion{
  4. process {
  5. value = "sas"
  6. }
  7. sas {
  8. origin{
  9. value = "/route"
  10. }
  11. destination{
  12. value = "/route"
  13. }
  14. extension{
  15. value = ".sas7bdat"
  16. }
  17. file{
  18. value = "mytable"
  19. }
  20. month{
  21. value = "201010,201011"
  22. }
  23. table{
  24. value = "tbl"
  25. }
  26. }
  27. }
  28.  
  29. spark2-submit --class com.bcp.coe.Main --master yarn --deploy-mode client --driver-memory 10G --driver-cores 8 --executor-memory 13G --executor-cores 4 --num-executors 10 --verbose --files properties.conf /home/s16746/ingestion-1.0-SNAPSHOT-jar-with-dependencies.jar --files application.conf
  30.  
  31. com.typesafe.config.ConfigException$Missing: No configuration setting found for key 'ingestion'
  32.  
  33. spark2-submit ...
  34. --conf spark.driver.extraClassPath=./
  35. --conf spark.executor.extraClassPath=./ // if you need to load config at executors
  36. ...
  37.  
  38. spark2-submit ...
  39. --conf spark.driver.extraJavaOptions=-Dconfig.file=./application.conf
  40. --conf spark.executor.extraJavaOptions=-Dconfig.file=./application.conf
  41. ...
Add Comment
Please, Sign In to add comment