Guest User

Untitled

a guest
Nov 18th, 2017
79
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 0.75 KB | None | 0 0
  1. echo '>> Start of Script'
  2. nodes=($( cat $PBS_NODEFILE | sort | uniq ))
  3. nnodes=${#nodes[@]}
  4. last=$(( $nnodes - 1 ))
  5.  
  6. export SPARK_HOME=/work/{user}/sparktest/spark/
  7. ssh ${nodes[0]} "cd ${SPARK_HOME}; ./sbin/start-master.sh"
  8. sparkmaster="spark://${nodes[0]}:7077"
  9. echo 'master created'
  10.  
  11. for i in $( seq 0 $last )
  12. do
  13. ssh ${nodes[$i]} "cd ${SPARK_HOME}; nohup ./sbin/start-slave ${sparkmaster} &> /work/{user}/sparktest/testSucess/nohup-${nodes[$i]}.out" &
  14. done
  15. echo 'Worker Started'
  16.  
  17. ${SPARK_HOME}/bin/spark-submit /work/{user}/sparktest/testSucess/sparkscript.py
  18. echo 'application submitted'
  19. echo ${nnodes[0]}
  20.  
  21. ssh ${nodes[0]} " cd ${SPARK_HOME}; ./sbin/stop-master.sh"
  22. for i in $( seq 0 $last )
  23. do
  24. ssh ${nodes[$i]} "cd ${SPARK_HOME}; ./sbin/stop-slaves.sh"
  25. done
Add Comment
Please, Sign In to add comment