Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- echo '>> Start of Script'
- nodes=($( cat $PBS_NODEFILE | sort | uniq ))
- nnodes=${#nodes[@]}
- last=$(( $nnodes - 1 ))
- export SPARK_HOME=/work/{user}/sparktest/spark/
- ssh ${nodes[0]} "cd ${SPARK_HOME}; ./sbin/start-master.sh"
- sparkmaster="spark://${nodes[0]}:7077"
- echo 'master created'
- for i in $( seq 0 $last )
- do
- ssh ${nodes[$i]} "cd ${SPARK_HOME}; nohup ./sbin/start-slave ${sparkmaster} &> /work/{user}/sparktest/testSucess/nohup-${nodes[$i]}.out" &
- done
- echo 'Worker Started'
- ${SPARK_HOME}/bin/spark-submit /work/{user}/sparktest/testSucess/sparkscript.py
- echo 'application submitted'
- echo ${nnodes[0]}
- ssh ${nodes[0]} " cd ${SPARK_HOME}; ./sbin/stop-master.sh"
- for i in $( seq 0 $last )
- do
- ssh ${nodes[$i]} "cd ${SPARK_HOME}; ./sbin/stop-slaves.sh"
- done
Add Comment
Please, Sign In to add comment