Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #!/bin/bash
- ### paste this file to user data field as text
- ### make sure to replace <master-private-ip> and <worker-private-ip1> with your own IP addresses. Replace <id_rsa.pub> with your own as well
- ### it works best when you launch a master first and configure it and then launch 1 slave with the id_rsa of the master. You can create an image of the slave and apply that image to as many slaves as needed. make sure to add slave private IPs to spark configuration
- apt-get update
- #apt-get install nano
- apt-get -y install apt install openjdk-8-jre-headless
- apt-get -y install scala
- apt-get -y install python3-pip
- apt-get -y install openssh-server openssh-client
- apt-get -y install curl
- curl -O http://www.gtlib.gatech.edu/pub/apache/spark/spark-3.0.0-preview2/spark-3.0.0-preview2-bin-hadoop2.7.tgz
- tar xvf spark-3.0.0-preview2-bin-hadoop2.7.tgz
- mv "/spark-3.0.0-preview2-bin-hadoop2.7/" /usr/local/spark
- ###master
- #echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/" | sudo tee /usr/local/spark/conf/spark-env.sh
- ###slaves
- echo '<id_rsa.pub>' | sudo tee -a ~/.ssh/authorized_keys
- ###Do this manually after EC2 instance is finished installing
- echo 'export PATH=/usr/local/spark/bin:$PATH' >> ~/.bash_profile
- source ~/.bash_profile
- ssh-keygen -t rsa -P "" -f ~/.ssh/id_rsa
- ###Master only:
- ###copy and paste the output to a text doc and save for slaves
- cat ~/.ssh/id_rsa.pub
- ###Master only:duplicate this line with as many slaves as needed
- sudo chmod -R a+rwx /usr/local/spark
- echo 'export SPARK_MASTER_HOST=<master-private-ip>' | sudo tee -a /usr/local/spark/conf/spark-env.sh
- echo '<worker-private-ip1>'' | sudo tee -a /usr/local/spark/conf/slaves
- ###Checks on master
- ### be able to connect to slaves
- #ssh -i ~/.ssh/id_rsa ubuntu@<worker-private-ip1>
- ### launch spark
- ### sudo sh /usr/local/spark/sbin/start-all.sh
- ### get the port for your security settings
- #ps aux|grep -i spark
- ### spark UI
- #publicDNS (IPv4):8080
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement