spark setup

#!/bin/bash

### paste this file to user data field as text

### make sure to replace <master-private-ip> and <worker-private-ip1> with your own IP addresses. Replace <id_rsa.pub> with your own as well

### it works best when you launch a master first and configure it and then launch 1 slave with the id_rsa of the master. You can create an image of the slave and apply that image to as many slaves as needed. make sure to add slave private IPs to spark configuration

apt-get update

#apt-get install nano

apt-get -y install apt install openjdk-8-jre-headless

apt-get -y install scala

apt-get -y install python3-pip

apt-get -y install openssh-server openssh-client

apt install curl

curl -O http://www.gtlib.gatech.edu/pub/apache/spark/spark-3.0.0-preview2/spark-3.0.0-preview2-bin-hadoop2.7.tgz

tar xvf spark-3.0.0-preview2-bin-hadoop2.7.tgz

mv "/spark-3.0.0-preview2-bin-hadoop2.7/" /usr/local/spark

###master: uncomment below when setting up master

#echo "export JAVA_HOME=/usr/lib/jvm/java-8-openjdk-amd64/" | sudo tee /usr/local/spark/conf/spark-env.sh


###slaves: uncomment below when setting up slave
echo '<id_rsa.pub>' | sudo tee -a ~/.ssh/authorized_keys



###########################################################################################################


###Do this manually after EC2 instance is finished installing

echo 'export PATH=/usr/local/spark/bin:$PATH' >> ~/.bash_profile

source ~/.bash_profile

ssh-keygen -t rsa -P "" -f ~/.ssh/id_rsa

###Master only:
###copy and paste the output to a text doc and save for slaves
cat ~/.ssh/id_rsa.pub
sudo chmod -R a+rwx /usr/local/spark
echo 'export SPARK_MASTER_HOST=<master-private-ip>' | sudo tee -a /usr/local/spark/conf/spark-env.sh
###Master only:duplicate this line with as many slaves as needed
echo '<worker-private-ip1>'' | sudo tee -a /usr/local/spark/conf/slaves



###########################################################################################################
###Checks on master

### be able to connect to slaves
#ssh -i ~/.ssh/id_rsa ubuntu@<worker-private-ip1>

### launch spark
### sudo sh /usr/local/spark/sbin/start-all.sh

### get the port for your security settings
#ps aux|grep -i spark

### spark UI
#publicDNS (IPv4):8080