Script

# Install java
sudo apt-get update
sudo apt-get install software-properties-common -y
sudo add-apt-repository ppa:webupd8team/java
sudo apt-get update

sudo echo oracle-java8-installer shared/accepted-oracle-license-v1-1 select true | /usr/bin/debconf-set-selections

sudo apt-get install oracle-java8-installer -y
sudo apt-get install oracle-java8-set-default -y

# Set JAVA_HOME and JRE_HOME
echo "export JAVA_HOME=/usr/lib/jvm/java-8-oracle" >> ~/.bashrc
echo "export JRE_HOME=/usr/lib/jvm/java-8-oracle/jre" >> ~/.bashrc

# install anaconda with python 3.6.5
sudo cd /tmp
sudo apt-get install curl -y
sudo curl -O https://repo.anaconda.com/archive/Anaconda3-5.2.0-Linux-x86_64.sh
sudo bash Anaconda3-5.2.0-Linux-x86_64.sh -b

# Install pyspark
echo "export PATH=/root/anaconda3/bin:$PATH" >> ~/.bashrc
sudo /root/anaconda3/bin/conda install -c conda-forge pyspark


sudo apt-get install openssh-server openssh-client -y
sudo /etc/init.d/ssh restart


cd /root && wget https://archive.apache.org/dist/spark/spark-2.3.1/spark-2.3.1-bin-hadoop2.7.tgz && tar xzf spark-2.3.1-bin-hadoop2.7.tgz

echo 'export PYSPARK_DRIVER_PYTHON=ipython3' >> ~/.bashrc
echo 'export PYSPARK_DRIVER_PYTHON_OPTS="notebook"' >> ~/.bashrc
echo 'export PATH="/home/louis/anaconda3/bin:$PATH"' >> ~/.bashrc
echo 'export SPARK_HOME="/root/spark-2.3.1-bin-hadoop2.7/"' >> ~/.bashrc
echo 'export PATH=$PATH:$SPARK_HOME/bin' >> ~/.bashrc