Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #!/bin/bash
- MBXSPARK_VERSION="0.4.8"
- yum -y remove gcc
- yum -y remove gcc
- yum -y install gcc
- yum -y install gcc-c++
- echo "installing gzip gunzip"
- yum -y install gzip gunzip
- aws s3 cp s3://mapbox-qubole/mbxspark/mbxspark-${MBXSPARK_VERSION}.tar.gz /tmp
- echo "copied mbxspark, now setting up python3 environment"
- source /usr/lib/hustler/bin/qubole-bash-lib.sh
- echo "sourcing qubole bash"
- echo "copying tecumseh"
- aws s3 cp s3://mapbox-qubole/tecumseh/tecumseh-0.0.1.tar.gz /tmp/tecumseh-src.tar.gz
- echo "gunzipping tecumseh"
- gunzip /tmp/tecumseh-src.tar.gz
- echo "decompressing tecumseh tar and copying src files"
- mkdir /tmp/tecumseh && tar xf /tmp/tecumseh-src.tar -C /tmp/tecumseh --strip-components 1
- sleep 3
- package_management_enabled=$(nodeinfo_feature tapp.ui_enable_package_management)
- use_spark=$(nodeinfo use_spark)
- py_package_name=$(nodeinfo quboled_env_python_package_name)
- echo "py package name is ${py_package_name} use spark is ${use_spark} and package management is ${package_management_enabled}"
- if [[ ${package_management_enabled} = "true" && ${use_spark} = "1" ]]; then
- while [[ ! -d "/usr/lib/envs/${py_package_name}" ]] && [[ "${#py_package_name}" -ne "0" ]]
- do
- echo "python3 environment not ready. Sleeping for 3 secs!"
- sleep 3
- done
- echo "python3 environment ($py_package_name) ready, sleeping more to be sure!"
- sleep 30
- ##Start Health Check to decomission unhealthy nodes###
- if [ ! -d "/usr/lib/envs/${py_package_name}" ] ; then
- nodename=$(hostname)
- echo "Env is still not ready! Aborting and decomissioning the current node gracefully"
- shutdown -h
- fi
- echo "installing private module mbxspark-${MBXSPARK_VERSION}"
- /usr/lib/envs/${py_package_name}/bin/pip install /tmp/mbxspark-${MBXSPARK_VERSION}.tar.gz
- # Upgrade some packages
- /usr/lib/envs/${py_package_name}/bin/pip install --upgrade boto3 requests
- #Installing setuptools
- /usr/lib/envs/${py_package_name}/bin/pip install setuptools
- #Installing tecumseh
- /usr/lib/envs/${py_package_name}/bin/pip install /tmp/tecumseh
- fi
Add Comment
Please, Sign In to add comment