API tools faq
paste
Advertisement
zamotivator

Untitled

zamotivator
Nov 30th, 2012
101
0
Never
Add comment
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Bash 13.48 KB | None | 0 0
raw download clone embed print report
  1. #!/bin/bash
  2.  
  3. set -eu
  4.  
  5. function print_help ()
  6. {
  7. echo <<EOF "Usage:
  8. Access to remote hosts:
  9.  deploy.sh access    <user> <password> <public_key> [host_list]
  10.  
  11. Prepare remote machines:
  12.  deploy.sh prepare_developer    <user> [host_list]
  13.  deploy.sh prepare_chroot       <user> [host_list]
  14.  deploy.sh prepare_runtime      <user> [host_list]
  15.  
  16. Build packages/repositories:
  17.  deploy.sh build         <packages_path>
  18.  deploy.sh build_deps    <packages_path>
  19.  
  20. SciDB control on remote machines:
  21.  deploy.sh scidb_install    <packages_path> [host_list]
  22.  deploy.sh scidb_remove     <packages_path> [host_list]
  23.  deploy.sh scidb_prepare    <user> [host_list]
  24.  deploy.sh scidb_start      <user> <coordinator>
  25.  
  26. Description:
  27.  access               provide access by ssh-key to [host_list] for <user>
  28.  
  29.  prepare_developer    prepare [host_list] on <user> for build SciDB
  30.  prepare_chroot       prepare [host_list] on <user> for build SciDB packages
  31.  prepare_runtime      prepare [host_list] on <user> for run SciDB
  32.  
  33.  build                build SciDB packages to <packages_path>
  34.  build_deps           build packages for dependencies to <packages_path>
  35.  
  36.  scidb_install        install SciDB on [host_list] (expects you are registered required repositories)
  37.  scidb_remove         remove SciDB from [host_list]
  38.  scidb_prepare        prepare for run SciDB cluster on [host_list] (first host would considered as coordinator)"
  39. EOF
  40. exit 1
  41. }
  42.  
  43. # detect directory where we run
  44. source_path=`dirname $0`
  45. source_path=`readlink -f ${source_path}/../`
  46. bin_path=${source_path}/deployment/common
  47. if [ -z "${bin_path}" ] ; then
  48.     bin_path=./common
  49. fi
  50. echo "Source path: ${source_path}"
  51. echo "Script common path: ${bin_path}"
  52.  
  53. function remote_no_pass ()
  54. {
  55. local username=${1}
  56. local password=${2}
  57. local hostname=${3}
  58. shift 3
  59. expect <<EOF
  60. set timeout -1
  61. spawn $@
  62. expect {
  63.   "${username}@${hostname}'s password:" { send "${password}\r"; exp_continue }
  64.   eof                                   { }  
  65. }
  66. EOF
  67. }
  68.  
  69. function remote ()
  70. {
  71. local username=${1}
  72. local password=${2}
  73. local hostname=${3}
  74. local files=${5-""}
  75. remote_no_pass "${username}" "${password}" "${hostname}" "scp -q -o StrictHostKeyChecking=no -r ${bin_path} ${username}@${hostname}:/tmp/deployment"
  76. if [ -n "${files}" ]; then
  77.     remote_no_pass "${username}" "${password}" "${hostname}" "scp -q -o StrictHostKeyChecking=no -r ${files} ${username}@${hostname}:/tmp/deployment"
  78. fi;
  79. remote_no_pass  "${username}" "${password}" "${hostname}" "ssh -o StrictHostKeyChecking=no ${username}@${hostname} \"cd /tmp/deployment && ${4}\""
  80. remote_no_pass "${username}" "${password}" "${hostname}" "ssh -o StrictHostKeyChecking=no ${username}@${hostname}"  "\"rm -rf /tmp/deployment\""
  81. }
  82.  
  83. function provide_access ()
  84. {
  85.     local username=${1}
  86.     local password="${2}"
  87.     local key=${3}
  88.     local hostname=${4}
  89.     if [ "${username}" != "root" ]; then
  90.     echo "Check for ${username} on ${hostname}"
  91.     remote root "${password}" "${hostname}" "./user_create.sh \\\"${username}\\\" \\\"${password}\\\""
  92.     fi;
  93.     echo "Provide access by ~/.ssh/id_rsa.pub to ${username}@${hostname}"
  94.     remote "${username}" "${password}" "${hostname}" "./user_access.sh \\\"${username}\\\" \\\"${key}\\\""
  95. }
  96.  
  97. function push_source ()
  98. {
  99.     local username=${1}
  100.     local result_path=`readlink -f ${2}`
  101.     local hostname=${3}
  102.     echo "Remove ${username}@${hostname}:${result_path}"
  103.     remote_no_pass "${username}" "" "${hostname}" "ssh -o StrictHostKeyChecking=no ${username}@${hostname}"  "\"rm -rf ${result_path}\""
  104.     echo "Copy ${source_path} to ${username}@${hostname}:${result_path}"
  105.     remote_no_pass "${username}" "" "${hostname}" "scp -q -o StrictHostKeyChecking=no -r ${source_path} ${username}@${hostname}:${result_path}"
  106. }
  107.  
  108.  
  109. function configure_rpm ()
  110. {
  111.     build_kind=centos-6.3-x86_64
  112.     suffix=rpm
  113.     kind=rpm
  114.     function package_info ()
  115.     {
  116.     rpm -qip ${1} | grep Name | awk '{print $3}'
  117.     }
  118.     install="yum install -y *.rpm"
  119.     remove="yum remove -y"
  120. }
  121.  
  122. function configure_deb ()
  123. {
  124.     build_kind=ubuntu-precise-amd64
  125.     suffix=deb
  126.     kind=deb
  127.     function package_info ()
  128.     {
  129.     dpkg -I ${1} | grep Package | awk '{print $2}'
  130.     }
  131.     install="dpkg -R -i ."
  132.     remove="apt-get remove -y"
  133. }
  134. function configure_package_manager ()
  135. {
  136.     local hostname=${1}
  137.     local with_redhat=${2}
  138.     FILE=/etc/issue
  139.     if [ "${hostname}" != "localhost" ]; then
  140.     remote_no_pass root "" "${hostname}" "scp -q -o StrictHostKeyChecking=no -r root@${hostname}:/etc/issue ./issue"
  141.     FILE=./issue
  142.     fi;
  143.     local OS=`${bin_path}/os_detect.sh ${FILE}`
  144.     if [ "${hostname}" != "localhost" ]; then
  145.     rm -f ./issue
  146.     fi;
  147.     case "${OS}" in
  148.     "CentOS 6.3")
  149.         configure_rpm
  150.         ;;
  151.     "RedHat 6.3")
  152.         if [ ${with_redhat} == 1 ]; then
  153.         configure_rpm
  154.         else
  155.         echo "We do not support build SciDB under RedHat 6.3. Please use CentOS 6.3 instead"
  156.         exit 1
  157.         fi;
  158.         ;;
  159.     "Ubuntu 12.04")
  160.         configure_deb
  161.         ;;
  162.     *)
  163.         echo "Not supported OS"
  164.         exit 1;
  165.         ;;
  166.     esac
  167. }
  168.  
  169. function push_and_pull_packages ()
  170. {
  171.     local username=${2}
  172.     local hostname=${3}
  173.     local push=${5}
  174.     configure_package_manager ${hostname} 1
  175.     local path_local=`readlink -f ${1}`
  176.     local path_remote=`readlink -f ${4}`
  177.     local scp_args_remote="${username}@${hostname}:${path_remote}/*"
  178.     if [ $push == 1 ]; then
  179.     remote_no_pass "${username}" "" "${hostname}" "rm -rf ${path_remote}"
  180.     remote_no_pass "${username}" "" "${hostname}" "mkdir -p ${path_remote}"
  181.     remote_no_pass "${username}" "" "${hostname}" "scp -q -o StrictHostKeyChecking=no ${path_local} ${scp_args_remote}"
  182.     else
  183.     rm -rf ${path_local}
  184.     mkdir -p ${path_local}
  185.     remote_no_pass "${username}" "" "${hostname}" "scp -q -o StrictHostKeyChecking=no ${scp_args_remote} ${path_local}"
  186.     fi;
  187. }
  188.  
  189. function build ()
  190. {
  191.     configure_package_manager "localhost" 0
  192.     local packages_path=`readlink -f ${1}`
  193.     rm -rf ${packages_path}
  194.     #(cd ${source_path}; cmake .; make -j5)
  195.     (cd ${source_path}; ./utils/make_packages.sh ${kind} chroot ${packages_path} ${build_kind})
  196.     #cp /tmp/scidb_packaging/chroot_result/* ${packages_path} || true
  197. }
  198.  
  199. function setup_ccache ()
  200. {
  201.     local username=${1}
  202.     local hostname=${2}
  203.     remote ${username} "" ${hostname} "./setup_ccache.sh"
  204. }
  205.  
  206. function setup_main_scidb_repository()
  207. {
  208.     local hostname=${1}
  209.     echo "Setup main SciDB repository on ${hostname}"
  210.     remote root "" ${hostname} "./setup_main_scidb_repository.sh"
  211. }
  212.  
  213. function stop_virtual_bridge_zero ()
  214. {
  215.     local hostname=${1}
  216.     remote root "" ${hostname} "./stop_virbr0.sh"
  217. }
  218.  
  219. function configure_postgresql ()
  220. {
  221.     local username=${1}
  222.     local hostname=${2}
  223.     remote root "" ${hostname} "./configure_postgresql.sh ${username} 127.0.0.1/8"
  224. }
  225.  
  226. function prepare_developer ()
  227. {
  228.     local username=${1}
  229.     local hostname=${2}
  230.     echo "Prepare ${username}@${hostname} for developer"
  231.     setup_main_scidb_repository "${hostname}"
  232.     remote root "" ${hostname} "./prepare_developer.sh"
  233.     setup_ccache ${username} ${hostname}
  234.     configure_postgresql ${username} ${hostname}
  235.     stop_virtual_bridge_zero "${hostname}"
  236. }
  237.  
  238. function prepare_chroot ()
  239. {
  240.     local username=${1}
  241.     local hostname=${2}
  242.     echo "Prepare for build SciDB packages in chroot on ${host}"
  243.     setup_main_scidb_repository "${hostname}"
  244.     remote root "" ${host} "./prepare_chroot.sh ${username}"
  245.     remote ${username} "" ${host} "./chroot_build.sh" "${source_path}/utils/chroot_build.py"
  246. }
  247.  
  248. function prepare_runtime ()
  249. {
  250.     local username=${1}
  251.     local hostname=${2}
  252.     setup_main_scidb_repository "${hostname}"
  253.     remote root "" ${hostname} "./prepare_runtime.sh"
  254.     stop_virtual_bridge_zero "${hostname}"
  255. }
  256.  
  257. function prepare_runtime_postgresql ()
  258. {
  259.     local username=${1}
  260.     local hostname=${2}
  261.     configure_postgresql ${username} ${hostname}
  262.     #echo "Perhaps you also need 'sudo -u postgres ./init-db.sh <user_name> <database_name> <user_password> <port=5432>'"
  263. }
  264.  
  265. function package_names()
  266. {
  267.     for file in $@; do
  268.     package_info ${file}
  269.     done;
  270. }
  271.  
  272. function scidb_remove()
  273. {
  274.     local hostname=${2}
  275.     configure_package_manager ${hostname} 1
  276.     local packages_path=`readlink -f ${1}`
  277.     local packages=`ls ${packages_path}/*.${kind} | xargs`
  278.     remote root "" "${hostname}" "${remove} `package_names ${packages} | xargs`"
  279. }
  280.  
  281. function scidb_install()
  282. {
  283.     local hostname=${2}
  284.     local with_coordinator=${3}
  285.     configure_package_manager ${hostname} 1
  286.     local packages_path=`readlink -f ${1}`
  287.     if [ ${with_coordinator} -eq 1 ]; then
  288.     local packages=`ls ${packages_path}/*.${kind} | xargs`
  289.     remote root "" "${hostname}" "${install}" "${packages}"
  290.     else
  291.     local packages=`ls ${packages_path}/*.${kind} | grep -v coord | xargs`
  292.     remote root "" "${hostname}" "${install}" "${packages}"
  293.     fi;
  294. }
  295.  
  296. function scidb_config ()
  297. {
  298. local username="${1}"
  299. local password="${2}"
  300. local coordinator="${3}"
  301. shift 3
  302. echo "[scidb]"
  303. echo "server-0=${coordinator},1"
  304. node_number=1
  305. for hostname in $@; do
  306.     echo "server-${node_number}=${hostname},2"
  307.     let node_number++
  308. done;
  309. echo "db_user=${username}"
  310. echo "db_passwd=${password}"
  311. echo "install_root=/opt/scidb/12.11"
  312. echo "pluginsdir=/opt/scidb/12.11/lib/scidb/plugins"
  313. echo "logconf=/opt/scidb/12.11/share/scidb/log4cxx.properties"
  314. echo "base-path=/home/scidb/database"
  315. echo "tmp-path=/tmp"
  316. echo "base-port=1239"
  317. echo "interface=eth0"
  318. echo "no-watchdog=false"
  319. }
  320.  
  321. function scidb_prepare_node ()
  322. {
  323.     local username=${1}
  324.     local hostname=${2}
  325.     remote ${username} "" ${hostname} "./scidb_prepare.sh"
  326.     remote root "" ${hostname} "cat config.ini > /opt/scidb/12.11/etc/config.ini" `readlink -f ./config.ini`
  327. }
  328.  
  329. function scidb_prepare ()
  330. {
  331.     local username=${1}
  332.     local coordinator=${2}
  333.     shift 2
  334.     local coordinator_key="`ssh -o StrictHostKeyChecking=no scidb@${coordinator} \"cat ~/.ssh/id_rsa.pub\"`"
  335.     scidb_config ${username} "scidb123" ${coordinator} "$@" | tee ./config.ini
  336.     scidb_prepare_node ${username} ${coordinator}
  337.     for hostname in $@; do
  338.     scidb_prepare_node ${username} ${hostname}
  339.     provide_access ${username} "" "${coordinator_key}" ${hostname}
  340.     done;
  341.     rm -f ./config.ini
  342.     remote ${username} "" ${coordinator} "./scidb_prepare_coordinator.sh ${username} scidb123 ${username}"
  343. }
  344.  
  345. function scidb_start ()
  346. {
  347.     local username=${1}
  348.     local coordinator=${2}
  349.     remote ${username} "" ${coordinator} "scidb.py startall ${username}"
  350. }
  351.  
  352. function prepare_httpd_cdash ()
  353. {
  354.     username=${1}
  355.     build_machine=${2}
  356.     remote root "" ${build_machine} "./prepare_httpd_cdash.sh ${username}"
  357. }
  358.  
  359. if [ $# -lt 1 ]; then
  360.     print_help
  361. fi
  362. action=$1
  363.  
  364. case ${action} in
  365.     access)
  366.     if [ $# -lt 4 ]; then
  367.         print_help
  368.     fi
  369.     user=${2}
  370.     password=${3}
  371.     key="${4}"
  372.     shift 4
  373.     for host in $@; do
  374.             provide_access ${user} "${password}" "${key}" "${host}"
  375.     done;
  376.     ;;
  377.     push_source)
  378.     if [ $# -lt 4 ]; then
  379.         print_help
  380.     fi
  381.     user=${2}
  382.     path=${3}
  383.     shift 3
  384.     for host in $@; do
  385.         push_source ${user} ${path} ${host}
  386.         done;
  387.     ;;
  388.     pull_packages)
  389.     if [ $# -lt 5 ]; then
  390.         print_help
  391.     fi
  392.     path_local=`readlink -f ${2}`
  393.     username=${3}
  394.     path_remote=`readlink -f ${4}`
  395.     shift 4
  396.     for hostname in $@; do
  397.         push_and_pull_packages ${path_local} ${username} ${hostname} ${path_remote} 0
  398.     done;
  399.     ;;
  400.     push_packages)
  401.     if [ $# -lt 5 ]; then
  402.         print_help
  403.     fi
  404.     path_local=`readlink -f ${2}`
  405.     username=${3}
  406.     path_remote=`readlink -f ${4}`
  407.     shift 4
  408.     for hostname in $@; do
  409.         push_and_pull_packages ${path_local} ${username} ${hostname} ${path_remote} 1
  410.     done;
  411.     ;;
  412.     prepare_developer)
  413.     if [ $# -lt 3 ]; then
  414.         print_help
  415.     fi
  416.     user=$2
  417.     shift 2
  418.     for host in $@; do
  419.         prepare_developer ${user} "${host}"
  420.     done;
  421.     ;;
  422.     prepare_chroot)
  423.     if [ $# -lt 3 ]; then
  424.         print_help
  425.     fi
  426.     user=$2
  427.     shift 2
  428.     for host in $@; do
  429.         prepare_developer ${user} "${host}"
  430.         prepare_chroot ${user} "${host}"
  431.     done;
  432.     ;;
  433.     prepare_runtime)
  434.     if [ $# -lt 3 ]; then
  435.         print_help
  436.     fi
  437.     username=$2
  438.     coordinator=${3}
  439.     echo "Coordinator IP: ${coordinator}"
  440.     shift 3
  441.     for hostname in ${coordinator} $@; do
  442.         prepare_runtime ${username} ${hostname}
  443.     done;
  444.     prepare_runtime_postgresql ${username} ${coordinator}
  445.     ;;
  446.     build)
  447.     if [ $# -ne 2 ]; then
  448.         print_help
  449.     fi
  450.     packages_path=${2}
  451.     build "${packages_path}"
  452.     ;;
  453.     build_deps)
  454.     if [ $# -ne 2 ]; then
  455.         print_help
  456.     fi
  457.     packages_path=${2}
  458.     echo "TODO build SciDB dependencies packages"
  459.     ;;
  460.     scidb_install)
  461.     if [ $# -lt 3 ]; then
  462.         print_help
  463.     fi
  464.     packages_path=${2}
  465.     coordinator=${3}
  466.     echo "Coordinator IP: ${coordinator}"
  467.     shift 3
  468.     scidb_install ${packages_path} ${coordinator} 1
  469.     for hostname in $@; do
  470.         scidb_install ${packages_path} ${hostname} 0
  471.     done;
  472.     ;;
  473.     scidb_remove)
  474.     if [ $# -lt 3 ]; then
  475.         print_help
  476.     fi
  477.     packages_path=${2}
  478.     shift 2
  479.     for host in $@; do
  480.         scidb_remove ${packages_path} ${host}
  481.     done;
  482.     ;;
  483.     scidb_prepare)
  484.     if [ $# -lt 3 ]; then
  485.         print_help
  486.     fi
  487.     username=${2}
  488.     shift 2
  489.     scidb_prepare ${username} $@
  490.     ;;
  491.     scidb_start)
  492.     if [ $# -lt 3 ]; then
  493.         print_help
  494.     fi
  495.     username=${2}
  496.     coordinator=${3}
  497.     scidb_start ${username} ${coordinator}
  498.     ;;
  499.     prepare_httpd_cdash)
  500.     if [ $# -lt 3 ]; then
  501.         print_help
  502.     fi;
  503.     username=${2}
  504.     shift 2
  505.     for hostname in $@; do
  506.         prepare_httpd_cdash ${username} ${hostname}
  507.     done;
  508.     ;;
  509.     *)
  510.     print_help
  511.     ;;
  512. esac
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement
Public Pastes
Advertisement
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy.  OK, I Understand
Not a member of Pastebin yet?
Sign Up, it unlocks many cool features!