Create a new version of paste:

####################################################################################################################################
#Configure additional cards NIC1 (192.168.0.22?) i NIC2 (10.10.10.1?)
#USE IT IN COMMAND LINE (without #):
#curl http://pastebin.com/raw/SNnhH3rx > xendeletelocalsr.sh; tr -d "\015" < xendeletelocalsr.sh > xendeletelocalsrnew.sh
#sh xendeletelocalsrnew.sh
#I have managed to configure 4 servers of beta3 dundee with glusterfs and ctdb and fully working ha without SPOF.
#I have four NICs: NIC0 management of xenserver 192.168.0.20*, NIC1 gluster vm client 192.168.0.22*, bond NIC2-3 10.10.10.1*, VIPs #from ctdb 192.168.0.21*. In /etc/hosts #I keep only 10.10.10.1x server ips and use them in glusterfs volume creation and in #consequence in glusterfs backend traffic.
#The main compilation script
#Removing local storage and configuring ctdb, /etc/hosts:
#http://pastebin.com/SNnhH3rx
#At the end you should:
#gluster peer probe servers
#and
#init_gluster3 "xen" "1" "2" "3" "vol0" 3 #replica 3 at the end!
#create SR nfs and iso, make vm and install xen tools and enable ha with 3 failovers.
#You are encouraged to send some patches or opinions newfuntek(at)gmail.com
#Here are some screenshots of the glusterfs sr in the xenserver pool:
#http://s17.postimg.org/3y47n8w27/glusterfsxenserv03.jpg
#http://s17.postimg.org/n4heqfcjz/glusterfsxenserv01.jpg
#http://s17.postimg.org/gs29gl9hr/glusterfsxenserv02.jpg
####################################################################################################################################
#Internet help
#Xenserver doc links
#http://www.poppelgaard.com/citrix-xenserver-6-5
#http://www.gluster.org/community/documentation/index.php/GlusterFS_Documentation
#xenserver tutorials and cheatsheets
#http://www.admin-magazine.com/HPC/Articles/GlusterFS
#http://www.slashroot.in/gfs-gluster-file-system-complete-tutorial-guide-for-an-administrator 
#https://virtualizationandstorage.wordpress.com/2010/11/15/xenserver-commands/
#http://krypted.com/unix/using-the-xensource-command-line-interface/
#http://funwithlinux.net/2013/02/glusterfs-tips-and-tricks-centos/ 
#http://xmodulo.com/category/xenserver
#compilation of xenserver
#https://discussions.citrix.com/topic/372069-does-libvmi-work-on-xenserver/ 
#glusterfs slides info
#http://rajesh-joseph.blogspot.com/2015/11/usenix-lisa-2015-tutorial-on-glusterfs.html 
#https://github.com/gluster/gluster-tutorial/blob/master/LISA-GlusterFS-Introduction.pdf 
#https://github.com/gluster/gluster-tutorial/blob/master/LISA-GlusterFS-Hands-on.pdf 
#compilation of glusterfs
#http://majentis.com/?p=319 
#różne możliwości glusterfs jak w RAID
#http://sysadm.pp.ua/linux/glusterfs-setup.html 
#glusterfs performance
#https://blog.secretisland.de/xenserver-mit-glusterfs/ 
#http://blog.dradmin.co.in/?tag=glusterfs-how-to
#https://gluster.readthedocs.org/en/latest/Administrator%20Guide/Managing%20Volumes/
#https://www.mail-archive.com/users@ovirt.org/msg31079.html
#http://www.gluster.org/community/documentation/index.php/Performance_Testing
#glusterfs on lvm
#https://support.rackspace.com/how-to/getting-started-with-glusterfs-considerations-and-installation/ 
#glusterfs profiling (delays in ops)
#https://gluster.readthedocs.org/en/latest/Administrator%20Guide/Monitoring%20Workload/ 
#xenserver glusterfs discussion - they said not possible ;)
#http://discussions.citrix.com/topic/366729-about-xenserver-glusterfs/page-2
#xenserver ha
#http://docs.citrix.com/de-de/xencenter/6-1/xs-xc-protection/xs-xc-pools-ha/xs-xc-pools-ha-about.html 
#http://xapi-project.github.io/features/HA/HA.html
#https://support.citrix.com/servlet/KbServlet/download/21018-102-664364/High%20Availability%20for%20Citrix%20XenServer.pdf
#https://xen-orchestra.com/blog/xenserver-and-vm-high-availability/
#https://discussions.citrix.com/topic/367150-ntp-ha-self-fencing/page-2#entry1884695
#http://discussions.citrix.com/topic/333343-need-help-interpreting-xha-logs/
#failure ha restart
#http://support.citrix.com/article/CTX128275
#http://citrixtechs.com/blog/help-my-citrix-xenserver-poolmaster-is-down-2/
#http://discussions.citrix.com/topic/292757-local-storage-unplugged-and-un-repairable/
#xenserver iscsi
#http://gluster.readthedocs.org/en/latest/Administrator%20Guide/GlusterFS%20iSCSI/#Running_the_target_on_the_gluster_client
#xenserver multipath
#http://docs.citrix.com/content/dam/docs/en-us/xenserver/xenserver-61/xs-design-multipathing-config.pdf
#xenserver iptables discussion
#http://discussions.citrix.com/topic/235974-cannot-add-new-nfs-virtual-disk-sr-can-add-iso-library-nfs-sr/page-2 
#rebalance nics (bond for glusterfs)
#http://www.gluster.org/pipermail/gluster-users/2014-November/019463.html
#http://www.gluster.org/pipermail/gluster-users/2014-November/019466.html
#multi nic splitnetwork for gluster in near future
#https://www.gluster.org/pipermail/gluster-users/2015-May/021815.html
#http://www.gluster.org/community/documentation/index.php/Features/SplitNetwork
#http://pl.atyp.us/hekafs.org/index.php/2013/01/split-and-secure-networks-for-glusterfs/
#host in vm inside vm config
#https://wiki.openstack.org/wiki/XenServer/VirtualBox
#reinstall xenserver host
#http://support.citrix.com/article/CTX136342
#discuss rejoin xenserver
#http://discussions.citrix.com/topic/303468-server-removed-from-pool-after-failure-now-that-it-is-back-it-can-not-rejoin-pool-and-emergency-master-reset-does-not-work/
#increase dom0 memory
#http://support.citrix.com/article/CTX134951
#find rpm 
#http://rpm.pbone.net/ 
#win admin
#win7-10: Net user administrator /active:yes 
#umount /dev/v/l 
#e2fsck -f /dev/v/l 
#resize2fs -p /dev/v/l 4G 
#lvreduce -L -8.9G /dev/

####################################################################################################################################
#turn off nfs i iptables 
chkconfig nfs off 
service nfs stop
rm -f /etc/exports
service ntpd restart
chkconfig ntpd on
service iptables stop
chkconfig iptables off
systemctl disable firewalld
systemctl stop firewalld

#turn off selinux
sed -i "s/SELINUX=enforcing/SELINUX=disabled/" /etc/sysconfig/selinux
#change the linux name to Centos for CEPH-DEPLOY
mv /etc/centos-release /etc/centos-release.xs -f
echo 'CentOS release 7.2 (Final)' > /etc/centos-release

#new package update lists
#https://discussions.citrix.com/topic/372069-does-libvmi-work-on-xenserver/
#yum install epel-release 
#http://www.tecmint.com/how-to-enable-epel-repository-for-rhel-centos-6-5/ 
#http://elrepo.org/tiki/tiki-index.php 
#rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org 
#rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-2.el7.elrepo.noarch.rpm 
#wget http://dl.fedoraproject.org/pub/epel/7/x86_64/e/epel-release-7-6.noarch.rpm
#rpm -ivh epel-release-7-6.noarch.rpm
#rpm -Uvh http://pkgs.repoforge.org/rpmforge-release/rpmforge-release-0.5.3-1.el7.rf.x86_64.rpm
#rpm -Uvh http://rpms.famillecollet.com/enterprise/remi-release-7.rpm
#rpm -Uvh http://repo.webtatic.com/yum/el7/webtatic-release.rpm
sed -i -e "/baseurl=http:\/\/www.uk/d" /etc/yum.repos.d/CentOS-Base.repo
sed -i -e "/mirrorlist/d" /etc/yum.repos.d/CentOS-Base.repo
sed -i -e "s/^#base/base/" /etc/yum.repos.d/CentOS-Base.repo
sed -i -e "s/enabled=0/enabled=1/" /etc/yum.repos.d/CentOS-Base.repo
sed -i -e "s/\$releasever/7/" /etc/yum.repos.d/CentOS-Base.repo
yum install epel-release -y
yum install centos-release-gluster -y
sed -i -e "s/enabled=0/enabled=1/" /etc/yum.repos.d/CentOS-Gluster-3.7.repo
sed -i -e "s/\$releasever/7/" /etc/yum.repos.d/CentOS-Gluster-3.7.repo
sed -i -e "s/buildlogs.centos.org\/centos\/7\/storage\/\$basearch\/gluster-3.7/buildlogs.centos.org\/centos\/7\/storage\/\$basearch\/gluster-3.8/" /etc/yum.repos.d/CentOS-Gluster-3.7.repo
#sed -i -e "s/enabled=0/enabled=1/" /etc/yum.repos.d/epel-testing.repo
yum clean all
yum repolist enabled
yum -y install deltarpm
#yum update --skip-broken -y
yum -y install vim-enhanced mc yum-utils curl e4fsprogs epel-rpm-macros
curl ix.io/client > /usr/local/bin/ix
chmod +x /usr/local/bin/ix

yum install -y glusterfs glusterfs-api-devel python-gluster ctdb ceph-deploy bind-utils xfsprogs git dnsmasq xfsprogs ansible
#yum install -y open-vm-tools ## TYLKO DLA MASZYN WIRTUALNYCH VMWARE

#run serwis glusterfs
systemctl unmask rpcbind.socket
service glusterd start
service glusterd status
service glusterfsd start
service glusterfsd status
chkconfig glusterd on
chkconfig glusterfsd on

wget http://halizard.org/release/noSAN-combined/halizard_nosan_installer_1.4.7 
chmod 755 halizard_nosan_installer_1.4.7  
sed -i 's/`uname -r`/3.10.0+2/' halizard_nosan_installer_1.4.7

#curl http://ix.io/oxr > /etc/ntp.conf.ix 
#curl http://ix.io/ojO > /etc/ntp.conf.ix 
#gluster10_4nodes
#curl http://ix.io/op8 > /etc/ntp.conf.ix
#gluster_200_201 
#curl http://ix.io/oKN > /etc/ntp.conf.ix 
#gluster_201_202_203 
curl http://ix.io/QLt > /etc/ntp.conf.ix 
#gluster10_3nodes
#curl http://ix.io/Tm5 > /etc/ntp.conf.ix
tr -d "\015" < /etc/ntp.conf.ix > /etc/ntp.conf 
echo OPTIONS="-u ntp:ntp -p /var/run/ntpd.pid -x" > /etc/sysconfig/ntpd 
echo SYNC_HWCLOCK=yes >> /etc/sysconfig/ntpd
hwclock --systohc
service ntpd stop 
service ntpd start 
ntpstat -s 
ntpq -p 
ntpstat -s

echo "#/opt/xensource/bin/xe-toolstack-restart" >> /etc/rc.d/rc.local
echo "service glusterd restart" >> /etc/rc.d/rc.local 
echo "service glusterfsd restart" >> /etc/rc.d/rc.local 
echo "service ctdb restart" >> /etc/rc.d/rc.local 
chmod 755  /etc/rc.d/rc.local 

echo "xe host-emergency-ha-disable force=true" > /usr/local/bin/restartfence 
echo "/opt/xensource/bin/xe-toolstack-restart" >> /usr/local/bin/restartfence 
echo "service glusterd restart" >> /usr/local/bin/restartfence 
echo "service glusterfsd restart" >> /usr/local/bin/restartfence 
echo "service ctdb restart" >> /usr/local/bin/restartfence  
chmod 755 /usr/local/bin/restartfence

echo "/opt/xensource/bin/xe-toolstack-restart" >> /usr/local/bin/restartoolxen
echo "service glusterd restart" >> /usr/local/bin/restartoolxen
echo "service glusterfsd restart" >> /usr/local/bin/restartoolxen
echo "service ctdb restart" >> /usr/local/bin/restartoolxen
chmod 755 /usr/local/bin/restartoolxen

echo "service glusterd restart" > /usr/local/bin/restartdodatki
echo "service glusterfsd restart" >> /usr/local/bin/restartdodatki
echo "service ctdb restart" >> /usr/local/bin/restartdodatki
chmod 755 /usr/local/bin/restartdodatki

echo "service glusterd restart" > /usr/local/bin/restartgluster
echo "service glusterfsd restart" >> /usr/local/bin/restartgluster
chmod 755 /usr/local/bin/restartgluster

#init_gluster4 "xen" "1" "2" "3" "4" "vol0" 4
#server="xen"; host01="1"; host02="2"; host03="3"; host04="4"; volume="vol0"; replica=4; 
cat <<__EOF__ > /usr/local/bin/init_gluster4
server=\$1  
host01=\$2  
host02=\$3  
host03=\$4
host04=\$5  
volume=\$6
replica=\$7
gluster peer status
#na dowolnym jednym wykonac 
#glusterfs dwa volumeny vol0 (iso) I vol1 (gfs) na sda3 i sda2  
#gluster volume stop \$volume force 
#gluster volume delete \$volume force 
gluster volume create \$volume replica \$replica \${server}\${host01}:/export/\${server}\${host01}-\$volume \${server}\${host02}:/export/\${server}\${host02}-\$volume \${server}\${host03}:/export/\${server}\${host03}-\$volume \${server}\${host04}:/export/\${server}\${host04}-\$volume force  
gluster volume set \$volume nfs.port 2049  
gluster volume set \$volume performance.cache-size 128MB  
gluster volume set \$volume performance.write-behind-window-size 4MB  
gluster volume set \$volume performance.io-thread-count 64  
gluster volume set \$volume performance.io-cache on  
gluster volume set \$volume performance.read-ahead on  
gluster volume start \$volume  
gluster volume info \$volume  
gluster volume status \$volume  
#montowanie NFS SR pod localhost lub VIP :/vol0 lub vol1  
__EOF__
chmod 755 /usr/local/bin/init_gluster4 

#init_gluster3 "xen" "1" "2" "3" "vol0" 3
#server="xen"; host01="1"; host02="2"; host03="3"; volume="vol0"; replica=3; 
cat <<__EOF__ > /usr/local/bin/init_gluster3
server=\$1  
host01=\$2  
host02=\$3  
host03=\$4
volume=\$5
replica=\$6
gluster peer status
#na dowolnym jednym wykonac 
#glusterfs dwa volumeny vol0 (iso) I vol1 (gfs) na sda3 i sda2  
#gluster volume stop \$volume force 
#gluster volume delete \$volume force 
gluster volume create \$volume replica \$replica \${server}\${host01}:/export/\${server}\${host01}-\$volume \${server}\${host02}:/export/\${server}\${host02}-\$volume \${server}\${host03}:/export/\${server}\${host03}-\$volume force  
gluster volume set \$volume nfs.port 2049  
gluster volume set \$volume performance.cache-size 128MB  
gluster volume set \$volume performance.write-behind-window-size 4MB  
gluster volume set \$volume performance.io-thread-count 64  
gluster volume set \$volume performance.io-cache on  
gluster volume set \$volume performance.read-ahead on  
gluster volume start \$volume  
gluster volume info \$volume  
gluster volume status \$volume  
#montowanie NFS SR pod localhost lub VIP :/vol0 lub vol1  
__EOF__
chmod 755 /usr/local/bin/init_gluster3

#init_gluster2 "xen" "1" "2" "vol0" 2
#server="xen"; host01="1"; host02="2"; volume="vol0"; replica=2; 
cat <<__EOF__ > /usr/local/bin/init_gluster2
server=\$1  
host01=\$2  
host02=\$3   
volume=\$4
replica=\$5
gluster peer status
#na dowolnym jednym wykonac 
#glusterfs dwa volumeny vol0 (iso) I vol1 (gfs) na sda3 i sda2  
#gluster volume stop \$volume force 
#gluster volume delete \$volume force 
gluster volume create \$volume replica \$replica \${server}\${host01}:/export/\${server}\${host01}-\$volume \${server}\${host02}:/export/\${server}\${host02}-\$volume force  
gluster volume set \$volume nfs.port 2049  
gluster volume set \$volume performance.cache-size 128MB  
gluster volume set \$volume performance.write-behind-window-size 4MB  
gluster volume set \$volume performance.io-thread-count 64  
gluster volume set \$volume performance.io-cache on  
gluster volume set \$volume performance.read-ahead on  
gluster volume start \$volume  
gluster volume info \$volume  
gluster volume status \$volume  
#montowanie NFS SR pod localhost lub VIP :/vol0 lub vol1  
__EOF__
chmod 755 /usr/local/bin/init_gluster2 

#init_brick "/dev/sdb1" "vol2"
cat <<__EOF__ > /usr/local/bin/init_brick
dev4gfs=\$1  
volume=\$2  
hname=\`hostname\` 
mkfs.ext4 -m 0 -j \$dev4gfs
tune2fs -O dir_index -o user_xattr \$dev4gfs
mkdir -p /export/\${hname}-\${volume}
echo  "\$dev4gfs /export/\${hname}-\${volume} ext4 rw,noatime,nodiratime,user_xattr,barrier=0,data=ordered 1 2" >> /etc/fstab
mount -a 
__EOF__
chmod 755 /usr/local/bin/init_brick

#reformat_brick "/dev/sda2" "vol1"
cat <<__EOF__ > /usr/local/bin/reformat_brick
dev4gfs=\$1  
volume=\$2  
hname=\`hostname\` 
umount /export/\${hname}-\${volume}
mkfs.ext4 -m 0 -j \$dev4gfs
tune2fs -O dir_index -o user_xattr \$dev4gfs
mount /export/\${hname}-\${volume}
__EOF__
chmod 755 /usr/local/bin/reformat_brick

#removeallandformat "/dev/sdb" 
cat <<__EOF__ > /usr/local/bin/removeallandformat
#!/bin/bash
# Script to automatically format all partitions on /dev/sda and create 
# a single partition for the whole disk
# Remove each partition
for v_partition in \`parted -s \$1 print|awk '/^ / {print \$1}'\`; do umount \$1\${v_partition}; parted -s \$1 rm \${v_partition}; done
# Find size of disk
v_disk=\`parted -s \$1 print|awk '/^Disk \/dev/ {print \$3}'| sed 's/[Mm][Bb]//'\`
# Create single partition
parted -s \$1 mkpart primary 1 \${v_disk}
# Format the partition
# mke2fs -T ext3 \${1}1
__EOF__
chmod 755  /usr/local/bin/removeallandformat

####################################################################################################################################
#ctdb need compiling: SCHED_FIFO does not run in xenserver - how to enable it?
yum install gcc-c++ autoconf rpm-build -y
yum install popt-devel libtalloc-devel libtdb-devel libtevent-devel -y 

mkdir rpmbuild/SPECS -p; curl http://ix.io/QMh > rpmbuild/SPECS/ctdb.spec 
wget https://download.samba.org/pub/ctdb/ctdb-2.5.6.tar.gz -P rpmbuild/SOURCES 
rpmbuild -bp rpmbuild/SPECS/ctdb.spec 
sed -i "s/SCHED_FIFO/SCHED_OTHER/"  rpmbuild/BUILD/ctdb-2.5.6/common/system_util.c  
sed -i "s/p.sched_priority = 1/p.sched_priority = 0/"  rpmbuild/BUILD/ctdb-2.5.6/common/system_util.c  
rpmbuild -bc --short-circuit  rpmbuild/SPECS/ctdb.spec --noclean 
rpmbuild -bi --short-circuit  rpmbuild/SPECS/ctdb.spec --noclean 
rpmbuild -bb --short-circuit  rpmbuild/SPECS/ctdb.spec --noclean 
rpm -ivh rpmbuild/RPMS/x86_64/ctdb-* --nodeps --force
mv rpmbuild rpmbuild-ctdb -f
sed -i "s/^# CTDB_LOGFILE/CTDB_LOGFILE/" /etc/sysconfig/ctdb 
sed -i "s/^# CTDB_NODES/CTDB_NODES/" /etc/sysconfig/ctdb  
sed -i "s/^CTDB_RECOVERY_LOCK/# CTDB_RECOVERY_LOCK/" /etc/sysconfig/ctdb 
sed -i 's/Restart=no/Restart=always/' /usr/lib/systemd/system/ctdb.service 
#service smb stop
#chkconfig smb off

#compiling module ceph.ko for cephfs
wget http://downloadns.citrix.com.edgesuite.net/11624/XenServer-7.0.0-binpkg.iso
mkdir iso1; mount XenServer-7.0.0-binpkg.iso iso1
yum localinstall iso1/x86_64/kernel-* -y
#wget http://downloadns.citrix.com.edgesuite.net/11620/XenServer-7.0.0-DDK.iso
wget http://downloadns.citrix.com.edgesuite.net/11623/XenServer-7.0.0-source.iso
mkdir iso; mount XenServer-7.0.0-source.iso iso
rpm -ivh iso/kernel-3.10.96-484.383030.src.rpm 
rpmbuild -bp rpmbuild/SPECS/kernel.spec 
sed -i 's/# CONFIG_CEPH_FS is not set/CONFIG_CEPH_FS=m/g' rpmbuild/BUILD/kernel-3.10.96/linux-3.10.96/.config 
rpmbuild -bc --short-circuit  rpmbuild/SPECS/kernel.spec --noclean 
rpmbuild -bi --short-circuit  rpmbuild/SPECS/kernel.spec --noclean 
rpmbuild -bb --short-circuit  rpmbuild/SPECS/kernel.spec --noclean 
rpm -ivh rpmbuild/RPMS/x86_64/kernel-* --nodeps --force
#scp  rpmbuild/RPMS/x86_64/kernel-* xen2:
#scp  rpmbuild/RPMS/x86_64/kernel-* xen3:
#rpm -ivh kernel-* --nodeps --force


####################################################################################################################################
##CEPH XENSERVER
#http://www.virtualtothecore.com/en/quickly-build-a-new-ceph-cluster-with-ceph-deploy-on-centos-7/
#ntpd, no selinux, ssh-copy-id to do
useradd -d /home/cephuser -m cephuser
echo "cephuser ALL = (root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/cephuser
chmod 0440 /etc/sudoers.d/cephuser
sed -i s'/Defaults requiretty/#Defaults requiretty'/g /etc/sudoers
git clone https://github.com/mstarikov/rbdsr.git
cd rbdsr
python ./install_rbdsr.py enable
####################################################################################################################################
#KONIEC