Instalação CEPH

############# FASE 0 - FEITO NA VM
# Puxar a VM de http://vmubuntu.setic.ufsc.br
# Extrair e identificar o arquivo .VDI
# Importar a VM no VirtualBox com:
## Use an existing virtual hard disk -> VDI extraido
## 4GB de RAM
## Um HD extra de 10GB
## Rede como Bridge (Settings > Network > Attached To > Bridged Adapter)

# Credencial padrao -  osboxes / osboxes.org
# Mudar a senha de root para cephserver
sudo su -
passwd

# Instalar pacotes basicos
apt install -y ssh vim
echo "PermitRootLogin yes" >> /etc/ssh/sshd_config

# Em /etc/cloud/cloud.cfg , mudar preserve_hostname: para True
echo date > /etc/machine-id
reboot

# Instalar Pacotes
wget -q -O- 'https://download.ceph.com/keys/release.asc' | sudo apt-key add -
echo deb http://download.ceph.com/debian-mimic/ $(lsb_release -sc) main | sudo tee /etc/apt/sources.list.d/ceph.list
apt update
apt install ceph-deploy ceph

############# FASE 1 - Criar o MON1
# Mudar o conteudos do /etc/hostname para o nome da maquina (mon1, osd1, osd2, osd3, etc)
hostnamectl set-hostname NOME

# Identificar os IPs dos servidores ( ifconfig|grep 150.162 ) e adicionar no /etc/hosts
150.162.x.y mon1 mds1
150.162.x.y osd1
150.162.x.y osd2
150.162.x.y osd3

# Troca de chaves
# No keygen, basta ir dando ENTER
ssh-keygen
ssh-copy-id root@mon1
ssh-copy-id root@osd1
ssh-copy-id root@osd2
ssh-copy-id root@osd3

# Testar se a Troca de chaves funcionou:
# O seguinte comando não deve pedir senha
ssh root@mon1 hostname
ssh root@osd1 hostname
ssh root@osd2 hostname
ssh root@osd3 hostname

### Configurações iniciais do cluster
cd /etc/ceph
ceph-deploy new mon1 --public-network  '150.162.X.0/24'
ceph-deploy install mon1
ceph-deploy mon create-initial
ceph-deploy mon create mon1

### testar se o Mon esta correto:
ceph -s
# Aparecerá algo assim
  cluster:
    id:     xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx
    health: HEALTH_OK

  services:
    mon: 1 daemons, quorum mon1
    mgr: no daemons active
    osd: 0 osds: 0 up, 0 in

  data:
    pools:   0 pools, 0 pgs
    objects: 0  objects, 0 B
    usage:   0 B used, 0 B / 0 B avail
    pgs:





############# FASE 2 - Criar o MGR1
ceph-deploy mgr create mon1
ceph mgr module enable dashboard
ceph config set mgr mgr/dashboard/ssl false
ceph config set mgr mgr/dashboard/mon1/server_port 7000
ceph config set mgr mgr/dashboard/mon1/server_addr ip-do-mon1
ceph dashboard set-login-credentials admin ceph

# Adicionar no /etc/ceph/ceph.conf
[mon]
mgr initial modules = dashboard

service ceph-mgr@mon1 restart
service ceph-mon@mon1 restart

### Testar se o MGR esta correto:
Acessar no browser http://150.162.x.y:7000

############# FASE 3 - Criar o OSDs
# No MON1 - Executar os comandos abaixo
cd /etc/ceph
ceph-deploy install osd1 osd2 osd3
ceph-deploy admin osd1 osd2 osd3

# No OSD1
# Criar uma particao
ceph-volume lvm create --data /dev/sdb


###  Criando os Pools Replicados
# No Mon1
ceph osd pool create fr-data 64 64
ceph osd pool create fr-metadata 64 64

# Comando de Teste
#ceph osd pool ls
#ceph df
#ceph osd pool get fr-data all
#ceph status

#### Fase 4 - Instalar o CEPHFS
# No Mon1
# Instalar o MDS1
ceph-deploy mds create mon1

# Criar o Sistema de Arquivos
ceph fs new fs_padrao fr-metadata fr-data

# Testar
ceph fs ls
# Deve responder
name: fs_padrao, metadata pool: fr-metadata, data pools: [fr-data ]


#### Fase 5 - Configurar o Cliente CEPHFS
mkdir /ceph
key=`ceph auth print_key client.admin`
mount -t ceph mon1:6789:/ /ceph/ -o name=admin,secret=$key
df -h | grep ceph

################################################################################################
################################################################################################
################################################################################################
################################################################################################
################################################################################################
################################################################################################


### Criando as Imagens
rbd create bloco1 --size 1G  -p br-teste --image-format=2
rbd feature -p br-teste disable bloco1 exclusive-lock,object-map,fast-diff,deep-flatten

### Montando a Imagem no MON1
# Adicionar no arquivo /etc/ceph/rbdmap
br-teste/bloco1 id=admin,keyring=/etc/ceph/ceph.client.admin.keyring
service rbdmap restart
#ls -lh /dev/rbd/
#mkfs.xfs /dev/rbd/bloco1
#mkdir /ceph
#mount /dev/rbd/bloco1 /ceph
#dd if=/dev/zero of=/ceph/zero.iso bs=1M count=100
#ceph df

### Alterando os Pools
#ceph status
#ceph osd pool get br-teste all
ceph osd pool set br-teste size 2
ceph osd pool set br-teste min_size 1
#ceph status

### Criando os Pools Erasure Code
ceph osd erasure-code-profile set erasure-2-1 ruleset-failure-domain=osd k=2 m=1
ceph osd pool create be-teste 128 128 erasure erasure-2-1

ceph osd tier add be-teste br-teste
ceph osd tier cache-mode br-teste writeback
ceph osd tier set-overlay be-teste br-teste

#ceph df

### Substituindo HDs
# Executando no HOST!!!
OSD_RUIM=3
DEV=/dev/sdb

ceph osd out $OSD_RUIM
while ! ceph osd safe-to-destroy $OSD_RUIM ; sleep 10; echo -n "." ; done ; echo; echo "Pode continuar...";echo
ceph osd destroy $OSD_RUIM --yes-i-really-mean-it
ceph-disk prepare --bluestore $DEV --osd-id $OSD_RUIM

### Editando o Crushmap
# Puxando
ceph osd getcrushmap -o crushmapdump
crushtool -d crushmapdump -o cm-decompilado.tier
vi cm-decompilado.tier

# Salvando
crushtool -c cm-decompilado.tier -o cm-compilado.tier
ceph osd setcrushmap -i cm-compilado.tier