hdf-2-apache-kafka

#MAKE USERS KAFKA and ZK?
#IF BROKER ID ERROR add to server.properties reserved.broker.max.id=1000000000
#copy your existing zoo.cfg and server.properties per server, you will reuse these.


#PREP for Apache Install on HDF Servers
# Important Note - Make sure to record each broker.id and put it into the config file. The current broker.id can be found in the $KAFKA_LOG_DIRS/metadata.properties

sudo yum install java-1.8.0-openjdk

#Make Kafka and ZK Dirs
mkdir /usr/local/kafka
mkdir /usr/local/zookeeper

#move tars to dirs
mv kafka_2.11-1.1.1.tgz /usr/local/kafka
mv zookeeper-3.4.12.tar.gz /usr/local/zookeeper

cd /usr/local/kafka
tar xvf kafka_2.11-1.1.1.tgz
ln -s kafka_2.11-1.1.1 prod
cp /root/server.properties  /usr/local/kafka/prod/config/
vi /usr/local/kafka/prod/config/server.properties # reserved.broker.max.id=1000000000

cd /usr/local/zookeeper
tar xvf zookeeper-3.4.12.tar.gz
ln -s zookeeper-3.4.12 prod
cp /root/zoo.cfg  /usr/local/zookeeper/prod/conf/


#########
#########
INIT.D SCRIPTS
#########

####
KAFKA
####
#KF = https://pastebin.com/xTdAnsJK
vi /etc/init.d/kafka-broker

#Edit bin/kafka-run-class.sh and set KAFKA_JMX_OPTS variable
vi /usr/local/kafka/prod/bin/kafka-run-class.sh

KAFKA_JMX_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Djava.net.preferIPv4Stack=true"

export KAFKA_HEAP_OPTS="-Xmx4g -Xms4g"

#Update bin/kafka-server-start.sh add the below line
vi /usr/local/kafka/prod/bin/kafka-server-start.sh
export JMX_PORT=9999

#UPDATE Kafka log4j to use /var/log/kafka
vi /usr/local/kafka/prod/config/log4j.properties
:%s/${kafka.logs.dir}/\/var\/log\/kafka/



####
ZOOKEEPER
####
vi /etc/init.d/zookeeper-server
#ZK = https://pastebin.com/Lfn0xmVN

####
#edit logging setup by updating the zkEnv.sh
# vi /usr/local/zookeeper/prod/bin/zkEnv.sh
if [ "x${ZOO_LOG_DIR}" = "x" ]
then
    ZOO_LOG_DIR="/var/log/zookeeper"
fi

if [ "x${ZOO_LOG4J_PROP}" = "x" ]
then
    ZOO_LOG4J_PROP="INFO,ROLLINGFILE"
fi
#######

chmod 700 /etc/init.d/kafka-broker
chmod 700 /etc/init.d/zookeeper-server

chkconfig --add zookeeper-server
chkconfig --add kafka-broker
chkconfig zookeeper-server on
chkconfig kafka-broker on


#Remove HDF
yum erase -y ambari-*
yum erase -y zookeeper*
yum erase -y kafka*