bhd

// Initialization Commands

start-dfs.sh
start-yarn.sh
jps

// Hadoop Basic FS Commands

hadoop fs -mkdir %folder_name%
hadoop fs -ls
hadoop fs -copyFromLocal %locallocation% %hadooplocation%
hadoop fs -copyToLocal %hadooplocation% %locallocation%

// Word Count Map & Reduce

cd myc
ls -l
vi map.c
cc -o mymapper map.c
cc -o myreduce reduce.c
ls -l
hadoop jar $HADOOP_HOME/share/hadoop/tools/lib/hadoop-streaming-2.9.0.jar -input input -output out1 -mapper mymapper -reducer myreduce -file $HOME/myc/mymapper -file $HOME/myc/myreduce
hadoop fs -ls out1
hadoop fs -copyToLocal out1/part-00000 out1.txt
nano out1.txt

// Start Hive - SQL based interface for map & reduce using hadoop

cd hive
ls -l
hive

// Hive Commands

create table student(name STRING, age INT, score FLOAT);
show tables;
describe student;
load data local inpath 'studentdata.csv' overwrite into table student;
select * from student;

// Start Pig

cd pig
pig -x local

// Pig Commands

A = load './pg20417.txt';
B = foreach A generate flatten(TOKENIZE((chararray)$0)) as word;
c = group B by word;
D = foreach C generate COUNT(B), group;
store D into './wordcount';
quit;
ls -l
cd wordcount
nano part-r-00000

// Sqoop - get data from external database into hadoop

cd sqoop
ls -l
hadoop fs -ls
sqoop import --table BhavData -- connect jdbc:mysql://54.70.8.241/StockData --username hduser --password hduser -m 1
hadoop fs -ls
hadoop fs -ls BhavData

// Flume - allows you to capture streaming data - general purpose tools not limited to hadoop
cd flime
cp ../data/*.txt
ls -l
flume-ng agent --conf $FLUME_HOME/conf/ -f $FLUME_HOME/conf/flume.conf -n a1 &
ls -l sink
cp pg20417.txt spool
ls -l spool
ls -l sink
cp pg5000.txt spool
ls -l spool
ls -l sink
jps
kill -g 3667

// Clear spool and sink

rm spool/*
rm sink/*
ls -l spool
ls -l sink