run_better_map_reduce

#!/bin/bash

set -x

cd /usr/local/hadoop/
bin/hdfs namenode -format
sbin/start-dfs.sh
bin/hdfs dfs -mkdir /user
bin/hdfs dfs -mkdir /user/domi

#bin/hdfs dfs -put /home/domi/Desktop/input_files/set01 wc_input
bin/hdfs dfs -put /home/domi/Desktop/input_files/set03 wc_input

#wc map reduce
bin/hadoop jar share/hadoop/tools/lib/hadoop-streaming-2.7.2.jar -input wc_input -output wc_output -mapper /home/domi/PycharmProjects/HadoopWordCount/better/wc/wc_mapper.py -reducer /home/domi/PycharmProjects/HadoopWordCount/better/wc/wc_reducer.py

#bin/hdfs dfs -cat wc_output/*
#read -p "Press [Enter] to confirm wc output..."

#all map reduce
bin/hadoop jar share/hadoop/tools/lib/hadoop-streaming-2.7.2.jar -input wc_input -output all_output -mapper /home/domi/PycharmProjects/HadoopWordCount/better/all/all_mapper.py -reducer /home/domi/PycharmProjects/HadoopWordCount/better/all/all_reducer.py
bin/hdfs dfs -cat all_output/* > /home/domi/all_output.txt
bin/hdfs dfs -put /home/domi/all_output.txt .
bin/hdfs dfs -cat all_output/*

#freq map reduce
bin/hadoop jar share/hadoop/tools/lib/hadoop-streaming-2.7.2.jar -input wc_output -output freq_output -mapper /home/domi/PycharmProjects/HadoopWordCount/better/freq/freq_mapper.py -reducer /home/domi/PycharmProjects/HadoopWordCount/better/freq/freq_reducer.py -cacheFile hdfs://localhost:9000/user/domi/all_output.txt#all_output.txt

bin/hdfs dfs -cat freq_output/*
read -p "Press [Enter] key to confirm freq output..."

#copy input files for freq - list of 200 most popular words in 6 languages
bin/hdfs dfs -put /home/domi/PycharmProjects/HadoopWordCount/better/most_frequent/en_200_most_frequent.txt .
bin/hdfs dfs -put /home/domi/PycharmProjects/HadoopWordCount/better/most_frequent/pl_200_most_frequent.txt .
bin/hdfs dfs -put /home/domi/PycharmProjects/HadoopWordCount/better/most_frequent/hu_200_most_frequent.txt .
bin/hdfs dfs -put /home/domi/PycharmProjects/HadoopWordCount/better/most_frequent/de_200_most_frequent.txt .
bin/hdfs dfs -put /home/domi/PycharmProjects/HadoopWordCount/better/most_frequent/sv_200_most_frequent.txt .
bin/hdfs dfs -put /home/domi/PycharmProjects/HadoopWordCount/better/most_frequent/nl_200_most_frequent.txt .

#lang mp reduce
bin/hadoop jar share/hadoop/tools/lib/hadoop-streaming-2.7.2.jar -input freq_output -output lang_output -mapper /home/domi/PycharmProjects/HadoopWordCount/better/lang/lang_mapper.py -reducer /home/domi/PycharmProjects/HadoopWordCount/better/lang/lang_reducer.py -cacheFile hdfs://localhost:9000/user/domi/en_200_most_frequent.txt#en_200_most_frequent.txt,hdfs://localhost:9000/user/domi/pl_200_most_frequent.txt#pl_200_most_frequent.txt,hdfs://localhost:9000/user/domi/hu_200_most_frequent.txt#hu_200_most_frequent.txt,hdfs://localhost:9000/user/domi/de_200_most_frequent.txt#de_200_most_frequent.txt,hdfs://localhost:9000/user/domi/sv_200_most_frequent.txt#sv_200_most_frequent.txt,hdfs://localhost:9000/user/domi/nl_200_most_frequent.txt#nl_200_most_frequent.txt


bin/hdfs dfs -cat lang_output/* > /home/domi/lang_output.txt
bin/hdfs dfs -cat lang_output/*

sbin/stop-dfs.sh
for x in `cd /etc/init.d ; ls hadoop*` ; do sudo service $x stop ; done
rm -rf /tmp/hadoop-domi/dfs/*