run_map_reduce.sh

#!/bin/bash

set -x

#en

cd /usr/local/hadoop/
bin/hdfs namenode -format

sbin/start-dfs.sh

bin/hdfs dfs -mkdir /user
bin/hdfs dfs -mkdir /user/domi

#bin/hdfs dfs -put /home/domi/Desktop/input_files/set01 wc_input
bin/hdfs dfs -put /home/domi/Desktop/input_files/set02 wc_input

bin/hadoop jar share/hadoop/tools/lib/hadoop-streaming-2.7.2.jar -input wc_input -output wc_output -mapper /home/domi/PycharmProjects/HadoopWordCount/wc/wc_mapper.py -reducer /home/domi/PycharmProjects/HadoopWordCount/wc/wc_reducer.py

bin/hdfs dfs -cat wc_output/* > /home/domi/wc_output.txt
#bin/hdfs dfs -cat wc_output/*

bin/hdfs dfs -put /home/domi/PycharmProjects/HadoopWordCount/most_frequent/en_200_most_frequent.txt .
bin/hdfs dfs -put /home/domi/PycharmProjects/HadoopWordCount/most_frequent/pl_200_most_frequent.txt .
bin/hdfs dfs -put /home/domi/PycharmProjects/HadoopWordCount/most_frequent/hu_200_most_frequent.txt .
bin/hdfs dfs -put /home/domi/PycharmProjects/HadoopWordCount/most_frequent/de_200_most_frequent.txt .
bin/hdfs dfs -put /home/domi/PycharmProjects/HadoopWordCount/most_frequent/sv_200_most_frequent.txt .
bin/hdfs dfs -put /home/domi/PycharmProjects/HadoopWordCount/most_frequent/nl_200_most_frequent.txt .

bin/hadoop jar share/hadoop/tools/lib/hadoop-streaming-2.7.2.jar -input wc_output -output lang_output -mapper /home/domi/PycharmProjects/HadoopWordCount/lang/lang_mapper.py -reducer /home/domi/PycharmProjects/HadoopWordCount/lang/lang_reducer.py -cacheFile hdfs://localhost:9000/user/domi/en_200_most_frequent.txt#en_200_most_frequent.txt,hdfs://localhost:9000/user/domi/pl_200_most_frequent.txt#pl_200_most_frequent.txt,hdfs://localhost:9000/user/domi/hu_200_most_frequent.txt#hu_200_most_frequent.txt,hdfs://localhost:9000/user/domi/de_200_most_frequent.txt#de_200_most_frequent.txt,hdfs://localhost:9000/user/domi/sv_200_most_frequent.txt#sv_200_most_frequent.txt,hdfs://localhost:9000/user/domi/nl_200_most_frequent.txt#nl_200_most_frequent.txt -verbose


bin/hdfs dfs -cat lang_output/* > /home/domi/lang_output.txt
bin/hdfs dfs -cat lang_output/*

sbin/stop-dfs.sh

for x in `cd /etc/init.d ; ls hadoop*` ; do sudo service $x stop ; done

rm -rf /tmp/hadoop-domi/dfs/*