#! /usr/bin/env bash


OUT_DIR="local1"
NUM_REDUCERS=5

hadoop dfs -rm -r -skipTrash ${OUT_DIR} > /dev/null

yarn jar /opt/cloudera/parcels/CDH/lib/hadoop-mapreduce/hadoop-streaming.jar \
    -D mapreduce.job.reducer=${NUM_REDUCERS} \
    -D mapred.job.name="MR1" \
    -files mapper1.py,reducer1.py \
    -mapper mapper1.py \
    -reducer reducer1.py \
    -input /data/ids_part \
    -output $OUT_DIR > /dev/null


for num in 'seq 0 (($NUM_REDUCERS - 1))'
do
    hdfs dfs -cat ${OUT_DIR}/part-0000$num | head
done