Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import org.apache.hadoop.conf.Configuration;
- import org.apache.hadoop.fs.Path;
- import org.apache.hadoop.io.IntWritable;
- import org.apache.hadoop.io.LongWritable;
- import org.apache.hadoop.io.Text;
- import org.apache.hadoop.mapreduce.Counter;
- import org.apache.hadoop.mapreduce.Reducer;
- import org.apache.hadoop.mapreduce.Mapper;
- import org.apache.hadoop.mapreduce.Job;
- import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
- import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
- import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
- import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
- public class WordCount {
- @SuppressWarnings("deprecation")
- public static void main(String[] args) throws Exception {
- if (args.length != 2) {
- System.err.println("Usage: WordCount <input path> <output path>");
- System.exit(-1); }
- System.out.println("In Driver now!");
- //Create configuration:
- Configuration conf = new Configuration(true);
- //Create Jobs
- Job job = new Job (conf, "word Count");
- job.setJarByClass(WordCount.class);
- job.setJobName("WordCount");
- //Setup MapReduce
- job.setMapperClass(WordMapperAvg.class);
- job.setReducerClass(SumReducer2.class);
- job.setNumReduceTasks(1);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
- //Input:
- FileInputFormat.addInputPath(job, new Path(args[0]));
- job.setInputFormatClass(TextInputFormat.class);
- //Output
- FileOutputFormat.setOutputPath(job, new Path(args[1]));
- job.setOutputFormatClass(TextOutputFormat.class);
- int code = job.waitForCompletion(true) ? 0 : 1;
- //Counter
- Counter mapperCounter = job.getCounters().findCounter(WordMapperAvg.MapCounters.MAP_RECORD_COUNTER);
- Counter reducerCounter = job.getCounters().findCounter(SumReducer2.ReducerCounters.REDUCER_RECORD_COUNTER);
- System.out.println("Total Num. of records processed in MAP:"+mapperCounter.getValue());
- System.out.println("Total num. of records processed in REDUCER:" +reducerCounter.getValue());
- System.exit(code);
- }
- }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement