Advertisement
Guest User

WordCount.java

a guest
Feb 19th, 2019
79
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Java 2.00 KB | None | 0 0
  1. import org.apache.hadoop.conf.Configuration;
  2. import org.apache.hadoop.fs.Path;
  3. import org.apache.hadoop.io.IntWritable;
  4. import org.apache.hadoop.io.LongWritable;
  5. import org.apache.hadoop.io.Text;
  6. import org.apache.hadoop.mapreduce.Counter;
  7. import org.apache.hadoop.mapreduce.Reducer;
  8. import org.apache.hadoop.mapreduce.Mapper;
  9. import org.apache.hadoop.mapreduce.Job;
  10. import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
  11. import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
  12. import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
  13. import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
  14.  
  15. public class WordCount {
  16. @SuppressWarnings("deprecation")
  17. public static void main(String[] args) throws Exception {
  18.  
  19.         if (args.length != 2) {
  20.        
  21.         System.err.println("Usage: WordCount <input path> <output path>");
  22.         System.exit(-1); }
  23.        
  24.         System.out.println("In Driver now!");
  25. //Create configuration:
  26. Configuration conf = new Configuration(true);
  27.  
  28. //Create Jobs
  29. Job job = new Job (conf, "word Count");
  30.  
  31. job.setJarByClass(WordCount.class);
  32. job.setJobName("WordCount");
  33. //Setup MapReduce
  34. job.setMapperClass(WordMapperAvg.class);
  35. job.setReducerClass(SumReducer2.class);
  36. job.setNumReduceTasks(1);
  37.  
  38. job.setOutputKeyClass(Text.class);
  39. job.setOutputValueClass(IntWritable.class);
  40.  
  41. //Input:
  42. FileInputFormat.addInputPath(job, new Path(args[0]));
  43. job.setInputFormatClass(TextInputFormat.class);
  44.  
  45. //Output
  46. FileOutputFormat.setOutputPath(job, new Path(args[1]));
  47. job.setOutputFormatClass(TextOutputFormat.class);
  48. int code = job.waitForCompletion(true) ? 0 : 1;
  49. //Counter
  50. Counter mapperCounter = job.getCounters().findCounter(WordMapperAvg.MapCounters.MAP_RECORD_COUNTER);
  51. Counter reducerCounter = job.getCounters().findCounter(SumReducer2.ReducerCounters.REDUCER_RECORD_COUNTER);
  52. System.out.println("Total Num. of records processed in MAP:"+mapperCounter.getValue());
  53. System.out.println("Total num. of records processed in REDUCER:" +reducerCounter.getValue());
  54. System.exit(code);     
  55.  
  56. }
  57. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement