Guest User

Untitled

a guest
Oct 21st, 2019
86
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. package org.myorg;
  2.  
  3. import java.io.IOException;
  4. import java.util.*;
  5.  
  6. import org.apache.hadoop.fs.Path;
  7. import org.apache.hadoop.conf.*;
  8. import org.apache.hadoop.io.*;
  9. import org.apache.hadoop.mapred.*;
  10. import org.apache.hadoop.util.*;
  11.  
  12. public class FindMax {
  13.  
  14. public static class Map extends MapReduceBase implements Mapper<LongWritable, Text, Text, IntWritable> {
  15. private IntWritable val = new IntWritable();
  16. private Text department = new Text();
  17.  
  18. public void map(LongWritable key, Text value, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
  19. String line = value.toString();
  20. StringTokenizer tokenizer = new StringTokenizer(line);
  21. while (tokenizer.hasMoreTokens()){
  22. department.set(tokenizer.nextToken());
  23. val.set(Integer.parseInt(tokenizer.nextToken()));
  24. }
  25. output.collect(department, val);
  26. }
  27. }
  28.  
  29. public static class Reduce extends MapReduceBase implements Reducer<Text, IntWritable, Text, IntWritable> {
  30.  
  31. public void reduce(Text key, Iterator<IntWritable> values, OutputCollector<Text, IntWritable> output, Reporter reporter) throws IOException {
  32. int maxx = 0;
  33. while (values.hasNext()) {
  34. int temp = values.next().get();
  35. if (maxx < temp){
  36. maxx = temp;
  37. }
  38. }
  39. output.collect(key, new IntWritable(maxx));
  40. }
  41. }
  42.  
  43. public static void main(String[] args) throws Exception {
  44. JobConf conf = new JobConf(FindMax.class);
  45. conf.setJobName("findmax");
  46.  
  47. conf.setOutputKeyClass(Text.class);
  48. conf.setOutputValueClass(IntWritable.class);
  49.  
  50. conf.setMapperClass(Map.class);
  51. conf.setCombinerClass(Reduce.class);
  52. conf.setReducerClass(Reduce.class);
  53.  
  54. conf.setInputFormat(TextInputFormat.class);
  55. conf.setOutputFormat(TextOutputFormat.class);
  56.  
  57. FileInputFormat.setInputPaths(conf, new Path(args[0]));
  58. FileOutputFormat.setOutputPath(conf, new Path(args[1]));
  59.  
  60. JobClient.runJob(conf);
  61. }
  62. }
RAW Paste Data