Advertisement
Guest User

Untitled

a guest
Feb 22nd, 2018
71
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 2.19 KB | None | 0 0
  1. package wordCount;
  2.  
  3.  
  4. import java.io.IOException;
  5.  
  6. import org.apache.hadoop.conf.Configuration;
  7.  
  8. import org.apache.hadoop.fs.Path;
  9.  
  10. import org.apache.hadoop.io.IntWritable;
  11.  
  12. import org.apache.hadoop.io.LongWritable;
  13.  
  14. import org.apache.hadoop.io.Text;
  15.  
  16. import org.apache.hadoop.mapreduce.Job;
  17.  
  18. import org.apache.hadoop.mapreduce.Mapper;
  19.  
  20. import org.apache.hadoop.mapreduce.Reducer;
  21.  
  22. import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
  23.  
  24. import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
  25.  
  26. import org.apache.hadoop.util.GenericOptionsParser;
  27.  
  28. import java.util.regex.Pattern;
  29.  
  30. import java.util.regex.Matcher;
  31.  
  32. public class WordCount {
  33.  
  34. public static void main(String [] args) throws Exception
  35.  
  36. {
  37.  
  38. Configuration c=new Configuration();
  39.  
  40. String[] files=new GenericOptionsParser(c,args).getRemainingArgs();
  41.  
  42. Path input=new Path(files[0]);
  43.  
  44. Path output=new Path(files[1]);
  45.  
  46. Job j=new Job(c,"wordcount");
  47.  
  48. j.setJarByClass(WordCount.class);
  49.  
  50. j.setMapperClass(MapForWordCount.class);
  51.  
  52. j.setReducerClass(ReduceForWordCount.class);
  53.  
  54. j.setOutputKeyClass(Text.class);
  55.  
  56. j.setOutputValueClass(IntWritable.class);
  57.  
  58. FileInputFormat.addInputPath(j, input);
  59.  
  60. FileOutputFormat.setOutputPath(j, output);
  61.  
  62. System.exit(j.waitForCompletion(true)?0:1);
  63.  
  64. }
  65.  
  66. public static class MapForWordCount extends Mapper<LongWritable, Text, Text, IntWritable>{
  67.  
  68. private static Pattern inputP = Pattern.compile("(.*) (\\d*)");
  69.  
  70. public void map(LongWritable key, Text value, Context con) throws IOException, InterruptedException
  71.  
  72. {
  73.  
  74. String line = value.toString();
  75. Matcher input = inputP.matcher(line);
  76.  
  77. if (input.matches()) {
  78. String cardSuit = input.group(1).toLowerCase();
  79. int cardValue = Integer.parseInt(input.group(2));
  80.  
  81. con.write(new Text(cardSuit), new IntWritable(cardValue));
  82.  
  83. }
  84.  
  85. }
  86. }
  87.  
  88.  
  89. public static class ReduceForWordCount extends Reducer<Text, IntWritable, Text, IntWritable>
  90.  
  91. {
  92.  
  93. public void reduce(Text word, Iterable<IntWritable> values, Context con) throws IOException, InterruptedException
  94.  
  95. {
  96.  
  97. int sum = 0;
  98.  
  99. for(IntWritable value : values)
  100.  
  101. {
  102.  
  103. sum += value.get();
  104.  
  105. }
  106.  
  107. con.write(word, new IntWritable(sum));
  108.  
  109. }
  110.  
  111. }
  112.  
  113. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement