Advertisement
Guest User

Untitled

a guest
Jul 1st, 2016
60
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 4.42 KB | None | 0 0
  1. import java.io.IOException;
  2. import java.util.Scanner;
  3. import java.util.ArrayList;
  4. import java.util.List;
  5.  
  6. import org.apache.hadoop.conf.Configuration;
  7. import org.apache.hadoop.fs.Path;
  8. import org.apache.hadoop.io.IntWritable;
  9. import org.apache.hadoop.io.LongWritable;
  10. import org.apache.hadoop.io.Text;
  11. import org.apache.hadoop.mapreduce.Job;
  12. import org.apache.hadoop.mapreduce.Mapper;
  13. import org.apache.hadoop.mapreduce.Reducer;
  14. import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
  15. import org.apache.hadoop.mapreduce.lib.input.TextInputFormat;
  16. import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
  17. import org.apache.hadoop.util.ToolRunner;
  18.  
  19. public class WordCount {
  20.  
  21. static int printUsage() {
  22. System.out.println("wordcount [-r <reduces>] <input> <output>");
  23. ToolRunner.printGenericCommandUsage(System.out);
  24. return -1;
  25. }
  26.  
  27. public static void main(String[] args) throws Exception {
  28.  
  29. List<String> otherArgs = new ArrayList<String>();
  30.  
  31. Configuration conf = new Configuration();
  32.  
  33. for(int i=0; i < args.length; ++i) {
  34. try {
  35. if ("-r".equals(args[i])) {
  36. conf.setInt("mapreduce.job.reduces", Integer.parseInt(args[++i]));
  37. } else {
  38. otherArgs.add(args[i]);
  39. }
  40. } catch (NumberFormatException except) {
  41. System.out.println("ERROR: Integer expected instead of " + args[i]);
  42. System.exit(printUsage());
  43. } catch (ArrayIndexOutOfBoundsException except) {
  44. System.out.println("ERROR: Required parameter missing from " +
  45. args[i-1]);
  46. System.exit(printUsage());
  47. }
  48. }
  49. // Make sure there are exactly 2 parameters left.
  50. if (otherArgs.size() != 2) {
  51. System.out.println("ERROR: Wrong number of parameters: " +
  52. otherArgs.size() + " instead of 2.");
  53. System.exit(printUsage());
  54. }
  55.  
  56. Path input = new Path(otherArgs.get(0));
  57. Path output =new Path(otherArgs.get(1));
  58.  
  59. Job job = Job.getInstance(conf);
  60. job.setJarByClass(WordCount.class);
  61. job.setJobName("WordCount");
  62.  
  63. FileInputFormat.addInputPath(job, input);
  64. FileOutputFormat.setOutputPath(job, output);
  65.  
  66. job.setMapperClass(MyMapper.class);
  67. //job.setCombinerClass(WordCount$MyReducer.class);
  68. job.setReducerClass(MyReducer.class);
  69.  
  70. // An InputFormat for plain text files.
  71. // Files are broken into lines. Either linefeed or carriage-return are used
  72. // to signal end of line. Keys are the position in the file, and values
  73. // are the line of text.
  74. job.setInputFormatClass(TextInputFormat.class);
  75.  
  76. job.setOutputKeyClass(Text.class);
  77. job.setOutputValueClass(IntWritable.class);
  78.  
  79. job.waitForCompletion(true);
  80.  
  81. }
  82.  
  83. public static class MyMapper extends Mapper<LongWritable, Text, Text, IntWritable>{
  84. private final static IntWritable one = new IntWritable(1);
  85. private Text word = new Text();
  86.  
  87. @Override
  88. protected void cleanup(Context context) throws IOException,
  89. InterruptedException {
  90. // TODO Auto-generated method stub
  91. super.cleanup(context);
  92. }
  93.  
  94. @Override
  95. protected void map(LongWritable key, Text value, Context context)
  96. throws IOException, InterruptedException {
  97. Scanner scanner = new Scanner(value.toString());
  98. scanner.useDelimiter(" ");
  99. while (scanner.hasNext()) {
  100.  
  101. // ToDo
  102.  
  103. }
  104. scanner.close();
  105. }
  106.  
  107. @Override
  108. public void run(Context context) throws IOException,
  109. InterruptedException {
  110. // TODO Auto-generated method stub
  111. super.run(context);
  112. }
  113.  
  114. @Override
  115. protected void setup(Context context) throws IOException,
  116. InterruptedException {
  117. // TODO Auto-generated method stub
  118. super.setup(context);
  119. }
  120.  
  121. }
  122.  
  123. public static class MyReducer extends Reducer<Text, IntWritable, Text, IntWritable>{
  124.  
  125. @Override
  126. protected void cleanup(Context context) throws IOException,
  127. InterruptedException {
  128. // TODO Auto-generated method stub
  129. super.cleanup(context);
  130. }
  131.  
  132. @Override
  133. protected void reduce(Text key, Iterable<IntWritable> values, Context context)
  134. throws IOException, InterruptedException {
  135.  
  136. // ToDo
  137. }
  138.  
  139. @Override
  140. public void run(Context arg0) throws IOException, InterruptedException {
  141. // TODO Auto-generated method stub
  142. super.run(arg0);
  143. }
  144.  
  145. @Override
  146. protected void setup(Context context) throws IOException,
  147. InterruptedException {
  148. // TODO Auto-generated method stub
  149. super.setup(context);
  150. }
  151. }
  152. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement