Want more features on Pastebin? Sign Up, it's FREE!
Guest

Proper use of Tool and Configured classes

By: a guest on Nov 26th, 2012  |  syntax: Java  |  size: 2.58 KB  |  views: 207  |  expires: Never
download  |  raw  |  embed  |  report abuse  |  print
This paste has a previous version, view the difference. Text below is selected. Please press Ctrl+C to copy to your clipboard. (⌘+C on Mac)
  1. import java.util.*;
  2. import java.io.IOException;
  3. import org.apache.hadoop.fs.Path;
  4. import org.apache.hadoop.conf.*;
  5. import org.apache.hadoop.mapred.*;
  6. import org.apache.hadoop.util.*;
  7. import org.apache.hadoop.io.*;
  8. import org.apache.hadoop.util.Tool;
  9. import org.apache.hadoop.util.ToolRunner;
  10. import org.apache.hadoop.mapred.JobConf;
  11. import org.apache.hadoop.mapred.JobClient;
  12. import org.apache.hadoop.conf.Configured;
  13. import org.apache.hadoop.conf.Configuration;
  14. import org.apache.hadoop.mapred.OutputCollector;
  15. import org.apache.hadoop.mapred.MapReduceBase;
  16. import org.apache.hadoop.mapred.Reducer;
  17. import org.apache.hadoop.mapred.Reporter;
  18. import org.apache.hadoop.mapred.Mapper;
  19. import org.apache.hadoop.mapred.FileInputFormat;
  20. import org.apache.hadoop.mapred.FileOutputFormat;
  21.  
  22.  
  23. public class linecount extends Configured implements Tool
  24. {
  25.         //define mapper class
  26.         public static class  maplc extends  MapReduceBase  implements  Mapper <LongWritable, Text, Text,IntWritable>
  27.         {
  28.                 IntWritable One= new IntWritable(1);
  29.                 Text Word =new Text( "Total No. of Lines");
  30.  
  31.                 //define map method
  32.                 public void map (LongWritable key, Text value,OutputCollector<Text,IntWritable> output,Reporter reporter) throws IOException
  33.                 {
  34.                                                 output.collect (Word,One);
  35.                 }
  36.         }
  37.         //define reducer class
  38.         public static class reducelc extends  MapReduceBase implements Reducer <Text,IntWritable, Text,IntWritable>
  39.         {
  40.                 //define reduce class
  41.                 public void reduce (Text key, Iterator<IntWritable> values,OutputCollector<Text,IntWritable> output,Reporter reporter)throws IOException
  42.                 {
  43.                                 int Sum=0;
  44.                                 while (values.hasNext())
  45.                                 {      
  46.                                         Sum+=values.next().get();
  47.                                 }
  48.                                 output.collect(key,new IntWritable(Sum));
  49.                 }
  50.         }
  51.  
  52.         @Override
  53.         public int run (String[] args )throws Exception {
  54.        
  55.        
  56.            
  57.         JobConf  conf = new JobConf(getConf());
  58.                 conf.setJarByClass(linecount.class);
  59.                 conf.setJobName("linecount");
  60.             conf.setOutputKeyClass(Text.class);
  61.                 conf.setOutputValueClass(IntWritable.class);
  62.                 conf.setMapperClass(maplc.class);
  63.                 conf.setReducerClass(reducelc.class);
  64.                 conf.setInputFormat(TextInputFormat.class);
  65.                 conf.setOutputFormat(TextOutputFormat.class);
  66.                 conf.set("mapred.job.queue.name", "hdmi-paypal");
  67.                 FileInputFormat.setInputPaths(conf, new Path(args[0]));
  68.                 FileOutputFormat.setOutputPath(conf, new Path(args[1]));
  69.  
  70.                 JobClient.runJob(conf);
  71.                 return 1;
  72.  
  73.         }
  74.         public static void main (String[] args ) throws Exception {
  75.                 int exitCode =ToolRunner.run(new linecount(),args);
  76.                 System.exit(exitCode);
  77.  
  78.          }
  79. }
clone this paste RAW Paste Data