Advertisement
Guest User

Argumments to Hadoop Command

a guest
Nov 24th, 2012
112
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Java 2.65 KB | None | 0 0
  1. import java.util.*;
  2. import java.io.IOException;
  3. import org.apache.hadoop.fs.Path;
  4. import org.apache.hadoop.conf.*;
  5. import org.apache.hadoop.mapred.*;
  6. import org.apache.hadoop.util.*;
  7. import org.apache.hadoop.io.*;
  8. import org.apache.hadoop.util.Tool;
  9. import org.apache.hadoop.util.ToolRunner;
  10. import org.apache.hadoop.mapred.JobConf;
  11. import org.apache.hadoop.mapred.JobClient;
  12. import org.apache.hadoop.conf.Configured;
  13. import org.apache.hadoop.conf.Configuration;
  14. import org.apache.hadoop.mapred.OutputCollector;
  15. import org.apache.hadoop.mapred.MapReduceBase;
  16. import org.apache.hadoop.mapred.Reducer;
  17. import org.apache.hadoop.mapred.Reporter;
  18. import org.apache.hadoop.mapred.Mapper;
  19. import org.apache.hadoop.mapred.FileInputFormat;
  20. import org.apache.hadoop.mapred.FileOutputFormat;
  21.  
  22.  
  23. public class linecount extends Configured implements Tool
  24. {
  25.     //define mapper class
  26.     public static class  maplc extends  MapReduceBase  implements  Mapper <LongWritable, Text, Text,IntWritable>
  27.     {
  28.         IntWritable One= new IntWritable(1);
  29.         Text Word =new Text( "Total No. of Lines");
  30.  
  31.         //define map method
  32.         public void map (LongWritable key, Text value,OutputCollector<Text,IntWritable> output,Reporter reporter) throws IOException
  33.         {
  34.                         output.collect (Word,One);
  35.         }
  36.     }
  37.     //define reducer class
  38.     public static class reducelc extends  MapReduceBase implements Reducer <Text,IntWritable, Text,IntWritable>
  39.     {
  40.         //define reduce class
  41.         public void reduce (Text key, Iterator<IntWritable> values,OutputCollector<Text,IntWritable> output,Reporter reporter)throws IOException
  42.         {
  43.                 int Sum=0;
  44.                 while (values.hasNext())
  45.                 {  
  46.                     Sum+=values.next().get();
  47.                 }
  48.                 output.collect(key,new IntWritable(Sum));
  49.         }
  50.     }
  51.  
  52.         @Override
  53.     public int run (String[] args )throws Exception {
  54.    
  55.    
  56.        
  57.         JobConf  conf = new JobConf(linecount.class);
  58.         conf.setJobName("linecount");
  59.         conf.setOutputKeyClass(Text.class);
  60.         conf.setOutputValueClass(IntWritable.class);
  61.         conf.setMapperClass(maplc.class);
  62.         conf.setReducerClass(reducelc.class);
  63.         conf.setInputFormat(TextInputFormat.class);
  64.         conf.setOutputFormat(TextOutputFormat.class);
  65.         conf.set("mapred.job.queue.name", "hdmi-paypal");
  66.         FileInputFormat.setInputPaths(conf, new Path(args[0]));
  67.         FileOutputFormat.setOutputPath(conf, new Path(args[1]));
  68.  
  69.         JobClient.runJob(conf);
  70.         return 1;
  71.  
  72.     }
  73.     public static void main (String[] args ) throws Exception {
  74.             int exitCode =ToolRunner.run(new linecount(),args);
  75.                 System.exit(exitCode);
  76.  
  77.          }
  78. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement