SHOW:
|
|
- or go back to the newest paste.
| 1 | import java.util.*; | |
| 2 | import java.io.IOException; | |
| 3 | import org.apache.hadoop.fs.Path; | |
| 4 | import org.apache.hadoop.conf.*; | |
| 5 | import org.apache.hadoop.mapred.*; | |
| 6 | import org.apache.hadoop.util.*; | |
| 7 | import org.apache.hadoop.io.*; | |
| 8 | import org.apache.hadoop.util.Tool; | |
| 9 | import org.apache.hadoop.util.ToolRunner; | |
| 10 | import org.apache.hadoop.mapred.JobConf; | |
| 11 | import org.apache.hadoop.mapred.JobClient; | |
| 12 | import org.apache.hadoop.conf.Configured; | |
| 13 | import org.apache.hadoop.conf.Configuration; | |
| 14 | import org.apache.hadoop.mapred.OutputCollector; | |
| 15 | import org.apache.hadoop.mapred.MapReduceBase; | |
| 16 | import org.apache.hadoop.mapred.Reducer; | |
| 17 | import org.apache.hadoop.mapred.Reporter; | |
| 18 | import org.apache.hadoop.mapred.Mapper; | |
| 19 | import org.apache.hadoop.mapred.FileInputFormat; | |
| 20 | import org.apache.hadoop.mapred.FileOutputFormat; | |
| 21 | ||
| 22 | ||
| 23 | public class linecount extends Configured implements Tool | |
| 24 | {
| |
| 25 | //define mapper class | |
| 26 | public static class maplc extends MapReduceBase implements Mapper <LongWritable, Text, Text,IntWritable> | |
| 27 | {
| |
| 28 | IntWritable One= new IntWritable(1); | |
| 29 | Text Word =new Text( "Total No. of Lines"); | |
| 30 | ||
| 31 | //define map method | |
| 32 | public void map (LongWritable key, Text value,OutputCollector<Text,IntWritable> output,Reporter reporter) throws IOException | |
| 33 | {
| |
| 34 | output.collect (Word,One); | |
| 35 | } | |
| 36 | } | |
| 37 | //define reducer class | |
| 38 | public static class reducelc extends MapReduceBase implements Reducer <Text,IntWritable, Text,IntWritable> | |
| 39 | {
| |
| 40 | //define reduce class | |
| 41 | public void reduce (Text key, Iterator<IntWritable> values,OutputCollector<Text,IntWritable> output,Reporter reporter)throws IOException | |
| 42 | {
| |
| 43 | int Sum=0; | |
| 44 | while (values.hasNext()) | |
| 45 | {
| |
| 46 | Sum+=values.next().get(); | |
| 47 | } | |
| 48 | output.collect(key,new IntWritable(Sum)); | |
| 49 | } | |
| 50 | } | |
| 51 | ||
| 52 | @Override | |
| 53 | public int run (String[] args )throws Exception {
| |
| 54 | ||
| 55 | ||
| 56 | ||
| 57 | - | JobConf conf = new JobConf(linecount.class); |
| 57 | + | JobConf conf = new JobConf(getConf()); |
| 58 | conf.setJarByClass(linecount.class); | |
| 59 | conf.setJobName("linecount");
| |
| 60 | conf.setOutputKeyClass(Text.class); | |
| 61 | conf.setOutputValueClass(IntWritable.class); | |
| 62 | conf.setMapperClass(maplc.class); | |
| 63 | conf.setReducerClass(reducelc.class); | |
| 64 | conf.setInputFormat(TextInputFormat.class); | |
| 65 | conf.setOutputFormat(TextOutputFormat.class); | |
| 66 | conf.set("mapred.job.queue.name", "hdmi-paypal");
| |
| 67 | FileInputFormat.setInputPaths(conf, new Path(args[0])); | |
| 68 | FileOutputFormat.setOutputPath(conf, new Path(args[1])); | |
| 69 | ||
| 70 | JobClient.runJob(conf); | |
| 71 | return 1; | |
| 72 | ||
| 73 | } | |
| 74 | public static void main (String[] args ) throws Exception {
| |
| 75 | int exitCode =ToolRunner.run(new linecount(),args); | |
| 76 | System.exit(exitCode); | |
| 77 | ||
| 78 | } | |
| 79 | } |
