Guest User

Untitled

a guest
May 21st, 2018
123
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 3.20 KB | None | 0 0
  1. package reader;
  2.  
  3. import org.apache.hadoop.conf.Configuration;
  4. import org.apache.hadoop.conf.Configured;
  5. import org.apache.hadoop.fs.Path;
  6. import org.apache.hadoop.hive.ql.io.sarg.PredicateLeaf;
  7. import org.apache.hadoop.hive.ql.io.sarg.SearchArgument;
  8. import org.apache.hadoop.hive.ql.io.sarg.SearchArgument.Builder;
  9. import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory;
  10. import org.apache.hadoop.io.IntWritable;
  11. import org.apache.hadoop.io.LongWritable;
  12. import org.apache.hadoop.io.NullWritable;
  13. import org.apache.hadoop.io.Text;
  14. import org.apache.hadoop.mapreduce.Job;
  15. import org.apache.hadoop.mapreduce.Mapper;
  16. import org.apache.hadoop.mapreduce.Reducer;
  17. import org.apache.hadoop.util.GenericOptionsParser;
  18. import org.apache.hadoop.util.Tool;
  19. import org.apache.hadoop.util.ToolRunner;
  20. import org.apache.orc.mapred.OrcStruct;
  21. import org.apache.orc.mapreduce.OrcInputFormat;
  22. import org.apache.orc.mapreduce.OrcOutputFormat;
  23. import java.io.IOException;
  24.  
  25.  
  26. public class Main extends Configured implements Tool
  27. {
  28.  
  29.  
  30.  
  31. public static class MyMapper extends Mapper<NullWritable, OrcStruct, LongWritable, Text>
  32. {
  33. private LongWritable one = new LongWritable(1);
  34.  
  35. @Override
  36. protected void map(NullWritable key, OrcStruct value, Context context)
  37. throws IOException, InterruptedException
  38. {
  39. String temp = value.getFieldValue(6).toString();
  40. if(temp.equalsIgnoreCase("0111000000"))
  41. {
  42. Text word = new Text();
  43. word.set(temp);
  44. context.write(one, word);
  45. }
  46. }
  47. }
  48.  
  49.  
  50.  
  51. public static class MyReducer extends Reducer<LongWritable, Text, LongWritable, IntWritable>
  52. {
  53. @SuppressWarnings("unused")
  54. public void reduce(LongWritable key, Iterable<Text> values, Context context)
  55. throws IOException, InterruptedException
  56. {
  57. int counter = 0;
  58. for(Text txt:values) counter++;
  59.  
  60. context.write(key, new IntWritable(counter));
  61. }
  62. }
  63.  
  64.  
  65.  
  66. @Override
  67. public int run(String[] args) throws Exception
  68. {
  69. Configuration conf = getConf();
  70. Job job = Job.getInstance(conf);
  71.  
  72. Builder builder = SearchArgumentFactory.newBuilder();
  73. String[]columns = {"lenb","locdate","lochour","normdate",
  74. "normhour","caller","called","duration","hash","dates"};
  75.  
  76. SearchArgument sarg =
  77. builder.startAnd().equals("called",PredicateLeaf.Type.STRING,
  78. "0111000000").end().build();
  79.  
  80. OrcInputFormat.setSearchArgument(conf, sarg, columns);
  81.  
  82. args = new GenericOptionsParser(conf, args).getRemainingArgs();
  83.  
  84. job.setJarByClass(Main.class);
  85. job.setMapperClass(MyMapper.class);
  86. job.setReducerClass(MyReducer.class);
  87. job.setMapOutputKeyClass(LongWritable.class);
  88. job.setMapOutputValueClass(Text.class);
  89.  
  90. job.setOutputKeyClass(LongWritable.class);
  91. job.setOutputValueClass(IntWritable.class);
  92.  
  93. OrcInputFormat.addInputPath(job, new Path(args[0]));
  94. OrcOutputFormat.setOutputPath(job, new Path(args[1]));
  95.  
  96. job.setInputFormatClass(OrcInputFormat.class);
  97.  
  98. return job.waitForCompletion(true)? 0:1;
  99.  
  100. }
  101.  
  102.  
  103.  
  104. public static void main(String[]args) throws Exception
  105. {
  106. long time1 = System.currentTimeMillis();
  107.  
  108. int exitCode = ToolRunner.run(new Main(), args);
  109.  
  110. long time2 = System.currentTimeMillis();
  111. System.out.println("Total time = "+ (time2 - time1)/1000);
  112.  
  113. System.exit(exitCode);
  114. }
  115.  
  116. }
Add Comment
Please, Sign In to add comment