Advertisement
Guest User

Untitled

a guest
May 24th, 2017
70
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Java 5 3.22 KB | None | 0 0
  1. import java.io.IOException;
  2. import java.lang.InterruptedException;
  3. import java.util.StringTokenizer;
  4.  
  5. import org.apache.hadoop.io.IntWritable;
  6. import org.apache.hadoop.io.Text;
  7. import org.apache.hadoop.conf.*;
  8. import org.apache.hadoop.conf.Configuration;
  9. import org.apache.hadoop.fs.Path;
  10. import org.apache.hadoop.mapreduce.Job;
  11. import org.apache.hadoop.mapreduce.Mapper;
  12. import org.apache.hadoop.mapreduce.Reducer;
  13. import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
  14. import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
  15. import org.apache.hadoop.util.GenericOptionsParser;
  16.  
  17. public class DeviceTraffic {
  18.  
  19. //public static String publisherId;
  20. Configuration conf = new Configuration();
  21. static class DeviceTrafficMapper
  22. extends Mapper<Object, Text, Text, IntWritable> {
  23. //private static final int MISSING = 9999;
  24.  
  25. public void map(Object key, Text value, Context context)
  26. throws IOException, InterruptedException {
  27. //Configuration mconf = new Configuration();
  28. //publisherId = mconf.get("PublisherId");
  29. String mpublisherId=conf.get("PublisherId");
  30. String line = value.toString();
  31. String[] splitValues = line.split("\t");
  32. System.out.println("Split Value size is "+String.valueOf(splitValues.length));
  33. if (splitValues!=null && splitValues.length>2)
  34. {
  35. String pubPaymentId=splitValues[24];
  36. String device;
  37. int impression;
  38. System.out.println("Publisher id is "+mpublisherId);
  39. System.out.println("Publisher Payment id is "+pubPaymentId);
  40. if (pubPaymentId.equals(mpublisherId))
  41. {
  42. if (splitValues[33]!=null && !splitValues[33].equals("") && !splitValues[33].equals("ELI"))
  43. {
  44. device = splitValues[33];
  45. //if (splitValues[20]!=null && !splitValues[20].equals("") && !splitValues[20].equals("Fraud") && !splitValues[20].equals("ELI"))
  46. impression = Integer.valueOf(splitValues[20]);
  47. //impression = 1;
  48. //else
  49. //impression = 0;
  50. context.write(new Text(value), new IntWritable(impression));
  51. }
  52. }
  53. else
  54. {
  55. context.write(new Text(mpublisherId), new IntWritable(1));
  56. }
  57. }
  58. }
  59. }
  60. static class DeviceTrafficReducer
  61. extends Reducer<Text, IntWritable, Text, IntWritable> {
  62. public void reduce(Text key, Iterable<IntWritable> values,
  63. Context context)
  64. throws IOException, InterruptedException {
  65. int sumValue = 0;
  66. //BufferedWriter writer = new BufferedWriter(new FileWriter("/home/hadoop/ads_deliveries/output.txt"));
  67. for (IntWritable value : values) {
  68. //writer.write(String.valueOf(value)+"\t");
  69. sumValue +=  value.get();
  70. }
  71. context.write(key, new IntWritable(sumValue));
  72. }
  73. }
  74.  
  75. public static void main(String[] args) throws Exception {
  76. if (args.length != 3) {
  77. System.err.println("Usage: PublisherUrl <publihser id> <input path> <output path>");
  78. System.exit(-1);
  79. }
  80. conf.setIfUnset("PublisherId",args[0]);
  81. //publisherId=args[0];
  82. System.out.println("Publisher ID in argument is "+args[0]);
  83. DeviceTraffic.setpubId(args[0]);
  84. Job job = new Job(conf);
  85. job.setJarByClass(DeviceTraffic.class);
  86. FileInputFormat.addInputPath(job, new Path(args[1]));
  87. FileOutputFormat.setOutputPath(job, new Path(args[2]));
  88. job.setMapperClass(DeviceTrafficMapper.class);
  89. job.setReducerClass(DeviceTrafficReducer.class);
  90. job.setOutputKeyClass(Text.class);
  91. job.setOutputValueClass(IntWritable.class);
  92. System.exit(job.waitForCompletion(true) ? 0 : 1);
  93. }
  94. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement