Advertisement
JunkPile77

Project1

Mar 21st, 2019
91
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Java 2.79 KB | None | 0 0
  1. package org.apache.hadoop.ramapo;
  2. import java.io.IOException;
  3. import java.util.StringTokenizer;
  4.  
  5. import org.apache.hadoop.conf.Configuration;
  6. import org.apache.hadoop.fs.Path;
  7. import org.apache.hadoop.io.Writable;
  8. import org.apache.hadoop.io.IntWritable;
  9. import org.apache.hadoop.io.LongWritable;
  10. import org.apache.hadoop.io.NullWritable;
  11. import org.apache.hadoop.io.Text;
  12. import org.apache.hadoop.mapreduce.Job;
  13. import org.apache.hadoop.mapreduce.Mapper;
  14. import org.apache.hadoop.mapreduce.Reducer;
  15. import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
  16. import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
  17.  
  18. // Input-> utr.txt -> ut.csv -> uu.csv
  19.  
  20. // first step: output is <userId, likedMovieId> (where liked is 3.0 or greater)
  21.  
  22. public class Project{
  23.   public static class MyMapper
  24.         extends Mapper<LongWritable, Text, IntWritable, Text> {
  25.             public void map (LongWritable key, Text value, Context context) throws IOException, InterruptedException{
  26.               String[] lines = value.toString().split(",");
  27.               Integer user = Integer.parseInt(lines[0]);
  28.               Integer movie = Integer.parseInt(lines[1]);
  29.               double rating = Double.parseDouble(lines[2]);
  30.  
  31.               context.write(new IntWritable(user), new Text(Integer.toString(movie) + "," + Double.toString(rating)));
  32.  
  33.  
  34.             }
  35.     }
  36.  
  37.  
  38.   public static class MyReducer
  39.         extends Reducer<IntWritable, Text, IntWritable, IntWritable> {
  40.             public void reduce (IntWritable key, Iterable<Text> value, Context context) throws IOException, InterruptedException{
  41.  
  42.               for(Text val : value) {
  43.                 String[] lines = val.toString().split(",");
  44.                 Integer movie = Integer.parseInt(lines[0]);
  45.                 Double rating = Double.parseDouble(lines[1]);
  46.  
  47.                 if (rating >= 3) {
  48.                   System.out.println(key.toString() + " " + String.valueOf(rating));
  49.                   context.write(key, new IntWritable(movie));
  50.                 }
  51.               }
  52.             }
  53.     }
  54.  
  55.   public static void main(String[] args) throws Exception {
  56.         Configuration conf = new Configuration();
  57.         Job job = Job.getInstance(conf, "Project");
  58.         job.setJarByClass(Project.class);
  59.         job.setMapperClass(MyMapper.class);
  60.         //job.setCombinerClass(MyReducer.class);
  61.         job.setReducerClass(MyReducer.class);
  62.         job.setMapOutputKeyClass(IntWritable.class);
  63.         job.setMapOutputValueClass(Text.class);
  64.         job.setOutputKeyClass(IntWritable.class);
  65.         job.setOutputValueClass(IntWritable.class);
  66.         FileInputFormat.addInputPath(job, new Path(args[0]));
  67.         FileOutputFormat.setOutputPath(job, new Path(args[1]));
  68.         System.exit(job.waitForCompletion(true) ? 0 : 1);
  69.   }
  70. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement