Advertisement
Guest User

Untitled

a guest
Jul 19th, 2018
122
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Java 3.61 KB | None | 0 0
  1. package bgp.d2distributed;
  2.  
  3. import java.io.IOException;
  4. import java.util.HashMap;
  5. import java.util.Iterator;
  6. import java.util.Map.Entry;
  7. import java.util.StringTokenizer;
  8. import org.apache.hadoop.conf.Configuration;
  9. import org.apache.hadoop.fs.Path;
  10. import org.apache.hadoop.io.LongWritable;
  11. import org.apache.hadoop.io.Text;
  12. import org.apache.hadoop.mapreduce.Job;
  13. import org.apache.hadoop.mapreduce.Mapper;
  14. import org.apache.hadoop.mapreduce.Reducer;
  15. import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
  16. import org.apache.hadoop.mapreduce.lib.input.NLineInputFormat;
  17. import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
  18.  
  19. public class D2D {
  20.     /**
  21.      * Mapper<KEYIN, VALUEIN, KEYOUT, VALUEOUT>
  22.      * KEYIN: chiave fittizia per rispettare il paradigma Mapper
  23.      * VALUEIN: Text (i due file)
  24.      * KEYOUT: k-mero
  25.      * VALUEOUT: prodotto delle occurrenze Si*Qi
  26.      */
  27.  
  28.     public static class PartialScoreMapper extends Mapper<Object, Text[], Text, LongWritable> {
  29.  
  30.         @Override
  31.         public void map(Object key, Text[] value, Context context) throws IOException, InterruptedException {
  32.             StringTokenizer stTokFile1 = new StringTokenizer(value[0].toString());
  33.             StringTokenizer stTokFile2 = new StringTokenizer(value[1].toString());
  34.             String kmerS="", kmerQ="";
  35.             long Si=0, Qi=0;
  36.             HashMap<String, Long> file1 = new HashMap<String, Long>();
  37.             HashMap<String, Long> file2 = new HashMap<String, Long>();
  38.             while(stTokFile1.hasMoreTokens() && stTokFile2.hasMoreTokens()) {
  39.                 kmerS = stTokFile1.nextToken();
  40.                 kmerQ = stTokFile2.nextToken();
  41.                 Si = Long.valueOf(stTokFile1.nextToken());
  42.                 Qi = Long.valueOf(stTokFile2.nextToken());
  43.                 file1.put(kmerS, Si);
  44.                 file2.put(kmerQ, Qi);
  45.             }
  46.             Iterator<Entry<String,Long>> it1 = file1.entrySet().iterator();
  47.             Entry<String, Long> currEntry = null;
  48.             String currKmer = "";
  49.             long currOcc1 = 0;
  50.             long currOcc2 = 0;
  51.             Long partialScore = new Long(0);
  52.             while(it1.hasNext()){
  53.                 currEntry = it1.next();
  54.                 currKmer = currEntry.getKey();
  55.                 currOcc1 = currEntry.getValue();
  56.                 if(file2.containsKey(currKmer)) {
  57.                     currOcc2 = file2.get(currKmer);
  58.                     partialScore = currOcc1*currOcc2;
  59.                     context.write(new Text(currKmer), new LongWritable(partialScore));
  60.                 }
  61.                 else
  62.                     context.write(new Text(currKmer), new LongWritable(0));        
  63.             }
  64.  
  65.         }
  66.     }
  67.  
  68.     public static class D2ScoreReducer extends Reducer<Text, LongWritable, Object, LongWritable> {
  69.  
  70.         private LongWritable result = new LongWritable();
  71.  
  72.         @Override
  73.         protected void reduce(Text kmer, Iterable<LongWritable> partialScore, Context context) throws IOException, InterruptedException {
  74.             long score = 0;
  75.             for (LongWritable pScore : partialScore) {
  76.                 score += pScore.get();
  77.             }
  78.             this.result.set(score);
  79.             context.write(kmer,this.result);
  80.         }
  81.     }
  82.  
  83.     public static void main(String[] args) throws IOException, ClassNotFoundException, InterruptedException {
  84.         Configuration conf = new Configuration();
  85.         System.out.println("Configuration conf = new Configuration();");
  86.         Job job = Job.getInstance(conf, "d2d");
  87.         System.out.println("Job job = Job.getInstance(conf, \"d2d\");");
  88.         job.setJarByClass(D2D.class);
  89.         System.out.println("job.setJarByClass(D2D.class);");
  90.         job.setMapperClass(PartialScoreMapper.class);
  91.         job.setReducerClass(D2ScoreReducer.class);
  92.         job.setOutputKeyClass(Object.class);
  93.         job.setOutputValueClass(LongWritable.class);
  94.         NLineInputFormat.setNumLinesPerSplit(job, 101);
  95.         FileInputFormat.addInputPath(job, new Path(args[0]));
  96.         FileOutputFormat.setOutputPath(job, new Path(args[1]));
  97.         System.exit(job.waitForCompletion(true) ? 0 : 1);
  98.     }
  99.  
  100.  
  101. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement