Advertisement
Guest User

Untitled

a guest
Nov 1st, 2014
161
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 2.49 KB | None | 0 0
  1. public class CSMRMapper extends Mapper<Text,VectorWritable,IntWritable,
  2. DocumentWritable >{
  3.  
  4. @Override
  5. public void map(Text key, VectorWritable value, Context context)
  6. throws IOException, InterruptedException{
  7. DocumentWritable p = new DocumentWritable(new Text(key.toString())
  8. ,new VectorWritable(value.get()));
  9. context.write(new IntWritable(1),p);
  10.  
  11. }
  12.  
  13. public class CSMRReducer extends Reducer<IntWritable, DocumentWritable ,Text,
  14. VectorArrayWritable> {
  15.  
  16. private ArrayList<DocumentWritable> al;
  17. private VectorWritable[] val;
  18. private VectorArrayWritable vaw;
  19.  
  20. @Override
  21. public void reduce(IntWritable key, Iterable<DocumentWritable> values,
  22. Context context) throws IOException, InterruptedException{
  23. al = new ArrayList();
  24. vaw = new VectorArrayWritable();
  25.  
  26. /* Storing each key-value pair (document) in a java.util.ArrayList */
  27. for (DocumentWritable v : values){
  28. al.add(new DocumentWritable(v.getKey(),v.getValue()));
  29. }
  30.  
  31. /* Generating all the possible combinations of documents */
  32. if (al.size()>0){
  33. for (int i=0;i<al.size();++i){
  34. for (int j=i+1;j<al.size();++j){
  35. val = new VectorWritable[2];
  36.  
  37. /* Generating the key for the current document pair with
  38. the format "doci_name@docj_name" */
  39. String k = al.get(i).getKey().toString()+
  40. "@"+al.get(j).getKey().toString();
  41.  
  42. //First Document (doci)
  43. val[0] = new VectorWritable(al.get(i).getValue().get());
  44. //Second Document (docj)
  45. val[1] = new VectorWritable(al.get(j).getValue().get());
  46. vaw.set(val);
  47.  
  48. context.write(new Text(k), vaw);
  49. }
  50. }
  51. }
  52. }
  53.  
  54. }
  55.  
  56. public class CosineSimilarityReducer
  57. extends Reducer<Text, VectorArrayWritable, Text, DoubleWritable>{
  58.  
  59. @Override
  60. public void reduce(Text key, Iterable<VectorArrayWritable> value, Context context)
  61. throws IOException, InterruptedException{
  62.  
  63. CosineDistanceMeasure cdm = new CosineDistanceMeasure();
  64. VectorWritable docX,docY;
  65. double cosine;
  66. for (VectorArrayWritable v : value){
  67. docX = (VectorWritable)v.get()[0];
  68. docY = (VectorWritable)v.get()[1];
  69. cosine = cdm.distance(docX.get(), docY.get());
  70. context.write(key, new DoubleWritable(cosine));
  71. }
  72.  
  73. }
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement