Advertisement
Guest User

Untitled

a guest
Jan 23rd, 2017
92
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 0.87 KB | None | 0 0
  1. //wczytanie plikow
  2.  
  3. val txt1 = sc.textFile("hdfs://grid223-20:9000/input/taglogsbig/huge10g.log")
  4. val txt2 = sc.textFile("hdfs://grid223-20:9000/input/taglogsbig/huge10g2.log")
  5. val txt3 = sc.textFile("hdfs://grid223-20:9000/input/taglogsbig/huge10g3.log")
  6.  
  7. //polaczenie plikow
  8.  
  9. val txt = txt1.union(txt2).union(txt3)
  10.  
  11. //wyliczenia czasowe dla naszego uzytkownika
  12.  
  13. val user13TimePerHost = txt.filter(x=>x.split("\t")(1) == "user39").map(x => (x.split("\t")(0),x.split("\t").last.toDouble)).reduceByKey(_+_)
  14.  
  15. //wyliczenia czasowe dla wszystkich uzytkownikow
  16.  
  17. val allUsersTimePerHost = txt.map(x => (x.split("\t")(0),x.split("\t").last.toDouble)).reduceByKey(_+_)
  18.  
  19. //polaczenie
  20.  
  21. val totalData = allUsersTimePerHost.join(user13TimePerHost)
  22.  
  23. //zwrocenie danych
  24.  
  25. val finalResult = totalData.map(x => (x._1,x._2._2*100/x._2._1)).collect()
  26. System.exit(0)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement