Advertisement
Guest User

Untitled

a guest
Aug 19th, 2019
104
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 2.30 KB | None | 0 0
  1. 1,ed23,jon,doe,director,usa
  2. 2,ed23,jon,doe,director,usa
  3. 3,er67,jake,Kogan,director,usa
  4. 4,er67,jake,Kogan,director,usa
  5. 5,dc10,Charls,Morg,manager,usa
  6. 6,kc56,patel,Kumar,associate,india
  7.  
  8. 2,ed23,jon,doe,director,usa
  9. 4,er67,jake,Kogan,director,usa
  10. 5,dc10,Charls,Morg,manager,usa
  11. 6,kc56,patel,Kumar,associate,india
  12.  
  13. $ cat input.txt
  14. ID, uid ,firstname ,lastname, grade , country n28
  15. 1 , ed23 , jon , doe , director , usa
  16. 2 , ed23 , jon , doe , director , usa
  17. 3 , er67 , jake , Kogan , director , usa
  18. 4 , er67 , jake , Kogan , director , usa
  19. 5 , dc10 , Charls ,Morg , manager , usa
  20. 6 , kc56 , patel ,Kumar , associate , india
  21.  
  22. $ awk -F' *, *' -v OFS=,
  23. 'NR==1 {$1=$1;$0=$0; print; next};
  24. {id=$1; $1=""; $0=$0; if (!seen[$0]++) {print id $0}}' input.txt
  25. ID,uid,firstname,lastname,grade,country n28
  26. 1,ed23,jon,doe,director,usa
  27. 3,er67,jake,Kogan,director,usa
  28. 5,dc10,Charls,Morg,manager,usa
  29. 6,kc56,patel,Kumar,associate,india
  30.  
  31. $ awk -F' *, *' -v OFS=,
  32. 'NR==1 {$1=$1;$0=$0",count";print;next};
  33. {id=$1; $1=""; $0=$0; seen[$0]++; ids[$0]=id};
  34. END { for (id in ids) {print ids[id] id, seen[id]} }' input.txt |
  35. sort -n
  36. ID,uid,firstname,lastname,grade,country n28,count
  37. 2,ed23,jon,doe,director,usa,2
  38. 4,er67,jake,Kogan,director,usa,2
  39. 5,dc10,Charls,Morg,manager,usa,1
  40. 6,kc56,patel,Kumar,associate,india,1
  41.  
  42. awk -F, '{X=""; for (i=2;i<29;i++) X=X " " $i;}
  43. seen[X]!=1 {print;}
  44. {seen[X]=1;}' < input
  45.  
  46. $ tac file | awk -F , '{ key = $0; sub("[^,]*,", "", key) } !seen[key]++' | tac
  47. 2,ed23,jon,doe,director,usa
  48. 4,er67,jake,Kogan,director,usa
  49. 5,dc10,Charls,Morg,manager,usa
  50. 6,kc56,patel,Kumar,associate,india
  51.  
  52. $ tac file | awk -F , '{ key = $0; sub("[^,]*,", "", key) } key != prev; { prev = key }' | tac
  53. 2,ed23,jon,doe,director,usa
  54. 4,er67,jake,Kogan,director,usa
  55. 5,dc10,Charls,Morg,manager,usa
  56. 6,kc56,patel,Kumar,associate,india
  57.  
  58. $ tr ',' 't' < temp/testfile | uniq -f 1 | tr 't' ','
  59. 1,ed23,jon,doe,director,usa
  60. 3,er67,jake,Kogan,director,usa
  61. 5,dc10,Charls,Morg,manager,usa
  62. 6,kc56,patel,Kumar,associate,india
  63.  
  64. $ tac temp/testfile|tr ',' 't' | uniq -f 1 | tr 't' ','|tac
  65. 2,ed23,jon,doe,director,usa
  66. 4,er67,jake,Kogan,director,usa
  67. 5,dc10,Charls,Morg,manager,usa
  68. 6,kc56,patel,Kumar,associate,india
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement