SHARE
TWEET

Untitled

a guest Aug 19th, 2019 63 Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. 1,ed23,jon,doe,director,usa
  2. 2,ed23,jon,doe,director,usa
  3. 3,er67,jake,Kogan,director,usa
  4. 4,er67,jake,Kogan,director,usa
  5. 5,dc10,Charls,Morg,manager,usa
  6. 6,kc56,patel,Kumar,associate,india
  7.      
  8. 2,ed23,jon,doe,director,usa
  9. 4,er67,jake,Kogan,director,usa
  10. 5,dc10,Charls,Morg,manager,usa
  11. 6,kc56,patel,Kumar,associate,india
  12.      
  13. $ cat input.txt
  14. ID, uid  ,firstname ,lastname,   grade    , country n28
  15. 1 , ed23 , jon     ,   doe   ,  director  ,  usa
  16. 2 , ed23 ,  jon     ,  doe   ,  director     , usa
  17. 3 , er67 ,  jake     , Kogan ,  director     , usa
  18. 4 , er67 ,  jake     , Kogan ,  director     , usa
  19. 5 , dc10 ,  Charls     ,Morg ,  manager      , usa
  20. 6 , kc56 ,  patel     ,Kumar ,  associate    , india
  21.      
  22. $ awk -F' *, *' -v OFS=,
  23.     'NR==1 {$1=$1;$0=$0; print; next};
  24.      {id=$1; $1=""; $0=$0; if (!seen[$0]++) {print id $0}}' input.txt
  25. ID,uid,firstname,lastname,grade,country n28
  26. 1,ed23,jon,doe,director,usa
  27. 3,er67,jake,Kogan,director,usa
  28. 5,dc10,Charls,Morg,manager,usa
  29. 6,kc56,patel,Kumar,associate,india
  30.      
  31. $ awk -F' *, *' -v OFS=,
  32.    'NR==1 {$1=$1;$0=$0",count";print;next};
  33.    {id=$1; $1=""; $0=$0; seen[$0]++; ids[$0]=id};
  34.    END { for (id in ids) {print ids[id] id, seen[id]} }' input.txt  |
  35.  sort -n
  36. ID,uid,firstname,lastname,grade,country n28,count
  37. 2,ed23,jon,doe,director,usa,2
  38. 4,er67,jake,Kogan,director,usa,2
  39. 5,dc10,Charls,Morg,manager,usa,1
  40. 6,kc56,patel,Kumar,associate,india,1
  41.      
  42. awk -F, '{X=""; for (i=2;i<29;i++) X=X " " $i;}
  43.      seen[X]!=1 {print;}
  44.      {seen[X]=1;}' < input
  45.      
  46. $ tac file | awk -F , '{ key = $0; sub("[^,]*,", "", key) } !seen[key]++' | tac
  47. 2,ed23,jon,doe,director,usa
  48. 4,er67,jake,Kogan,director,usa
  49. 5,dc10,Charls,Morg,manager,usa
  50. 6,kc56,patel,Kumar,associate,india
  51.      
  52. $ tac file | awk -F , '{ key = $0; sub("[^,]*,", "", key) } key != prev; { prev = key }' | tac
  53. 2,ed23,jon,doe,director,usa
  54. 4,er67,jake,Kogan,director,usa
  55. 5,dc10,Charls,Morg,manager,usa
  56. 6,kc56,patel,Kumar,associate,india
  57.      
  58. $ tr ',' 't' < temp/testfile | uniq -f 1 | tr 't' ','
  59. 1,ed23,jon,doe,director,usa
  60. 3,er67,jake,Kogan,director,usa
  61. 5,dc10,Charls,Morg,manager,usa
  62. 6,kc56,patel,Kumar,associate,india
  63.      
  64. $ tac temp/testfile|tr ',' 't' | uniq -f 1 | tr 't' ','|tac
  65. 2,ed23,jon,doe,director,usa
  66. 4,er67,jake,Kogan,director,usa
  67. 5,dc10,Charls,Morg,manager,usa
  68. 6,kc56,patel,Kumar,associate,india
RAW Paste Data
We use cookies for various purposes including analytics. By continuing to use Pastebin, you agree to our use of cookies as described in the Cookies Policy. OK, I Understand
 
Top