Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- 1,ed23,jon,doe,director,usa
- 2,ed23,jon,doe,director,usa
- 3,er67,jake,Kogan,director,usa
- 4,er67,jake,Kogan,director,usa
- 5,dc10,Charls,Morg,manager,usa
- 6,kc56,patel,Kumar,associate,india
- 2,ed23,jon,doe,director,usa
- 4,er67,jake,Kogan,director,usa
- 5,dc10,Charls,Morg,manager,usa
- 6,kc56,patel,Kumar,associate,india
- $ cat input.txt
- ID, uid ,firstname ,lastname, grade , country n28
- 1 , ed23 , jon , doe , director , usa
- 2 , ed23 , jon , doe , director , usa
- 3 , er67 , jake , Kogan , director , usa
- 4 , er67 , jake , Kogan , director , usa
- 5 , dc10 , Charls ,Morg , manager , usa
- 6 , kc56 , patel ,Kumar , associate , india
- $ awk -F' *, *' -v OFS=,
- 'NR==1 {$1=$1;$0=$0; print; next};
- {id=$1; $1=""; $0=$0; if (!seen[$0]++) {print id $0}}' input.txt
- ID,uid,firstname,lastname,grade,country n28
- 1,ed23,jon,doe,director,usa
- 3,er67,jake,Kogan,director,usa
- 5,dc10,Charls,Morg,manager,usa
- 6,kc56,patel,Kumar,associate,india
- $ awk -F' *, *' -v OFS=,
- 'NR==1 {$1=$1;$0=$0",count";print;next};
- {id=$1; $1=""; $0=$0; seen[$0]++; ids[$0]=id};
- END { for (id in ids) {print ids[id] id, seen[id]} }' input.txt |
- sort -n
- ID,uid,firstname,lastname,grade,country n28,count
- 2,ed23,jon,doe,director,usa,2
- 4,er67,jake,Kogan,director,usa,2
- 5,dc10,Charls,Morg,manager,usa,1
- 6,kc56,patel,Kumar,associate,india,1
- awk -F, '{X=""; for (i=2;i<29;i++) X=X " " $i;}
- seen[X]!=1 {print;}
- {seen[X]=1;}' < input
- $ tac file | awk -F , '{ key = $0; sub("[^,]*,", "", key) } !seen[key]++' | tac
- 2,ed23,jon,doe,director,usa
- 4,er67,jake,Kogan,director,usa
- 5,dc10,Charls,Morg,manager,usa
- 6,kc56,patel,Kumar,associate,india
- $ tac file | awk -F , '{ key = $0; sub("[^,]*,", "", key) } key != prev; { prev = key }' | tac
- 2,ed23,jon,doe,director,usa
- 4,er67,jake,Kogan,director,usa
- 5,dc10,Charls,Morg,manager,usa
- 6,kc56,patel,Kumar,associate,india
- $ tr ',' 't' < temp/testfile | uniq -f 1 | tr 't' ','
- 1,ed23,jon,doe,director,usa
- 3,er67,jake,Kogan,director,usa
- 5,dc10,Charls,Morg,manager,usa
- 6,kc56,patel,Kumar,associate,india
- $ tac temp/testfile|tr ',' 't' | uniq -f 1 | tr 't' ','|tac
- 2,ed23,jon,doe,director,usa
- 4,er67,jake,Kogan,director,usa
- 5,dc10,Charls,Morg,manager,usa
- 6,kc56,patel,Kumar,associate,india
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement