Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #!/bin/bash
- _filename=$1
- _outputfile="frobnicated"
- #For each file:
- # get a list of non-unique ids
- _non_unique_ids=$(cut -d',' -f1 "${_filename}" | sort | uniq -d)
- #
- # next, split your file into 2 piles
- # use grep -v -F here. -F lets you use your nui's as a single big filter.
- # This works even for monstrous files.
- _unique_entries=$(grep -v -F "${_non_unique_ids}" "${_filename}")
- _dupe_entries=$(grep -F "${_non_unique_ids}" "${_filename}")
- # dump your unique entries to a file
- echo "$_unique_entries" > "$_outputfile"
- # just for testing, so you can see for yourself
- echo "###### below were dupes" >> "$_outputfile"
- # pick only the first entry that has a dupe ID
- for _dupe in $_non_unique_ids ; do
- # if you want to use something other than just grabbing the first dupe
- # entry, you'll have to put it here.
- _match=$(grep "${_dupe}" <<< "${_dupe_entries}" | head -n1)
- if [[ -n "${_match}" ]] ; then
- echo "${_match}" >> "$_outputfile"
- fi
- done
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement