Advertisement
metalx1000

Convert Voters Tab tsv to CSV and import into mysql

Apr 2nd, 2016
706
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Bash 2.59 KB | None | 0 0
  1. #!/bin/bash
  2.  
  3. #convert voters tsv to csv for mysql import
  4. #get files from http://flvoters.com/downloads.html
  5. #example wget "http://flvoters.com/download/20160229/CLL_20160307.txt"
  6. county="CLL"
  7.  
  8. echo "Converting file to CSV format"
  9. #add head line
  10. echo '"id","lname","name suffix","fname","mname","no address","address","address2","city","state","zip","mail1","mail2","mail3","mailcity","mailstate","mailzip","mailcounty","gender","race","dob","rdate","party","precinct","Precinct Group","Precinct Split","Precinct Suffix","Voter Status","Congressional District","House District","Senate District","commission","School","area code","phone","email"' > data.csv
  11.  
  12. let lines=$(wc -l $county_*.txt |awk '{print $1}')
  13.  
  14. sed 's/   / /g' $county_*.txt|\
  15.   sed 's/\t/","/g'|\
  16.   sed "s/$county\",\"/\"/g"|\
  17.   while read line;
  18.   do
  19.     echo "$line\"" >> data.csv
  20.   done
  21.  
  22.  
  23. echo "[[Compressing]]"
  24. zip data.csv.zip data.csv
  25.  
  26. echo "To import:"
  27. echo "==========="
  28.  
  29. cat << EOF
  30. DROP TABLE voters;
  31. CREATE TABLE voters(
  32.    id                     INTEGER  NOT NULL PRIMARY KEY
  33.   ,lname                  TEXT NOT NULL
  34.   ,name_suffix            TEXT
  35.   ,fname                  TEXT NOT NULL
  36.   ,mname                  TEXT NOT NULL
  37.   ,no_address             TEXT NOT NULL
  38.   ,address                TEXT NOT NULL
  39.   ,address2               TEXT
  40.   ,city                   TEXT NOT NULL
  41.   ,state                  TEXT
  42.   ,zip                    INTEGER  NOT NULL
  43.   ,mail1                  TEXT
  44.   ,mail2                  TEXT
  45.   ,mail3                  TEXT
  46.   ,mailcity               TEXT
  47.   ,mailstate              TEXT
  48.   ,mailzip                INTEGER
  49.   ,mailcounty             TEXT
  50.   ,gender                 VARCHAR(1) NOT NULL
  51.   ,race                   INTEGER  NOT NULL
  52.   ,dob                    TEXT  NOT NULL
  53.   ,rdate                  TEXT  NOT NULL
  54.   ,party                  VARCHAR(3) NOT NULL
  55.   ,precinct               INTEGER  NOT NULL
  56.   ,Precinct_Group         BIT  NOT NULL
  57.   ,Precinct_Split         NUMERIC(5,1) NOT NULL
  58.   ,Precinct_Suffix        VARCHAR(1)
  59.   ,Voter_Status           VARCHAR(3) NOT NULL
  60.   ,Congressional_District INTEGER  NOT NULL
  61.   ,House_District         INTEGER  NOT NULL
  62.   ,Senate_District        INTEGER  NOT NULL
  63.   ,commission             INTEGER  NOT NULL
  64.   ,School                 INTEGER  NOT NULL
  65.   ,area_code              INTEGER
  66.   ,phone                  INTEGER
  67.   ,email                  TEXT
  68.   ,FIELD37                TEXT
  69. );
  70. EOF
  71.  
  72. echo "LOAD DATA INFILE '$PWD/data.csv' INTO TABLE voters FIELDS TERMINATED BY ',' ENCLOSED BY '\"' LINES TERMINATED BY '\n' IGNORE 1 LINES;"
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement