sub_o

zenxv's fetch.py

Dec 2nd, 2020
882
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
  1. #! /usr/bin/env python
  2.  
  3. import warnings
  4. warnings. filterwarnings('ignore')
  5.  
  6. import pandas as pd
  7. import requests
  8.  
  9. def collapse_percent_per_party(results_by_candidate, candidates):
  10.     percent_per_party = {}
  11.     for candidate, count in results_by_candidate.items():
  12.         party                    = candidates [candidate] ['party']
  13.         percent_per_party[party] = percent_per_party.get(party, 0) + count
  14.  
  15.     return percent_per_party
  16.  
  17. states = [
  18.     'Alaska', 'Alabama', 'Arkansas', 'Arizona', 'California', 'Colorado',
  19.     'Connecticut', 'Delaware', 'Florida', 'Georgia',
  20.     'Hawaii', 'Iowa', 'Idaho', 'Illinois', 'Indiana', 'Kansas', 'Kentucky',
  21.     'Louisiana', 'Massachusetts', 'Maryland', 'Maine', 'Michigan',
  22.     'Minnesota', 'Missouri', 'Mississippi', 'Montana', 'North Carolina',
  23.     'North Dakota', 'Nebraska', 'New Hampshire', 'New Jersey', 'New Mexico',
  24.     'Nevada', 'New York', 'Ohio', 'Oklahoma', 'Oregon', 'Pennsylvania',
  25.     'Rhode Island', 'South Carolina', 'South Dakota', 'Tennessee', 'Texas',
  26.     'Utah', 'Virginia', 'Vermont', 'Washington', 'Wisconsin',
  27.     'West Virginia', 'Wyoming',
  28. ]
  29.  
  30. all_results = {}
  31. for state in states:
  32.     print('Fetching Results from {}'.format(state) )
  33.     formatted_state = state. lower().replace(' ', '-')
  34.     state_url = requests.get('https://static01.nyt.com/elections-assets/2020/data/api/2020-11-03/race-page/{}/president.json'.format(formatted_state) ).json()
  35.     all_results[formatted_state] = state_url
  36.  
  37.  
  38.  
  39. records = []
  40. for state, state_url in all_results.items():
  41.     race = state_url['data'] ['races'] [0]
  42.  
  43.     for candidate in race['candidates']:
  44.         if candidate['party_id'] == 'republican':
  45.             candidate['party'] = 'rep'
  46.         elif candidate['party_id'] == 'democrat':
  47.             candidate['party'] = 'dem'
  48.         else:
  49.             candidate['party'] = 'trd'
  50.  
  51.     candidates = { candidate['candidate_key']: candidate for candidate in race['candidates'] }
  52.  
  53.     for data_point in race['timeseries']:
  54.         data_point['state'] = state
  55.         data_point['expected_votes'] = race['tot_exp_vote']
  56.  
  57.  
  58.         percentage_votes = collapse_percent_per_party(data_point['vote_shares'], candidates)
  59.         for party in ['rep', 'dem', 'trd']:
  60.             data_point['vote_shares_{}'.format(party)] = percentage_votes.get(party, 0)
  61.  
  62.         data_point.pop( 'vote_shares')
  63.         records. append(data_point)
  64.  
  65. time_series_df = pd.DataFrame. from_records(records)
  66. time_series_df.to_csv('/users/apple/documents/Edison0112.csv', encoding='utf-8')
RAW Paste Data