zenxv's fetch.py

#! /usr/bin/env python

import warnings
warnings. filterwarnings('ignore')

import pandas as pd
import requests

def collapse_percent_per_party(results_by_candidate, candidates):
    percent_per_party = {}
    for candidate, count in results_by_candidate.items():
        party                    = candidates [candidate] ['party']
        percent_per_party[party] = percent_per_party.get(party, 0) + count

    return percent_per_party

states = [
    'Alaska', 'Alabama', 'Arkansas', 'Arizona', 'California', 'Colorado',
    'Connecticut', 'Delaware', 'Florida', 'Georgia',
    'Hawaii', 'Iowa', 'Idaho', 'Illinois', 'Indiana', 'Kansas', 'Kentucky',
    'Louisiana', 'Massachusetts', 'Maryland', 'Maine', 'Michigan',
    'Minnesota', 'Missouri', 'Mississippi', 'Montana', 'North Carolina',
    'North Dakota', 'Nebraska', 'New Hampshire', 'New Jersey', 'New Mexico',
    'Nevada', 'New York', 'Ohio', 'Oklahoma', 'Oregon', 'Pennsylvania',
    'Rhode Island', 'South Carolina', 'South Dakota', 'Tennessee', 'Texas',
    'Utah', 'Virginia', 'Vermont', 'Washington', 'Wisconsin',
    'West Virginia', 'Wyoming',
]

all_results = {}
for state in states:
    print('Fetching Results from {}'.format(state) )
    formatted_state = state. lower().replace(' ', '-')
    state_url = requests.get('https://static01.nyt.com/elections-assets/2020/data/api/2020-11-03/race-page/{}/president.json'.format(formatted_state) ).json()
    all_results[formatted_state] = state_url



records = []
for state, state_url in all_results.items():
    race = state_url['data'] ['races'] [0]

    for candidate in race['candidates']:
        if candidate['party_id'] == 'republican':
            candidate['party'] = 'rep'
        elif candidate['party_id'] == 'democrat':
            candidate['party'] = 'dem'
        else:
            candidate['party'] = 'trd'

    candidates = { candidate['candidate_key']: candidate for candidate in race['candidates'] }

    for data_point in race['timeseries']:
        data_point['state'] = state
        data_point['expected_votes'] = race['tot_exp_vote']


        percentage_votes = collapse_percent_per_party(data_point['vote_shares'], candidates)
        for party in ['rep', 'dem', 'trd']:
            data_point['vote_shares_{}'.format(party)] = percentage_votes.get(party, 0)

        data_point.pop( 'vote_shares')
        records. append(data_point)

time_series_df = pd.DataFrame. from_records(records)
time_series_df.to_csv('/users/apple/documents/Edison0112.csv', encoding='utf-8')