import requests
import json
import time

base_url = 'https://bitnodes.earn.com/api/v1/snapshots/'
timestamp_dict = {}
stop_date = 1551398088

def collect_timestamps_dict(page):
	print('Collecting page', page, '...')
	data = requests.get(url = base_url, params = {'page':page,'limit':100}).json()
	next_page = data['next']
	for d in data['results']:
		timestamp = d['timestamp']
		url = d['url']
		print(timestamp, url)
		timestamp_dict[timestamp] = url
		if int(timestamp) <= stop_date:
			return
	if timestamp > stop_date:
		time.sleep(1)
		collect_timestamps_dict(page + 1)

def collect_specific_snapshot(snapshot):
	print('Collecting snapshot', snapshot, '...')
	raw_data = requests.get(url = base_url + str(snapshot)).json()
	saved_ip = ''
	nodes = raw_data['nodes']
	for n in nodes:
		if ']' in n:
			n = n.split(']:')[0].replace('[','')
		else:
			n = n.split(':')[0]
		saved_ip = saved_ip + n + '\n'
	open('2019-03-' + snapshot + '.txt', 'w').write(saved_ip)

# First, collect all timestamps as json file
collect_timestamps_dict(1)
with open('timestamps.json', 'w') as fp:
    json.dump(timestamp_dict, fp)

# Then collect all snapshot as json file
count = 0
with open('timestamps.json') as f:
	data = json.load(f)
	for d in data:
		count += 1
		time.sleep(1)
		print(count)
		collect_specific_snapshot(d)