from bs4 import BeautifulSoup
import requests

output = open('bob.txt','a+', encoding='utf-8')
headers = {
    "Cookie": "intercom-id-ynnn5d6q=19c62e81-30f6-4610-8919-1401a97b54f9; __cfduid=dc642a4610cc9527d0c9d234d04cb34751545244738; _ga=GA1.2.1288742159.1553910371; site24x7rumID=3269752668489886.1566647311221; __zlcmid=sTiGOr2yM1lpij; site24x7RUMSnapshotId=1558962251056.false; EduSiteCookie=bf6c659f-af11-4ec4-8fc0-ac377d8e4abb"
}

url, page = "https://open.kattis.com/problems?page=", 0
print("Reading page: ", flush=True, end='')
while True:
    req = requests.get(url + str(page), headers=headers)
    soup = BeautifulSoup(req.text, 'html.parser')
    print(f"{page}..", flush=True,end='')
    results = 0
    page += 1
    for row in soup.find('table').find('tbody').find_all('tr'):
        results += 1
        vals, cl = row.find_all('td'), row.get('class')
        output.write(','.join([vals[x].text for x in range(9)]) + ',' + (str(cl[1]) if len(cl) > 1 else 'unsolved') + '\n')
    if results == 0:
        break
print('done')
output.flush()
output.close()