FW_Scraper.py

#! /usr/bin/env python3

from bs4 import BeautifulSoup as bs4
import requests
import os
import time

# Making a folder in which to put the threads' html files
if not os.path.exists("Threads"):
    os.makedirs("Threads")

# Setting user agent to standard Firefox-on-Windows because the site blocks the script otherwise
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 6.2; Win64; x64; rv:55.0) Gecko/20100101 Firefox/55.0',
}

# Checking to see how many pages there are total so the main loop can have a definite end
pages_check = requests.get("https://www.fatwallet.com/forums/search/results.php?&type=forums&query=&sort=&match=titleop&forum=52&active=all&newsort=date", headers=headers)
pages_soup = bs4(pages_check.text, 'html.parser')
pages_buttons= pages_soup.select('.standardPageSection .pageLink')
pages = int(pages_buttons[-1].text.replace(',',''))

# Checks file to see last page completed. Used for resuming in the event that the download was stopped
if os.path.isfile('latest_page_done.txt'):
    start_page = int(open('latest_page_done.txt', 'r').read()) + 1
else:
    start_page = 1

print('Beginning download at page {}'.format(start_page))

# Main loop for downloading threads
for p in range(start_page,pages+1):
    res = requests.get("https://www.fatwallet.com/forums/search/results.php?page="+str(p)+"&type=forums&query=&sort=&match=titleop&forum=52&active=all&newsort=date", headers=headers)
    soup = bs4(res.text, 'html.parser')
    threads = soup.select('.forumTopicListBNS .forumTopicTitle')
    for i in threads:
        threadid = i['href'].split('/')[-1]
        if not os.path.isfile('Threads/'+threadid+'.html'):
            print('Downloading Thread: ID {}'.format(threadid))
            threadlink = 'https://www.fatwallet.com/forums/textthread.php?catid=52&threadid='+threadid
            res2 = requests.get(threadlink, headers=headers)
            with open('Threads/'+threadid+'.html', 'w') as f:
                f.write(res2.text)
            time.sleep(0.3)
    print('Page {} finished; Starting page {}'.format(p, p+1))
    with open("latest_page_done.txt", 'w') as f:
        f.write(str(p))