D卡修改

# -*- coding: utf-8 -*-

import requests
import csv
import random
import json
import pickle
from fake_useragent import UserAgent
from time import sleep


ua = UserAgent()
headers = {
'User-Agent': ua.random,
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
'Accept-Language': 'h-TW,zh;q=0.9,en-US;q=0.8,en;q=0.7,vi;q=0.6,zh-CN;q=0.5',
'Accept-Encoding': 'gzip, deflate, br',
'Prefer': 'safe',
'Content-Type': 'application/x-www-form-urlencoded',
'Connection': 'keep-alive',
'Cookie': 'dcsrd=0lpqI2DEQ-OCic1WrT8pUwH6;',
'Upgrade-Insecure-Requests': '1',
'Pragma': 'no-cache',
'Cache-Control': 'no-cache'
                                       }



折扣 = []
打折 = []
免運 = []
代言 = []

before_api = ''
i = 0
while True:
    try:
        api = 'https://www.dcard.tw/service/api/v2/forums/buyonline/posts?popular=false&limit=100&' + before_api
        response = requests.get(api,headers=headers)
        # response = scraper.get(api,headers=headers).text
        posts = response.json()
        # posts = json.loads(response)
        i += len(posts)
        #print('1')
        for post in posts:
            pid = post['id']
            date = post['createdAt']

            post_url = f'https://www.dcard.tw/service/api/v2/posts/{pid}'
            response = requests.get(post_url,headers=headers)
            article = response.json()
            #print('2')

            comments = []
            comment_url = f'https://www.dcard.tw/service/api/v2/posts/{pid}/comments?popular=false&limit=100'
            response2 = requests.get(comment_url,headers=headers)
            all_comments = response2.json()
            #print(all_comments);
            for comment in all_comments:
                try:
                    comment_content = comment['content'].replace(',','，')
                    comments.append(comment_content)
                except:
                    pass

            title = article['title'].replace(',','，')
            content = article['content'].replace(',','，')
            content = ' '.join(content.split())

            has_shopee = False
            if '蝦皮' in content or '蝦皮' in title:
                has_shopee = True

            if '折扣' in title:
                折扣.append([title,has_shopee,content,comments])

            if '打折' in title:
                打折.append([title,has_shopee,content,comments])

            if '免運' in title:
                免運.append([title,has_shopee,content,comments])

            if '代言' in title:
                代言.append([title,has_shopee,content,comments])

        before_api = f'before={pid}'
        print(f'\r {i} , 折扣 {len(折扣)} , 免運 {len(免運)}, 代言 {len(代言)}, 打折 {len(打折)}',end='')

        if i>= 30000:
            break
    except Exception as e:
        print(e)
        break