crawler

import sys
from bs4 import BeautifulSoup
import requests
import urllib.request

base_url = 'https://news.naver.com/main/main.nhn?mode=LSD&mid=shm&sid1='
category = [100, 101, 102, 103, 104, 105] #정치, 경제, 사회, 생활문화, 세계, 생활과학
category = [100]

for i in category:
    URL = base_url + str(i)
    response = urllib.request.urlopen(URL)
    soup = BeautifulSoup(response, "html.parser")
    #results = soup.select("#section_body .photo a")
    results = soup.select("#section_body")
    print(results)
    #print(response)
#    print(soup)
    #print(URL)
#    for result in results:
#        print("제목 : ", result.attrs["title"])
#        url_article = result.attrs["href"]
#        response = urllib.request.urlopen(url_article)
#        soup_article = BeautifulSoup(response, "html.parser")
#        content = soup_article.select_one("#articleBodyContents")