import re
import requests
import time
import pymongo
from pprint import pprint
from bs4 import BeautifulSoup

def mongo_connect():
    client = pymongo.MongoClient('mongodb://172.17.0.2:27017/')
    db = client['mydb']
    collections = db['news']
    collections.create_index('url', unique=True)
    return collections

def get_article(url, title):
    try:
        r = requests.get(url)
    except requests.exceptions.ConnectionError:
        time.sleep(5)
        try:
            r = requests.get(url)
        except requests.exceptions.ConnectionError:
            return False

    if r.status_code == 200:
        soup = BeautifulSoup(r.content, "html5lib")
        result = {}
        result['title'] = title
        result['url'] = url
        result['date'] = soup.find('time').text
        result['content'] = ''
        tmp_content = soup.find('div', {'class': 'side-article txt-article'}).findAll('p')
        for i in tmp_content:
            result['content'] = result['content']+" "+i.text

        return result
    else:
        return False

def get_index(url):
    r = requests.get(url)
    if r.status_code == 200:
        soup = BeautifulSoup(r.content, "html5lib")
        data = soup.find('div', {'class': 'lsi'}).findAll('h3')
        for i in data:
            result = {}
            result['title'] = re.sub(r'\t|\n', '', i.text).strip()
            result['link'] = i.find('a')['href']
            yield result
    else:
        yield None

def main():
    collections = mongo_connect()
    for i in get_index('http://jabar.tribunnews.com/'):
        data = get_article(i['link'], i['title'])
        try:
            collections.insert_one(data).inserted_id
            pprint(data)
            print
        except pymongo.errors.DuplicateKeyError:
            print('data sudah ada')


if __name__ == '__main__':
    main()