Advertisement
Guest User

Untitled

a guest
May 10th, 2017
126
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 5.56 KB | None | 0 0
  1. # -*- coding:utf-8 -*-
  2.  
  3. from peewee import (CharField, DateTimeField, TextField, datetime as peewee_datetime)
  4.  
  5. import time
  6. import datetime
  7.  
  8. import requests
  9. from bs4 import BeautifulSoup as Soup
  10.  
  11.  
  12. class SendMessageController(object):
  13.     def __init__(self, token):
  14.         self._token = token
  15.  
  16.     def call(self, url, id, text):
  17.         response = requests.get(self._make_url(url, self._token, chat_id=id, text=unicode(text)), verify=False)
  18.         if response.status_code != 200:
  19.             raise Exception('Message doesn\'t send')
  20.  
  21.     def _make_url(self, url, token, **kwargs):
  22.         data = u'&'.join([u'%s=%s' % (key, value) for key, value in kwargs.items()])
  23.         return url + token + '/sendMessage?' + data
  24.  
  25. # Telegram params
  26.  
  27. TELEGRAM_BOT_URL = 'https://api.telegram.org/bot'
  28. TELEGRAM_BOT_TOKEN = ''
  29. TELEGRAM_ID = 24000000 # id чата или пользователя которому отправлять уведомления
  30.  
  31. DB_CONFIG = dict(
  32.     user='',
  33.     password='',
  34.     host='localhost',
  35.     port=5432,
  36.     database=''
  37. )
  38.  
  39. db = PooledPostgresqlExtDatabase(**DB_CONFIG)
  40. db.commit_select = True
  41. db.autorollback = True
  42.  
  43.  
  44. class BaseModel(Model):
  45.     class Meta:
  46.         database = db
  47.  
  48. class Flat(BaseModel):
  49.     class Meta:
  50.         db_table = 'flat'
  51.  
  52.     title = CharField()
  53.     url = TextField()
  54.     price = CharField()
  55.     offer_datetime = CharField()
  56.     created = DateTimeField(default=peewee_datetime.datetime.now())
  57.  
  58.  
  59. def execute(**kwargs):
  60.     required_attrs = ("float_price_from", "float_price_to", "float_number_of_rooms_to",
  61.                       "private_business", "district_id")
  62.     for attr in required_attrs:
  63.         if attr not in kwargs:
  64.             raise Exception('Missed "%s" argument in "%s"' % (attr, kwargs))
  65.  
  66.     url = 'http://olx.ua/nedvizhimost/arenda-kvartir/dolgosrochnaya-arenda-kvartir/kiev/?' \
  67.           'search[filter_float_price:from]=%s&search[filter_float_price:to]=%s&search' \
  68.           '[filter_float_number_of_rooms:to]=%s&search[private_business]=%s&search[district_id]=%s' % \
  69.           (kwargs["float_price_from"], kwargs["float_price_to"], kwargs["float_number_of_rooms_to"],
  70.            kwargs["private_business"], kwargs["district_id"])
  71.     send_message = SendMessageController(TELEGRAM_BOT_TOKEN)
  72.     response = None
  73.     try:
  74.         response = requests.get(url)
  75.     except Exception as e:
  76.         send_message.call(TELEGRAM_BOT_URL, TELEGRAM_ID, e)
  77.  
  78.     if response.status_code != 200:
  79.         send_message.call(TELEGRAM_BOT_URL, TELEGRAM_ID, 'Unexpected response code=%s' % response.status_code)
  80.         raise Exception('Unexpected response code=%s' % response.status_code)
  81.  
  82.     result = Soup(response.text, 'html.parser')
  83.  
  84.     offer_list = result.find_all('td', 'offer')
  85.  
  86.     if not offer_list:
  87.         raise Exception('Found 0 offers')
  88.  
  89.     if not Flat.table_exists():
  90.         Flat.create_table()
  91.  
  92.     for offer in offer_list:
  93.         url = offer.find_all('div', 'space rel')[0].h3.a['href']
  94.         if url.find("#") != -1:
  95.             url = url[:url.find("#")]
  96.  
  97.         if not Flat.select().where(Flat.url == url):
  98.             time.sleep(0.2)
  99.             # выбираем предложения только по опр району
  100.             district_full = offer.find('p', 'color-9 lheight16 marginbott5').small.span.text.strip()
  101.             if len(district_full.split(' ')) <= 1:
  102.                 continue
  103.             district = district_full.split(' ')[1]
  104.             offer_dist_id = 0
  105.             for d in kiev_districts:
  106.                 if d.name == district:
  107.                     offer_dist_id = d.id
  108.                     break
  109.                 else:
  110.                     offer_dist_id = 0
  111.             if offer_dist_id != kwargs["district_id"]:
  112.                 continue
  113.  
  114.             title = offer.find_all('div', 'space rel')[0].h3.a.strong.text
  115.             price = offer.find('p', 'price').strong.text
  116.             date = offer.find('p', 'color-9 lheight16 marginbott5 x-normal').text
  117.             date = prepare_date(date)
  118.  
  119.             Flat.create(
  120.                 url=url,
  121.                 title=title,
  122.                 price=price,
  123.                 offer_datetime=date
  124.             )
  125.             telegram_text = u'Новое объявление\n%s\nЦена(%s) Дата добавления(%s)\nТекущая дата(%s)Ссылка(%s)' % \
  126.                             (title, price, date, datetime.datetime.now(), url)
  127.             send_message.call(TELEGRAM_BOT_URL, TELEGRAM_ID, telegram_text)
  128.  
  129. def prepare_date(date):
  130.     parsed_date = date.split()
  131.     return ' '.join(parsed_date)
  132.  
  133.  
  134. class District(object):
  135.     def __init__(self, name, dist_id):
  136.         self.name = name
  137.         self.id = dist_id
  138.  
  139. kiev_districts = [
  140.     District(u'Голосеевский', 1),
  141.     District(u'Дарницкий', 3),
  142.     District(u'Деснянский', 5),
  143.     District(u'Днепровский', 7),
  144.     District(u'Оболонский', 9),
  145.     District(u'Печерский', 11),
  146.     District(u'Подольский', 13),
  147.     District(u'Святошинский', 15),
  148.     District(u'Соломенский', 17),
  149.     District(u'Шевченковский', 19),
  150. ]
  151.  
  152. if __name__ == '__main__':
  153.     ("float_price_from", "float_price_to", "float_number_of_rooms_to",
  154.                       "private_business", "district_id")
  155.     params = dict(
  156.         float_price_from=4000,
  157.         float_price_to=10000,
  158.         float_number_of_rooms_to=2,
  159.         private_business='private', # business
  160.         district_id=9)
  161.  
  162.     while True:
  163.         execute(**params)
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement