Advertisement
Guest User

Untitled

a guest
Nov 16th, 2018
179
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
Python 3.33 KB | None | 0 0
  1. import multiprocessing
  2. import sqlite3
  3. import time
  4. import vk_api
  5. from numpy import array_split
  6. from pony.orm import *
  7.  
  8. conn = sqlite3.connect('input_db.sqlite')
  9. c = conn.cursor()
  10.  
  11. query = c.execute('SELECT count(*) FROM Market WHERE count >= 5;')
  12. count_of_rows = query.fetchone()[0]
  13.  
  14. # Output database
  15.  
  16. db = Database()
  17.  
  18.  
  19. class Market(db.Entity):
  20.     market_id = Required(int)
  21.     count = Required(int)
  22.     is_active = Required(bool)
  23.     site = Optional(str)
  24.  
  25.  
  26. db.bind(provider='sqlite', filename='output_db.sqlite', create_db=True)
  27. db.generate_mapping(create_tables=True)
  28.  
  29.  
  30. ACCESS_TOKENS = [
  31.     '3d6b3c183d6b3c183d6b3c18c93d0d7f4433d6b3d6b3c1866c3d332dfc8cb94247e89df',
  32.     '68426fa8d7466cd656dfe04078fd4cc4c48a097ad6a7e496a38a70783f09a269447018b7a8633097d1ba0',
  33.     '8e8d9603bb8e77361e1dc1f42634be64a2fa7b4fccb2c6399dadb38184b429b4a0cdafa36e2d152d84b85',
  34.     'd21c87ec1a4d1252e37a565d30c86110c5aa3ea3031cdb3ab19bb5b770357bf91c239c74b3adc6bd5871d',
  35.     '8a61bac950054afb63d74c08e598d444a56a3b1fb7d3f606805eaee57ce3b14ea189f4ca5d792762ea8ce'
  36. ]
  37.  
  38. NUM_PROCS = len(ACCESS_TOKENS)
  39.  
  40.  
  41. def club_is_active(vk_club_id, access_token):
  42.  
  43.     vk_session = vk_api.VkApi(token=access_token)
  44.     vk = vk_session.get_api()
  45.     try:
  46.         posts = vk.wall.get(owner_id=-1 * vk_club_id, count=2)
  47.     except:
  48.         return False
  49.  
  50.     if len(posts['items']) == 0:
  51.         return False
  52.  
  53.     post = None
  54.  
  55.     if len(posts['items']) == 1:
  56.         post = posts['items'][0]
  57.  
  58.     if len(posts['items']) == 2:
  59.         if 'is_pinned' in posts['items'][0]:
  60.             post = posts['items'][1]
  61.         else:
  62.             post = posts['items'][0]
  63.  
  64.     post_time = post['date']
  65.     now_time = int(time.time())
  66.  
  67.     max_delta_secs = 2592000  # 30 days in secs
  68.  
  69.     delta_time = now_time - post_time
  70.  
  71.     if delta_time > max_delta_secs:
  72.         return False
  73.     else:
  74.         return True
  75.  
  76.  
  77. def get_club_site(vk_club_id, access_token):
  78.     vk_session = vk_api.VkApi(token=access_token)
  79.     vk = vk_session.get_api()
  80.  
  81.     try:
  82.         club = vk.groups.getById(group_id=vk_club_id, fields='site')
  83.     except:
  84.         return None
  85.  
  86.     return None if club[0]['site'] == '' else club[0]['site']
  87.  
  88.  
  89. def parse_chunk(chunk, access_token):
  90.     for row in chunk:
  91.         vk_club_id = int(row[1])
  92.         products_in_market_count = int(row[2])
  93.  
  94.         _club_is_active = club_is_active(vk_club_id, access_token)
  95.         club_site = get_club_site(vk_club_id, access_token)
  96.  
  97.         if not club_site:
  98.             club_site = ''
  99.  
  100.         with db_session:
  101.             m = Market(market_id=vk_club_id, count=products_in_market_count,
  102.                        is_active=_club_is_active, site=club_site)
  103.             commit()
  104.  
  105.         print('club: {} products: {} active: {} site: {}'.format(
  106.             vk_club_id, products_in_market_count, _club_is_active, club_site)
  107.         )
  108.  
  109.  
  110. if __name__ == "__main__":
  111.     query = c.execute('SELECT * FROM Market WHERE count >= 5')
  112.     result = query.fetchall()
  113.  
  114.     chunks = array_split(result, NUM_PROCS)
  115.  
  116.     processes = []
  117.     i = 0
  118.     for chunk in chunks:
  119.         process = multiprocessing.Process(
  120.             target=parse_chunk, args=(chunk, ACCESS_TOKENS[i]))
  121.         processes.append(process)
  122.         process.start()
  123.         i += 1
  124.  
  125.     for process in processes:
  126.         process.join()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement