Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import multiprocessing
- import sqlite3
- import time
- import vk_api
- from numpy import array_split
- from pony.orm import *
- conn = sqlite3.connect('input_db.sqlite')
- c = conn.cursor()
- query = c.execute('SELECT count(*) FROM Market WHERE count >= 5;')
- count_of_rows = query.fetchone()[0]
- # Output database
- db = Database()
- class Market(db.Entity):
- market_id = Required(int)
- count = Required(int)
- is_active = Required(bool)
- site = Optional(str)
- db.bind(provider='sqlite', filename='output_db.sqlite', create_db=True)
- db.generate_mapping(create_tables=True)
- ACCESS_TOKENS = [
- '3d6b3c183d6b3c183d6b3c18c93d0d7f4433d6b3d6b3c1866c3d332dfc8cb94247e89df',
- '68426fa8d7466cd656dfe04078fd4cc4c48a097ad6a7e496a38a70783f09a269447018b7a8633097d1ba0',
- '8e8d9603bb8e77361e1dc1f42634be64a2fa7b4fccb2c6399dadb38184b429b4a0cdafa36e2d152d84b85',
- 'd21c87ec1a4d1252e37a565d30c86110c5aa3ea3031cdb3ab19bb5b770357bf91c239c74b3adc6bd5871d',
- '8a61bac950054afb63d74c08e598d444a56a3b1fb7d3f606805eaee57ce3b14ea189f4ca5d792762ea8ce'
- ]
- NUM_PROCS = len(ACCESS_TOKENS)
- def club_is_active(vk_club_id, access_token):
- vk_session = vk_api.VkApi(token=access_token)
- vk = vk_session.get_api()
- try:
- posts = vk.wall.get(owner_id=-1 * vk_club_id, count=2)
- except:
- return False
- if len(posts['items']) == 0:
- return False
- post = None
- if len(posts['items']) == 1:
- post = posts['items'][0]
- if len(posts['items']) == 2:
- if 'is_pinned' in posts['items'][0]:
- post = posts['items'][1]
- else:
- post = posts['items'][0]
- post_time = post['date']
- now_time = int(time.time())
- max_delta_secs = 2592000 # 30 days in secs
- delta_time = now_time - post_time
- if delta_time > max_delta_secs:
- return False
- else:
- return True
- def get_club_site(vk_club_id, access_token):
- vk_session = vk_api.VkApi(token=access_token)
- vk = vk_session.get_api()
- try:
- club = vk.groups.getById(group_id=vk_club_id, fields='site')
- except:
- return None
- return None if club[0]['site'] == '' else club[0]['site']
- def parse_chunk(chunk, access_token):
- for row in chunk:
- vk_club_id = int(row[1])
- products_in_market_count = int(row[2])
- _club_is_active = club_is_active(vk_club_id, access_token)
- club_site = get_club_site(vk_club_id, access_token)
- if not club_site:
- club_site = ''
- with db_session:
- m = Market(market_id=vk_club_id, count=products_in_market_count,
- is_active=_club_is_active, site=club_site)
- commit()
- print('club: {} products: {} active: {} site: {}'.format(
- vk_club_id, products_in_market_count, _club_is_active, club_site)
- )
- if __name__ == "__main__":
- query = c.execute('SELECT * FROM Market WHERE count >= 5')
- result = query.fetchall()
- chunks = array_split(result, NUM_PROCS)
- processes = []
- i = 0
- for chunk in chunks:
- process = multiprocessing.Process(
- target=parse_chunk, args=(chunk, ACCESS_TOKENS[i]))
- processes.append(process)
- process.start()
- i += 1
- for process in processes:
- process.join()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement