Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- # флаги для синхронизации
- _source_end = False
- _fetcher_end = False
- _parser_end = False
- async def proto_authors_source(loop):
- logger.info('SOURCE AuthorsList UP')
- async with aiomysql.create_pool(
- host=utils.DB['HOST'],
- port=utils.DB['PORT'],
- user=utils.DB['USER'],
- password=utils.DB['PASSWORD'],
- db=utils.DB['USEDB'],
- charset='utf8',
- loop=loop
- ) as pool:
- async with pool.get() as conn:
- async with conn.cursor() as cur:
- await cur.execute(_proto_authors_source_query)
- while True:
- _author = await cur.fetchone()
- if not _author:
- break
- logger.debug('SOURCE: %s' % repr(_author))
- global _dbg_counter_source
- _dbg_counter_source += 1
- logger.info('SOURCE: counter #%s' % _dbg_counter_source)
- await taskq.put(_author[0])
- global _source_end
- _source_end = True
- logger.info('SOURCE DOWN')
- async def fetcher_task(): # TODO: TO SLOW
- logger.info('FETCHER UP')
- async with aiohttp.ClientSession() as session:
- while True:
- _link = await taskq.get()
- url = utils.samlibjoin(_link)
- logger.debug("fetcher taskq.get() '%s'" % url)
- html = await fetch(session, url)
- await parserq.put( (_link, html, ) )
- if taskq.empty() and _source_end:
- global _fetcher_end
- _fetcher_end = True
- break
- logger.info('FETCHER DOWN')
- async def parser_task():
- logger.info('PARSER UP')
- while True:
- try:
- _link, html = await parserq.get()
- _name, _descr, _upd, _tsize, _tcount = parsers.SI_parse_index_counters(html)
- logger.debug("parser %s" % ([_link, _name, _descr, _upd, _tsize, _tcount]))
- await dbwq.put( (_link, _name, _descr, _upd, _tsize, _tcount, ) )
- global _dbg_counter_parser
- _dbg_counter_parser += 1
- if parserq.empty() and _fetcher_end:
- global _parser_end
- _parser_end = True
- break
- except GeneratorExit:
- raise GeneratorExit
- except:
- logger.error("PARSER ERROR!")
- logger.error("exc_info() -> '%s': '%s'" % sys.exc_info()[:2])
- # input('PRESS <ENTER>') # TODO: temporary
- logger.info('PARSER DOWN')
- async def dbwriter_task(loop):
- logger.info('DBWRITER UP')
- async with aiomysql.create_pool(
- host=utils.DB['HOST'],
- port=utils.DB['PORT'],
- user=utils.DB['USER'],
- password=utils.DB['PASSWORD'],
- db=utils.DB['USEDB'],
- charset='utf8',
- loop=loop
- ) as pool:
- async with pool.get() as conn:
- async with conn.cursor() as cur:
- while True:
- _parsed = await dbwq.get()
- if not _parsed[3] == 'NULL':
- _lastupd = utils.escape_quote(datetime.strptime(_parsed[3], '%d/%m/%Y').strftime('%Y.%m.%d'))
- else:
- _lastupd = 'NULL'
- _data = (
- utils.escape_quote(_parsed[1]),
- utils.escape_quote(_parsed[2]),
- utils.escape_quote(_parsed[5]),
- utils.escape_quote(_parsed[4]),
- _lastupd,
- utils.escape_quote(_parsed[0]),
- )
- _q = _proto_author_update % _data
- logger.debug("dbwriter query data:\n%s" % _q)
- await cur.execute(_q)
- await conn.commit()
- if dbwq.empty() and _parser_end:
- break
- logger.info('DBWRITER DOWN')
- async def timeout_control(loop, timeout=20, check_period=10):
- logger.info("TIMEOUT CONTROL UP")
- while True:
- await asyncio.sleep(check_period)
- if taskq.empty() and parserq.empty() and dbwq.empty() and _source_end:
- logger.info("TIMEOUT CONTROL | waiting for completion...")
- await loop.shutdown_asyncgens()
- await asyncio.sleep(timeout)
- logger.info("TIMEOUT CONTROL | close event loop")
- loop.stop()
- break
- logger.info("TIMEOUT CONTROL DOWN")
Advertisement
Add Comment
Please, Sign In to add comment