Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- import hashlib
- from google.appengine.api import memcache, datastore_errors
- from google.appengine.datastore.datastore_query import Cursor
- from flask import request
- from ndb import tasklets
- class BasePager(object):
- def __init__(self, **kwargs):
- try:
- self.page = int(kwargs.pop('page', request.args.get('page', 1)))
- except ValueError:
- self.page = 1
- if self.page < 1:
- self.page = 1
- @property
- def has_prev(self):
- return self.page > 1
- @property
- def prev_page(self):
- if self.has_prev:
- return self.page - 1
- else:
- return self.page
- @property
- def next_page(self):
- if self.has_next:
- return self.page + 1
- else:
- return self.page
- def __nonzero__(self):
- return self.has_prev or self.has_next
- class Pager(BasePager):
- def __init__(self, **kwargs):
- self.query = kwargs.pop('query')
- self.lifetime = kwargs.pop('lifetime', 3600)
- super(Pager, self).__init__(**kwargs)
- def paginate(self, page_size=20, **q_options):
- if self.page > 1:
- cursor, more = self._get_from_cache(self.page - 1)
- if not cursor:
- self.page, cursor, _ = self._get_max_avail_page(page_size)
- else:
- cursor = None
- res, cursor, more = self._fetch_page(
- page_size, start_cursor=cursor, **q_options)
- if cursor:
- self._add_to_cache(self.page, cursor, more)
- self.has_next = more
- return res, cursor, more
- def _fetch_page(self, *args, **q_options):
- return self.query.fetch_page(*args, **q_options)
- @property
- def _query_id(self):
- if not hasattr(self, '__query_id'):
- hsh = hashlib.md5()
- hsh.update(repr(self.query))
- self.__query_id = hsh.hexdigest()
- return self.__query_id
- def _get_cache_key(self, page):
- return '%s%s' % (self._query_id, page)
- def _add_to_cache(self, page, cursor, more):
- value = cursor.to_bytes() + str(int(more))
- memcache.set(self._get_cache_key(page), value,
- namespace=self.__class__.__name__)
- def _get_from_cache(self, page):
- value = memcache.get(self._get_cache_key(page),
- namespace=self.__class__.__name__)
- if not value:
- return None, None
- cursor = Cursor.from_bytes(value[:-1])
- more = bool(int(value[-1:]))
- return cursor, more
- def _get_max_avail_page(self, page_size):
- # set limit to 1000 results to prevent abuse
- prev_cursor = None
- for page in xrange(1, 1000 / page_size):
- cursor, more = self._get_from_cache(page)
- if not cursor:
- res, cursor, more = self._fetch_page(
- page_size, start_cursor=prev_cursor, keys_only=True)
- if cursor:
- self._add_to_cache(page, cursor, more)
- if not more:
- return page, prev_cursor, more
- prev_cursor = cursor
- else:
- return page, cursor, more
- def filter_page(query, func, page_size=20, **q_options):
- return filter_page_async(query, func, page_size, **q_options).get_result()
- @tasklets.tasklet
- def filter_page_async(query, func, page_size=20, **q_options):
- assert not q_options.get('keys_only'), 'Filter expects model instance'
- q_options.setdefault('batch_size', 2 * page_size)
- q_options.setdefault('produce_cursors', True)
- limit = 2 * page_size + 1
- cursor = q_options.pop('start_cursor', None)
- results = []
- for i in xrange(10):
- it = query.iter(limit=limit, start_cursor=cursor, **q_options)
- while (yield it.has_next_async()):
- res = it.next()
- if func(res):
- results.append(res)
- if len(results) >= page_size:
- break
- if len(results) >= page_size:
- break
- try:
- cursor = it.cursor_after()
- except datastore_errors.BadArgumentError:
- break
- try:
- cursor = it.cursor_after()
- except datastore_errors.BadArgumentError:
- cursor = None
- raise tasklets.Return(results, cursor,
- it.probably_has_next())
- class FilterPager(Pager):
- def __init__(self, **kwargs):
- self.func = kwargs.pop('func')
- super(FilterPager, self).__init__(**kwargs)
- def _fetch_page(self, *args, **q_options):
- q_options['keys_only'] = False
- return filter_page(self.query, self.func, *args, **q_options)
Add Comment
Please, Sign In to add comment