Guest User

Untitled

a guest
Jan 24th, 2018
87
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 4.56 KB | None | 0 0
  1. import hashlib
  2.  
  3. from google.appengine.api import memcache, datastore_errors
  4. from google.appengine.datastore.datastore_query import Cursor
  5. from flask import request
  6. from ndb import tasklets
  7.  
  8.  
  9. class BasePager(object):
  10. def __init__(self, **kwargs):
  11. try:
  12. self.page = int(kwargs.pop('page', request.args.get('page', 1)))
  13. except ValueError:
  14. self.page = 1
  15. if self.page < 1:
  16. self.page = 1
  17.  
  18. @property
  19. def has_prev(self):
  20. return self.page > 1
  21.  
  22. @property
  23. def prev_page(self):
  24. if self.has_prev:
  25. return self.page - 1
  26. else:
  27. return self.page
  28.  
  29. @property
  30. def next_page(self):
  31. if self.has_next:
  32. return self.page + 1
  33. else:
  34. return self.page
  35.  
  36. def __nonzero__(self):
  37. return self.has_prev or self.has_next
  38.  
  39.  
  40. class Pager(BasePager):
  41. def __init__(self, **kwargs):
  42. self.query = kwargs.pop('query')
  43. self.lifetime = kwargs.pop('lifetime', 3600)
  44. super(Pager, self).__init__(**kwargs)
  45.  
  46. def paginate(self, page_size=20, **q_options):
  47. if self.page > 1:
  48. cursor, more = self._get_from_cache(self.page - 1)
  49. if not cursor:
  50. self.page, cursor, _ = self._get_max_avail_page(page_size)
  51. else:
  52. cursor = None
  53.  
  54. res, cursor, more = self._fetch_page(
  55. page_size, start_cursor=cursor, **q_options)
  56. if cursor:
  57. self._add_to_cache(self.page, cursor, more)
  58. self.has_next = more
  59.  
  60. return res, cursor, more
  61.  
  62. def _fetch_page(self, *args, **q_options):
  63. return self.query.fetch_page(*args, **q_options)
  64.  
  65. @property
  66. def _query_id(self):
  67. if not hasattr(self, '__query_id'):
  68. hsh = hashlib.md5()
  69. hsh.update(repr(self.query))
  70. self.__query_id = hsh.hexdigest()
  71. return self.__query_id
  72.  
  73. def _get_cache_key(self, page):
  74. return '%s%s' % (self._query_id, page)
  75.  
  76. def _add_to_cache(self, page, cursor, more):
  77. value = cursor.to_bytes() + str(int(more))
  78. memcache.set(self._get_cache_key(page), value,
  79. namespace=self.__class__.__name__)
  80.  
  81. def _get_from_cache(self, page):
  82. value = memcache.get(self._get_cache_key(page),
  83. namespace=self.__class__.__name__)
  84. if not value:
  85. return None, None
  86. cursor = Cursor.from_bytes(value[:-1])
  87. more = bool(int(value[-1:]))
  88. return cursor, more
  89.  
  90. def _get_max_avail_page(self, page_size):
  91. # set limit to 1000 results to prevent abuse
  92. prev_cursor = None
  93. for page in xrange(1, 1000 / page_size):
  94. cursor, more = self._get_from_cache(page)
  95. if not cursor:
  96. res, cursor, more = self._fetch_page(
  97. page_size, start_cursor=prev_cursor, keys_only=True)
  98. if cursor:
  99. self._add_to_cache(page, cursor, more)
  100. if not more:
  101. return page, prev_cursor, more
  102. prev_cursor = cursor
  103. else:
  104. return page, cursor, more
  105.  
  106.  
  107. def filter_page(query, func, page_size=20, **q_options):
  108. return filter_page_async(query, func, page_size, **q_options).get_result()
  109.  
  110.  
  111. @tasklets.tasklet
  112. def filter_page_async(query, func, page_size=20, **q_options):
  113. assert not q_options.get('keys_only'), 'Filter expects model instance'
  114.  
  115. q_options.setdefault('batch_size', 2 * page_size)
  116. q_options.setdefault('produce_cursors', True)
  117.  
  118. limit = 2 * page_size + 1
  119. cursor = q_options.pop('start_cursor', None)
  120.  
  121. results = []
  122. for i in xrange(10):
  123. it = query.iter(limit=limit, start_cursor=cursor, **q_options)
  124. while (yield it.has_next_async()):
  125. res = it.next()
  126. if func(res):
  127. results.append(res)
  128. if len(results) >= page_size:
  129. break
  130. if len(results) >= page_size:
  131. break
  132. try:
  133. cursor = it.cursor_after()
  134. except datastore_errors.BadArgumentError:
  135. break
  136.  
  137. try:
  138. cursor = it.cursor_after()
  139. except datastore_errors.BadArgumentError:
  140. cursor = None
  141. raise tasklets.Return(results, cursor,
  142. it.probably_has_next())
  143.  
  144.  
  145. class FilterPager(Pager):
  146. def __init__(self, **kwargs):
  147. self.func = kwargs.pop('func')
  148. super(FilterPager, self).__init__(**kwargs)
  149.  
  150. def _fetch_page(self, *args, **q_options):
  151. q_options['keys_only'] = False
  152. return filter_page(self.query, self.func, *args, **q_options)
Add Comment
Please, Sign In to add comment