Advertisement
Guest User

Untitled

a guest
Sep 17th, 2019
103
0
Never
Not a member of Pastebin yet? Sign Up, it unlocks many cool features!
text 1.46 KB | None | 0 0
  1. # file: core/management/commands/bench_po.py
  2.  
  3. from timeit import default_timer as timer
  4. from memory_profiler import profile
  5. from django.core.management.base import BaseCommand
  6. from core.models import CoreData
  7.  
  8.  
  9. class Command(BaseCommand):
  10. def add_arguments(self, parser):
  11. parser.add_argument('batch_size', type=int)
  12.  
  13. @profile
  14. def iterate_all(self):
  15. qs = CoreData.objects.all()
  16. counter = 0
  17. for counter, o in enumerate(qs.iterator(chunk_size=self.batch_size)):
  18. o
  19. print(f'Found {counter}')
  20.  
  21. @profile
  22. def get_all_batch(self):
  23. qs = CoreData.objects.all()
  24. counter = 0
  25. for counter, o in enumerate(qs):
  26. o
  27. print(f'Found {counter}')
  28.  
  29. @profile
  30. def get_chanked_batch(self):
  31. qs = CoreData.objects.all()
  32. counter = 0
  33.  
  34. def batch_qs(qs, batch_size=1000):
  35. total = qs.count()
  36. for start in range(0, total, batch_size):
  37. end = min(start + batch_size, total)
  38. yield qs[start:end]
  39.  
  40. for batch in batch_qs(qs, batch_size=self.batch_size):
  41. for document in batch:
  42. document
  43. counter += 1
  44.  
  45. print(f'Found {counter}')
  46.  
  47. def handle(self, *args, **options):
  48. self.batch_size = options['batch_size']
  49. start = timer()
  50. self.iterate_all()
  51. print(timer() - start)
  52.  
  53.  
  54. # self.get_all_batch()
  55. # self.get_chanked_batch()
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement